1.开始处理联赛次级赛季

foot-api/module/elem/pojo/League.go

@@ -12,6 +12,7 @@ type League struct {
 	//联赛名称
 	Name string `xorm:" comment('联赛名称') index"`
 	ShortName string `xorm:" comment('联赛名简称') index"`
+	ShortUrl string `xorm:" comment('联赛入口路径') index"`
 
 	//联赛级别
 	Level       int `xorm:" comment('联赛级别') index"`

foot-api/module/elem/pojo/LeagueSub.go → foot-api/module/elem/pojo/LeagueStub.go

@@ -8,13 +8,17 @@ import (
 联赛子表表
  */
 //不管是从哪个平台抓取的数据，都使用win007的联赛的ID数据
-type LeagueSub struct {
+type LeagueStub struct {
 	//联赛id
 	LeagueId string  `xorm:" comment('LeagueId') index"`
-	//次级名称
-	Name string `xorm:" comment('次级名称') index"`
-
+	//年份,使用,间隔
+	Season string `xorm:" comment('Season')"`
+	//subs
+	SubIds []string `xorm:" comment('Subs')"`
+	//--
+	Round int `xorm:" comment('最大的回合数')"`
 
 	pojo.BasePojo `xorm:"extends"`
 }
 
+

foot-core/module/elem/service/LeagueStubService.go

+package service
+
+import (
+	"tesou.io/platform/foot-parent/foot-api/common/base"
+	"tesou.io/platform/foot-parent/foot-api/module/elem/pojo"
+	"tesou.io/platform/foot-parent/foot-core/common/base/service/mysql"
+)
+
+//联赛表
+type LeagueStubService struct {
+	mysql.BaseService
+}
+
+func (this *LeagueStubService) Exist(e *pojo.LeagueStub) (string, bool) {
+	//sql_build := strings.Builder{}
+	//eventMatchDateStr := e.EventMatchDate.Format("2006-01-02 15:04:05")
+	//sql_build.WriteString(" MatchId = '" + e.MatchId + "' AND TeamId = '" + e.TeamId + "' AND EventMatchDate = '" + eventMatchDateStr + "'")
+	//temp := &pojo.BFFutureEvent{}
+	//var id string
+	//exist, err := mysql.GetEngine().Where(sql_build.String()).Get(temp)
+	//if err != nil {
+	//	base.Log.Error("Exist:", err)
+	//}
+	//if exist {
+	//	id = temp.Id
+	//}
+	//return id, exist
+	return "",false
+}
+
+func (this *LeagueStubService) FindById(id string) *pojo.League {
+	league := new(pojo.League)
+	league.Id = id
+	_, err := mysql.GetEngine().Get(league)
+	if err != nil {
+		base.Log.Info("FindById:", err)
+	}
+	return league
+}
+
+func (this *LeagueStubService) FindByName(name string) *pojo.League {
+	league := new(pojo.League)
+	league.Name = name
+	_, err := mysql.GetEngine().Get(league)
+	if err != nil {
+		base.Log.Info("FindByName:", err)
+	}
+	return league
+}

foot-spider/module/win007/proc/LeagueProcesser.go

@@ -79,6 +79,7 @@ func (this *LeagueProcesser) Process(p *page.Page) {
 		league.Name = lName
 		league.Sid = sId
 		league.SName = sName
+		league.ShortUrl = lUrl
 		this.league_list = append(this.league_list,league)
 	})
 }

foot-spider/module/win007/proc/LeagueStubProcesser.go

+package proc
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"github.com/hu17889/go_spider/core/common/page"
+	"github.com/hu17889/go_spider/core/pipeline"
+	"github.com/hu17889/go_spider/core/spider"
+	"strings"
+	"tesou.io/platform/foot-parent/foot-api/common/base"
+	entity2 "tesou.io/platform/foot-parent/foot-api/module/elem/pojo"
+	service2 "tesou.io/platform/foot-parent/foot-core/module/elem/service"
+	"tesou.io/platform/foot-parent/foot-spider/module/win007"
+	"tesou.io/platform/foot-parent/foot-spider/module/win007/down"
+)
+
+type LeagueStubProcesser struct {
+	service2.LeagueService
+	service2.CompService
+	//联赛数据
+	league_list           []*entity2.League
+	sUrl_Id map[string]string
+	sUrl_Name map[string]string
+}
+
+func GetLeagueStubProcesser() *LeagueStubProcesser {
+	return &LeagueStubProcesser{}
+}
+
+func (this *LeagueStubProcesser) Startup() {
+	//初始化参数值
+	this.league_list = make([]*entity2.League, 0)
+	this.sUrl_Id = make(map[string]string)
+	this.sUrl_Name = make(map[string]string)
+	newSpider := spider.NewSpider(this, "LeagueStubProcesser")
+	//sid 数据
+	sid_stat_url := "http://m.win007.com/info.htm#section0";
+	document, _ := GetDocument(sid_stat_url)
+	document.Find("a[href*='sid']").Each(func(i int, selection *goquery.Selection) {
+		sUrl, _ := selection.Attr("href")
+		sId := strings.Split(sUrl, "sid=")[1]
+		sName := strings.TrimSpace(selection.Text())
+		base.Log.Info("sId:", sId, ",sName:", sName, ",sUrl:"+sUrl)
+		this.sUrl_Id[win007.WIN007_BASE_URL+sUrl] = sId
+		this.sUrl_Name[win007.WIN007_BASE_URL+sUrl] = sName
+		newSpider = newSpider.AddUrl(win007.WIN007_BASE_URL+sUrl, "html")
+	})
+	newSpider.SetDownloader(down.NewMWin007Downloader())
+	newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole())
+	newSpider.SetSleepTime("rand",100,2000)
+	newSpider.SetThreadnum(1).Run()
+}
+
+func (this *LeagueStubProcesser) Process(p *page.Page) {
+	request := p.GetRequest()
+	if !p.IsSucc() {
+		base.Log.Error("URL:,", request.Url, p.Errormsg())
+		return
+	}
+
+	rawText := p.GetBodyStr()
+	if rawText == "" {
+		base.Log.Error("rawText:为空.url:", request.Url)
+		return
+	}
+
+	sUrl := request.Url
+	sId := this.sUrl_Id[sUrl]
+	sName := this.sUrl_Name[sUrl]
+
+	p.GetHtmlParser().Find("a.gameItem[href*='info'][href*='htm']").Each(func(i int, selection *goquery.Selection) {
+		lUrl, _ := selection.Attr("href")
+		l_arr := strings.Split(lUrl, "/")
+		lId_suffix := l_arr[len(l_arr)-1]
+		lId := strings.ReplaceAll(lId_suffix,".htm","")
+		lName := strings.TrimSpace(selection.Text())
+		base.Log.Info("lId:", lId, ",lName:", lName, ",lUrl:"+lUrl)
+		league := new(entity2.League)
+		league.Id = lId
+		league.Name = lName
+		league.Sid = sId
+		league.SName = sName
+		league.ShortUrl = lUrl
+		this.league_list = append(this.league_list,league)
+	})
+}
+
+func (this *LeagueStubProcesser) Finish() {
+	base.Log.Info("联赛解析完成,执行入库 \r\n")
+
+	league_list_slice := make([]interface{}, 0)
+	league_modify_list_slice := make([]interface{}, 0)
+	for _, v := range this.league_list {
+		if nil == v {
+			continue
+		}
+		exists := this.LeagueService.ExistById(v.Id)
+		if exists {
+			league_modify_list_slice = append(league_modify_list_slice,v)
+			continue
+		}
+		league_list_slice = append(league_list_slice, v)
+	}
+	this.LeagueService.SaveList(league_list_slice)
+	this.LeagueService.ModifyList(league_modify_list_slice)
+}