1.xx

FOOT000.go

@@ -128,10 +128,6 @@ func main() {
 	case "delvoid\n", "delvoid":
 		new(service4.AnalyService).DelTovoidData()
 	case "auto\n", "auto":
-		go func() {
-			time.Sleep(6 * time.Hour)
-			new(service4.AnalyService).DelTovoidData()
-		}()
 		go func() {
 			for {
 				base.Log.Info("--------全量数据更新开始运行--------")
@@ -166,7 +162,7 @@ func main() {
 		//		time.Sleep(time.Duration(pubService.CycleTime()) * time.Minute)
 		//	}
 		//}()
-		time.Sleep(1 * time.Second)
+		time.Sleep(10 * time.Second)
 		for {
 			matchLastService := new(service3.MatchLastService)
 			matchLasts := matchLastService.FindNear()

FOOT000Cmd.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 	"tesou.io/platform/foot-parent/foot-api/common/base"
 	launch2 "tesou.io/platform/foot-parent/foot-core/launch"
-	service4 "tesou.io/platform/foot-parent/foot-core/module/analy/service"
 	service2 "tesou.io/platform/foot-parent/foot-core/module/core/service"
 	"tesou.io/platform/foot-parent/foot-core/module/leisu/constants"
 	"tesou.io/platform/foot-parent/foot-core/module/leisu/service"
@@ -141,11 +140,6 @@ HEAD:
 		goto HEAD
 	case "auto\n", "auto\r\n":
 		go func() {
-			go func() {
-				time.Sleep(24 * time.Hour)
-				new(service4.AnalyService).DelTovoidData()
-			}()
-
 			for {
 				base.Log.Info("--------全量数据更新开始运行--------")
 				//1.安装数据库
@@ -179,7 +173,7 @@ HEAD:
 		//		time.Sleep(time.Duration(pubService.CycleTime()) * time.Minute)
 		//	}
 		//}()
-		time.Sleep(1 * time.Second)
+		time.Sleep(10 * time.Second)
 		go func() {
 			for {
 				matchLastService := new(service3.MatchLastService)

FS000.go

@@ -2,21 +2,23 @@ package main
 
 import (
 	"fmt"
-	"tesou.io/platform/foot-parent/foot-spider/launch"
 	"time"
 )
 
+var SingleThread bool
+
 func main() {
 	now := time.Now()
 	temp_year, _ := time.ParseDuration("-8760h")
 	add := now.Add(temp_year)
-	parse:= now.Format( "2006")
-	parse2 := add.Format( "2006")
+	parse := now.Format("2006")
+	parse2 := add.Format("2006")
 	fmt.Println(parse)
 	fmt.Println(parse2)
 	//launch.Spider_match_his("2019")
-	launch.Spider_History()
+	//launch.Spider_History()
 	//launch.Spider_leagueSeason()
 
-}
+	fmt.Println(SingleThread)
 
+}

foot-spider/launch/FS202BaseFaceApplication.go

@@ -26,6 +26,7 @@ func Spider_baseFace(spiderAll bool) {
 
 	processer := proc.GetBaseFaceProcesser()
 	processer.MatchLastList = matchLasts
+	processer.SingleThread = true
 	processer.Startup()
 }
 

foot-spider/launch/FS300AsiaLastApplication.go

@@ -42,6 +42,7 @@ func Spider_asiaLastNew(spiderAll bool) {
 
 	processer := proc.GetAsiaLastNewProcesser()
 	processer.MatchLastList = matchLasts
+	processer.SingleThread = true
 	processer.Startup()
 }
 

foot-spider/launch/FS400EuroLastApplication.go

@@ -41,6 +41,7 @@ func Spider_euroLast() {
 	processer := proc.GetEuroLastProcesser()
 	processer.MatchLastList = matchLasts
 	processer.CompWin007Ids = compIds
+	processer.SingleThread = true
 	processer.Startup()
 }
 

foot-spider/launch/FS401EuroHisApplication.go

@@ -42,6 +42,7 @@ func Spider_euroHis() {
 	processer := proc.GetEuroTrackProcesser()
 	processer.CompWin007Ids = compIds
 	processer.MatchLastList = matchLasts
+	processer.SingleThread = true
 	processer.Startup()
 
 }
@@ -105,6 +106,7 @@ func Spider_euroHis_Incomplete() {
 	processer := proc.GetEuroTrackProcesser()
 	processer.CompWin007Ids = compIds
 	processer.MatchLastList = matchLasts
+	processer.SingleThread = true
 	processer.Startup()
 
 }

foot-spider/module/win007/proc/AsiaLastNewProcesser.go

@@ -21,7 +21,8 @@ type AsiaLastNewProcesser struct {
 	service.AsiaLastService
 	service.AsiaHisService
 	service.AsiaTrackService
-
+	//是否是单线程
+	SingleThread bool
 	MatchLastList      []*pojo.MatchLast
 	Win007idMatchidMap map[string]string
 }
@@ -47,10 +48,13 @@ func (this *AsiaLastNewProcesser) Startup() {
 	var newSpider *spider.Spider
 	for i, v := range this.MatchLastList {
 
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread && i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			processer = GetAsiaLastNewProcesser()
 			processer.Setup(this)
 			newSpider = spider.NewSpider(processer, "AsiaLastNewProcesser"+strconv.Itoa(i))
+		}else{
+			processer = this
+			newSpider = spider.NewSpider(processer, "AsiaLastNewProcesser"+strconv.Itoa(i))
 		}
 
 		temp_flag := v.Ext[win007.MODULE_FLAG]
@@ -63,7 +67,7 @@ func (this *AsiaLastNewProcesser) Startup() {
 
 		url := strings.Replace(win007.WIN007_ASIAODD_NEW_URL_PATTERN, "${matchId}", win007_id, 1)
 		newSpider = newSpider.AddUrl(url, "json")
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread && i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			newSpider.SetDownloader(down.NewMAsiaLastApiDownloader())
 			newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole())
 			newSpider.SetSleepTime("rand", 1000, 20000)

foot-spider/module/win007/proc/AsiaLastProcesser.go

@@ -20,7 +20,8 @@ import (
 type AsiaLastProcesser struct {
 	service.AsiaLastService
 	service.AsiaHisService
-
+	//是否是单线程
+	SingleThread bool
 	MatchLastList      []*pojo.MatchLast
 	Win007idMatchidMap map[string]string
 }

foot-spider/module/win007/proc/BaseFaceProcesser.go

@@ -26,7 +26,8 @@ type BaseFaceProcesser struct {
 	service.BFJinService
 	service.BFFutureEventService
 	service2.LeagueService
-
+	//是否是单线程
+	SingleThread bool
 	MatchLastList      []*pojo.MatchLast
 	Win007idMatchidMap map[string]string
 }
@@ -52,10 +53,13 @@ func (this *BaseFaceProcesser) Startup() {
 	var newSpider *spider.Spider
 	for i, v := range this.MatchLastList {
 
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread && i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			processer = GetBaseFaceProcesser()
 			processer.Setup(this)
 			newSpider = spider.NewSpider(processer, "BaseFaceProcesser"+strconv.Itoa(i))
+		}else{
+			processer = this
+			newSpider = spider.NewSpider(processer, "BaseFaceProcesser"+strconv.Itoa(i))
 		}
 
 		temp_flag := v.Ext[win007.MODULE_FLAG]
@@ -68,7 +72,7 @@ func (this *BaseFaceProcesser) Startup() {
 
 		url := strings.Replace(win007.WIN007_BASE_FACE_URL_PATTERN, "${matchId}", win007_id, 1)
 		newSpider = newSpider.AddUrl(url, "html")
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread && i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			newSpider.SetDownloader(down.NewMWin007Downloader())
 			newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole())
 			newSpider.SetSleepTime("rand", 1000, 20000)

foot-spider/module/win007/proc/EuroLastProcesser.go

@@ -25,6 +25,8 @@ type EuroLastProcesser struct {
 	service2.EuroLastService
 	service2.EuroHisService
 	//入参
+	//是否是单线程
+	SingleThread bool
 	MatchLastList      []*pojo.MatchLast
 	//博彩公司对应的win007id
 	CompWin007Ids      []string
@@ -53,10 +55,13 @@ func (this *EuroLastProcesser) Startup() {
 	var newSpider *spider.Spider
 	for i, v := range this.MatchLastList {
 
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread && i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			processer = GetEuroLastProcesser()
 			processer.Setup(this)
 			newSpider = spider.NewSpider(processer, "EuroLastProcesser"+strconv.Itoa(i))
+		}else{
+			processer = this
+			newSpider = spider.NewSpider(processer, "EuroLastProcesser"+strconv.Itoa(i))
 		}
 
 		temp_flag := v.Ext[win007.MODULE_FLAG]
@@ -69,7 +74,7 @@ func (this *EuroLastProcesser) Startup() {
 
 		url := strings.Replace(win007.WIN007_EUROODD_URL_PATTERN, "${matchId}", win007_id, 1)
 		newSpider = newSpider.AddUrl(url, "html")
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread && i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			newSpider.SetDownloader(down.NewMWin007Downloader())
 			newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole())
 			newSpider.SetSleepTime("rand", 100, 2000)

foot-spider/module/win007/proc/EuroTrackProcesser.go

@@ -24,6 +24,8 @@ type EuroTrackProcesser struct {
 	service2.EuroHisService
 	service2.EuroTrackService
 	//入参
+	//是否是单线程
+	SingleThread bool
 	MatchLastList []*pojo.MatchLast
 	//博彩公司对应的win007id
 	CompWin007Ids      []string
@@ -52,10 +54,13 @@ func (this *EuroTrackProcesser) Startup() {
 	var newSpider *spider.Spider
 	for i, v := range this.MatchLastList {
 
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread &&i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			processer = GetEuroTrackProcesser()
 			processer.Setup(this)
 			newSpider = spider.NewSpider(processer, "EuroTrackProcesser"+strconv.Itoa(i))
+		}else{
+			processer = this
+			newSpider = spider.NewSpider(processer, "EuroTrackProcesser"+strconv.Itoa(i))
 		}
 
 		temp_flag := v.Ext[win007.MODULE_FLAG]
@@ -70,7 +75,7 @@ func (this *EuroTrackProcesser) Startup() {
 			url := strings.Replace(base_url, "${cId}", v, 1)
 			newSpider = newSpider.AddUrl(url, "html")
 		}
-		if i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
+		if !this.SingleThread && i%10000 == 0 { //10000个比赛一个spider,一个赛季大概有30万场比赛,最多30spider
 			newSpider.SetDownloader(down.NewMWin007Downloader())
 			newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole())
 			newSpider.SetSleepTime("rand", 1000, 20000)

foot-spider/module/win007/proc/LeagueProcesser.go

@@ -17,6 +17,8 @@ import (
 type LeagueProcesser struct {
 	service2.LeagueService
 	service2.CompService
+	//是否是单线程
+	SingleThread bool
 	//联赛数据
 	league_list []*entity2.League
 	sUrl_Id     map[string]string

foot-spider/module/win007/proc/LeagueSeasonProcesser.go

@@ -18,7 +18,8 @@ type LeagueSeasonProcesser struct {
 	service2.LeagueService
 	service2.LeagueSeasonService
 	service2.LeagueSubService
-
+	//是否是单线程
+	SingleThread bool
 	//联赛次级数据
 	leagueSeason_list []*pojo.LeagueSeason
 	leagueSub_list    []*pojo.LeagueSub

foot-spider/module/win007/proc/MatchAnalyProcesser.go

@@ -16,6 +16,8 @@ import (
 )
 
 type MatchAnalyProcesser struct {
+	//是否是单线程
+	SingleThread bool
 	//博彩公司对应的win007id
 	MatchLastList      []*pojo.MatchLast
 	Win007idMatchidMap map[string]string

foot-spider/module/win007/proc/MatchHisProcesser.go

@@ -23,7 +23,8 @@ type MatchHisProcesser struct {
 	service2.LeagueSeasonService
 	service2.LeagueSubService
 	LeagueSeasonProcesser
-
+	//是否是单线程
+	SingleThread bool
 	Season string
 	//比赛数据
 	MatchHis_list []*pojo2.MatchHis

foot-spider/module/win007/proc/MatchLastProcesser.go

@@ -22,6 +22,8 @@ type MatchLastProcesser struct {
 	service.MatchHisService
 	service2.LeagueService
 	service2.CompService
+	//是否是单线程
+	SingleThread bool
 	//抓取的url
 	MatchlastUrl string
 	//联赛数据