package main

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"learngo/crawler/engine"
	"learngo/crawler/zhenai/paser"
	"net/http"
	"regexp"
)

func main() {
	engine.Run(
		engine.Request{
			Url:        "https://www.zhenai.com/zhenghun",
			ParserFunc: paser.ParseCityList,
		})
}

func mains() {
	resp, err := http.Get("https://www.zhenai.com/zhenghun")

	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		fmt.Println(" Error: status code ", resp.StatusCode)
		return
	}
	// 自动发现见面编码
	e := determineEncoding(resp.Body)
	// 把内容以GBK编码读取
	utf8Reader := transform.NewReader(resp.Body, e.NewDecoder())

	all, err := ioutil.ReadAll(utf8Reader)
	if err != nil {
		panic(err)
	}

	//fmt.Printf("%s\n", all)

	printCityList(all)

}

func determineEncoding(r io.Reader) encoding.Encoding {

	bytes, err := bufio.NewReader(r).Peek(1024)
	if err != nil {
		panic(err)
	}
	e, name, certain := charset.DetermineEncoding(
		bytes,
		"",
	)
	fmt.Printf(" name %s certain %s", name, certain)
	return e
}

func printCityList(contents []byte) {

	re := regexp.MustCompile(`<a href="(http://www.zhenai.com/zhenghun/[0-9a-z]+)"[^>]*>([^<]+)</a>`)
	mastches := re.FindAllSubmatch(contents, -1)
	fmt.Printf("查找%s 城市", len(mastches))
	for _, m := range mastches {
		fmt.Printf("City: %s ,URL: %s\n", m[2], m[1])
	}
}
