From 0957b419c17ea54cc0b5c2406645cba9ad778792 Mon Sep 17 00:00:00 2001 From: Lion Date: Fri, 2 Dec 2022 18:29:39 +0800 Subject: [PATCH] load segments while editor init --- binding/golang/go.mod | 2 + binding/golang/go.sum | 2 + maker/golang/main.go | 48 +++++++-------------- maker/golang/xdb/editor.go | 85 ++++++++++++++++++++++++++++++++++---- maker/golang/xdb/maker.go | 41 ++++-------------- maker/golang/xdb/util.go | 42 +++++++++++++++++++ 6 files changed, 148 insertions(+), 72 deletions(-) diff --git a/binding/golang/go.mod b/binding/golang/go.mod index f27eeae..2bb4dc1 100644 --- a/binding/golang/go.mod +++ b/binding/golang/go.mod @@ -3,3 +3,5 @@ module github.com/lionsoul2014/ip2region/binding/golang go 1.17 require github.com/mitchellh/go-homedir v1.1.0 + +require github.com/yookoala/realpath v1.0.0 // indirect diff --git a/binding/golang/go.sum b/binding/golang/go.sum index ae38d14..837982a 100644 --- a/binding/golang/go.sum +++ b/binding/golang/go.sum @@ -1,2 +1,4 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ= +github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= diff --git a/maker/golang/main.go b/maker/golang/main.go index 780c187..f8549cd 100644 --- a/maker/golang/main.go +++ b/maker/golang/main.go @@ -246,55 +246,35 @@ func testBench() { } var count, errCount, tStart = 0, 0, time.Now() - var scanner = bufio.NewScanner(handle) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n")) - var ps = strings.SplitN(l, "|", 3) - if len(ps) != 3 { - fmt.Printf("invalid ip segment line `%s`\n", l) - return - } - - sip, err := xdb.CheckIP(ps[0]) - if err != nil { - fmt.Printf("check start ip `%s`: %s\n", ps[0], err) - return - } - - eip, err := xdb.CheckIP(ps[1]) - if err != nil { - fmt.Printf("check end ip `%s`: %s\n", ps[1], err) - return - } - - if sip > eip { - fmt.Printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]) - return - } - + var iErr = xdb.IterateSegments(handle, nil, func(sip uint32, eip uint32, region *string) error { + var s = *region + var l = fmt.Sprintf("%d|%d|%s", sip, eip, s) fmt.Printf("try to bench segment: `%s`\n", l) mip := xdb.MidIP(sip, eip) for _, ip := range []uint32{sip, xdb.MidIP(sip, mip), mip, xdb.MidIP(mip, eip), eip} { fmt.Printf("|-try to bench ip '%s' ... ", xdb.Long2IP(ip)) - region, _, err := searcher.Search(ip) + r, _, err := searcher.Search(ip) if err != nil { - fmt.Printf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) - return + return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) } // check the region info count++ - if region != ps[2] { + if r != s { errCount++ - fmt.Printf(" --[Failed] (%s != %s)\n", region, ps[2]) + fmt.Printf(" --[Failed] (%s != %s)\n", r, s) if ignoreError == false { - return + return fmt.Errorf("") } } else { fmt.Printf(" --[Ok]\n") } } + return nil + }) + if iErr != nil { + fmt.Printf("%s", err) + return } fmt.Printf("Bench finished, {count: %d, failed: %d, took: %s}\n", count, errCount, time.Since(tStart)) @@ -381,6 +361,8 @@ func edit() { continue } fmt.Printf("PutFile(%s): Ok\n", file) + } else if len(cmd) > 0 { + help() } } } diff --git a/maker/golang/xdb/editor.go b/maker/golang/xdb/editor.go index 9a59870..ea9417e 100644 --- a/maker/golang/xdb/editor.go +++ b/maker/golang/xdb/editor.go @@ -6,31 +6,102 @@ package xdb -import "container/list" +import ( + "container/list" + "fmt" + "os" + "path/filepath" + "time" +) type Editor struct { // source ip file - srcFile string - dstFile string + srcHandle *os.File + dstHandle *os.File // region info - region map[int]string + // @Note: 2^32 items at most + region map[uint32]string + rIndex uint32 // segments list segments *list.List } func NewEditor(srcFile string, dstFile string) (*Editor, error) { + // check the src and dst file + srcPath, err := filepath.Abs(srcFile) + if err != nil { + return nil, err + } + + dstPath, err := filepath.Abs(dstFile) + if err != nil { + return nil, err + } + + if srcPath == dstPath { + return nil, fmt.Errorf("src_path(%s) = dst_path(%s)", srcFile, dstFile) + } + + srcHandle, err := os.OpenFile(srcPath, os.O_RDONLY, 0600) + if err != nil { + return nil, err + } + + dstHandle, err := os.OpenFile(dstPath, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + e := &Editor{ - srcFile: srcFile, - dstFile: dstFile, + srcHandle: srcHandle, + dstHandle: dstHandle, - region: map[int]string{}, + region: map[uint32]string{}, + rIndex: uint32(0), segments: list.New(), } + + // load the segments + if err = e.loadSegments(); err != nil { + return nil, fmt.Errorf("failed load segments: %s", err) + } + return e, nil } +// Load all the segments from the source file +func (e *Editor) loadSegments() error { + var last *Segment = nil + var tStart = time.Now() + + var err = IterateSegments(e.srcHandle, nil, func(sip uint32, eip uint32, region *string) error { + var str = *region + var seg = &Segment{ + StartIP: sip, + EndIP: eip, + Region: str, + } + + // check the continuity of the data segment + if last != nil { + if last.EndIP+1 != seg.StartIP { + return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str) + } + } + + last = seg + return nil + }) + if err != nil { + return fmt.Errorf("failed to load segments: %s", err) + } + + fmt.Printf("all segments loaded, length: %d, elapsed: %s\n", e.segments.Len(), time.Since(tStart)) + return nil +} + func (e *Editor) Put(ip string) error { return nil } diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index b3871df..1ddd38e 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -50,12 +50,10 @@ package xdb import ( - "bufio" "encoding/binary" "fmt" "log" "os" - "strings" "time" ) @@ -140,50 +138,29 @@ func (m *Maker) loadSegments() error { var last *Segment = nil var tStart = time.Now() - var scanner = bufio.NewScanner(m.srcHandle) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n")) + var err = IterateSegments(m.srcHandle, func(l string) { log.Printf("load segment: `%s`", l) - - var ps = strings.SplitN(l, "|", 3) - if len(ps) != 3 { - return fmt.Errorf("invalid ip segment line `%s`", l) - } - - sip, err := CheckIP(ps[0]) - if err != nil { - return fmt.Errorf("check start ip `%s`: %s", ps[0], err) - } - - eip, err := CheckIP(ps[1]) - if err != nil { - return fmt.Errorf("check end ip `%s`: %s", ps[1], err) - } - - if sip > eip { - return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) - } - - if len(ps[2]) < 1 { - return fmt.Errorf("empty region info in segment line `%s`", l) - } - + }, func(sip uint32, eip uint32, region *string) error { + var str = *region var seg = &Segment{ StartIP: sip, EndIP: eip, - Region: ps[2], + Region: str, } // check the continuity of the data segment if last != nil { if last.EndIP+1 != seg.StartIP { - return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, ps[0]) + return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str) } } m.segments = append(m.segments, seg) last = seg + return nil + }) + if err != nil { + return fmt.Errorf("failed to load segments: %s", err) } log.Printf("all segments loaded, length: %d, elapsed: %s", len(m.segments), time.Since(tStart)) diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index df01c5b..7bc17e6 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -5,7 +5,9 @@ package xdb import ( + "bufio" "fmt" + "os" "strconv" "strings" ) @@ -45,6 +47,46 @@ func MidIP(sip uint32, eip uint32) uint32 { return uint32((uint64(sip) + uint64(eip)) >> 1) } +func IterateSegments(handle *os.File, before func(l string), cb func(sip uint32, eip uint32, region *string) error) error { + var scanner = bufio.NewScanner(handle) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n")) + if before != nil { + before(l) + } + + var ps = strings.SplitN(l, "|", 3) + if len(ps) != 3 { + return fmt.Errorf("invalid ip segment line `%s`", l) + } + + sip, err := CheckIP(ps[0]) + if err != nil { + return fmt.Errorf("check start ip `%s`: %s", ps[0], err) + } + + eip, err := CheckIP(ps[1]) + if err != nil { + return fmt.Errorf("check end ip `%s`: %s", ps[1], err) + } + + if sip > eip { + return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) + } + + if len(ps[2]) < 1 { + return fmt.Errorf("empty region info in segment line `%s`", l) + } + + if err = cb(sip, eip, &ps[2]); err != nil { + return err + } + } + + return nil +} + func CheckSegments(segList []*Segment) error { var last *Segment for _, seg := range segList { -- GitLab