client_internal_test.go 2.3 KB
Newer Older
H
Helin Wang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
package master

import (
	"fmt"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/PaddlePaddle/Paddle/go/connection"
	"github.com/PaddlePaddle/recordio"
)

const (
	totalTask    = 20
	chunkPerTask = 10
)

func init() {
	log.SetLevel(log.ErrorLevel)
}

type TestAddresser string

func (a TestAddresser) Address() string {
	return string(a)
}

35 36 37 38 39 40 41 42 43 44 45 46 47
type myStore struct {
	buf []byte
}

func (m *myStore) Save(b []byte) error {
	m.buf = b
	return nil
}

func (m *myStore) Load() ([]byte, error) {
	return m.buf, nil
}

H
Helin Wang 已提交
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
func TestGetFinishTask(t *testing.T) {
	const path = "/tmp/master_client_test_0"

	l, err := net.Listen("tcp", ":0")
	if err != nil {
		panic(err)
	}

	ss := strings.Split(l.Addr().String(), ":")
	p, err := strconv.Atoi(ss[len(ss)-1])
	if err != nil {
		panic(err)
	}

	go func(l net.Listener) {
63 64 65 66 67
		s, err := NewService(&myStore{}, chunkPerTask, time.Second, 1)
		if err != nil {
			panic(err)
		}

H
Helin Wang 已提交
68
		server := rpc.NewServer()
69
		err = server.Register(s)
H
Helin Wang 已提交
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
		if err != nil {
			panic(err)
		}

		mux := http.NewServeMux()
		mux.Handle(rpc.DefaultRPCPath, server)
		err = http.Serve(l, mux)
		if err != nil {
			panic(err)
		}
	}(l)

	f, err := os.Create(path)
	if err != nil {
		panic(err)
	}

	for i := 0; i < totalTask*chunkPerTask; i++ {
		w := recordio.NewWriter(f, -1, -1)
		w.Write(nil)
		// call Close to force RecordIO writing a chunk.
		w.Close()
	}
	f.Close()

H
Helin Wang 已提交
95
	// Manually intialize client to avoid calling c.getRecords()
H
Helin Wang 已提交
96 97 98 99 100 101 102 103 104 105
	c := &Client{}
	c.conn = connection.New()
	go c.monitorMaster(TestAddresser(fmt.Sprintf(":%d", p)))
	c.SetDataset([]string{path})

	checkOnePass := func(i int) {
		var tasks []Task
		for idx := 0; idx < totalTask; idx++ {
			task, err := c.getTask()
			if err != nil {
H
Helin Wang 已提交
106
				t.Fatalf("Error: %v, pass: %d\n", err, i)
H
Helin Wang 已提交
107 108 109 110 111 112
			}
			tasks = append(tasks, task)
		}

		_, err = c.getTask()
		if err == nil {
H
Helin Wang 已提交
113
			t.Fatalf("Should get error, pass: %d\n", i)
H
Helin Wang 已提交
114 115 116 117
		}

		err = c.taskFinished(tasks[0].ID)
		if err != nil {
H
Helin Wang 已提交
118
			t.Fatalf("Error: %v, pass: %d\n", err, i)
H
Helin Wang 已提交
119 120 121 122 123 124 125 126 127 128 129
		}
		tasks = tasks[1:]
		task, err := c.getTask()
		if err != nil {
			t.Fatal(err)
		}
		tasks = append(tasks, task)

		for _, task := range tasks {
			err = c.taskFinished(task.ID)
			if err != nil {
H
Helin Wang 已提交
130
				t.Fatalf("Error: %v, pass: %d\n", err, i)
H
Helin Wang 已提交
131 132 133 134 135 136 137 138
			}
		}
	}

	for i := 0; i < 10; i++ {
		checkOnePass(i)
	}
}