cube agent init comit

cube/CMakeLists.txt

@@ -16,3 +16,4 @@ add_subdirectory(cube-server)
 add_subdirectory(cube-api)
 add_subdirectory(cube-builder)
 add_subdirectory(cube-transfer)
+add_subdirectory(cube-agent)

cube/cube-agent/CMakeLists.txt

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+project(cube-agent Go)
+
+include(cmake/golang.cmake)
+
+ExternalGoProject_Add(agent-docopt-go github.com/docopt/docopt-go)
+ExternalGoProject_Add(agent-logex github.com/Badangel/logex)
+ExternalGoProject_Add(agent-pipeline github.com/Badangel/pipeline)
+
+add_subdirectory(src)

cube/cube-agent/cmake/CMakeDetermineGoCompiler.cmake

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+if(NOT CMAKE_Go_COMPILER)
+  if(NOT $ENV{GO_COMPILER} STREQUAL "")
+    get_filename_component(CMAKE_Go_COMPILER_INIT $ENV{GO_COMPILER} PROGRAM PROGRAM_ARGS CMAKE_Go_FLAGS_ENV_INIT)
+
+    if(CMAKE_Go_FLAGS_ENV_INIT)
+      set(CMAKE_Go_COMPILER_ARG1 "${CMAKE_Go_FLAGS_ENV_INIT}" CACHE STRING "First argument to Go compiler")
+    endif()
+
+    if(NOT EXISTS ${CMAKE_Go_COMPILER_INIT})
+      message(SEND_ERROR "Could not find compiler set in environment variable GO_COMPILER:\n$ENV{GO_COMPILER}.")
+    endif()
+
+  endif()
+
+  set(Go_BIN_PATH
+    $ENV{GOPATH}
+    $ENV{GOROOT}
+    $ENV{GOROOT}/../bin
+    $ENV{GO_COMPILER}
+    /usr/bin
+    /usr/local/bin
+    )
+
+  if(CMAKE_Go_COMPILER_INIT)
+    set(CMAKE_Go_COMPILER ${CMAKE_Go_COMPILER_INIT} CACHE PATH "Go Compiler")
+  else()
+    find_program(CMAKE_Go_COMPILER
+      NAMES go
+      PATHS ${Go_BIN_PATH}
+    )
+    EXEC_PROGRAM(${CMAKE_Go_COMPILER} ARGS version OUTPUT_VARIABLE GOLANG_VERSION)
+    STRING(REGEX MATCH "go[0-9]+.[0-9]+.[0-9]+[ /A-Za-z0-9]*" VERSION "${GOLANG_VERSION}")
+    message("-- The Golang compiler identification is ${VERSION}")
+    message("-- Check for working Golang compiler: ${CMAKE_Go_COMPILER}")
+  endif()
+
+endif()
+
+mark_as_advanced(CMAKE_Go_COMPILER)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CMakeGoCompiler.cmake.in
+  ${CMAKE_PLATFORM_INFO_DIR}/CMakeGoCompiler.cmake @ONLY)
+
+set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER")
\ No newline at end of file

cube/cube-agent/cmake/CMakeGoCompiler.cmake.in

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+set(CMAKE_Go_COMPILER "@CMAKE_Go_COMPILER@")
+set(CMAKE_Go_COMPILER_LOADED 1)
+
+set(CMAKE_Go_SOURCE_FILE_EXTENSIONS go)
+set(CMAKE_Go_LINKER_PREFERENCE 40)
+set(CMAKE_Go_OUTPUT_EXTENSION .o)
+set(CMAKE_Go_OUTPUT_EXTENSION_REPLACE 1)
+set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER")

cube/cube-agent/cmake/CMakeGoInformation.cmake

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+if(NOT CMAKE_Go_COMPILE_OBJECT)
+  set(CMAKE_Go_COMPILE_OBJECT "go tool compile -l -N -o <OBJECT> <SOURCE> ")
+endif()
+
+if(NOT CMAKE_Go_LINK_EXECUTABLE)
+  set(CMAKE_Go_LINK_EXECUTABLE "go tool link -o <TARGET> <OBJECTS>  ")
+endif()
\ No newline at end of file

cube/cube-agent/cmake/CMakeTestGoCompiler.cmake

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+set(CMAKE_Go_COMPILER_WORKS 1 CACHE INTERNAL "")
\ No newline at end of file

cube/cube-agent/cmake/golang.cmake

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+set(GOPATH "${CMAKE_CURRENT_LIST_DIR}/../")
+file(MAKE_DIRECTORY ${GOPATH})
+
+function(ExternalGoProject_Add TARG)
+  add_custom_target(${TARG} env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get ${ARGN})
+endfunction(ExternalGoProject_Add)
+
+function(add_go_executable NAME)
+  file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
+  add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
+    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build
+    -o "${CMAKE_CURRENT_BINARY_DIR}/${NAME}"
+    ${CMAKE_GO_FLAGS} ${GO_SOURCE}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+
+  add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN})
+  install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${NAME} DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/bin)
+endfunction(add_go_executable)
+
+
+function(ADD_GO_LIBRARY NAME BUILD_TYPE)
+  if(BUILD_TYPE STREQUAL "STATIC")
+    set(BUILD_MODE -buildmode=c-archive)
+    set(LIB_NAME "lib${NAME}.a")
+  else()
+    set(BUILD_MODE -buildmode=c-shared)
+    if(APPLE)
+      set(LIB_NAME "lib${NAME}.dylib")
+    else()
+      set(LIB_NAME "lib${NAME}.so")
+    endif()
+  endif()
+
+  file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
+  add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
+    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
+    -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
+    ${CMAKE_GO_FLAGS} ${GO_SOURCE}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+
+  add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN})
+
+  if(NOT BUILD_TYPE STREQUAL "STATIC")
+    install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/bin)
+  endif()
+endfunction(ADD_GO_LIBRARY)
\ No newline at end of file

cube/cube-agent/src/CMakeLists.txt

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+set(SOURCE_FILE cube-agent.go)
+add_go_executable(cube-agent ${SOURCE_FILE})
+add_dependencies(cube-agent agent-docopt-go)
+add_dependencies(cube-agent agent-logex)
+add_dependencies(cube-agent agent-pipeline)

cube/cube-agent/src/agent/define.go

+package agent
+
+import (
+	"errors"
+	_ "github.com/Badangel/logex"
+	"github.com/Badangel/pipeline"
+	"os/exec"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+var (
+	Dir           string
+	WorkerNum     int
+	QueueCapacity int32
+	MasterHost    []string
+	MasterPort    []string
+
+	TestHostname string
+	TestIdc      string
+	ShardLock    sync.RWMutex
+
+	CmdWorkPool   *WorkPool
+	CmdWorkFilter sync.Map
+)
+
+type (
+	Status struct {
+		Status  string `json:"status"`
+		Version string `json:"version"`
+	}
+
+	MasterResp struct {
+		Success string `json:"success"`
+		Message string `json:"message"`
+		Data    string `json:"data"`
+	}
+
+	ShardInfo struct {
+		DictName   string
+		ShardSeq   int
+		SlotIdList string
+		DataDir    string
+		Service    string `json:"service,omitempty"`
+		Libcube    string `json:"libcube,omitempty"`
+	}
+
+	CubeResp struct {
+		Status     int    `json:"status"`
+		CurVersion string `json:"cur_version"`
+		BgVersion  string `json:"bg_version"`
+	}
+)
+
+var BUILTIN_STATUS = Status{"RUNNING", "3.0.0.1"}
+
+var ShardInfoMap map[string]map[string]*ShardInfo
+var disks []string
+
+func GetMaster(master string) (host, port string, err error) {
+	if len(ShardInfoMap) < 1 {
+		return "", "", errors.New("empty master list.")
+	}
+	if master == "" {
+		return MasterHost[0], MasterPort[0], nil
+	}
+	if _, ok := ShardInfoMap[master]; ok {
+		m := strings.Split(master, ":")
+		if len(m) != 2 {
+			return MasterHost[0], MasterPort[0], nil
+		}
+		return m[0], m[1], nil
+	} else {
+		return MasterHost[0], MasterPort[0], nil
+	}
+}
+
+func init() {
+	dfCmd := "df -h | grep -E '/home|/ssd'"
+	stdout, _, err := pipeline.Run(exec.Command("/bin/bash", "-c", dfCmd))
+
+	if err == nil && stdout.String() != "" {
+		t := strings.TrimSpace(stdout.String())
+		diskLi := strings.Split(t, "\n")
+		for _, diskStr := range diskLi {
+			disk := strings.Fields(diskStr)
+			usedPercent, _ := strconv.Atoi(strings.TrimRight(disk[4], "%"))
+			if usedPercent <= 40 {
+				disks = append(disks, disk[5])
+			}
+		}
+	}
+
+	if len(disks) == 0 {
+		disks = append(disks, "/home")
+	}
+
+	//logex.Debugf("available disks found: (%+v)", disks)
+}

cube/cube-agent/src/agent/http.go

+//-*- coding:utf-8; indent-tabs-mode:nil; -*-
+
+package agent
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Badangel/logex"
+)
+
+type handlerFunc func(subpath string, m map[string]string, b []byte) (string, string, error)
+
+var ( // key = subpath; eg: path="/checker/job", key="job"
+	getHandler    map[string]handlerFunc
+	putHandler    map[string]handlerFunc
+	deleteHandler map[string]handlerFunc
+	postHandler   map[string]handlerFunc
+)
+
+func StartHttp(addr string) error {
+
+	// init handlers:
+	initGetHandlers()
+	initPostHandlers()
+
+	http.HandleFunc("/agent/", handleRest)
+	logex.Notice("start http ", addr)
+	return http.ListenAndServe(addr, nil)
+}
+
+func handleRest(w http.ResponseWriter, r *http.Request) {
+	var (
+		req_log string
+		status int32
+	)
+	time_begin := time.Now()
+
+	cont_type := make([]string, 1, 1)
+	cont_type[0] = "application/json"
+	header := w.Header()
+	header["Content-Type"] = cont_type
+	w.Header().Add("Access-Control-Allow-Origin", "*")
+
+	m := parseHttpKv(r)
+	b, _ := ioutil.ReadAll(r.Body)
+
+	req_log = fmt.Sprintf("handle %v %v %v from %v, len(m)=%v, m=%+v",
+		r.Method, r.URL.Path, r.URL.RawQuery, r.RemoteAddr, len(m), m)
+
+	api := r.URL.Path
+
+	var showHandler map[string]handlerFunc
+	switch r.Method {
+	case "GET":
+		showHandler = getHandler
+	case "POST": // create
+		showHandler = postHandler
+	case "PUT": // update
+		showHandler = putHandler
+	case "DELETE":
+		showHandler = deleteHandler
+	default:
+		logex.Warningf(`{"error":1, "message":"unsupport method %v"}`, r.Method)
+	}
+
+	handler, ok := showHandler[api]
+
+	if !ok {
+		key_list := make([]string, 0, len(showHandler))
+		for key := range showHandler {
+			key_list = append(key_list, key)
+		}
+		status = 2
+		fmt.Fprintf(w, `{"success":"%v", "message":"wrong api", "method":"%s", "api":"%s", "api_list":"%v"}`,
+			status, r.Method, api, key_list)
+
+		logex.Noticef(`%v, time=%v, status=%v`, 
+			req_log, time.Now().Sub(time_begin).Nanoseconds()/1000000, status)
+		return
+	}
+
+	var s string
+	rst, handle_log, err := handler(api, m, b)
+	if err == nil {
+		status = 0
+		s = fmt.Sprintf(`{"success":"%v", "message":"query ok", "data":%s}`, status, rst)
+	} else {
+		status = 255
+		s = fmt.Sprintf(`{"success":"%v", "message":%v, "data":%s}`, 
+			status, quote(err.Error()), rst)
+	}
+
+	if isJsonDict(s) {
+		fmt.Fprintln(w, s)
+	}else {
+		logex.Fatalf("invalid json: %v", s)
+	}
+
+	if err == nil {
+		logex.Noticef(`%v, time=%v, status=%v, handle_log=%v`, 
+			req_log, time.Now().Sub(time_begin).Nanoseconds()/1000000, 
+			status, quote(handle_log))
+	} else {
+		logex.Noticef(`%v, time=%v, status=%v, err=%v, handle_log=%v`, 
+			req_log, time.Now().Sub(time_begin).Nanoseconds()/1000000, 
+			status, quote(err.Error()), quote(handle_log))
+	}
+}
+
+func parseHttpKv(r *http.Request) map[string]string {
+	r.ParseForm()
+	m := make(map[string]string)
+	for k, v := range r.Form {
+		switch k {
+		case "user": // remove @baidu.com for user
+			m[k] = strings.Split(v[0], "@")[0]
+		default:
+			m[k] = v[0]
+		}
+	}
+	
+	// allow passing hostname for debug
+	if _, ok := m["hostname"]; !ok{
+		ip := r.RemoteAddr[:strings.Index(r.RemoteAddr, ":")]
+		m["hostname"], _ = getHostname(ip)
+	}
+	return m
+}
+
+// restReq sends a restful request to requrl and returns response body.
+func restReq(method, requrl string, timeout int, kv *map[string]string) (string, error) {
+	logex.Debug("####restReq####")
+	logex.Debug(*kv)
+	data := url.Values{}
+	if kv != nil {
+		for k, v := range *kv {
+			logex.Trace("req set:", k, v)
+			data.Set(k, v)
+		}
+	}
+	if method == "GET" || method == "DELETE" {
+		requrl = requrl + "?" + data.Encode()
+		data = url.Values{}
+	}
+
+	logex.Notice(method, requrl)
+	req, err := http.NewRequest(method, requrl, bytes.NewBufferString(data.Encode()))
+	if err != nil {
+		logex.Warning("NewRequest failed:", err)
+		return "", err
+	}
+	if method == "POST" || method == "PUT" {
+		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+		req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
+	}
+
+	client := &http.Client{}
+	client.Timeout = time.Duration(timeout) * time.Second
+	resp, err := client.Do(req)
+	if err != nil {
+		logex.Warning("Do failed:", err)
+		return "", err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		logex.Warning("resp status: " + resp.Status)
+		return "", errors.New("resp status: " + resp.Status)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	return string(body), err
+}

cube/cube-agent/src/agent/http_get.go

+//-*- coding:utf-8; indent-tabs-mode:nil; -*-
+package agent
+
+import (
+	"fmt"
+	"encoding/json"
+)
+
+func initGetHandlers() {
+	getHandler = map[string]handlerFunc{
+		"/agent/status":           GetStatus,
+	}
+}
+
+func GetStatus(subpath string, m map[string]string, b []byte) (string, string, error) {
+	b, err := json.Marshal(BUILTIN_STATUS);
+	if err != nil {
+		return quote(""), "", fmt.Errorf("json marshal failed, %v", err)
+    }
+
+	return string(b), "", err
+}
\ No newline at end of file

cube/cube-agent/src/agent/http_post.go

+//-*- coding:utf-8; indent-tabs-mode:nil; -*-
+package agent
+
+import (
+	"fmt"
+	"encoding/json"
+	"github.com/Badangel/logex"
+)
+
+func initPostHandlers() {
+	postHandler = map[string]handlerFunc{
+		"/agent/cmd":           PostCmd,
+	}
+}
+
+func PostCmd(subpath string, m map[string]string, b []byte) (string, string, error) {
+	var work Work
+	err := json.Unmarshal(b, &work)
+	if err != nil {
+		logex.Warningf("Unmarshal from %s error (+%v)", string(b), err)
+		return quote(""), "", fmt.Errorf("Work json unmarshal work failed, %v", err)
+	}
+
+	if _, ok := CmdWorkFilter.Load(work.Token()); ok {
+		logex.Warningf("Another work with same token is doing. Token(%s)", work.Token())
+		return quote(""), "", fmt.Errorf("Another work with same key is doing.", err)
+	}
+	
+	CmdWorkFilter.Store(work.Token(), true)
+	err = work.DoWork()
+	CmdWorkFilter.Delete(work.Token())
+	if err != nil {
+		return quote(""), "", fmt.Errorf("Do work failed.", err)
+	}
+
+	return quote(""), "", err
+}
\ No newline at end of file

cube/cube-agent/src/agent/util.go

+package agent
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Badangel/logex"
+)
+
+// restReq sends a restful request to requrl and returns response body.
+func RestReq(method, requrl string, timeout int, kv *map[string]string) (string, error) {
+	data := url.Values{}
+	if kv != nil {
+		for k, v := range *kv {
+			//logex.Trace("req set:", k, v)
+			data.Set(k, v)
+		}
+	}
+	if method == "GET" || method == "DELETE" {
+		requrl = requrl + "?" + data.Encode()
+		data = url.Values{}
+	}
+
+	//logex.Notice(method, requrl)
+	req, err := http.NewRequest(method, requrl, bytes.NewBufferString(data.Encode()))
+	if err != nil {
+		logex.Warning("NewRequest failed:", err)
+		return "", err
+	}
+	if method == "POST" || method == "PUT" {
+		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+		req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
+	}
+
+	client := &http.Client{}
+	client.Timeout = time.Duration(timeout) * time.Second
+	resp, err := client.Do(req)
+	if err != nil {
+		logex.Warning("Do failed:", err)
+		return "", err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		logex.Warning("resp status: " + resp.Status)
+		return "", errors.New("resp status: " + resp.Status)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	return string(body), err
+}
+
+// restReq sends a restful request to requrl and returns response body as json.
+func JsonReq(method, requrl string, timeout int, kv *map[string]string,
+	out interface{}) error {
+	s, err := RestReq(method, requrl, timeout, kv)
+	logex.Debugf("json request method:[%v], requrl:[%s], timeout:[%v], map[%v], out_str:[%s]", method, requrl, timeout, kv, s)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal([]byte(s), out)
+}
+
+func GetHdfsMeta(src string) (master, ugi, path string, err error) {
+	//src = "hdfs://root:rootpasst@st1-inf-platform0.st01.baidu.com:54310/user/mis_user/news_dnn_ctr_cube_1/1501836820/news_dnn_ctr_cube_1_part54.tar"
+	//src = "hdfs://st1-inf-platform0.st01.baidu.com:54310/user/mis_user/news_dnn_ctr_cube_1/1501836820/news_dnn_ctr_cube_1_part54.tar"
+
+	ugiBegin := strings.Index(src, "//")
+	ugiPos := strings.LastIndex(src, "@")
+	if ugiPos != -1 && ugiBegin != -1 {
+		ugi = src[ugiBegin+2 : ugiPos]
+	}
+	src1 := strings.Replace(strings.Replace(src, "hdfs://", "", 1), ugi, "", 1)
+	if ugi != "" {
+		src1 = src1[1:]
+	}
+	pos := strings.Index(src1, "/")
+	if pos != -1 {
+		master = src1[0:pos]
+		path = src1[pos:]
+	} else {
+		logex.Warningf("failed to get the master or path for (%s)", src)
+		err = errors.New("invalid master or path found")
+	}
+	logex.Debugf("parse the (%s) succ, master is %s, ugi is (%s), path is %s", src, master, ugi, path)
+	return
+}
+
+func getHostIp() (string, error) {
+	if addrs, err := net.InterfaceAddrs(); err == nil {
+		for _, addr := range addrs {
+			ips := addr.String()
+			logex.Debugf("get host ip: %v", ips)
+			if strings.HasPrefix(ips, "127") {
+				continue
+			} else {
+				list := strings.Split(ips, "/")
+				if len(list) != 2 {
+					continue
+				}
+				return list[0], nil
+			}
+		}
+	}
+	return "unkown ip", errors.New("get host ip failed")
+}
+
+func getHostname(ip string) (hostname string, err error) {
+	if hostnames, err := net.LookupAddr(ip); err != nil {
+		hostname = ip
+		//logex.Warningf("cannot find the hostname of ip (%s), error (%v)", ip, err)
+	} else {
+		if len(hostnames) > 0 {
+			hostname = hostnames[0][:strings.LastIndex(hostnames[0], ".baidu.com.")]
+		} else {
+			hostname = ip
+		}
+	}
+
+	return hostname, err
+}
+
+func GetLocalHostname() (hostname string, err error) {
+	if ip, err := getHostIp(); err == nil {
+		return getHostname(ip)
+	} else {
+		return "unkown ip", err
+	}
+}
+
+func GetLocalHostnameCmd() (hostname string, err error) {
+	cmd := "hostname"
+	stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
+	if stdout != "" && err == nil {
+		hostname := strings.TrimSpace(stdout)
+		index := strings.LastIndex(hostname, ".baidu.com")
+		if index > 0 {
+			return hostname[:strings.LastIndex(hostname, ".baidu.com")], nil
+		} else {
+			return hostname, nil
+		}
+	} else {
+		logex.Debugf("using hostname cmd failed. err:%v", err)
+		return GetLocalHostname()
+	}
+}
+
+// quote quotes string for json output. eg: s="123", quote(s)=`"123"`
+func quote(s string) string {
+    return fmt.Sprintf("%q", s)
+}
+
+// quoteb quotes byte array for json output.
+func quoteb(b []byte) string {
+    return quote(string(b))
+}
+
+// quotea quotes string array for json output
+func quotea(a []string) string {
+    b, _ := json.Marshal(a)
+    return string(b)
+}
+
+func isJsonDict(s string) bool {
+    var js map[string]interface{}
+    return json.Unmarshal([]byte(s), &js) == nil
+}

cube/cube-agent/src/agent/work_pool.go

+package agent
+
+import (
+	"errors"
+	"fmt"
+	"sync/atomic"
+	"sync"
+)
+
+type (
+	workType struct {
+		poolWorker    PoolWorker
+		resultChannel chan error
+	}
+
+	WorkPool struct {
+		queueChannel    chan workType
+		workChannel     chan PoolWorker
+		queuedWorkNum   int32
+		activeWorkerNum int32
+		queueCapacity   int32
+		workFilter      sync.Map
+	}
+)
+
+type PoolWorker interface {
+	Token() string
+	DoWork()
+}
+
+func NewWorkPool(workerNum int, queueCapacity int32) *WorkPool {
+	workPool := WorkPool{
+		queueChannel:    make(chan workType),
+		workChannel:     make(chan PoolWorker, queueCapacity),
+		queuedWorkNum:   0,
+		activeWorkerNum: 0,
+		queueCapacity:   queueCapacity,
+	}
+
+	for i := 0; i < workerNum; i++ {
+		go workPool.startWorkRoutine()
+	}
+
+	go workPool.startQueueRoutine()
+
+	return &workPool
+}
+
+func (workPool *WorkPool) startWorkRoutine() {
+	for {
+		select {
+		case work := <-workPool.workChannel:
+			workPool.doWork(work)
+			break
+		}
+	}
+}
+
+func (workPool *WorkPool) startQueueRoutine() {
+	for {
+		select {
+		case queueItem := <-workPool.queueChannel:
+			if atomic.AddInt32(&workPool.queuedWorkNum, 0) == workPool.queueCapacity {
+				queueItem.resultChannel <- fmt.Errorf("work pool fulled with %v pending works", QueueCapacity)
+				continue
+			}
+
+			atomic.AddInt32(&workPool.queuedWorkNum, 1)
+
+			workPool.workChannel <- queueItem.poolWorker
+
+			queueItem.resultChannel <- nil
+
+			break
+		}
+	}
+}
+
+func (workPool *WorkPool) doWork(poolWorker PoolWorker) {
+	defer atomic.AddInt32(&workPool.activeWorkerNum, -1)
+	defer workPool.workFilter.Delete(poolWorker.Token())
+
+	atomic.AddInt32(&workPool.queuedWorkNum, -1)
+	atomic.AddInt32(&workPool.activeWorkerNum, 1)
+
+	poolWorker.DoWork()
+}
+
+func (workPool *WorkPool) PostWorkWithToken(poolWorker PoolWorker) (err error) {
+	if _, ok := workPool.workFilter.Load(poolWorker.Token()); ok {
+		return errors.New("another work with same key is doing.")
+	}
+	workPool.workFilter.Store(poolWorker.Token(), true)
+	return workPool.PostWork(poolWorker)
+}
+
+func (workPool *WorkPool) PostWork(poolWorker PoolWorker) (err error) {
+	work := workType{poolWorker, make(chan error)}
+
+	defer close(work.resultChannel)
+
+	workPool.queueChannel <- work
+
+	err = <-work.resultChannel
+
+	return err
+}

cube/cube-agent/src/cube-agent.go

+package main
+
+import (
+	"agent"
+	"fmt"
+	"github.com/docopt/docopt-go"
+	"github.com/Badangel/logex"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+)
+
+func main() {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	agent.Dir, _ = filepath.Abs(filepath.Dir(os.Args[0]))
+	usage := fmt.Sprintf(`Usage: ./m_master [options]
+
+Options:
+  -n WORKERNUM    set worker num.
+  -q QUEUENUM     set queue num.
+  -P LISTEN_PORT  agent listen port
+
+Log options:
+  -l LOG_LEVEL      set log level, values: 0,1,2,4,8,16. [default: 16]
+  --log_dir=DIR     set log output dir. [default: ./log]
+  --log_name=NAME   set log name. [default: m_agent]`, agent.Dir)
+
+	opts, err := docopt.Parse(usage, nil, true, "Cube Agent Checker 1.0.0", false)
+	if err != nil {
+		fmt.Println("ERROR:", err)
+		os.Exit(1)
+	}
+
+	log_level, _ := strconv.Atoi(opts["-l"].(string))
+	log_name := opts["--log_name"].(string)
+	log_dir := opts["--log_dir"].(string)
+	logex.SetLevel(getLogLevel(log_level))
+	if err := logex.SetUpFileLogger(log_dir, log_name, nil); err != nil {
+		fmt.Println("ERROR:", err)
+	}
+
+	logex.Notice("--- NEW SESSION -------------------------")
+	logex.Notice(">>> log_level:", log_level)
+
+	agent.WorkerNum = 10
+	if opts["-n"] != nil {
+		n, err := strconv.Atoi(opts["-n"].(string))
+		if err == nil {
+			agent.WorkerNum = n
+		}
+	}
+
+	agent.QueueCapacity = 20
+	if opts["-q"] != nil {
+		q, err := strconv.Atoi(opts["-q"].(string))
+		if err == nil {
+			agent.QueueCapacity = int32(q)
+		}
+	}
+
+	agent.CmdWorkPool = agent.NewWorkPool(agent.WorkerNum, agent.QueueCapacity)
+
+	if opts["-P"] == nil {
+		logex.Fatalf("ERROR: -P LISTEN PORT must be set!")
+		os.Exit(255)
+	}
+
+	agentPort := opts["-P"].(string)
+	logex.Notice(">>> starting server...")
+	addr := ":" + agentPort
+
+	if agent.StartHttp(addr) != nil {
+		logex.Noticef("cant start http(addr=%v). quit.", addr)
+		os.Exit(0)
+	}
+}
+
+func getLogLevel(log_level int) logex.Level {
+	switch log_level {
+	case 16:
+		return logex.DEBUG
+	case 8:
+		return logex.TRACE
+	case 4:
+		return logex.NOTICE
+	case 2:
+		return logex.WARNING
+	case 1:
+		return logex.FATAL
+	case 0:
+		return logex.NONE
+	}
+	return logex.DEBUG
+}