Merge pull request #124 from PaddlePaddle/merge_longteng

 add server python API

CMakeLists.txt

@@ -46,11 +46,11 @@ set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING
 
 set(THIRD_PARTY_BUILD_TYPE Release)
 
-option(WITH_AVX     "Compile Paddle Serving with AVX intrinsics"    ${AVX_FOUND})
-option(WITH_MKL     "Compile Paddle Serving with MKL support."      ${AVX_FOUND})
-option(WITH_GPU     "Compile Paddle Serving with NVIDIA GPU"        ${CUDA_FOUND})
-option(CLIENT_ONLY  "Compile client libraries and demos only"       FALSE)
-option(WITH_ELASTIC_CTR "Compile ELASITC-CTR solution"              FALSE)
+option(WITH_AVX	    "Compile Paddle Serving with AVX intrinsics"    OFF)
+option(WITH_MKL	    "Compile Paddle Serving with MKL support."      OFF)
+option(WITH_GPU	    "Compile Paddle Serving with NVIDIA GPU"        OFF)
+option(CLIENT_ONLY  "Compile client libraries and demos only"       OFF)
+option(WITH_ELASTIC_CTR "Compile ELASITC-CTR solution"              OFF)
 
 set(WITH_MKLML ${WITH_MKL})
 if (NOT DEFINED WITH_MKLDNN)
@@ -128,14 +128,10 @@ if (NOT CLIENT_ONLY)
 endif()
 
 add_subdirectory(core)
+
 if(NOT CLIENT_ONLY)
 add_subdirectory(paddle_inference)
 endif()
 
-if(CLIENT_ONLY)
 add_subdirectory(python)
-set(PYTHON_INCLUDE_DIR ${PYTHON_INCLUDE})
-set(PYTHON_LIBRARIES ${PYTHON_LIB})
-endif()
-
-add_subdirectory(examples)
+#add_subdirectory(examples)

cmake/paddlepaddle.cmake

@@ -36,7 +36,7 @@ SET(PADDLE_VERSION "latest")
 if (WITH_GPU)
     SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}-gpu-cuda${CUDA_VERSION_MAJOR}-cudnn7-avx-mkl")
 else()
-    if (AVX_FOUND)
+    if (WITH_AVX)
         if (WITH_MKLML)
             SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}-cpu-avx-mkl")
         else()
@@ -62,10 +62,13 @@ ExternalProject_Add(
     INSTALL_COMMAND
         ${CMAKE_COMMAND} -E copy_directory ${PADDLE_DOWNLOAD_DIR}/paddle/include ${PADDLE_INSTALL_DIR}/include &&
         ${CMAKE_COMMAND} -E copy_directory ${PADDLE_DOWNLOAD_DIR}/paddle/lib ${PADDLE_INSTALL_DIR}/lib &&
-        ${CMAKE_COMMAND} -E copy_directory ${PADDLE_DOWNLOAD_DIR}/third_party ${PADDLE_INSTALL_DIR}/third_party &&
-        ${CMAKE_COMMAND} -E copy ${PADDLE_INSTALL_DIR}/third_party/install/mkldnn/lib/libmkldnn.so.0 ${PADDLE_INSTALL_DIR}/third_party/install/mkldnn/lib/libmkldnn.so
+        ${CMAKE_COMMAND} -E copy_directory ${PADDLE_DOWNLOAD_DIR}/third_party ${PADDLE_INSTALL_DIR}/third_party 
 )
 
+if (WITH_MKLML)
+   file(COPY ${PADDLE_INSTALL_DIR}/third_party/install/mkldnn/lib/libmkldnn.so.0 DESTINATION ${PADDLE_INSTALL_DIR}/third_party/install/mkldnn/lib/libmkldnn.so FOLLOW_SYMLINK_CHAIN)
+endif()
+
 INCLUDE_DIRECTORIES(${PADDLE_INCLUDE_DIR})
 SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${PADDLE_INSTALL_DIR}/third_party/install/mklml/lib")
 LINK_DIRECTORIES(${PADDLE_INSTALL_DIR}/third_party/install/mklml/lib)
@@ -73,6 +76,9 @@ LINK_DIRECTORIES(${PADDLE_INSTALL_DIR}/third_party/install/mklml/lib)
 SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${PADDLE_INSTALL_DIR}/third_party/install/mkldnn/lib")
 LINK_DIRECTORIES(${PADDLE_INSTALL_DIR}/third_party/install/mkldnn/lib)
 
+ADD_LIBRARY(openblas STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET openblas PROPERTY IMPORTED_LOCATION ${PADDLE_INSTALL_DIR}/third_party/install/openblas/lib/libopenblas.a)
+
 ADD_LIBRARY(paddle_fluid STATIC IMPORTED GLOBAL)
 SET_PROPERTY(TARGET paddle_fluid PROPERTY IMPORTED_LOCATION ${PADDLE_INSTALL_DIR}/lib/libpaddle_fluid.a)
 

cmake/paddlepaddle.cmake_from_source

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+INCLUDE(ExternalProject)
+
+SET(PADDLE_SOURCES_DIR ${THIRD_PARTY_PATH}/Paddle)
+SET(PADDLE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/Paddle/)
+SET(PADDLE_INCLUDE_DIR "${PADDLE_INSTALL_DIR}/include" CACHE PATH "PaddlePaddle include directory." FORCE)
+SET(PADDLE_LIBRARIES "${PADDLE_INSTALL_DIR}/lib/libpaddle_fluid.a" CACHE FILEPATH "Paddle library." FORCE)
+
+INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir)
+
+# Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
+set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/glog")
+
+message( "WITH_GPU = ${WITH_GPU}")
+
+# If minimal .a is need, you can set  WITH_DEBUG_SYMBOLS=OFF
+ExternalProject_Add(
+    extern_paddle
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    # TODO(wangguibao): change to de newst repo when they changed.
+    GIT_REPOSITORY  "https://github.com/PaddlePaddle/Paddle"
+    GIT_TAG         "v1.5.1"
+    PREFIX          ${PADDLE_SOURCES_DIR}
+    UPDATE_COMMAND  ""
+    BINARY_DIR ${CMAKE_BINARY_DIR}/Paddle
+    CMAKE_ARGS      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+                    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+                    -DCMAKE_INSTALL_PREFIX=${PADDLE_INSTALL_DIR}
+                    -DCMAKE_INSTALL_LIBDIR=${PADDLE_INSTALL_DIR}/lib
+                    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                    -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+                    -DCMAKE_PREFIX_PATH=${prefix_path}
+                    -DCMAKE_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}
+                    -DWITH_SWIG_PY=OFF
+                    -DWITH_PYTHON=OFF
+                    -DWITH_MKL=OFF
+                    -DWITH_AVX=OFF
+                    -DWITH_MKLDNN=OFF
+                    -DWITH_GPU=OFF
+                    -DWITH_FLUID_ONLY=ON
+                    -DWITH_TESTING=OFF
+                    -DWITH_DISTRIBUTE=OFF
+                    -DON_INFER=ON
+                    ${EXTERNAL_OPTIONAL_ARGS}
+    LIST_SEPARATOR |
+    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PADDLE_INSTALL_DIR}
+                     -DCMAKE_INSTALL_LIBDIR:PATH=${PADDLE_INSTALL_DIR}/lib
+                     -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+                     -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+    BUILD_COMMAND $(MAKE)
+    INSTALL_COMMAND $(MAKE) fluid_lib_dist
+)
+
+ExternalProject_Get_Property(extern_paddle BINARY_DIR)
+ADD_LIBRARY(paddle_fluid STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET paddle_fluid PROPERTY IMPORTED_LOCATION ${BINARY_DIR}/fluid_install_dir/paddle/fluid/inference/libpaddle_fluid.a)
+
+LIST(APPEND external_project_dependencies paddle)
+
+ADD_LIBRARY(snappystream STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET snappystream PROPERTY IMPORTED_LOCATION ${BINARY_DIR}/fluid_install_dir/third_party/install/snappystream/lib/libsnappystream.a)
+
+ADD_LIBRARY(openblas STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET openblas PROPERTY IMPORTED_LOCATION ${BINARY_DIR}/fluid_install_dir/third_party/install/openblas/lib/libopenblas.a)
+
+ADD_LIBRARY(xxhash STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET xxhash PROPERTY IMPORTED_LOCATION ${BINARY_DIR}/fluid_install_dir/third_party/install/xxhash/lib/libxxhash.a)
+
+LIST(APPEND paddle_depend_libs
+        snappystream
+        snappy
+        xxhash
+	openblas)

core/CMakeLists.txt

@@ -25,4 +25,5 @@ endif()
 
 if (NOT CLIENT_ONLY)
 add_subdirectory(predictor)
+add_subdirectory(general-server)
 endif()

core/configure/CMakeLists.txt

@@ -29,11 +29,43 @@ FILE(GLOB inc ${CMAKE_CURRENT_BINARY_DIR}/*.pb.h)
 install(FILES ${inc}
         DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/configure)
 
+py_proto_compile(general_model_config_py_proto SRCS proto/general_model_config.proto)
+add_custom_target(general_model_config_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
+add_dependencies(general_model_config_py_proto general_model_config_py_proto_init)
+
+if (CLIENT_ONLY)
 py_proto_compile(sdk_configure_py_proto SRCS proto/sdk_configure.proto)
 add_custom_target(sdk_configure_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
 add_dependencies(sdk_configure_py_proto sdk_configure_py_proto_init)
 add_custom_command(TARGET sdk_configure_py_proto POST_BUILD
-		COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving/proto
-		COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving/proto
-		COMMENT "Copy generated python proto into directory paddle_serving/proto."
+		COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
+		COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
+		COMMENT "Copy generated python proto into directory paddle_serving_client/proto."
+		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+add_custom_command(TARGET general_model_config_py_proto POST_BUILD
+                COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
+                COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto
+                COMMENT "Copy generated general_model_config proto file into directory paddle_serving_client/proto."
+                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+endif()
+
+if (NOT CLIENT_ONLY)
+py_proto_compile(server_config_py_proto SRCS proto/server_configure.proto)
+add_custom_target(server_config_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
+add_dependencies(server_config_py_proto server_config_py_proto_init)
+add_custom_command(TARGET server_config_py_proto POST_BUILD
+		COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
+		COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
+		COMMENT "Copy generated python proto into directory paddle_serving_server/proto."
+		WORKING_DIRECTORY ${CMAKE_CURRENT_BINRARY_DIR})
+
+add_custom_command(TARGET general_model_config_py_proto POST_BUILD
+		COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
+		COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto
+		COMMENT "Copy generated general_model_config proto file into directory paddle_serving_server/proto."
 		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+endif()
+

core/configure/include/configure_parser.h

@@ -19,13 +19,17 @@
 namespace baidu {
 namespace paddle_serving {
 namespace configure {
-int read_proto_conf(const std::string &conf_path,
-                    const std::string &conf_file,
-                    google::protobuf::Message *conf);
 
-int write_proto_conf(google::protobuf::Message *message,
-                     const std::string &output_path,
-                     const std::string &output_file);
+  int read_proto_conf(const std::string &conf_full_path,
+                      google::protobuf::Message *conf);
+  
+  int read_proto_conf(const std::string &conf_path,
+                      const std::string &conf_file,
+                      google::protobuf::Message *conf);
+  
+  int write_proto_conf(google::protobuf::Message *message,
+                       const std::string &output_path,
+                       const std::string &output_file);
 
 }  // namespace configure
 }  // namespace paddle_serving

core/configure/proto/general_model_config.proto

@@ -16,14 +16,16 @@ syntax = "proto2";
 package baidu.paddle_serving.configure;
 
 message FeedVar {
-  required string name = 1;
-  required bool is_lod_tensor = 2;
-  required int32 feed_type = 3;
-  repeated int32 shape = 4;
+  optional string name = 1;
+  optional string alias_name = 2;
+  optional bool is_lod_tensor = 3 [ default = false ];
+  optional int32 feed_type = 4 [ default = 0 ];
+  repeated int32 shape = 5;
 }
 message FetchVar {
-  required string name = 1;
-  repeated int32 shape = 2;
+  optional string name = 1;
+  optional string alias_name = 2;
+  repeated int32 shape = 3;
 }
 message GeneralModelConfig {
   repeated FeedVar feed_var = 1;

core/configure/src/configure_parser.cpp

@@ -31,6 +31,24 @@ namespace baidu {
 namespace paddle_serving {
 namespace configure {
 
+int read_proto_conf(const std::string &conf_file_full_path,
+                    google::protobuf::Message *conf) {
+  int fd = open(conf_file_full_path.c_str(), O_RDONLY);
+  if (fd == -1) {
+    LOG(WARNING) << "File not found: " << conf_file_full_path.c_str();
+    return -1;
+  }
+
+  google::protobuf::io::FileInputStream input(fd);
+  bool success = google::protobuf::TextFormat::Parse(&input, conf);
+  close(fd);
+  if (!success) {
+    return -1;
+  }
+
+  return 0;
+}
+
 int read_proto_conf(const std::string &conf_path,
                     const std::string &conf_file,
                     google::protobuf::Message *conf) {

core/cube/cube-agent/src/CMakeLists.txt

-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-
-set(SOURCE_FILE cube-agent.go)
-add_go_executable(cube-agent ${SOURCE_FILE})
-add_dependencies(cube-agent agent-docopt-go)
-add_dependencies(cube-agent agent-logex)
-add_dependencies(cube-agent agent-pipeline)

core/cube/cube-agent/src/agent/define.go

-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package agent
-
-import (
-	"errors"
-	_ "github.com/Badangel/logex"
-	"strings"
-	"sync"
-)
-
-var (
-	Dir           string
-	WorkerNum     int
-	QueueCapacity int32
-	MasterHost    []string
-	MasterPort    []string
-
-	TestHostname string
-	TestIdc      string
-	ShardLock    sync.RWMutex
-
-	CmdWorkPool   *WorkPool
-	CmdWorkFilter sync.Map
-)
-
-type (
-	Status struct {
-		Status  string `json:"status"`
-		Version string `json:"version"`
-	}
-
-	MasterResp struct {
-		Success string `json:"success"`
-		Message string `json:"message"`
-		Data    string `json:"data"`
-	}
-
-	ShardInfo struct {
-		DictName   string
-		ShardSeq   int
-		SlotIdList string
-		DataDir    string
-		Service    string `json:"service,omitempty"`
-		Libcube    string `json:"libcube,omitempty"`
-	}
-
-	CubeResp struct {
-		Status     int    `json:"status"`
-		CurVersion string `json:"cur_version"`
-		BgVersion  string `json:"bg_version"`
-	}
-)
-
-var BUILTIN_STATUS = Status{"RUNNING", "3.0.0.1"}
-
-var ShardInfoMap map[string]map[string]*ShardInfo
-var disks []string
-
-func GetMaster(master string) (host, port string, err error) {
-	if len(ShardInfoMap) < 1 {
-		return "", "", errors.New("empty master list.")
-	}
-	if master == "" {
-		return MasterHost[0], MasterPort[0], nil
-	}
-	if _, ok := ShardInfoMap[master]; ok {
-		m := strings.Split(master, ":")
-		if len(m) != 2 {
-			return MasterHost[0], MasterPort[0], nil
-		}
-		return m[0], m[1], nil
-	} else {
-		return MasterHost[0], MasterPort[0], nil
-	}
-}

core/cube/cube-agent/src/agent/http.go

-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package agent
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/Badangel/logex"
-)
-
-type handlerFunc func(subpath string, m map[string]string, b []byte) (string, string, error)
-
-var ( // key = subpath; eg: path="/checker/job", key="job"
-	getHandler    map[string]handlerFunc
-	putHandler    map[string]handlerFunc
-	deleteHandler map[string]handlerFunc
-	postHandler   map[string]handlerFunc
-)
-
-func StartHttp(addr string) error {
-
-	// init handlers:
-	initGetHandlers()
-	initPostHandlers()
-
-	http.HandleFunc("/agent/", handleRest)
-	logex.Notice("start http ", addr)
-	return http.ListenAndServe(addr, nil)
-}
-
-func handleRest(w http.ResponseWriter, r *http.Request) {
-	var (
-		req_log string
-		status  int32
-	)
-	time_begin := time.Now()
-
-	cont_type := make([]string, 1, 1)
-	cont_type[0] = "application/json"
-	header := w.Header()
-	header["Content-Type"] = cont_type
-	w.Header().Add("Access-Control-Allow-Origin", "*")
-
-	m := parseHttpKv(r)
-	b, _ := ioutil.ReadAll(r.Body)
-
-	req_log = fmt.Sprintf("handle %v %v %v from %v, len(m)=%v, m=%+v",
-		r.Method, r.URL.Path, r.URL.RawQuery, r.RemoteAddr, len(m), m)
-
-	api := r.URL.Path
-
-	var showHandler map[string]handlerFunc
-	switch r.Method {
-	case "GET":
-		showHandler = getHandler
-	case "POST": // create
-		showHandler = postHandler
-	case "PUT": // update
-		showHandler = putHandler
-	case "DELETE":
-		showHandler = deleteHandler
-	default:
-		logex.Warningf(`{"error":1, "message":"unsupport method %v"}`, r.Method)
-	}
-
-	handler, ok := showHandler[api]
-
-	if !ok {
-		key_list := make([]string, 0, len(showHandler))
-		for key := range showHandler {
-			key_list = append(key_list, key)
-		}
-		status = 2
-		fmt.Fprintf(w, `{"success":"%v", "message":"wrong api", "method":"%s", "api":"%s", "api_list":"%v"}`,
-			status, r.Method, api, key_list)
-
-		logex.Noticef(`%v, time=%v, status=%v`,
-			req_log, time.Now().Sub(time_begin).Nanoseconds()/1000000, status)
-		return
-	}
-
-	var s string
-	rst, handle_log, err := handler(api, m, b)
-	if err == nil {
-		status = 0
-		s = fmt.Sprintf(`{"success":"%v", "message":"query ok", "data":%s}`, status, rst)
-	} else {
-		status = 255
-		s = fmt.Sprintf(`{"success":"%v", "message":%v, "data":%s}`,
-			status, quote(err.Error()), rst)
-	}
-
-	if isJsonDict(s) {
-		fmt.Fprintln(w, s)
-	} else {
-		logex.Fatalf("invalid json: %v", s)
-	}
-
-	if err == nil {
-		logex.Noticef(`%v, time=%v, status=%v, handle_log=%v`,
-			req_log, time.Now().Sub(time_begin).Nanoseconds()/1000000,
-			status, quote(handle_log))
-	} else {
-		logex.Noticef(`%v, time=%v, status=%v, err=%v, handle_log=%v`,
-			req_log, time.Now().Sub(time_begin).Nanoseconds()/1000000,
-			status, quote(err.Error()), quote(handle_log))
-	}
-}
-
-func parseHttpKv(r *http.Request) map[string]string {
-	r.ParseForm()
-	m := make(map[string]string)
-	for k, v := range r.Form {
-		switch k {
-		case "user": // remove @baidu.com for user
-			m[k] = strings.Split(v[0], "@")[0]
-		default:
-			m[k] = v[0]
-		}
-	}
-
-	// allow passing hostname for debug
-	if _, ok := m["hostname"]; !ok {
-		ip := r.RemoteAddr[:strings.Index(r.RemoteAddr, ":")]
-		m["hostname"], _ = getHostname(ip)
-	}
-	return m
-}
-
-// restReq sends a restful request to requrl and returns response body.
-func restReq(method, requrl string, timeout int, kv *map[string]string) (string, error) {
-	logex.Debug("####restReq####")
-	logex.Debug(*kv)
-	data := url.Values{}
-	if kv != nil {
-		for k, v := range *kv {
-			logex.Trace("req set:", k, v)
-			data.Set(k, v)
-		}
-	}
-	if method == "GET" || method == "DELETE" {
-		requrl = requrl + "?" + data.Encode()
-		data = url.Values{}
-	}
-
-	logex.Notice(method, requrl)
-	req, err := http.NewRequest(method, requrl, bytes.NewBufferString(data.Encode()))
-	if err != nil {
-		logex.Warning("NewRequest failed:", err)
-		return "", err
-	}
-	if method == "POST" || method == "PUT" {
-		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-		req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
-	}
-
-	client := &http.Client{}
-	client.Timeout = time.Duration(timeout) * time.Second
-	resp, err := client.Do(req)
-	if err != nil {
-		logex.Warning("Do failed:", err)
-		return "", err
-	}
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		logex.Warning("resp status: " + resp.Status)
-		return "", errors.New("resp status: " + resp.Status)
-	}
-
-	body, err := ioutil.ReadAll(resp.Body)
-	return string(body), err
-}

core/cube/cube-agent/src/agent/util.go

-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package agent
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/Badangel/logex"
-)
-
-// restReq sends a restful request to requrl and returns response body.
-func RestReq(method, requrl string, timeout int, kv *map[string]string) (string, error) {
-	data := url.Values{}
-	if kv != nil {
-		for k, v := range *kv {
-			//logex.Trace("req set:", k, v)
-			data.Set(k, v)
-		}
-	}
-	if method == "GET" || method == "DELETE" {
-		requrl = requrl + "?" + data.Encode()
-		data = url.Values{}
-	}
-
-	//logex.Notice(method, requrl)
-	req, err := http.NewRequest(method, requrl, bytes.NewBufferString(data.Encode()))
-	if err != nil {
-		logex.Warning("NewRequest failed:", err)
-		return "", err
-	}
-	if method == "POST" || method == "PUT" {
-		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-		req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
-	}
-
-	client := &http.Client{}
-	client.Timeout = time.Duration(timeout) * time.Second
-	resp, err := client.Do(req)
-	if err != nil {
-		logex.Warning("Do failed:", err)
-		return "", err
-	}
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		logex.Warning("resp status: " + resp.Status)
-		return "", errors.New("resp status: " + resp.Status)
-	}
-
-	body, err := ioutil.ReadAll(resp.Body)
-	return string(body), err
-}
-
-// restReq sends a restful request to requrl and returns response body as json.
-func JsonReq(method, requrl string, timeout int, kv *map[string]string,
-	out interface{}) error {
-	s, err := RestReq(method, requrl, timeout, kv)
-	logex.Debugf("json request method:[%v], requrl:[%s], timeout:[%v], map[%v], out_str:[%s]", method, requrl, timeout, kv, s)
-	if err != nil {
-		return err
-	}
-	return json.Unmarshal([]byte(s), out)
-}
-
-func GetHdfsMeta(src string) (master, ugi, path string, err error) {
-	//src = "hdfs://root:rootpasst@st1-inf-platform0.st01.baidu.com:54310/user/mis_user/news_dnn_ctr_cube_1/1501836820/news_dnn_ctr_cube_1_part54.tar"
-	//src = "hdfs://st1-inf-platform0.st01.baidu.com:54310/user/mis_user/news_dnn_ctr_cube_1/1501836820/news_dnn_ctr_cube_1_part54.tar"
-
-	ugiBegin := strings.Index(src, "//")
-	ugiPos := strings.LastIndex(src, "@")
-	if ugiPos != -1 && ugiBegin != -1 {
-		ugi = src[ugiBegin+2 : ugiPos]
-	}
-	src1 := strings.Replace(strings.Replace(src, "hdfs://", "", 1), ugi, "", 1)
-	if ugi != "" {
-		src1 = src1[1:]
-	}
-	pos := strings.Index(src1, "/")
-	if pos != -1 {
-		master = src1[0:pos]
-		path = src1[pos:]
-	} else {
-		logex.Warningf("failed to get the master or path for (%s)", src)
-		err = errors.New("invalid master or path found")
-	}
-	logex.Debugf("parse the (%s) succ, master is %s, ugi is (%s), path is %s", src, master, ugi, path)
-	return
-}
-
-func getHostIp() (string, error) {
-	if addrs, err := net.InterfaceAddrs(); err == nil {
-		for _, addr := range addrs {
-			ips := addr.String()
-			logex.Debugf("get host ip: %v", ips)
-			if strings.HasPrefix(ips, "127") {
-				continue
-			} else {
-				list := strings.Split(ips, "/")
-				if len(list) != 2 {
-					continue
-				}
-				return list[0], nil
-			}
-		}
-	}
-	return "unkown ip", errors.New("get host ip failed")
-}
-
-func getHostname(ip string) (hostname string, err error) {
-	if hostnames, err := net.LookupAddr(ip); err != nil {
-		hostname = ip
-		//logex.Warningf("cannot find the hostname of ip (%s), error (%v)", ip, err)
-	} else {
-		if len(hostnames) > 0 {
-			hostname = hostnames[0]
-		} else {
-			hostname = ip
-		}
-	}
-
-	return hostname, err
-}
-
-func GetLocalHostname() (hostname string, err error) {
-	if ip, err := getHostIp(); err == nil {
-		return getHostname(ip)
-	} else {
-		return "unkown ip", err
-	}
-}
-
-func GetLocalHostnameCmd() (hostname string, err error) {
-	cmd := "hostname"
-	stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-	if stdout != "" && err == nil {
-		hostname := strings.TrimSpace(stdout)
-		index := strings.LastIndex(hostname, ".baidu.com")
-		if index > 0 {
-			return hostname[:strings.LastIndex(hostname, ".baidu.com")], nil
-		} else {
-			return hostname, nil
-		}
-	} else {
-		logex.Debugf("using hostname cmd failed. err:%v", err)
-		return GetLocalHostname()
-	}
-}
-
-// quote quotes string for json output. eg: s="123", quote(s)=`"123"`
-func quote(s string) string {
-	return fmt.Sprintf("%q", s)
-}
-
-// quoteb quotes byte array for json output.
-func quoteb(b []byte) string {
-	return quote(string(b))
-}
-
-// quotea quotes string array for json output
-func quotea(a []string) string {
-	b, _ := json.Marshal(a)
-	return string(b)
-}
-
-func isJsonDict(s string) bool {
-	var js map[string]interface{}
-	return json.Unmarshal([]byte(s), &js) == nil
-}

core/cube/cube-agent/src/agent/work.go

-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package agent
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"github.com/Badangel/logex"
-	"github.com/Badangel/pipeline"
-	"os"
-	"os/exec"
-	"path"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-)
-
-const (
-	COMMAND_DOWNLOAD = "download"
-	COMMAND_RELOAD   = "reload"
-	COMMAND_SWITCH   = "enable"
-	COMMAND_ROLLBACK = "rollback"
-	COMMAND_CHECK    = "check"
-	COMMAND_CLEAR    = "clear"
-	COMMAND_POP      = "pop"
-
-	RETRY_TIMES                   = 3
-	REQUEST_MASTER_TIMEOUT_SECOND = 60
-	MAX_DOWN_CO                   = 7
-
-	RELOAD_RETRY_TIMES           = 3
-	RELOAD_RETRY_INTERVAL_SECOND = 10
-
-	DOWNLOAD_DONE_MARK_FILE = ".download_done"
-)
-
-type VersionInfo struct {
-	Version string
-	Depend  string
-	Source  string
-}
-
-type Work struct {
-	DictName          string        `json:"dict_name"`
-	ShardSeq          int           `json:"shard_seq"`
-	DeployPath        string        `json:"deploy_path"`
-	Command           string        `json:"command"`
-	Version           string        `json:"version"`
-	Depend            string        `json:"depend"`
-	Source            string        `json:"source"`
-	Mode              string        `json:"mode"`
-	DictMode          string        `json:"dict_mode"`
-	Port              string        `json:"port"`
-	bRollback         bool          `json:"b_rollback"`
-	RollbackInfo      []VersionInfo `json:"rollback_info"`
-	Status            string        `json:"status"`
-	FinishStatus      string        `json:"finish_status"`
-	Service           string        `json:"service,omitempty"`
-	VersionSign       string        `json:"version_sign,omitempty"`
-	MasterAddress     string        `json:"master_address,omitempty"`
-	ActiveVersionList string        `json:"active_version_list,omitempty"`
-}
-
-func (work *Work) Token() string {
-	return work.DictName + strconv.Itoa(work.ShardSeq) + work.Service
-}
-
-func (work *Work) Valid() bool {
-	if work.Command == "" ||
-		work.Version == "" ||
-		work.Depend == "" {
-		return false
-	}
-	return true
-}
-
-func (work *Work) DoWork() error {
-	var err error
-	if !work.Valid() {
-		err = errors.New("Work is invalid")
-		return err
-	}
-	switch work.Command {
-	case COMMAND_DOWNLOAD:
-		err = work.Download()
-	case COMMAND_RELOAD:
-		err = work.Reload()
-	case COMMAND_SWITCH:
-		err = work.Enable()
-	case COMMAND_CHECK:
-		err = work.Check()
-	case COMMAND_CLEAR:
-		err = work.Clear()
-	case COMMAND_POP:
-		err = work.Pop()
-	default:
-		logex.Debugf("Invalid command %s received", work.Command)
-		err = errors.New("Invalid command.")
-	}
-	return err
-}
-
-func GetDownloadDirs(dictName, service, version, depend, deployPath string, shardSeq,
-	split int) ([]string, error) {
-	dirs := make([]string, 0, split)
-	if deployPath == "" {
-		return dirs, errors.New("Invalid Deploy path")
-	}
-	parentDir := getParentDir(version, depend)
-	if split < 2 {
-		disk := path.Join(deployPath, "cube_data")
-		if service == "" {
-			dirs = append(dirs, path.Join(disk, strconv.Itoa(shardSeq), parentDir))
-		} else {
-			dirs = append(dirs, path.Join(disk, strconv.Itoa(shardSeq), parentDir+"-"+dictName))
-		}
-	} else {
-		for i := 0; i < split; i++ {
-			disk := path.Join(deployPath, "cube_data")
-			if service == "" {
-				dirs = append(dirs, path.Join(disk, strconv.Itoa(shardSeq), strconv.Itoa(i), parentDir))
-			} else {
-				dirs = append(dirs, path.Join(disk, strconv.Itoa(shardSeq),
-					parentDir+"-"+dictName))
-			}
-		}
-	}
-
-	return dirs, nil
-}
-
-func GetDataLinkDirs(dictName, service, version, depend, deployPath string, shardSeq,
-	split int) []string {
-	dirs := make([]string, 0, split)
-	parentDir := getParentDir(version, depend)
-	if split < 2 {
-		disk := path.Join(deployPath, "data")
-		if service == "" {
-			dirs = append(dirs, path.Join(disk, parentDir))
-		}
-	} else {
-		for i := 0; i < split; i++ {
-			disk := path.Join(deployPath, "data")
-			if service == "" {
-				dirs = append(dirs, path.Join(disk, strconv.Itoa(i), parentDir))
-			}
-		}
-	}
-	return dirs
-}
-
-func (work *Work) Download() (err error) {
-	err = DoDownload(work.DictName, work.Service, work.Version, work.Depend, work.Mode, work.Source,
-		work.DeployPath, work.ShardSeq)
-
-	if err != nil {
-		logex.Warningf("download error, failed to download %s, dir is %s, error is (+%v)", work.Source, work.DeployPath, err)
-		return
-	}
-
-	if work.Service == "" {
-		err = UnTar(work.DictName, work.Service, work.Version, work.Depend, work.Source,
-			work.DeployPath, work.ShardSeq)
-
-		if err == nil {
-			dataPath := path.Join(work.DeployPath, "data")
-
-			// remove all old links
-			if work.Mode == "base" || len(work.RollbackInfo) != 0 {
-				cmd := fmt.Sprintf("ls -l %s | grep -E 'data.|index.' | awk '{print $9}'", dataPath)
-				stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-				if err == nil && stdout != "" {
-					fileNameLi := strings.Split(strings.TrimSpace(stdout), "\n")
-					for _, file := range fileNameLi {
-						err = os.Remove(path.Join(dataPath, file))
-						logex.Debugf("os.Remove(%s) error (%+v) ", path.Join(dataPath, file), err)
-					}
-				}
-			}
-
-			// create symbolic link to the version rollbacked
-			err = CreateSymlink(work.DictName, work.Service, work.Version, work.Depend, dataPath,
-				work.DeployPath, work.ShardSeq, len(strings.Split(work.Source, ";")))
-		} else {
-			logex.Warningf("download error, failed to untar for %s, dir is %s, error is (+%v)", work.Source, work.DeployPath, err)
-		}
-	}
-
-	if err == nil {
-		// clear history data
-		work.clearData()
-		work.clearLink()
-	} else {
-		logex.Warningf("create symlink failed, error is (+%v)", err)
-	}
-
-	return
-}
-
-func (work *Work) clearData() (err error) {
-	split := len(strings.Split(work.Source, ";"))
-	downloadDirs, err := GetDownloadDirs(work.DictName, work.Service, work.Version, work.Depend,
-		work.DeployPath, work.ShardSeq, split)
-	if err != nil {
-		logex.Warningf("clearData failed, error is (+%v)", err)
-		return
-	}
-	for _, downloadDir := range downloadDirs {
-		parentDir, _ := filepath.Split(downloadDir)
-
-		cmd := fmt.Sprintf("ls -l %s | grep -v %s | awk '{print $9}'", parentDir, work.Depend)
-
-		stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-		if err != nil || stdout == "" || work.Service != "" {
-			cmd = fmt.Sprintf("find %s -type d -ctime +1 -print | xargs -i rm -rf {}", parentDir)
-			_, _, err = RetryCmd(cmd, RETRY_TIMES)
-		} else {
-			rmList := strings.Split(strings.TrimSpace(stdout), "\n")
-			for i := 0; i < len(rmList); i++ {
-				if rmList[i] == "" {
-					continue
-				}
-				cmd = fmt.Sprintf("rm -rf %s/%s*", parentDir, rmList[i])
-				_, _, err = RetryCmd(cmd, RETRY_TIMES)
-			}
-		}
-	}
-
-	return
-}
-
-func (work *Work) clearPatchData() (err error) {
-	if work.Service != "" {
-		return
-	}
-	split := len(strings.Split(work.Source, ";"))
-	downloadDirs, err := GetDownloadDirs(work.DictName, work.Service, work.Version, work.Depend,
-		work.DeployPath, work.ShardSeq, split)
-	if err != nil {
-		logex.Warningf("clearPatchData failed, error is (+%v)", err)
-		return
-	}
-	for _, downloadDir := range downloadDirs {
-		parentDir, _ := filepath.Split(downloadDir)
-		cmd := fmt.Sprintf("ls -l %s | grep %s_ | awk '{print $9}'", parentDir, work.Depend)
-		stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-		if err == nil && stdout != "" {
-			rmList := strings.Split(strings.TrimSpace(stdout), "\n")
-			for i := 0; i < len(rmList); i++ {
-				if rmList[i] == "" {
-					continue
-				}
-				cmd = fmt.Sprintf("rm -rf %s/%s*", parentDir, rmList[i])
-				_, _, err = RetryCmd(cmd, RETRY_TIMES)
-			}
-		}
-	}
-
-	return
-}
-
-func (work *Work) clearLink() (err error) {
-	if work.Service != "" {
-		return
-	}
-	split := len(strings.Split(work.Source, ";"))
-	dataLinkDirs := GetDataLinkDirs(work.DictName, work.Service, work.Version, work.Depend,
-		work.DeployPath, work.ShardSeq, split)
-	for _, linkDir := range dataLinkDirs {
-		parentDir, _ := filepath.Split(linkDir)
-		cmd := fmt.Sprintf("ls -l %s | grep -v %s | awk '{print $9}'", parentDir, work.Depend)
-
-		stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-		if err != nil || stdout == "" {
-			cmd = fmt.Sprintf("find %s -type d -ctime +1 -print | xargs -i rm -rf {}", parentDir)
-			_, _, err = RetryCmd(cmd, RETRY_TIMES)
-		} else {
-			rmList := strings.Split(strings.TrimSpace(stdout), "\n")
-			for i := 0; i < len(rmList); i++ {
-				if rmList[i] == "" {
-					continue
-				}
-				cmd = fmt.Sprintf("rm -rf %s/%s*", parentDir, rmList[i])
-				_, _, err = RetryCmd(cmd, RETRY_TIMES)
-			}
-		}
-	}
-
-	return
-}
-
-func (work *Work) clearPatchLink() (err error) {
-	if work.Service != "" {
-		return
-	}
-	split := len(strings.Split(work.Source, ";"))
-	dataLinkDirs := GetDataLinkDirs(work.DictName, work.Service, work.Version, work.Depend,
-		work.DeployPath, work.ShardSeq, split)
-	for _, linkDir := range dataLinkDirs {
-		parentDir, _ := filepath.Split(linkDir)
-		cmd := fmt.Sprintf("ls -l %s | grep %s_ | awk '{print $9}'", parentDir, work.Depend)
-
-		stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-		if err == nil && stdout != "" {
-			rmList := strings.Split(strings.TrimSpace(stdout), "\n")
-			for i := 0; i < len(rmList); i++ {
-				if rmList[i] == "" {
-					continue
-				}
-				cmd = fmt.Sprintf("rm -rf %s/%s*", parentDir, rmList[i])
-				_, _, err = RetryCmd(cmd, RETRY_TIMES)
-			}
-		}
-	}
-
-	return
-}
-
-func UnTar(dictName, service, version, depend, source, deployPath string, shardSeq int) (err error) {
-	sources := strings.Split(source, ";")
-	downloadDirs, err := GetDownloadDirs(dictName, service, version, depend, deployPath, shardSeq,
-		len(sources))
-	if err != nil {
-		logex.Warningf("UnTar failed, error is (+%v)", err)
-		return
-	}
-	for i := 0; i < len(sources); i++ {
-		fileName := GetFileName(sources[i])
-		untarCmd := fmt.Sprintf("tar xvf %s -C %s", path.Join(downloadDirs[i], fileName), downloadDirs[i])
-		_, _, err = RetryCmd(untarCmd, RETRY_TIMES)
-	}
-
-	return
-}
-
-func CreateSymlink(dictName, service, version, depend, dataPath, deployPath string, shardSeq,
-	split int) (err error) {
-	downloadDirs, err := GetDownloadDirs(dictName, service, version, depend, deployPath, shardSeq, split)
-	if err != nil {
-		logex.Warningf("CreateSymlink failed, error is (+%v)", err)
-	}
-	for i, downloadDir := range downloadDirs {
-		cmd := fmt.Sprintf("ls -l %s | grep -E 'data.|index.' | awk '{print $NF}'", downloadDir)
-		stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-
-		if err == nil && stdout != "" {
-			fileNameLi := strings.Split(strings.TrimSpace(stdout), "\n")
-			versionDir := getParentDir(version, depend)
-			versionFile := path.Join(dataPath, "VERSION")
-			dataSubPath := ""
-			if split > 1 {
-				dataSubPath = path.Join(dataPath, strconv.Itoa(i), versionDir)
-			} else {
-				dataSubPath = path.Join(dataPath, versionDir)
-			}
-			if err = os.MkdirAll(dataSubPath, 0755); err != nil {
-				// return err
-				logex.Warningf("os.Mkdir %s failed, err:[%v]", dataSubPath, err)
-			}
-			if dataSubPath != "" {
-				cmd = fmt.Sprintf("find %s/.. -type d -ctime +5 -print | xargs -i rm -rf {}", dataSubPath)
-				_, _, err = RetryCmd(cmd, RETRY_TIMES)
-			}
-			for _, file := range fileNameLi {
-				dataLink := ""
-				tempDataPath := ""
-				if split > 1 {
-					dataLink = path.Join(dataPath, strconv.Itoa(i), file)
-					tempDataPath = path.Join(dataPath, strconv.Itoa(i))
-				} else {
-					dataLink = path.Join(dataPath, file)
-					tempDataPath = dataPath
-				}
-				cmd = fmt.Sprintf("rm -rf %s", dataLink)
-				_, stderr, _ := RetryCmd(cmd, RETRY_TIMES)
-				logex.Noticef("rm -rf %s, err:[%s]", dataLink, stderr)
-
-				// create new symlink
-				err = os.Symlink(path.Join(downloadDir, file), dataLink)
-				logex.Noticef("os.Symlink %s %s return (%+v)", path.Join(downloadDir, file), dataLink, err)
-				fmt.Println("os.Symlink: ", path.Join(downloadDir, file), dataLink, err)
-				cmd = fmt.Sprintf("cp -d %s/index.* %s/", tempDataPath, dataSubPath)
-				_, stderr, _ = RetryCmd(cmd, RETRY_TIMES)
-				logex.Noticef("cp -d index Symlink to version dir %s, err:[%s]", dataSubPath, stderr)
-				cmd = fmt.Sprintf("cp -d %s/data.* %s/", tempDataPath, dataSubPath)
-				_, stderr, _ = RetryCmd(cmd, RETRY_TIMES)
-				logex.Noticef("cp -d data Symlink to version dir %s, err:[%s]", dataSubPath, stderr)
-			}
-			cmd = fmt.Sprintf("echo %s > %s", versionDir, versionFile)
-			if _, _, err = RetryCmd(cmd, RETRY_TIMES); err != nil {
-				return err
-			}
-		}
-	}
-
-	return
-}
-
-func (work *Work) CheckToReload() bool {
-	statusCmd := fmt.Sprintf("curl -s -d '{\"cmd\":\"status\"}' http://127.0.0.1:%s/ControlService/cmd", work.Port)
-	stdout, _, _ := RetryCmd(statusCmd, RETRY_TIMES)
-	var resp CubeResp
-	json.Unmarshal([]byte(stdout), &resp)
-	version := getParentDir(work.Version, work.Depend)
-
-	if resp.CurVersion == "" && resp.BgVersion == "" {
-		logex.Noticef("cube version empty")
-		return true
-	}
-	if resp.CurVersion == version || resp.BgVersion == version {
-		logex.Noticef("cube version has matched. version: %s", version)
-		return false
-	}
-	return true
-}
-
-func (work *Work) Reload() (err error) {
-	if work.Port == "" {
-		err = errors.New("Reload with invalid port.")
-		return
-	}
-	if !work.CheckToReload() {
-		work.writeStatus("finish_reload", "succ")
-		return
-	}
-	work.writeStatus("prepare_reload", "")
-
-	var stdout string
-	versionPath := getParentDir(work.Version, work.Depend)
-	bgLoadCmd := "bg_load_base"
-	if work.Mode == "delta" {
-		bgLoadCmd = "bg_load_patch"
-	}
-	if work.ActiveVersionList == "" {
-		work.ActiveVersionList = "[]"
-	}
-	for i := 0; i < RELOAD_RETRY_TIMES; i++ {
-		reloadCmd := fmt.Sprintf("curl -o /dev/null -s -w %%{http_code} -d '{\"cmd\":\"%s\",\"version_path\":\"/%s\"}' http://127.0.0.1:%s/ControlService/cmd", bgLoadCmd, versionPath, work.Port)
-		fmt.Println("reload: ", reloadCmd)
-		stdout, _, _ = RetryCmd(reloadCmd, 1)
-		fmt.Println("reload stdout: ", stdout)
-		if strings.TrimSpace(stdout) == "200" {
-			logex.Debugf("bg_load_base return succ")
-			break
-		} else {
-			logex.Warning("bg_load_base return failed")
-			time.Sleep(RELOAD_RETRY_INTERVAL_SECOND * time.Second)
-		}
-	}
-
-	if strings.TrimSpace(stdout) == "200" {
-		work.writeStatus("finish_reload", "succ")
-	} else {
-		work.writeStatus("finish_reload", "failed")
-		err = errors.New("reload failed.")
-	}
-
-	return
-}
-
-func (work *Work) Clear() (err error) {
-	work.Service = ""
-
-	var stdout string
-	var clearCmd string
-	for i := 0; i < RETRY_TIMES; i++ {
-		clearCmd = fmt.Sprintf("curl -o /dev/null -s -w %%{http_code} -d '{\"cmd\":\"clear\",\"table_name\":\"%s\"}' http://127.0.0.1:%s/NodeControlService/cmd", work.DictName, work.Port)
-		fmt.Println("clear: ", clearCmd)
-		stdout, _, _ = RetryCmd(clearCmd, 1)
-		fmt.Println("clear stdout: ", stdout)
-		if strings.TrimSpace(stdout) == "200" {
-			logex.Debugf("clear return succ")
-			break
-		} else {
-			logex.Warning("clear return failed")
-			time.Sleep(RELOAD_RETRY_INTERVAL_SECOND * time.Second)
-		}
-	}
-
-	if strings.TrimSpace(stdout) == "200" {
-		err = work.writeStatus("succ", "")
-	} else {
-		err = work.writeStatus("failed", "")
-	}
-
-	return
-}
-
-func (work *Work) Check() (err error) {
-	if work.Service != "" || work.VersionSign == "" {
-		return
-	}
-	var dataLinkDirs []string
-	split := len(strings.Split(work.Source, ";"))
-	dataLinkDirs = GetDataLinkDirs(work.DictName, work.Service, work.Version, work.Depend,
-		work.DeployPath, work.ShardSeq, split)
-
-	if _, t_err := os.Stat(work.DeployPath); os.IsNotExist(t_err) {
-		logex.Noticef("check DeployPath[%s] not exists.", work.DeployPath)
-		return
-	}
-
-	check_succ := true
-	for _, linkDir := range dataLinkDirs {
-		parentDir, _ := filepath.Split(linkDir)
-
-		cmd := fmt.Sprintf("ls -l %s | grep %s | awk '{print $9}' | grep -v data | grep -v index", parentDir, work.Depend)
-
-		stdout, _, err := RetryCmd(cmd, RETRY_TIMES)
-		if err != nil || stdout == "" {
-			check_succ = false
-			break
-		} else {
-			versionList := strings.Split(strings.TrimSpace(stdout), "\n")
-			logex.Noticef("calc ver_sign for [%v]", versionList)
-
-			var version_sign string
-			var version string
-			for i := 0; i < len(versionList); i++ {
-				split_index := strings.Index(versionList[i], "_")
-				if split_index > 0 && split_index < len(versionList[i]) {
-					version = versionList[i][split_index+1:]
-				} else {
-					version = versionList[i]
-				}
-				if version_sign == "" {
-					version_sign = fmt.Sprintf("%x", md5.Sum([]byte(version)))
-				} else {
-					version_sign = fmt.Sprintf("%x", md5.Sum([]byte(version_sign)))
-				}
-			}
-
-			if version_sign != work.VersionSign {
-				logex.Warningf("version_sign check failed. real[%v] expect[%v]", version_sign, work.VersionSign)
-				check_succ = false
-				break
-			}
-		}
-	}
-
-	if !check_succ {
-		work.clearPatchData()
-		work.clearPatchLink()
-		master_host, master_port, _ := GetMaster(work.MasterAddress)
-		cmd := fmt.Sprintf("cd %s && export STRATEGY_DIR=%s && ./downloader -h %s -p %s -d %s -s %d",
-			work.DeployPath, work.DeployPath, master_host, master_port, work.DictName, work.ShardSeq)
-		_, _, err = RetryCmd(cmd, RETRY_TIMES)
-	}
-
-	return
-}
-
-func (work *Work) Enable() (err error) {
-	if work.Port == "" {
-		err = errors.New("Enable with invalid port")
-		return
-	}
-	var stdout string
-	var cmd string
-	versionPath := getParentDir(work.Version, work.Depend)
-	for i := 0; i < RELOAD_RETRY_TIMES; i++ {
-		if work.Service != "" {
-			cmd = fmt.Sprintf("curl -o /dev/null -s -w %%{http_code} -d '{\"cmd\":\"reload_model\",\"version\":\"%s-%s\",\"dict_name\":\"%s\"}' http://127.0.0.1:%s/ControlService/cmd",
-				versionPath, work.DictName, work.DictName, work.Port)
-		} else {
-			cmd = fmt.Sprintf("curl -o /dev/null -s -w %%{http_code} -d '{\"cmd\":\"enable\",\"version\":\"%s\"}' http://127.0.0.1:%s/ControlService/cmd", versionPath, work.Port)
-		}
-		stdout, _, _ = RetryCmd(cmd, 1)
-
-		if strings.TrimSpace(stdout) == "200" {
-			logex.Debugf("enable return succ for %s, work dir is %s", work.Source, work.DeployPath)
-			break
-		} else {
-			logex.Warningf("enable return failed for %s, work dir is %s, error is (%+v)", work.Source, work.DeployPath, err)
-			time.Sleep(RELOAD_RETRY_INTERVAL_SECOND * time.Second)
-		}
-	}
-
-	if strings.TrimSpace(stdout) == "200" {
-		err = work.writeStatus("succ", "")
-	} else {
-		err = work.writeStatus("failed", "")
-	}
-
-	if work.Service == "" {
-		cmd = fmt.Sprintf("curl -o /dev/null -s -w %%{http_code} -d '{\"cmd\":\"bg_unload\"}' http://127.0.0.1:%s/ControlService/cmd", work.Port)
-		stdout, _, _ = RetryCmd(cmd, RETRY_TIMES)
-		if strings.TrimSpace(stdout) == "200" {
-			logex.Debugf("unload return succ")
-		} else {
-			logex.Warning("unload return failed")
-		}
-	}
-
-	RemoveStateFile(work.DictName, work.ShardSeq, work.Service)
-
-	return
-}
-
-func (work *Work) Pop() (err error) {
-	var stdout string
-	var cmd string
-	if work.ActiveVersionList == "" {
-		work.ActiveVersionList = "[]"
-	}
-	for i := 0; i < RELOAD_RETRY_TIMES; i++ {
-		cmd = fmt.Sprintf("curl -o /dev/null -s -w %%{http_code} -d '{\"cmd\":\"pop\",\"table_name\":\"%s\",\"active_versions\":%v}' http://127.0.0.1:%s/NodeControlService/cmd", work.DictName, work.ActiveVersionList, work.Port)
-		fmt.Println("pop: ", cmd)
-		stdout, _, _ = RetryCmd(cmd, 1)
-		fmt.Println("pop stdout: ", stdout)
-		if strings.TrimSpace(stdout) == "200" {
-			logex.Debugf("pop return succ")
-			break
-		} else {
-			logex.Warning("pop return failed")
-			time.Sleep(RELOAD_RETRY_INTERVAL_SECOND * time.Second)
-		}
-	}
-
-	if strings.TrimSpace(stdout) == "200" {
-		err = work.writeStatus("succ", "")
-	} else {
-		err = work.writeStatus("failed", "")
-	}
-
-	RemoveStateFile(work.DictName, work.ShardSeq, work.Service)
-	return
-}
-
-func writeStateFile(dictName string, shardSeq int, service, state string) {
-	stateFile := fmt.Sprintf(".state_%s_%d", dictName, shardSeq)
-	if service != "" {
-		stateFile = stateFile + "_" + service
-	}
-
-	cmd := fmt.Sprintf("echo '%s' > %s/state/%s", state, Dir, stateFile)
-	if _, _, err := RetryCmd(cmd, RETRY_TIMES); err != nil {
-		logex.Warningf("%s error (%+v)", cmd, err)
-	}
-}
-
-func RemoveStateFile(dictName string, shardSeq int, service string) {
-	stateFile := fmt.Sprintf(".state_%s_%d", dictName, shardSeq)
-	if service != "" {
-		stateFile = stateFile + "_" + service
-	}
-
-	cmd := fmt.Sprintf("rm -f %s/state/%s", Dir, stateFile)
-	if _, _, err := RetryCmd(cmd, RETRY_TIMES); err != nil {
-		logex.Warningf("%s error (%+v)", cmd, err)
-	}
-}
-
-func (work *Work) writeStatus(status string, finishStatus string) (err error) {
-	work.Status = status
-	work.FinishStatus = finishStatus
-	state, _ := json.Marshal(work)
-	writeStateFile(work.DictName, work.ShardSeq, work.Service, string(state))
-	return
-}
-
-func DoDownloadIndividual(source, downloadDir string, isService bool, timeOut int, ch chan error, wg *sync.WaitGroup) {
-	err := errors.New("DoDownloadIndividual start")
-	for i := 0; i < RETRY_TIMES; i++ {
-		err = FtpDownload(source, downloadDir, timeOut)
-		if err == nil {
-			logex.Debugf("download %s to %s succ", source, downloadDir)
-			if !isService {
-				err = FtpDownload(source+".md5", downloadDir, timeOut)
-			}
-		} else {
-			logex.Warningf("download error , source %s, downloadDir %s, err (%+v)", source, downloadDir, err)
-			continue
-		}
-
-		if err == nil && isService {
-			// touch download_succ file
-			cmd := fmt.Sprintf("touch %s", path.Join(downloadDir, DOWNLOAD_DONE_MARK_FILE))
-			RetryCmd(cmd, RETRY_TIMES)
-			break
-		}
-
-		// download md5 file succ, md5check
-		if err == nil {
-			// md5sum -c
-			fileName := GetFileName(source)
-			err = checkMd5(path.Join(downloadDir, fileName), path.Join(downloadDir, fileName+".md5"))
-			logex.Warningf("md5sum check %s %s return (%+v)", path.Join(downloadDir, fileName), path.Join(downloadDir, fileName+".md5"), err)
-			if err == nil {
-				// touch download_succ file
-				cmd := fmt.Sprintf("touch %s", path.Join(downloadDir, DOWNLOAD_DONE_MARK_FILE))
-				RetryCmd(cmd, RETRY_TIMES)
-				logex.Debugf("md5sum ok, source is %s, dir is %s", source, downloadDir)
-				break
-			} else {
-				logex.Warningf("md5sum error, source is %s, dir is %s", source, downloadDir)
-				continue
-			}
-		} else {
-			logex.Warningf("download %s return (%+v)", source+".md5", err)
-			continue
-		}
-	}
-
-	ch <- err
-	wg.Done()
-}
-
-func checkSources(source string) ([]string, error) {
-	sources := strings.Split(source, ";")
-	for i := 0; i < len(sources); i++ {
-		if sources[i] == "" || (!strings.HasPrefix(sources[i], "ftp://") && !strings.HasPrefix(sources[i], "http://")) {
-			return sources, errors.New("Invalid sources")
-		}
-	}
-	return sources, nil
-}
-
-func DoDownload(dictName, service, version, depend, mode, source, deployPath string,
-	shardSeq int) (err error) {
-	sources, err := checkSources(source)
-	if err != nil {
-		logex.Warningf("checkSources %s return (%+v)", source, err)
-		return
-	}
-	downloadDirs, err := GetDownloadDirs(dictName, service, version, depend, deployPath, shardSeq,
-		len(sources))
-	if err != nil {
-		logex.Warningf("GetDownloadDirs %s return (%+v)", source, err)
-		return
-	}
-	version_suffix := ""
-	if service != "" {
-		version_suffix = version_suffix + "-" + dictName
-	}
-	if !checkToDownload(downloadDirs) {
-		cmd := fmt.Sprintf("cd %s/cube_data && echo %s > VERSION && cp VERSION VERSION-%s",
-			deployPath, getParentDir(version, depend)+version_suffix, dictName)
-		_, _, err = RetryCmd(cmd, RETRY_TIMES)
-		logex.Debugf("echo VERSION cmd:[%s] err:[%v]", cmd, err)
-		return
-	}
-
-	ch := make(chan error, len(sources))
-	wg := sync.WaitGroup{}
-	j := 0
-	numCo := 0
-	for ; j < len(sources); j++ {
-		if numCo >= MAX_DOWN_CO {
-			wg.Wait()
-			logex.Noticef("DoDownload co down.")
-			numCo = 0
-		}
-		numCo += 1
-		wg.Add(1)
-		time.Sleep(2000 * time.Millisecond)
-		timeOut := 900
-		if mode == "base" {
-			timeOut = 3600
-		}
-		go DoDownloadIndividual(sources[j], downloadDirs[j], (service != ""), timeOut, ch, &wg)
-	}
-	wg.Wait()
-	close(ch)
-	for err = range ch {
-		if err != nil {
-			return
-		}
-	}
-	cmd := fmt.Sprintf("cd %s/cube_data && echo %s > VERSION && cp VERSION VERSION-%s",
-		deployPath, getParentDir(version, depend)+version_suffix, dictName)
-	_, _, err = RetryCmd(cmd, RETRY_TIMES)
-	logex.Debugf("echo VERSION cmd:[%s] err:[%v]", cmd, err)
-	return
-}
-
-func FtpDownload(source string, dest string, timeOut int) (err error) {
-	dlCmd := fmt.Sprintf("wget --quiet --level=100 -P %s %s --limit-rate=10240k", dest, source)
-	fmt.Println(dlCmd)
-
-	_, _, err = RetryCmdWithSleep(dlCmd, RETRY_TIMES)
-	return
-}
-
-func checkToDownload(downloadDirs []string) bool {
-	for _, v := range downloadDirs {
-		if _, err := os.Stat(path.Join(v, DOWNLOAD_DONE_MARK_FILE)); err != nil {
-			logex.Noticef("check [%v] not exists.", v)
-			return true
-		}
-	}
-
-	return false
-}
-
-// simple hash
-func getDownloadDisk(dictName string, shardSeq int) string {
-	index := len(dictName) * shardSeq % len(disks)
-
-	return disks[index]
-}
-
-func getParentDir(version string, depend string) (dir string) {
-	if version == depend {
-		dir = depend
-	} else {
-		dir = depend + "_" + version
-	}
-
-	return
-}
-
-func GetFileName(source string) (fileName string) {
-	s := strings.Split(source, "/")
-	fileName = s[len(s)-1]
-
-	return
-}
-
-func checkMd5(file string, fileMd5 string) (err error) {
-	cmd := fmt.Sprintf("md5sum %s | awk '{print $1}'", file)
-	stdout, _, _ := pipeline.Run(exec.Command("/bin/sh", "-c", cmd))
-	real_md5 := stdout.String()
-	cmd = fmt.Sprintf("cat %s | awk '{print $1}'", fileMd5)
-	stdout, _, _ = pipeline.Run(exec.Command("/bin/sh", "-c", cmd))
-	given_md5 := stdout.String()
-
-	if real_md5 != given_md5 {
-		logex.Warningf("checkMd5 failed real_md5[%s] given_md5[%s]", real_md5, given_md5)
-		err = errors.New("checkMd5 failed")
-	}
-
-	return
-}
-
-func RetryCmd(cmd string, retryTimes int) (stdoutStr string, stderrStr string, err error) {
-	for i := 0; i < retryTimes; i++ {
-		stdout, stderr, e := pipeline.Run(exec.Command("/bin/sh", "-c", cmd))
-		stdoutStr = stdout.String()
-		stderrStr = stderr.String()
-		err = e
-
-		logex.Debugf("cmd %s, stdout: %s, stderr: %s, err: (%+v)", cmd, stdoutStr, stderrStr, err)
-		if err == nil {
-			break
-		}
-	}
-
-	return
-}
-
-func RetryCmdWithSleep(cmd string, retryTimes int) (stdoutStr string, stderrStr string, err error) {
-	for i := 0; i < retryTimes; i++ {
-		stdout, stderr, e := pipeline.Run(exec.Command("/bin/sh", "-c", cmd))
-		stdoutStr = stdout.String()
-		stderrStr = stderr.String()
-		err = e
-
-		logex.Debugf("cmd %s, stdout: %s, stderr: %s, err: (%+v)", cmd, stdoutStr, stderrStr, err)
-		if err == nil {
-			break
-		}
-		time.Sleep(10000 * time.Millisecond)
-	}
-
-	return
-}

core/cube/cube-agent/src/agent/work_pool.go

-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package agent
-
-import (
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-)
-
-type (
-	workType struct {
-		poolWorker    PoolWorker
-		resultChannel chan error
-	}
-
-	WorkPool struct {
-		queueChannel    chan workType
-		workChannel     chan PoolWorker
-		queuedWorkNum   int32
-		activeWorkerNum int32
-		queueCapacity   int32
-		workFilter      sync.Map
-	}
-)
-
-type PoolWorker interface {
-	Token() string
-	DoWork()
-}
-
-func NewWorkPool(workerNum int, queueCapacity int32) *WorkPool {
-	workPool := WorkPool{
-		queueChannel:    make(chan workType),
-		workChannel:     make(chan PoolWorker, queueCapacity),
-		queuedWorkNum:   0,
-		activeWorkerNum: 0,
-		queueCapacity:   queueCapacity,
-	}
-
-	for i := 0; i < workerNum; i++ {
-		go workPool.startWorkRoutine()
-	}
-
-	go workPool.startQueueRoutine()
-
-	return &workPool
-}
-
-func (workPool *WorkPool) startWorkRoutine() {
-	for {
-		select {
-		case work := <-workPool.workChannel:
-			workPool.doWork(work)
-			break
-		}
-	}
-}
-
-func (workPool *WorkPool) startQueueRoutine() {
-	for {
-		select {
-		case queueItem := <-workPool.queueChannel:
-			if atomic.AddInt32(&workPool.queuedWorkNum, 0) == workPool.queueCapacity {
-				queueItem.resultChannel <- fmt.Errorf("work pool fulled with %v pending works", QueueCapacity)
-				continue
-			}
-
-			atomic.AddInt32(&workPool.queuedWorkNum, 1)
-
-			workPool.workChannel <- queueItem.poolWorker
-
-			queueItem.resultChannel <- nil
-
-			break
-		}
-	}
-}
-
-func (workPool *WorkPool) doWork(poolWorker PoolWorker) {
-	defer atomic.AddInt32(&workPool.activeWorkerNum, -1)
-	defer workPool.workFilter.Delete(poolWorker.Token())
-
-	atomic.AddInt32(&workPool.queuedWorkNum, -1)
-	atomic.AddInt32(&workPool.activeWorkerNum, 1)
-
-	poolWorker.DoWork()
-}
-
-func (workPool *WorkPool) PostWorkWithToken(poolWorker PoolWorker) (err error) {
-	if _, ok := workPool.workFilter.Load(poolWorker.Token()); ok {
-		return errors.New("another work with same key is doing.")
-	}
-	workPool.workFilter.Store(poolWorker.Token(), true)
-	return workPool.PostWork(poolWorker)
-}
-
-func (workPool *WorkPool) PostWork(poolWorker PoolWorker) (err error) {
-	work := workType{poolWorker, make(chan error)}
-
-	defer close(work.resultChannel)
-
-	workPool.queueChannel <- work
-
-	err = <-work.resultChannel
-
-	return err
-}

core/cube/cube-agent/src/cube-agent.go

-// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"agent"
-	"fmt"
-	"github.com/Badangel/logex"
-	"github.com/docopt/docopt-go"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strconv"
-)
-
-func main() {
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	agent.Dir, _ = filepath.Abs(filepath.Dir(os.Args[0]))
-	usage := fmt.Sprintf(`Usage: ./m_master [options]
-
-Options:
-  -n WORKERNUM    set worker num.
-  -q QUEUENUM     set queue num.
-  -P LISTEN_PORT  agent listen port
-
-Log options:
-  -l LOG_LEVEL      set log level, values: 0,1,2,4,8,16. [default: 16]
-  --log_dir=DIR     set log output dir. [default: ./log]
-  --log_name=NAME   set log name. [default: m_agent]`, agent.Dir)
-
-	opts, err := docopt.Parse(usage, nil, true, "Cube Agent Checker 1.0.0", false)
-	if err != nil {
-		fmt.Println("ERROR:", err)
-		os.Exit(1)
-	}
-
-	log_level, _ := strconv.Atoi(opts["-l"].(string))
-	log_name := opts["--log_name"].(string)
-	log_dir := opts["--log_dir"].(string)
-	logex.SetLevel(getLogLevel(log_level))
-	if err := logex.SetUpFileLogger(log_dir, log_name, nil); err != nil {
-		fmt.Println("ERROR:", err)
-	}
-
-	logex.Notice("--- NEW SESSION -------------------------")
-	logex.Notice(">>> log_level:", log_level)
-
-	agent.WorkerNum = 10
-	if opts["-n"] != nil {
-		n, err := strconv.Atoi(opts["-n"].(string))
-		if err == nil {
-			agent.WorkerNum = n
-		}
-	}
-
-	agent.QueueCapacity = 20
-	if opts["-q"] != nil {
-		q, err := strconv.Atoi(opts["-q"].(string))
-		if err == nil {
-			agent.QueueCapacity = int32(q)
-		}
-	}
-
-	agent.CmdWorkPool = agent.NewWorkPool(agent.WorkerNum, agent.QueueCapacity)
-
-	if opts["-P"] == nil {
-		logex.Fatalf("ERROR: -P LISTEN PORT must be set!")
-		os.Exit(255)
-	}
-
-	agentPort := opts["-P"].(string)
-	logex.Notice(">>> starting server...")
-	addr := ":" + agentPort
-
-	if agent.StartHttp(addr) != nil {
-		logex.Noticef("cant start http(addr=%v). quit.", addr)
-		os.Exit(0)
-	}
-}
-
-func getLogLevel(log_level int) logex.Level {
-	switch log_level {
-	case 16:
-		return logex.DEBUG
-	case 8:
-		return logex.TRACE
-	case 4:
-		return logex.NOTICE
-	case 2:
-		return logex.WARNING
-	case 1:
-		return logex.FATAL
-	case 0:
-		return logex.NONE
-	}
-	return logex.DEBUG
-}

core/general-client/include/general_model.h

@@ -45,7 +45,7 @@ class PredictorClient {
   PredictorClient() {}
   ~PredictorClient() {}
 
-  void init(const std::string& client_conf);
+  int init(const std::string& client_conf);
 
   void set_predictor_conf(const std::string& conf_path,
                           const std::string& conf_file);

core/general-client/src/general_model.cpp

@@ -27,45 +27,42 @@ using baidu::paddle_serving::predictor::general_model::FetchInst;
 namespace baidu {
 namespace paddle_serving {
 namespace general_model {
+using configure::GeneralModelConfig;
 
-void PredictorClient::init(const std::string &conf_file) {
-  _conf_file = conf_file;
-  std::ifstream fin(conf_file);
-  if (!fin) {
-    LOG(ERROR) << "Your inference conf file can not be found";
-    exit(-1);
-  }
-  _feed_name_to_idx.clear();
-  _fetch_name_to_idx.clear();
-  _shape.clear();
-  int feed_var_num = 0;
-  int fetch_var_num = 0;
-  fin >> feed_var_num >> fetch_var_num;
-  std::string name;
-  std::string fetch_var_name;
-  int shape_num = 0;
-  int dim = 0;
-  int type_value = 0;
-  for (int i = 0; i < feed_var_num; ++i) {
-    fin >> name;
-    _feed_name_to_idx[name] = i;
-    fin >> shape_num;
-    std::vector<int> tmp_feed_shape;
-    for (int j = 0; j < shape_num; ++j) {
-      fin >> dim;
-      tmp_feed_shape.push_back(dim);
+int PredictorClient::init(const std::string &conf_file) {
+  try {
+    GeneralModelConfig model_config;
+    if (configure::read_proto_conf(conf_file.c_str(),
+                                   &model_config) != 0) {
+      LOG(ERROR) << "Failed to load general model config"
+                 << ", file path: " << conf_file;
+      return -1;
+    }
+    _feed_name_to_idx.clear();
+    _fetch_name_to_idx.clear();
+    _shape.clear();
+    int feed_var_num = model_config.feed_var_size();
+    int fetch_var_num = model_config.fetch_var_size();
+    for (int i = 0; i < feed_var_num; ++i) {
+      _feed_name_to_idx[model_config.feed_var(i).alias_name()] = i;
+      std::vector<int> tmp_feed_shape;
+      for (int j = 0; j < model_config.feed_var(i).shape_size(); ++j) {
+        tmp_feed_shape.push_back(model_config.feed_var(i).shape(j));
+      }
+      _type.push_back(model_config.feed_var(i).feed_type());
+      _shape.push_back(tmp_feed_shape);
     }
-    fin >> type_value;
-    _type.push_back(type_value);
-    _shape.push_back(tmp_feed_shape);
-  }
 
-  for (int i = 0; i < fetch_var_num; ++i) {
-    fin >> name;
-    fin >> fetch_var_name;
-    _fetch_name_to_idx[name] = i;
-    _fetch_name_to_var_name[name] = fetch_var_name;
+    for (int i = 0; i < fetch_var_num; ++i) {
+      _fetch_name_to_idx[model_config.fetch_var(i).alias_name()] = i;
+      _fetch_name_to_var_name[model_config.fetch_var(i).alias_name()] =
+          model_config.fetch_var(i).name();
+    }
+  } catch (std::exception& e) {
+    LOG(ERROR) << "Failed load general model config" << e.what();
+    return -1;
   }
+  return 0;
 }
 
 void PredictorClient::set_predictor_conf(const std::string &conf_path,

core/general-client/src/pybind_general_model.cpp

@@ -33,7 +33,7 @@ PYBIND11_MODULE(serving_client, m) {
       .def(py::init())
       .def("init",
            [](PredictorClient &self, const std::string &conf) {
-             self.init(conf);
+             return self.init(conf);
            })
       .def("set_predictor_conf",
            [](PredictorClient &self,

core/general-server/CMakeLists.txt

+include_directories(SYSTEM  ${CMAKE_CURRENT_LIST_DIR}/../kvdb/include)
+include(op/CMakeLists.txt)
+include(proto/CMakeLists.txt)
+add_executable(serving ${serving_srcs})
+add_dependencies(serving pdcodegen fluid_cpu_engine pdserving paddle_fluid
+        opencv_imgcodecs cube-api)
+if (WITH_GPU)
+    add_dependencies(serving fluid_gpu_engine)
+endif()
+target_include_directories(serving PUBLIC
+        ${CMAKE_CURRENT_BINARY_DIR}/../../core/predictor
+        )
+
+if(WITH_GPU)
+    target_link_libraries(serving -Wl,--whole-archive fluid_gpu_engine
+            -Wl,--no-whole-archive)
+endif()
+
+target_link_libraries(serving -Wl,--whole-archive fluid_cpu_engine
+        -Wl,--no-whole-archive)
+
+target_link_libraries(serving paddle_fluid ${paddle_depend_libs})
+
+target_link_libraries(serving pdserving)
+target_link_libraries(serving cube-api)
+
+target_link_libraries(serving kvdb rocksdb)
+
+if(WITH_GPU)
+    target_link_libraries(serving ${CUDA_LIBRARIES})
+endif()
+
+if(WITH_MKL)
+    target_link_libraries(serving -liomp5 -lmklml_intel -lmkldnn -lpthread -lcrypto -lm -lrt -lssl -ldl -lz -lbz2)
+else()
+    target_link_libraries(serving openblas -lpthread -lcrypto -lm -lrt -lssl -ldl -lz -lbz2)
+endif()
+
+install(TARGETS serving
+        RUNTIME DESTINATION
+        ${PADDLE_SERVING_INSTALL_DIR}/demo/serving/bin)
+install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/conf DESTINATION
+        ${PADDLE_SERVING_INSTALL_DIR}/demo/serving/)
+
+FILE(GLOB inc ${CMAKE_CURRENT_BINARY_DIR}/*.pb.h)
+install(FILES ${inc}
+        DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/serving)
+
+if (${WITH_MKL})
+install(FILES
+        ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mklml/lib/libmklml_intel.so
+        ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mklml/lib/libiomp5.so
+        ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mkldnn/lib/libmkldnn.so.0
+        DESTINATION
+        ${PADDLE_SERVING_INSTALL_DIR}/demo/serving/bin)
+endif()

core/general-server/op/CMakeLists.txt

+FILE(GLOB op_srcs ${CMAKE_CURRENT_LIST_DIR}/*.cpp)
+LIST(APPEND serving_srcs ${op_srcs})

examples/demo-serving/op/general_infer_op.cpp → core/general-server/op/general_infer_op.cpp

@@ -12,15 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "examples/demo-serving/op/general_infer_op.h"
 #include <algorithm>
 #include <iostream>
 #include <memory>
 #include <sstream>
+#include "core/general-server/op/general_infer_op.h"
+#include "core/general-server/op/general_reader_op.h"
 #include "core/predictor/framework/infer.h"
 #include "core/predictor/framework/memory.h"
 #include "core/predictor/framework/resource.h"
-#include "examples/demo-serving/op/general_reader_op.h"
+
 
 namespace baidu {
 namespace paddle_serving {

examples/demo-serving/op/general_infer_op.h → core/general-server/op/general_infer_op.h

@@ -23,7 +23,7 @@
 #else
 #include "paddle_inference_api.h"  // NOLINT
 #endif
-#include "examples/demo-serving/general_model_service.pb.h"
+#include "core/general-server/general_model_service.pb.h"
 
 namespace baidu {
 namespace paddle_serving {

examples/demo-serving/op/general_reader_op.cpp → core/general-server/op/general_reader_op.cpp

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "examples/demo-serving/op/general_reader_op.h"
 #include <algorithm>
 #include <iostream>
 #include <memory>
 #include <sstream>
+#include "core/general-server/op/general_reader_op.h"
 #include "core/predictor/framework/infer.h"
 #include "core/predictor/framework/memory.h"
 
@@ -34,6 +34,8 @@ int conf_check(const Request *req,
                const std::shared_ptr<PaddleGeneralModelConfig> &model_config) {
   int var_num = req->insts(0).tensor_array_size();
   if (var_num != model_config->_feed_type.size()) {
+    VLOG(2) << "var num: " << var_num;
+    VLOG(2) << "model config var num: " << model_config->_feed_type.size();
     LOG(ERROR) << "feed var number not match.";
     return -1;
   }
@@ -84,7 +86,7 @@ int GeneralReaderOp::inference() {
   }
 
   int var_num = req->insts(0).tensor_array_size();
-  VLOG(3) << "var num: " << var_num;
+  VLOG(2) << "var num: " << var_num;
   // read config
 
   LOG(INFO) << "start to call load general model_conf op";
@@ -112,7 +114,7 @@ int GeneralReaderOp::inference() {
   paddle::PaddleTensor lod_tensor;
   for (int i = 0; i < var_num; ++i) {
     elem_type[i] = req->insts(0).tensor_array(i).elem_type();
-    VLOG(3) << "var[" << i << "] has elem type: " << elem_type[i];
+    VLOG(2) << "var[" << i << "] has elem type: " << elem_type[i];
     if (elem_type[i] == 0) {  // int64
       elem_size[i] = sizeof(int64_t);
       lod_tensor.dtype = paddle::PaddleDType::INT64;
@@ -124,17 +126,17 @@ int GeneralReaderOp::inference() {
     if (req->insts(0).tensor_array(i).shape(0) == -1) {
       lod_tensor.lod.resize(1);
       lod_tensor.lod[0].push_back(0);
-      VLOG(3) << "var[" << i << "] is lod_tensor";
+      VLOG(2) << "var[" << i << "] is lod_tensor";
     } else {
       lod_tensor.shape.push_back(batch_size);
       capacity[i] = 1;
       for (int k = 0; k < req->insts(0).tensor_array(i).shape_size(); ++k) {
         int dim = req->insts(0).tensor_array(i).shape(k);
-        VLOG(3) << "shape for var[" << i << "]: " << dim;
+        VLOG(2) << "shape for var[" << i << "]: " << dim;
         capacity[i] *= dim;
         lod_tensor.shape.push_back(dim);
       }
-      VLOG(3) << "var[" << i << "] is tensor, capacity: " << capacity[i];
+      VLOG(2) << "var[" << i << "] is tensor, capacity: " << capacity[i];
     }
     if (i == 0) {
       lod_tensor.name = "words";
@@ -149,19 +151,19 @@ int GeneralReaderOp::inference() {
       for (int j = 0; j < batch_size; ++j) {
         const Tensor &tensor = req->insts(j).tensor_array(i);
         int data_len = tensor.data_size();
-        VLOG(3) << "tensor size for var[" << i << "]: " << tensor.data_size();
+        VLOG(2) << "tensor size for var[" << i << "]: " << tensor.data_size();
         int cur_len = in->at(i).lod[0].back();
-        VLOG(3) << "current len: " << cur_len;
+        VLOG(2) << "current len: " << cur_len;
         in->at(i).lod[0].push_back(cur_len + data_len);
-        VLOG(3) << "new len: " << cur_len + data_len;
+        VLOG(2) << "new len: " << cur_len + data_len;
       }
       in->at(i).data.Resize(in->at(i).lod[0].back() * elem_size[i]);
       in->at(i).shape = {in->at(i).lod[0].back(), 1};
-      VLOG(3) << "var[" << i
+      VLOG(2) << "var[" << i
               << "] is lod_tensor and len=" << in->at(i).lod[0].back();
     } else {
       in->at(i).data.Resize(batch_size * capacity[i] * elem_size[i]);
-      VLOG(3) << "var[" << i
+      VLOG(2) << "var[" << i
               << "] is tensor and capacity=" << batch_size * capacity[i];
     }
   }
@@ -198,14 +200,14 @@ int GeneralReaderOp::inference() {
     }
   }
 
-  VLOG(3) << "read data from client success";
+  VLOG(2) << "read data from client success";
   // print request
   std::ostringstream oss;
   int64_t *example = reinterpret_cast<int64_t *>((*in)[0].data.data());
   for (int i = 0; i < 10; i++) {
     oss << *(example + i) << " ";
   }
-  VLOG(3) << "head element of first feed var : " << oss.str();
+  VLOG(2) << "head element of first feed var : " << oss.str();
   //
   return 0;
 }

examples/demo-serving/op/general_reader_op.h → core/general-server/op/general_reader_op.h

@@ -25,8 +25,8 @@
 #endif
 #include <string>
 #include "core/predictor/framework/resource.h"
-#include "examples/demo-serving/general_model_service.pb.h"
-#include "examples/demo-serving/load_general_model_service.pb.h"
+#include "core/general-server/general_model_service.pb.h"
+#include "core/general-server/load_general_model_service.pb.h"
 
 namespace baidu {
 namespace paddle_serving {

core/general-server/proto/CMakeLists.txt

+LIST(APPEND protofiles
+	${CMAKE_CURRENT_LIST_DIR}/load_general_model_service.proto
+	${CMAKE_CURRENT_LIST_DIR}/general_model_service.proto
+)
+
+PROTOBUF_GENERATE_SERVING_CPP(TRUE PROTO_SRCS PROTO_HDRS ${protofiles})
+LIST(APPEND serving_srcs ${PROTO_SRCS})

core/cube/cube-agent/src/agent/http_post.go → core/general-server/proto/general_model_service.proto

@@ -12,39 +12,37 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package agent
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/Badangel/logex"
-)
-
-func initPostHandlers() {
-	postHandler = map[string]handlerFunc{
-		"/agent/cmd": PostCmd,
-	}
-}
-
-func PostCmd(subpath string, m map[string]string, b []byte) (string, string, error) {
-	var work Work
-	err := json.Unmarshal(b, &work)
-	if err != nil {
-		logex.Warningf("Unmarshal from %s error (+%v)", string(b), err)
-		return quote(""), "", fmt.Errorf("Work json unmarshal work failed, %v", err)
-	}
-
-	if _, ok := CmdWorkFilter.Load(work.Token()); ok {
-		logex.Warningf("Another work with same token is doing. Token(%s)", work.Token())
-		return quote(""), "", fmt.Errorf("Another work with same key is doing.", err)
-	}
-
-	CmdWorkFilter.Store(work.Token(), true)
-	err = work.DoWork()
-	CmdWorkFilter.Delete(work.Token())
-	if err != nil {
-		return quote(""), "", fmt.Errorf("Do work failed.", err)
-	}
-
-	return quote(""), "", err
-}
+syntax = "proto2";
+import "pds_option.proto";
+import "builtin_format.proto";
+package baidu.paddle_serving.predictor.general_model;
+
+option cc_generic_services = true;
+
+message Tensor {
+  repeated bytes data = 1;
+  optional int32 elem_type = 2;
+  repeated int32 shape = 3;
+};
+
+message FeedInst {
+  repeated Tensor tensor_array = 1;
+};
+
+message FetchInst {
+  repeated Tensor tensor_array = 1;
+};
+
+message Request {
+  repeated FeedInst insts = 1;
+};
+
+message Response {
+  repeated FetchInst insts = 1;
+};
+
+service GeneralModelService {
+  rpc inference(Request) returns (Response);
+  rpc debug(Request) returns (Response);
+  option (pds.options).generate_impl = true;
+};

core/cube/cube-agent/src/agent/http_get.go → core/general-server/proto/load_general_model_service.proto

@@ -12,24 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package agent
+syntax = "proto2";
+import "pds_option.proto";
+package baidu.paddle_serving.predictor.load_general_model_service;
 
-import (
-	"encoding/json"
-	"fmt"
-)
+option cc_generic_services = true;
 
-func initGetHandlers() {
-	getHandler = map[string]handlerFunc{
-		"/agent/status": GetStatus,
-	}
-}
+message RequestAndResponse {
+  required int32 a = 1;
+  required float b = 2;
+};
 
-func GetStatus(subpath string, m map[string]string, b []byte) (string, string, error) {
-	b, err := json.Marshal(BUILTIN_STATUS)
-	if err != nil {
-		return quote(""), "", fmt.Errorf("json marshal failed, %v", err)
-	}
-
-	return string(b), "", err
-}
+service LoadGeneralModelService {
+  rpc inference(RequestAndResponse) returns (RequestAndResponse);
+  rpc debug(RequestAndResponse) returns (RequestAndResponse);
+  option (pds.options).generate_impl = true;
+};

core/predictor/common/constant.cpp

@@ -38,12 +38,12 @@ DEFINE_int32(
     0,
     "Number of pthreads that server runs on, not change if this value <= 0");
 DEFINE_int32(reload_interval_s, 10, "");
-DEFINE_bool(enable_model_toolkit, false, "enable model toolkit");
+DEFINE_bool(enable_model_toolkit, true, "enable model toolkit");
 DEFINE_string(enable_protocol_list, "baidu_std", "set protocol list");
 DEFINE_bool(enable_cube, false, "enable cube");
 DEFINE_string(general_model_path, "./conf", "");
 DEFINE_string(general_model_file, "general_model.prototxt", "");
-DEFINE_bool(enable_general_model, false, "enable general model");
+DEFINE_bool(enable_general_model, true, "enable general model");
 
 const char* START_OP_NAME = "startup_op";
 }  // namespace predictor

core/predictor/framework/resource.cpp

@@ -155,8 +155,11 @@ int Resource::initialize(const std::string& path, const std::string& file) {
 // model config
 int Resource::general_model_initialize(const std::string& path,
                                        const std::string& file) {
+  VLOG(2) << "general model path: " << path;
+  VLOG(2) << "general model file: " << file;
   if (!FLAGS_enable_general_model) {
-    return 0;
+    LOG(ERROR) << "general model is not enabled";
+    return -1;
   }
   ResourceConf resource_conf;
   if (configure::read_proto_conf(path, file, &resource_conf) != 0) {
@@ -183,6 +186,8 @@ int Resource::general_model_initialize(const std::string& path,
 
   _config.reset(new PaddleGeneralModelConfig());
   int feed_var_num = model_config.feed_var_size();
+  VLOG(2) << "load general model config";
+  VLOG(2) << "feed var num: " << feed_var_num;
   _config->_feed_name.resize(feed_var_num);
   _config->_feed_type.resize(feed_var_num);
   _config->_is_lod_feed.resize(feed_var_num);
@@ -190,15 +195,23 @@ int Resource::general_model_initialize(const std::string& path,
   _config->_feed_shape.resize(feed_var_num);
   for (int i = 0; i < feed_var_num; ++i) {
     _config->_feed_name[i] = model_config.feed_var(i).name();
+    VLOG(2) << "feed var[" << i << "]: "
+            << _config->_feed_name[i];
     _config->_feed_type[i] = model_config.feed_var(i).feed_type();
+    VLOG(2) << "feed type[" << i << "]: "
+            << _config->_feed_type[i];
+
     if (model_config.feed_var(i).is_lod_tensor()) {
+      VLOG(2) << "var[" << i << "] is lod tensor";
       _config->_feed_shape[i] = {-1};
       _config->_is_lod_feed[i] = true;
     } else {
+      VLOG(2) << "var[" << i << "] is tensor";
       _config->_capacity[i] = 1;
       _config->_is_lod_feed[i] = false;
       for (int j = 0; j < model_config.feed_var(i).shape_size(); ++j) {
         int32_t dim = model_config.feed_var(i).shape(j);
+        VLOG(2) << "var[" << i << "].shape[" << i << "]: " << dim;
         _config->_feed_shape[i].push_back(dim);
         _config->_capacity[i] *= dim;
       }

core/predictor/src/pdserving.cpp

@@ -126,10 +126,6 @@ int main(int argc, char** argv) {
     return 0;
   }
 
-  if (!FLAGS_g) {
-    google::SetCommandLineOption("flagfile", "conf/gflags.conf");
-  }
-
   google::ParseCommandLineFlags(&argc, &argv, true);
 
   g_change_server_port();

core/sdk-cpp/include/common.h

@@ -55,6 +55,7 @@
 
 #include "core/configure/include/configure_parser.h"
 #include "core/configure/sdk_configure.pb.h"
+#include "core/configure/general_model_config.pb.h"
 
 #include "core/sdk-cpp/include/utils.h"
 

examples/demo-serving/CMakeLists.txt

@@ -71,8 +71,14 @@ target_link_libraries(serving kvdb rocksdb)
 if(WITH_GPU)
     target_link_libraries(serving ${CUDA_LIBRARIES})
 endif()
-target_link_libraries(serving -liomp5 -lmklml_intel -lmkldnn -lpthread
-        -lcrypto -lm -lrt -lssl -ldl -lz -lbz2)
+
+if(WITH_MKL)
+    message("lalalala: " ${WITH_MKL})
+    target_link_libraries(serving -liomp5 -lmklml_intel -lmkldnn -lpthread -lcrypto -lm -lrt -lssl -ldl -lz -lbz2)
+else()
+    message("hehehehe: " ${WITH_MKL})
+    target_link_libraries(serving openblas -lpthread -lcrypto -lm -lrt -lssl -ldl -lz -lbz2)
+endif()
 
 install(TARGETS serving
         RUNTIME DESTINATION
@@ -85,10 +91,10 @@ install(FILES ${inc}
         DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/serving)
 
 if (${WITH_MKL})
-    install(FILES
-            ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mklml/lib/libmklml_intel.so
-                ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mklml/lib/libiomp5.so
-                ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mkldnn/lib/libmkldnn.so.0
-                DESTINATION
-                ${PADDLE_SERVING_INSTALL_DIR}/demo/serving/bin)
+install(FILES
+        ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mklml/lib/libmklml_intel.so
+        ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mklml/lib/libiomp5.so
+        ${CMAKE_BINARY_DIR}/third_party/install/Paddle/third_party/install/mkldnn/lib/libmkldnn.so.0
+        DESTINATION
+        ${PADDLE_SERVING_INSTALL_DIR}/demo/serving/bin)
 endif()

examples/demo-serving/conf/general_model.prototxt

-feed_var {
-    name: "words"
-    is_lod_tensor: true
-    feed_type: 0
+is_lod_feed: true
+is_lod_feed: false
+is_lod_feed: true
+feed_type: 1
+feed_type: 0
+feed_type: 1
+
+feed_shape {
     shape: -1
 }
-feed_var {
-    name: "label"
-    is_lod_tensor: false
-    feed_type: 0
-    shape: 1    
-}
-fetch_var {
-    name: "cost"
-    shape: 1
-}
-fetch_var {
-    name: "acc"
+
+feed_shape {
     shape: 1
-}
-fetch_var {
-    name: "prediction"
     shape: 2
+    shape: 3
+}
+
+feed_shape {
+    shape: -1
 }

examples/demo-serving/conf/gflags.conf

 --enable_model_toolkit
 --enable_cube=false
 --enable_general_model=true
+--general_model_path=./conf
+--general_model_file=general_model.prototxt

examples/demo-serving/conf/resource.prototxt

 model_toolkit_path: "./conf/"
 model_toolkit_file: "model_toolkit.prototxt"
 cube_config_file: "./conf/cube.conf"
-general_model_path: "./conf/"
-general_model_file: "general_model.prototxt"
-

examples/demo-serving/conf/service.prototxt

@@ -40,6 +40,6 @@ services {
   workflows: "workflow9"
 }
 services {
-  name: "GeneralModelService"
-  workflows: "workflow11"
-}
+  name: "LoadGeneralModelService"
+  workflows: "workflow10"
+}
\ No newline at end of file

examples/demo-serving/conf/workflow.prototxt

@@ -95,24 +95,7 @@ workflows {
   name: "workflow10"
   workflow_type: "Sequence"
   nodes {
-    name: "general_model_op"
-    type: "GeneralModelOp"
-  }
-}
-
-workflows {
-  name: "workflow11"
-  workflow_type: "Sequence"
-  nodes {
-    name: "general_reader_op"
-    type: "GeneralReaderOp"
-  }
-  nodes {
-    name: "general_infer_op"
-    type: "GeneralInferOp"
-    dependencies {
-      name: "general_reader_op"
-      mode: "RO"
-    }
+    name: "load_general_model_conf_op"
+    type: "LoadGeneralModelConfOp"
   }
 }

examples/demo-serving/kvdb/CURRENT

-MANIFEST-000020

examples/demo-serving/kvdb/IDENTITY

-13d73780-de4f-4b8c-9040-34e5adc9f9ae

examples/demo-serving/kvdb/LOG.old.1578712592954203

-2020/01/10-09:39:03.152794 7f45d8eaa440 RocksDB version: 6.2.4
-2020/01/10-09:39:03.152835 7f45d8eaa440 Git sha rocksdb_build_git_sha:3513d4e93f8530ac44cc0c1efea04be4259c6938
-2020/01/10-09:39:03.152838 7f45d8eaa440 Compile date Dec 23 2019
-2020/01/10-09:39:03.152842 7f45d8eaa440 DB SUMMARY
-2020/01/10-09:39:03.152858 7f45d8eaa440 SST files in kvdb dir, Total Num: 0, files: 
-2020/01/10-09:39:03.152861 7f45d8eaa440 Write Ahead Log file in kvdb: 
-2020/01/10-09:39:03.152864 7f45d8eaa440                         Options.error_if_exists: 0
-2020/01/10-09:39:03.152866 7f45d8eaa440                       Options.create_if_missing: 1
-2020/01/10-09:39:03.152867 7f45d8eaa440                         Options.paranoid_checks: 1
-2020/01/10-09:39:03.152868 7f45d8eaa440                                     Options.env: 0x1bed2ac0
-2020/01/10-09:39:03.152870 7f45d8eaa440                                Options.info_log: 0x206faac0
-2020/01/10-09:39:03.152871 7f45d8eaa440                Options.max_file_opening_threads: 16
-2020/01/10-09:39:03.152872 7f45d8eaa440                              Options.statistics: (nil)
-2020/01/10-09:39:03.152874 7f45d8eaa440                               Options.use_fsync: 0
-2020/01/10-09:39:03.152875 7f45d8eaa440                       Options.max_log_file_size: 0
-2020/01/10-09:39:03.152876 7f45d8eaa440                  Options.max_manifest_file_size: 1073741824
-2020/01/10-09:39:03.152877 7f45d8eaa440                   Options.log_file_time_to_roll: 0
-2020/01/10-09:39:03.152879 7f45d8eaa440                       Options.keep_log_file_num: 1000
-2020/01/10-09:39:03.152880 7f45d8eaa440                    Options.recycle_log_file_num: 0
-2020/01/10-09:39:03.152881 7f45d8eaa440                         Options.allow_fallocate: 1
-2020/01/10-09:39:03.152882 7f45d8eaa440                        Options.allow_mmap_reads: 0
-2020/01/10-09:39:03.152883 7f45d8eaa440                       Options.allow_mmap_writes: 0
-2020/01/10-09:39:03.152884 7f45d8eaa440                        Options.use_direct_reads: 0
-2020/01/10-09:39:03.152886 7f45d8eaa440                        Options.use_direct_io_for_flush_and_compaction: 0
-2020/01/10-09:39:03.152887 7f45d8eaa440          Options.create_missing_column_families: 0
-2020/01/10-09:39:03.152890 7f45d8eaa440                              Options.db_log_dir: 
-2020/01/10-09:39:03.152891 7f45d8eaa440                                 Options.wal_dir: kvdb
-2020/01/10-09:39:03.152892 7f45d8eaa440                Options.table_cache_numshardbits: 6
-2020/01/10-09:39:03.152893 7f45d8eaa440                      Options.max_subcompactions: 1
-2020/01/10-09:39:03.152894 7f45d8eaa440                  Options.max_background_flushes: -1
-2020/01/10-09:39:03.152896 7f45d8eaa440                         Options.WAL_ttl_seconds: 0
-2020/01/10-09:39:03.152897 7f45d8eaa440                       Options.WAL_size_limit_MB: 0
-2020/01/10-09:39:03.152898 7f45d8eaa440             Options.manifest_preallocation_size: 4194304
-2020/01/10-09:39:03.152900 7f45d8eaa440                     Options.is_fd_close_on_exec: 1
-2020/01/10-09:39:03.152901 7f45d8eaa440                   Options.advise_random_on_open: 1
-2020/01/10-09:39:03.152902 7f45d8eaa440                    Options.db_write_buffer_size: 0
-2020/01/10-09:39:03.152903 7f45d8eaa440                    Options.write_buffer_manager: 0x206fab20
-2020/01/10-09:39:03.152905 7f45d8eaa440         Options.access_hint_on_compaction_start: 1
-2020/01/10-09:39:03.152906 7f45d8eaa440  Options.new_table_reader_for_compaction_inputs: 0
-2020/01/10-09:39:03.152907 7f45d8eaa440           Options.random_access_max_buffer_size: 1048576
-2020/01/10-09:39:03.152908 7f45d8eaa440                      Options.use_adaptive_mutex: 0
-2020/01/10-09:39:03.152909 7f45d8eaa440                            Options.rate_limiter: (nil)
-2020/01/10-09:39:03.152911 7f45d8eaa440     Options.sst_file_manager.rate_bytes_per_sec: 0
-2020/01/10-09:39:03.152912 7f45d8eaa440                       Options.wal_recovery_mode: 2
-2020/01/10-09:39:03.152913 7f45d8eaa440                  Options.enable_thread_tracking: 0
-2020/01/10-09:39:03.152914 7f45d8eaa440                  Options.enable_pipelined_write: 0
-2020/01/10-09:39:03.152922 7f45d8eaa440         Options.allow_concurrent_memtable_write: 1
-2020/01/10-09:39:03.152924 7f45d8eaa440      Options.enable_write_thread_adaptive_yield: 1
-2020/01/10-09:39:03.152925 7f45d8eaa440             Options.write_thread_max_yield_usec: 100
-2020/01/10-09:39:03.152926 7f45d8eaa440            Options.write_thread_slow_yield_usec: 3
-2020/01/10-09:39:03.152927 7f45d8eaa440                               Options.row_cache: None
-2020/01/10-09:39:03.152928 7f45d8eaa440                              Options.wal_filter: None
-2020/01/10-09:39:03.152930 7f45d8eaa440             Options.avoid_flush_during_recovery: 0
-2020/01/10-09:39:03.152931 7f45d8eaa440             Options.allow_ingest_behind: 0
-2020/01/10-09:39:03.152932 7f45d8eaa440             Options.preserve_deletes: 0
-2020/01/10-09:39:03.152934 7f45d8eaa440             Options.two_write_queues: 0
-2020/01/10-09:39:03.152935 7f45d8eaa440             Options.manual_wal_flush: 0
-2020/01/10-09:39:03.152936 7f45d8eaa440             Options.atomic_flush: 0
-2020/01/10-09:39:03.152937 7f45d8eaa440             Options.avoid_unnecessary_blocking_io: 0
-2020/01/10-09:39:03.152938 7f45d8eaa440             Options.max_background_jobs: 2
-2020/01/10-09:39:03.152939 7f45d8eaa440             Options.max_background_compactions: -1
-2020/01/10-09:39:03.152941 7f45d8eaa440             Options.avoid_flush_during_shutdown: 0
-2020/01/10-09:39:03.152942 7f45d8eaa440           Options.writable_file_max_buffer_size: 1048576
-2020/01/10-09:39:03.152943 7f45d8eaa440             Options.delayed_write_rate : 16777216
-2020/01/10-09:39:03.152944 7f45d8eaa440             Options.max_total_wal_size: 0
-2020/01/10-09:39:03.152945 7f45d8eaa440             Options.delete_obsolete_files_period_micros: 21600000000
-2020/01/10-09:39:03.152947 7f45d8eaa440                   Options.stats_dump_period_sec: 600
-2020/01/10-09:39:03.152948 7f45d8eaa440                 Options.stats_persist_period_sec: 600
-2020/01/10-09:39:03.152949 7f45d8eaa440                 Options.stats_history_buffer_size: 1048576
-2020/01/10-09:39:03.152950 7f45d8eaa440                          Options.max_open_files: -1
-2020/01/10-09:39:03.152952 7f45d8eaa440                          Options.bytes_per_sync: 0
-2020/01/10-09:39:03.152953 7f45d8eaa440                      Options.wal_bytes_per_sync: 0
-2020/01/10-09:39:03.152954 7f45d8eaa440                   Options.strict_bytes_per_sync: 0
-2020/01/10-09:39:03.152955 7f45d8eaa440       Options.compaction_readahead_size: 0
-2020/01/10-09:39:03.152956 7f45d8eaa440 Compression algorithms supported:
-2020/01/10-09:39:03.152975 7f45d8eaa440 	kZSTDNotFinalCompression supported: 0
-2020/01/10-09:39:03.152981 7f45d8eaa440 	kZSTD supported: 0
-2020/01/10-09:39:03.152982 7f45d8eaa440 	kXpressCompression supported: 0
-2020/01/10-09:39:03.152983 7f45d8eaa440 	kLZ4HCCompression supported: 0
-2020/01/10-09:39:03.152984 7f45d8eaa440 	kLZ4Compression supported: 0
-2020/01/10-09:39:03.152986 7f45d8eaa440 	kBZip2Compression supported: 1
-2020/01/10-09:39:03.152987 7f45d8eaa440 	kZlibCompression supported: 1
-2020/01/10-09:39:03.152988 7f45d8eaa440 	kSnappyCompression supported: 0
-2020/01/10-09:39:03.152991 7f45d8eaa440 Fast CRC32 supported: Supported on x86
-2020/01/10-09:39:03.153040 7f45d8eaa440 [/db_impl_open.cc:242] Creating manifest 1 
-2020/01/10-09:39:03.155451 7f45d8eaa440 [/version_set.cc:4053] Recovering from manifest file: kvdb/MANIFEST-000001
-2020/01/10-09:39:03.155527 7f45d8eaa440 [/column_family.cc:482] --------------- Options for column family [default]:
-2020/01/10-09:39:03.155531 7f45d8eaa440               Options.comparator: leveldb.BytewiseComparator
-2020/01/10-09:39:03.155533 7f45d8eaa440           Options.merge_operator: None
-2020/01/10-09:39:03.155535 7f45d8eaa440        Options.compaction_filter: None
-2020/01/10-09:39:03.155536 7f45d8eaa440        Options.compaction_filter_factory: None
-2020/01/10-09:39:03.155537 7f45d8eaa440         Options.memtable_factory: SkipListFactory
-2020/01/10-09:39:03.155539 7f45d8eaa440            Options.table_factory: BlockBasedTable
-2020/01/10-09:39:03.155577 7f45d8eaa440            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x1e67c230)
-  cache_index_and_filter_blocks: 0
-  cache_index_and_filter_blocks_with_high_priority: 0
-  pin_l0_filter_and_index_blocks_in_cache: 0
-  pin_top_level_index_and_filter: 1
-  index_type: 0
-  data_block_index_type: 0
-  index_shortening: 1
-  data_block_hash_table_util_ratio: 0.750000
-  hash_index_allow_collision: 1
-  checksum: 1
-  no_block_cache: 0
-  block_cache: 0x206f8040
-  block_cache_name: LRUCache
-  block_cache_options:
-    capacity : 8388608
-    num_shard_bits : 4
-    strict_capacity_limit : 0
-    memory_allocator : None
-    high_pri_pool_ratio: 0.000
-  block_cache_compressed: (nil)
-  persistent_cache: (nil)
-  block_size: 4096
-  block_size_deviation: 10
-  block_restart_interval: 16
-  index_block_restart_interval: 1
-  metadata_block_size: 4096
-  partition_filters: 0
-  use_delta_encoding: 1
-  filter_policy: nullptr
-  whole_key_filtering: 1
-  verify_compression: 0
-  read_amp_bytes_per_bit: 0
-  format_version: 2
-  enable_index_compression: 1
-  block_align: 0
-2020/01/10-09:39:03.155593 7f45d8eaa440        Options.write_buffer_size: 67108864
-2020/01/10-09:39:03.155594 7f45d8eaa440  Options.max_write_buffer_number: 2
-2020/01/10-09:39:03.155596 7f45d8eaa440          Options.compression: NoCompression
-2020/01/10-09:39:03.155598 7f45d8eaa440                  Options.bottommost_compression: Disabled
-2020/01/10-09:39:03.155599 7f45d8eaa440       Options.prefix_extractor: nullptr
-2020/01/10-09:39:03.155600 7f45d8eaa440   Options.memtable_insert_with_hint_prefix_extractor: nullptr
-2020/01/10-09:39:03.155602 7f45d8eaa440             Options.num_levels: 7
-2020/01/10-09:39:03.155603 7f45d8eaa440        Options.min_write_buffer_number_to_merge: 1
-2020/01/10-09:39:03.155604 7f45d8eaa440     Options.max_write_buffer_number_to_maintain: 0
-2020/01/10-09:39:03.155606 7f45d8eaa440            Options.bottommost_compression_opts.window_bits: -14
-2020/01/10-09:39:03.155607 7f45d8eaa440                  Options.bottommost_compression_opts.level: 32767
-2020/01/10-09:39:03.155608 7f45d8eaa440               Options.bottommost_compression_opts.strategy: 0
-2020/01/10-09:39:03.155610 7f45d8eaa440         Options.bottommost_compression_opts.max_dict_bytes: 0
-2020/01/10-09:39:03.155611 7f45d8eaa440         Options.bottommost_compression_opts.zstd_max_train_bytes: 0
-2020/01/10-09:39:03.155612 7f45d8eaa440                  Options.bottommost_compression_opts.enabled: false
-2020/01/10-09:39:03.155614 7f45d8eaa440            Options.compression_opts.window_bits: -14
-2020/01/10-09:39:03.155615 7f45d8eaa440                  Options.compression_opts.level: 32767
-2020/01/10-09:39:03.155616 7f45d8eaa440               Options.compression_opts.strategy: 0
-2020/01/10-09:39:03.155617 7f45d8eaa440         Options.compression_opts.max_dict_bytes: 0
-2020/01/10-09:39:03.155619 7f45d8eaa440         Options.compression_opts.zstd_max_train_bytes: 0
-2020/01/10-09:39:03.155620 7f45d8eaa440                  Options.compression_opts.enabled: false
-2020/01/10-09:39:03.155621 7f45d8eaa440      Options.level0_file_num_compaction_trigger: 4
-2020/01/10-09:39:03.155622 7f45d8eaa440          Options.level0_slowdown_writes_trigger: 20
-2020/01/10-09:39:03.155624 7f45d8eaa440              Options.level0_stop_writes_trigger: 36
-2020/01/10-09:39:03.155625 7f45d8eaa440                   Options.target_file_size_base: 67108864
-2020/01/10-09:39:03.155626 7f45d8eaa440             Options.target_file_size_multiplier: 1
-2020/01/10-09:39:03.155627 7f45d8eaa440                Options.max_bytes_for_level_base: 268435456
-2020/01/10-09:39:03.155629 7f45d8eaa440                      Options.snap_refresh_nanos: 0
-2020/01/10-09:39:03.155630 7f45d8eaa440 Options.level_compaction_dynamic_level_bytes: 0
-2020/01/10-09:39:03.155631 7f45d8eaa440          Options.max_bytes_for_level_multiplier: 10.000000
-2020/01/10-09:39:03.155633 7f45d8eaa440 Options.max_bytes_for_level_multiplier_addtl[0]: 1
-2020/01/10-09:39:03.155635 7f45d8eaa440 Options.max_bytes_for_level_multiplier_addtl[1]: 1
-2020/01/10-09:39:03.155641 7f45d8eaa440 Options.max_bytes_for_level_multiplier_addtl[2]: 1
-2020/01/10-09:39:03.155642 7f45d8eaa440 Options.max_bytes_for_level_multiplier_addtl[3]: 1
-2020/01/10-09:39:03.155644 7f45d8eaa440 Options.max_bytes_for_level_multiplier_addtl[4]: 1
-2020/01/10-09:39:03.155645 7f45d8eaa440 Options.max_bytes_for_level_multiplier_addtl[5]: 1
-2020/01/10-09:39:03.155646 7f45d8eaa440 Options.max_bytes_for_level_multiplier_addtl[6]: 1
-2020/01/10-09:39:03.155647 7f45d8eaa440       Options.max_sequential_skip_in_iterations: 8
-2020/01/10-09:39:03.155648 7f45d8eaa440                    Options.max_compaction_bytes: 1677721600
-2020/01/10-09:39:03.155650 7f45d8eaa440                        Options.arena_block_size: 8388608
-2020/01/10-09:39:03.155651 7f45d8eaa440   Options.soft_pending_compaction_bytes_limit: 68719476736
-2020/01/10-09:39:03.155652 7f45d8eaa440   Options.hard_pending_compaction_bytes_limit: 274877906944
-2020/01/10-09:39:03.155653 7f45d8eaa440       Options.rate_limit_delay_max_milliseconds: 100
-2020/01/10-09:39:03.155655 7f45d8eaa440                Options.disable_auto_compactions: 0
-2020/01/10-09:39:03.155656 7f45d8eaa440                        Options.compaction_style: kCompactionStyleLevel
-2020/01/10-09:39:03.155658 7f45d8eaa440                          Options.compaction_pri: kMinOverlappingRatio
-2020/01/10-09:39:03.155659 7f45d8eaa440 Options.compaction_options_universal.size_ratio: 1
-2020/01/10-09:39:03.155661 7f45d8eaa440 Options.compaction_options_universal.min_merge_width: 2
-2020/01/10-09:39:03.155662 7f45d8eaa440 Options.compaction_options_universal.max_merge_width: 4294967295
-2020/01/10-09:39:03.155663 7f45d8eaa440 Options.compaction_options_universal.max_size_amplification_percent: 200
-2020/01/10-09:39:03.155664 7f45d8eaa440 Options.compaction_options_universal.compression_size_percent: -1
-2020/01/10-09:39:03.155666 7f45d8eaa440 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
-2020/01/10-09:39:03.155667 7f45d8eaa440 Options.compaction_options_fifo.max_table_files_size: 1073741824
-2020/01/10-09:39:03.155668 7f45d8eaa440 Options.compaction_options_fifo.allow_compaction: 0
-2020/01/10-09:39:03.155670 7f45d8eaa440                   Options.table_properties_collectors: 
-2020/01/10-09:39:03.155671 7f45d8eaa440                   Options.inplace_update_support: 0
-2020/01/10-09:39:03.155672 7f45d8eaa440                 Options.inplace_update_num_locks: 10000
-2020/01/10-09:39:03.155673 7f45d8eaa440               Options.memtable_prefix_bloom_size_ratio: 0.000000
-2020/01/10-09:39:03.155675 7f45d8eaa440               Options.memtable_whole_key_filtering: 0
-2020/01/10-09:39:03.155676 7f45d8eaa440   Options.memtable_huge_page_size: 0
-2020/01/10-09:39:03.155677 7f45d8eaa440                           Options.bloom_locality: 0
-2020/01/10-09:39:03.155678 7f45d8eaa440                    Options.max_successive_merges: 0
-2020/01/10-09:39:03.155679 7f45d8eaa440                Options.optimize_filters_for_hits: 0
-2020/01/10-09:39:03.155681 7f45d8eaa440                Options.paranoid_file_checks: 0
-2020/01/10-09:39:03.155682 7f45d8eaa440                Options.force_consistency_checks: 0
-2020/01/10-09:39:03.155683 7f45d8eaa440                Options.report_bg_io_stats: 0
-2020/01/10-09:39:03.155684 7f45d8eaa440                               Options.ttl: 0
-2020/01/10-09:39:03.155685 7f45d8eaa440          Options.periodic_compaction_seconds: 0
-2020/01/10-09:39:03.156268 7f45d8eaa440 [/version_set.cc:4267] Recovered from manifest file:kvdb/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
-2020/01/10-09:39:03.156273 7f45d8eaa440 [/version_set.cc:4276] Column family [default] (ID 0), log number is 0
-2020/01/10-09:39:03.158204 7f45d8eaa440 DB pointer 0x206f97b0
-2020/01/10-09:39:03.158523 7f455a5f6700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/10-09:39:03.158563 7f455a5f6700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 0.0 total, 0.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **

examples/demo-serving/kvdb/LOG.old.1578715678727481

-2020/01/11-11:16:32.954673 7f422a851440 RocksDB version: 6.2.4
-2020/01/11-11:16:32.954711 7f422a851440 Git sha rocksdb_build_git_sha:3513d4e93f8530ac44cc0c1efea04be4259c6938
-2020/01/11-11:16:32.954714 7f422a851440 Compile date Dec 23 2019
-2020/01/11-11:16:32.954717 7f422a851440 DB SUMMARY
-2020/01/11-11:16:32.954744 7f422a851440 CURRENT file:  CURRENT
-2020/01/11-11:16:32.954746 7f422a851440 IDENTITY file:  IDENTITY
-2020/01/11-11:16:32.954750 7f422a851440 MANIFEST file:  MANIFEST-000001 size: 13 Bytes
-2020/01/11-11:16:32.954752 7f422a851440 SST files in kvdb dir, Total Num: 0, files: 
-2020/01/11-11:16:32.954754 7f422a851440 Write Ahead Log file in kvdb: 000003.log size: 0 ; 
-2020/01/11-11:16:32.954756 7f422a851440                         Options.error_if_exists: 0
-2020/01/11-11:16:32.954757 7f422a851440                       Options.create_if_missing: 1
-2020/01/11-11:16:32.954758 7f422a851440                         Options.paranoid_checks: 1
-2020/01/11-11:16:32.954760 7f422a851440                                     Options.env: 0x1bed2ac0
-2020/01/11-11:16:32.954761 7f422a851440                                Options.info_log: 0x1f4720f0
-2020/01/11-11:16:32.954762 7f422a851440                Options.max_file_opening_threads: 16
-2020/01/11-11:16:32.954764 7f422a851440                              Options.statistics: (nil)
-2020/01/11-11:16:32.954765 7f422a851440                               Options.use_fsync: 0
-2020/01/11-11:16:32.954766 7f422a851440                       Options.max_log_file_size: 0
-2020/01/11-11:16:32.954768 7f422a851440                  Options.max_manifest_file_size: 1073741824
-2020/01/11-11:16:32.954769 7f422a851440                   Options.log_file_time_to_roll: 0
-2020/01/11-11:16:32.954770 7f422a851440                       Options.keep_log_file_num: 1000
-2020/01/11-11:16:32.954771 7f422a851440                    Options.recycle_log_file_num: 0
-2020/01/11-11:16:32.954772 7f422a851440                         Options.allow_fallocate: 1
-2020/01/11-11:16:32.954774 7f422a851440                        Options.allow_mmap_reads: 0
-2020/01/11-11:16:32.954775 7f422a851440                       Options.allow_mmap_writes: 0
-2020/01/11-11:16:32.954776 7f422a851440                        Options.use_direct_reads: 0
-2020/01/11-11:16:32.954777 7f422a851440                        Options.use_direct_io_for_flush_and_compaction: 0
-2020/01/11-11:16:32.954778 7f422a851440          Options.create_missing_column_families: 0
-2020/01/11-11:16:32.954780 7f422a851440                              Options.db_log_dir: 
-2020/01/11-11:16:32.954782 7f422a851440                                 Options.wal_dir: kvdb
-2020/01/11-11:16:32.954783 7f422a851440                Options.table_cache_numshardbits: 6
-2020/01/11-11:16:32.954784 7f422a851440                      Options.max_subcompactions: 1
-2020/01/11-11:16:32.954785 7f422a851440                  Options.max_background_flushes: -1
-2020/01/11-11:16:32.954787 7f422a851440                         Options.WAL_ttl_seconds: 0
-2020/01/11-11:16:32.954788 7f422a851440                       Options.WAL_size_limit_MB: 0
-2020/01/11-11:16:32.954789 7f422a851440             Options.manifest_preallocation_size: 4194304
-2020/01/11-11:16:32.954790 7f422a851440                     Options.is_fd_close_on_exec: 1
-2020/01/11-11:16:32.954791 7f422a851440                   Options.advise_random_on_open: 1
-2020/01/11-11:16:32.954793 7f422a851440                    Options.db_write_buffer_size: 0
-2020/01/11-11:16:32.954794 7f422a851440                    Options.write_buffer_manager: 0x1f472150
-2020/01/11-11:16:32.954795 7f422a851440         Options.access_hint_on_compaction_start: 1
-2020/01/11-11:16:32.954796 7f422a851440  Options.new_table_reader_for_compaction_inputs: 0
-2020/01/11-11:16:32.954797 7f422a851440           Options.random_access_max_buffer_size: 1048576
-2020/01/11-11:16:32.954798 7f422a851440                      Options.use_adaptive_mutex: 0
-2020/01/11-11:16:32.954799 7f422a851440                            Options.rate_limiter: (nil)
-2020/01/11-11:16:32.954801 7f422a851440     Options.sst_file_manager.rate_bytes_per_sec: 0
-2020/01/11-11:16:32.954802 7f422a851440                       Options.wal_recovery_mode: 2
-2020/01/11-11:16:32.954809 7f422a851440                  Options.enable_thread_tracking: 0
-2020/01/11-11:16:32.954810 7f422a851440                  Options.enable_pipelined_write: 0
-2020/01/11-11:16:32.954811 7f422a851440         Options.allow_concurrent_memtable_write: 1
-2020/01/11-11:16:32.954812 7f422a851440      Options.enable_write_thread_adaptive_yield: 1
-2020/01/11-11:16:32.954813 7f422a851440             Options.write_thread_max_yield_usec: 100
-2020/01/11-11:16:32.954815 7f422a851440            Options.write_thread_slow_yield_usec: 3
-2020/01/11-11:16:32.954816 7f422a851440                               Options.row_cache: None
-2020/01/11-11:16:32.954817 7f422a851440                              Options.wal_filter: None
-2020/01/11-11:16:32.954818 7f422a851440             Options.avoid_flush_during_recovery: 0
-2020/01/11-11:16:32.954819 7f422a851440             Options.allow_ingest_behind: 0
-2020/01/11-11:16:32.954820 7f422a851440             Options.preserve_deletes: 0
-2020/01/11-11:16:32.954822 7f422a851440             Options.two_write_queues: 0
-2020/01/11-11:16:32.954823 7f422a851440             Options.manual_wal_flush: 0
-2020/01/11-11:16:32.954824 7f422a851440             Options.atomic_flush: 0
-2020/01/11-11:16:32.954825 7f422a851440             Options.avoid_unnecessary_blocking_io: 0
-2020/01/11-11:16:32.954826 7f422a851440             Options.max_background_jobs: 2
-2020/01/11-11:16:32.954827 7f422a851440             Options.max_background_compactions: -1
-2020/01/11-11:16:32.954828 7f422a851440             Options.avoid_flush_during_shutdown: 0
-2020/01/11-11:16:32.954830 7f422a851440           Options.writable_file_max_buffer_size: 1048576
-2020/01/11-11:16:32.954831 7f422a851440             Options.delayed_write_rate : 16777216
-2020/01/11-11:16:32.954832 7f422a851440             Options.max_total_wal_size: 0
-2020/01/11-11:16:32.954833 7f422a851440             Options.delete_obsolete_files_period_micros: 21600000000
-2020/01/11-11:16:32.954834 7f422a851440                   Options.stats_dump_period_sec: 600
-2020/01/11-11:16:32.954836 7f422a851440                 Options.stats_persist_period_sec: 600
-2020/01/11-11:16:32.954837 7f422a851440                 Options.stats_history_buffer_size: 1048576
-2020/01/11-11:16:32.954838 7f422a851440                          Options.max_open_files: -1
-2020/01/11-11:16:32.954839 7f422a851440                          Options.bytes_per_sync: 0
-2020/01/11-11:16:32.954841 7f422a851440                      Options.wal_bytes_per_sync: 0
-2020/01/11-11:16:32.954842 7f422a851440                   Options.strict_bytes_per_sync: 0
-2020/01/11-11:16:32.954843 7f422a851440       Options.compaction_readahead_size: 0
-2020/01/11-11:16:32.954844 7f422a851440 Compression algorithms supported:
-2020/01/11-11:16:32.954856 7f422a851440 	kZSTDNotFinalCompression supported: 0
-2020/01/11-11:16:32.954862 7f422a851440 	kZSTD supported: 0
-2020/01/11-11:16:32.954863 7f422a851440 	kXpressCompression supported: 0
-2020/01/11-11:16:32.954864 7f422a851440 	kLZ4HCCompression supported: 0
-2020/01/11-11:16:32.954866 7f422a851440 	kLZ4Compression supported: 0
-2020/01/11-11:16:32.954867 7f422a851440 	kBZip2Compression supported: 1
-2020/01/11-11:16:32.954868 7f422a851440 	kZlibCompression supported: 1
-2020/01/11-11:16:32.954869 7f422a851440 	kSnappyCompression supported: 0
-2020/01/11-11:16:32.954871 7f422a851440 Fast CRC32 supported: Supported on x86
-2020/01/11-11:16:32.955074 7f422a851440 [/version_set.cc:4053] Recovering from manifest file: kvdb/MANIFEST-000001
-2020/01/11-11:16:32.955134 7f422a851440 [/column_family.cc:482] --------------- Options for column family [default]:
-2020/01/11-11:16:32.955138 7f422a851440               Options.comparator: leveldb.BytewiseComparator
-2020/01/11-11:16:32.955139 7f422a851440           Options.merge_operator: None
-2020/01/11-11:16:32.955141 7f422a851440        Options.compaction_filter: None
-2020/01/11-11:16:32.955142 7f422a851440        Options.compaction_filter_factory: None
-2020/01/11-11:16:32.955143 7f422a851440         Options.memtable_factory: SkipListFactory
-2020/01/11-11:16:32.955155 7f422a851440            Options.table_factory: BlockBasedTable
-2020/01/11-11:16:32.955188 7f422a851440            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x1df01130)
-  cache_index_and_filter_blocks: 0
-  cache_index_and_filter_blocks_with_high_priority: 0
-  pin_l0_filter_and_index_blocks_in_cache: 0
-  pin_top_level_index_and_filter: 1
-  index_type: 0
-  data_block_index_type: 0
-  index_shortening: 1
-  data_block_hash_table_util_ratio: 0.750000
-  hash_index_allow_collision: 1
-  checksum: 1
-  no_block_cache: 0
-  block_cache: 0x1d62c180
-  block_cache_name: LRUCache
-  block_cache_options:
-    capacity : 8388608
-    num_shard_bits : 4
-    strict_capacity_limit : 0
-    memory_allocator : None
-    high_pri_pool_ratio: 0.000
-  block_cache_compressed: (nil)
-  persistent_cache: (nil)
-  block_size: 4096
-  block_size_deviation: 10
-  block_restart_interval: 16
-  index_block_restart_interval: 1
-  metadata_block_size: 4096
-  partition_filters: 0
-  use_delta_encoding: 1
-  filter_policy: nullptr
-  whole_key_filtering: 1
-  verify_compression: 0
-  read_amp_bytes_per_bit: 0
-  format_version: 2
-  enable_index_compression: 1
-  block_align: 0
-2020/01/11-11:16:32.955193 7f422a851440        Options.write_buffer_size: 67108864
-2020/01/11-11:16:32.955194 7f422a851440  Options.max_write_buffer_number: 2
-2020/01/11-11:16:32.955196 7f422a851440          Options.compression: NoCompression
-2020/01/11-11:16:32.955198 7f422a851440                  Options.bottommost_compression: Disabled
-2020/01/11-11:16:32.955199 7f422a851440       Options.prefix_extractor: nullptr
-2020/01/11-11:16:32.955200 7f422a851440   Options.memtable_insert_with_hint_prefix_extractor: nullptr
-2020/01/11-11:16:32.955202 7f422a851440             Options.num_levels: 7
-2020/01/11-11:16:32.955203 7f422a851440        Options.min_write_buffer_number_to_merge: 1
-2020/01/11-11:16:32.955204 7f422a851440     Options.max_write_buffer_number_to_maintain: 0
-2020/01/11-11:16:32.955205 7f422a851440            Options.bottommost_compression_opts.window_bits: -14
-2020/01/11-11:16:32.955207 7f422a851440                  Options.bottommost_compression_opts.level: 32767
-2020/01/11-11:16:32.955208 7f422a851440               Options.bottommost_compression_opts.strategy: 0
-2020/01/11-11:16:32.955209 7f422a851440         Options.bottommost_compression_opts.max_dict_bytes: 0
-2020/01/11-11:16:32.955210 7f422a851440         Options.bottommost_compression_opts.zstd_max_train_bytes: 0
-2020/01/11-11:16:32.955211 7f422a851440                  Options.bottommost_compression_opts.enabled: false
-2020/01/11-11:16:32.955213 7f422a851440            Options.compression_opts.window_bits: -14
-2020/01/11-11:16:32.955214 7f422a851440                  Options.compression_opts.level: 32767
-2020/01/11-11:16:32.955215 7f422a851440               Options.compression_opts.strategy: 0
-2020/01/11-11:16:32.955216 7f422a851440         Options.compression_opts.max_dict_bytes: 0
-2020/01/11-11:16:32.955218 7f422a851440         Options.compression_opts.zstd_max_train_bytes: 0
-2020/01/11-11:16:32.955219 7f422a851440                  Options.compression_opts.enabled: false
-2020/01/11-11:16:32.955220 7f422a851440      Options.level0_file_num_compaction_trigger: 4
-2020/01/11-11:16:32.955221 7f422a851440          Options.level0_slowdown_writes_trigger: 20
-2020/01/11-11:16:32.955223 7f422a851440              Options.level0_stop_writes_trigger: 36
-2020/01/11-11:16:32.955224 7f422a851440                   Options.target_file_size_base: 67108864
-2020/01/11-11:16:32.955225 7f422a851440             Options.target_file_size_multiplier: 1
-2020/01/11-11:16:32.955226 7f422a851440                Options.max_bytes_for_level_base: 268435456
-2020/01/11-11:16:32.955228 7f422a851440                      Options.snap_refresh_nanos: 0
-2020/01/11-11:16:32.955229 7f422a851440 Options.level_compaction_dynamic_level_bytes: 0
-2020/01/11-11:16:32.955230 7f422a851440          Options.max_bytes_for_level_multiplier: 10.000000
-2020/01/11-11:16:32.955232 7f422a851440 Options.max_bytes_for_level_multiplier_addtl[0]: 1
-2020/01/11-11:16:32.955238 7f422a851440 Options.max_bytes_for_level_multiplier_addtl[1]: 1
-2020/01/11-11:16:32.955240 7f422a851440 Options.max_bytes_for_level_multiplier_addtl[2]: 1
-2020/01/11-11:16:32.955241 7f422a851440 Options.max_bytes_for_level_multiplier_addtl[3]: 1
-2020/01/11-11:16:32.955242 7f422a851440 Options.max_bytes_for_level_multiplier_addtl[4]: 1
-2020/01/11-11:16:32.955243 7f422a851440 Options.max_bytes_for_level_multiplier_addtl[5]: 1
-2020/01/11-11:16:32.955244 7f422a851440 Options.max_bytes_for_level_multiplier_addtl[6]: 1
-2020/01/11-11:16:32.955245 7f422a851440       Options.max_sequential_skip_in_iterations: 8
-2020/01/11-11:16:32.955246 7f422a851440                    Options.max_compaction_bytes: 1677721600
-2020/01/11-11:16:32.955248 7f422a851440                        Options.arena_block_size: 8388608
-2020/01/11-11:16:32.955249 7f422a851440   Options.soft_pending_compaction_bytes_limit: 68719476736
-2020/01/11-11:16:32.955250 7f422a851440   Options.hard_pending_compaction_bytes_limit: 274877906944
-2020/01/11-11:16:32.955251 7f422a851440       Options.rate_limit_delay_max_milliseconds: 100
-2020/01/11-11:16:32.955252 7f422a851440                Options.disable_auto_compactions: 0
-2020/01/11-11:16:32.955254 7f422a851440                        Options.compaction_style: kCompactionStyleLevel
-2020/01/11-11:16:32.955256 7f422a851440                          Options.compaction_pri: kMinOverlappingRatio
-2020/01/11-11:16:32.955257 7f422a851440 Options.compaction_options_universal.size_ratio: 1
-2020/01/11-11:16:32.955258 7f422a851440 Options.compaction_options_universal.min_merge_width: 2
-2020/01/11-11:16:32.955259 7f422a851440 Options.compaction_options_universal.max_merge_width: 4294967295
-2020/01/11-11:16:32.955260 7f422a851440 Options.compaction_options_universal.max_size_amplification_percent: 200
-2020/01/11-11:16:32.955262 7f422a851440 Options.compaction_options_universal.compression_size_percent: -1
-2020/01/11-11:16:32.955263 7f422a851440 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
-2020/01/11-11:16:32.955264 7f422a851440 Options.compaction_options_fifo.max_table_files_size: 1073741824
-2020/01/11-11:16:32.955265 7f422a851440 Options.compaction_options_fifo.allow_compaction: 0
-2020/01/11-11:16:32.955267 7f422a851440                   Options.table_properties_collectors: 
-2020/01/11-11:16:32.955268 7f422a851440                   Options.inplace_update_support: 0
-2020/01/11-11:16:32.955269 7f422a851440                 Options.inplace_update_num_locks: 10000
-2020/01/11-11:16:32.955270 7f422a851440               Options.memtable_prefix_bloom_size_ratio: 0.000000
-2020/01/11-11:16:32.955272 7f422a851440               Options.memtable_whole_key_filtering: 0
-2020/01/11-11:16:32.955273 7f422a851440   Options.memtable_huge_page_size: 0
-2020/01/11-11:16:32.955274 7f422a851440                           Options.bloom_locality: 0
-2020/01/11-11:16:32.955275 7f422a851440                    Options.max_successive_merges: 0
-2020/01/11-11:16:32.955276 7f422a851440                Options.optimize_filters_for_hits: 0
-2020/01/11-11:16:32.955277 7f422a851440                Options.paranoid_file_checks: 0
-2020/01/11-11:16:32.955278 7f422a851440                Options.force_consistency_checks: 0
-2020/01/11-11:16:32.955280 7f422a851440                Options.report_bg_io_stats: 0
-2020/01/11-11:16:32.955281 7f422a851440                               Options.ttl: 0
-2020/01/11-11:16:32.955282 7f422a851440          Options.periodic_compaction_seconds: 0
-2020/01/11-11:16:32.955848 7f422a851440 [/version_set.cc:4267] Recovered from manifest file:kvdb/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
-2020/01/11-11:16:32.955853 7f422a851440 [/version_set.cc:4276] Column family [default] (ID 0), log number is 0
-2020/01/11-11:16:32.955918 7f422a851440 EVENT_LOG_v1 {"time_micros": 1578712592955905, "job": 1, "event": "recovery_started", "log_files": [3]}
-2020/01/11-11:16:32.955931 7f422a851440 [/db_impl_open.cc:597] Recovering log #3 mode 2
-2020/01/11-11:16:32.955994 7f422a851440 [/version_set.cc:3546] Creating manifest 5
-2020/01/11-11:16:32.957004 7f422a851440 EVENT_LOG_v1 {"time_micros": 1578712592956998, "job": 1, "event": "recovery_finished"}
-2020/01/11-11:16:32.958911 7f422a851440 DB pointer 0x1f470de0
-2020/01/11-11:16:32.959218 7f41ac7f3700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-11:16:32.959255 7f41ac7f3700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 0.0 total, 0.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-2020/01/11-11:26:32.959466 7f41ac7f3700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-11:26:32.959522 7f41ac7f3700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 600.0 total, 600.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 600.0 total, 600.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 600.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-2020/01/11-11:36:32.959737 7f41ac7f3700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-11:36:32.959786 7f41ac7f3700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 1200.0 total, 600.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 1200.0 total, 600.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 1200.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-2020/01/11-11:46:32.960005 7f41ac7f3700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-11:46:32.960058 7f41ac7f3700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 1800.0 total, 600.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 1800.0 total, 600.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 1800.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-2020/01/11-11:56:32.960277 7f41ac7f3700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-11:56:32.960328 7f41ac7f3700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 2400.0 total, 600.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 2400.0 total, 600.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 2400.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **

examples/demo-serving/kvdb/LOG.old.1578716053197010

-2020/01/11-12:07:58.727896 7effc50af440 RocksDB version: 6.2.4
-2020/01/11-12:07:58.727932 7effc50af440 Git sha rocksdb_build_git_sha:3513d4e93f8530ac44cc0c1efea04be4259c6938
-2020/01/11-12:07:58.727935 7effc50af440 Compile date Dec 23 2019
-2020/01/11-12:07:58.727938 7effc50af440 DB SUMMARY
-2020/01/11-12:07:58.727964 7effc50af440 CURRENT file:  CURRENT
-2020/01/11-12:07:58.727966 7effc50af440 IDENTITY file:  IDENTITY
-2020/01/11-12:07:58.727969 7effc50af440 MANIFEST file:  MANIFEST-000005 size: 59 Bytes
-2020/01/11-12:07:58.727971 7effc50af440 SST files in kvdb dir, Total Num: 0, files: 
-2020/01/11-12:07:58.727973 7effc50af440 Write Ahead Log file in kvdb: 000006.log size: 0 ; 
-2020/01/11-12:07:58.727975 7effc50af440                         Options.error_if_exists: 0
-2020/01/11-12:07:58.727977 7effc50af440                       Options.create_if_missing: 1
-2020/01/11-12:07:58.727978 7effc50af440                         Options.paranoid_checks: 1
-2020/01/11-12:07:58.727979 7effc50af440                                     Options.env: 0x1bed2ac0
-2020/01/11-12:07:58.727981 7effc50af440                                Options.info_log: 0x20e963c0
-2020/01/11-12:07:58.727982 7effc50af440                Options.max_file_opening_threads: 16
-2020/01/11-12:07:58.727983 7effc50af440                              Options.statistics: (nil)
-2020/01/11-12:07:58.727984 7effc50af440                               Options.use_fsync: 0
-2020/01/11-12:07:58.727986 7effc50af440                       Options.max_log_file_size: 0
-2020/01/11-12:07:58.727987 7effc50af440                  Options.max_manifest_file_size: 1073741824
-2020/01/11-12:07:58.727988 7effc50af440                   Options.log_file_time_to_roll: 0
-2020/01/11-12:07:58.727989 7effc50af440                       Options.keep_log_file_num: 1000
-2020/01/11-12:07:58.727990 7effc50af440                    Options.recycle_log_file_num: 0
-2020/01/11-12:07:58.727992 7effc50af440                         Options.allow_fallocate: 1
-2020/01/11-12:07:58.727993 7effc50af440                        Options.allow_mmap_reads: 0
-2020/01/11-12:07:58.727994 7effc50af440                       Options.allow_mmap_writes: 0
-2020/01/11-12:07:58.727995 7effc50af440                        Options.use_direct_reads: 0
-2020/01/11-12:07:58.727996 7effc50af440                        Options.use_direct_io_for_flush_and_compaction: 0
-2020/01/11-12:07:58.727997 7effc50af440          Options.create_missing_column_families: 0
-2020/01/11-12:07:58.728000 7effc50af440                              Options.db_log_dir: 
-2020/01/11-12:07:58.728001 7effc50af440                                 Options.wal_dir: kvdb
-2020/01/11-12:07:58.728002 7effc50af440                Options.table_cache_numshardbits: 6
-2020/01/11-12:07:58.728003 7effc50af440                      Options.max_subcompactions: 1
-2020/01/11-12:07:58.728004 7effc50af440                  Options.max_background_flushes: -1
-2020/01/11-12:07:58.728006 7effc50af440                         Options.WAL_ttl_seconds: 0
-2020/01/11-12:07:58.728007 7effc50af440                       Options.WAL_size_limit_MB: 0
-2020/01/11-12:07:58.728008 7effc50af440             Options.manifest_preallocation_size: 4194304
-2020/01/11-12:07:58.728009 7effc50af440                     Options.is_fd_close_on_exec: 1
-2020/01/11-12:07:58.728010 7effc50af440                   Options.advise_random_on_open: 1
-2020/01/11-12:07:58.728011 7effc50af440                    Options.db_write_buffer_size: 0
-2020/01/11-12:07:58.728012 7effc50af440                    Options.write_buffer_manager: 0x20e96420
-2020/01/11-12:07:58.728014 7effc50af440         Options.access_hint_on_compaction_start: 1
-2020/01/11-12:07:58.728015 7effc50af440  Options.new_table_reader_for_compaction_inputs: 0
-2020/01/11-12:07:58.728016 7effc50af440           Options.random_access_max_buffer_size: 1048576
-2020/01/11-12:07:58.728017 7effc50af440                      Options.use_adaptive_mutex: 0
-2020/01/11-12:07:58.728018 7effc50af440                            Options.rate_limiter: (nil)
-2020/01/11-12:07:58.728020 7effc50af440     Options.sst_file_manager.rate_bytes_per_sec: 0
-2020/01/11-12:07:58.728021 7effc50af440                       Options.wal_recovery_mode: 2
-2020/01/11-12:07:58.728028 7effc50af440                  Options.enable_thread_tracking: 0
-2020/01/11-12:07:58.728029 7effc50af440                  Options.enable_pipelined_write: 0
-2020/01/11-12:07:58.728030 7effc50af440         Options.allow_concurrent_memtable_write: 1
-2020/01/11-12:07:58.728031 7effc50af440      Options.enable_write_thread_adaptive_yield: 1
-2020/01/11-12:07:58.728032 7effc50af440             Options.write_thread_max_yield_usec: 100
-2020/01/11-12:07:58.728034 7effc50af440            Options.write_thread_slow_yield_usec: 3
-2020/01/11-12:07:58.728035 7effc50af440                               Options.row_cache: None
-2020/01/11-12:07:58.728036 7effc50af440                              Options.wal_filter: None
-2020/01/11-12:07:58.728038 7effc50af440             Options.avoid_flush_during_recovery: 0
-2020/01/11-12:07:58.728039 7effc50af440             Options.allow_ingest_behind: 0
-2020/01/11-12:07:58.728040 7effc50af440             Options.preserve_deletes: 0
-2020/01/11-12:07:58.728041 7effc50af440             Options.two_write_queues: 0
-2020/01/11-12:07:58.728042 7effc50af440             Options.manual_wal_flush: 0
-2020/01/11-12:07:58.728043 7effc50af440             Options.atomic_flush: 0
-2020/01/11-12:07:58.728044 7effc50af440             Options.avoid_unnecessary_blocking_io: 0
-2020/01/11-12:07:58.728046 7effc50af440             Options.max_background_jobs: 2
-2020/01/11-12:07:58.728047 7effc50af440             Options.max_background_compactions: -1
-2020/01/11-12:07:58.728048 7effc50af440             Options.avoid_flush_during_shutdown: 0
-2020/01/11-12:07:58.728049 7effc50af440           Options.writable_file_max_buffer_size: 1048576
-2020/01/11-12:07:58.728050 7effc50af440             Options.delayed_write_rate : 16777216
-2020/01/11-12:07:58.728051 7effc50af440             Options.max_total_wal_size: 0
-2020/01/11-12:07:58.728052 7effc50af440             Options.delete_obsolete_files_period_micros: 21600000000
-2020/01/11-12:07:58.728054 7effc50af440                   Options.stats_dump_period_sec: 600
-2020/01/11-12:07:58.728055 7effc50af440                 Options.stats_persist_period_sec: 600
-2020/01/11-12:07:58.728056 7effc50af440                 Options.stats_history_buffer_size: 1048576
-2020/01/11-12:07:58.728057 7effc50af440                          Options.max_open_files: -1
-2020/01/11-12:07:58.728058 7effc50af440                          Options.bytes_per_sync: 0
-2020/01/11-12:07:58.728060 7effc50af440                      Options.wal_bytes_per_sync: 0
-2020/01/11-12:07:58.728061 7effc50af440                   Options.strict_bytes_per_sync: 0
-2020/01/11-12:07:58.728062 7effc50af440       Options.compaction_readahead_size: 0
-2020/01/11-12:07:58.728063 7effc50af440 Compression algorithms supported:
-2020/01/11-12:07:58.728066 7effc50af440 	kZSTDNotFinalCompression supported: 0
-2020/01/11-12:07:58.728069 7effc50af440 	kZSTD supported: 0
-2020/01/11-12:07:58.728070 7effc50af440 	kXpressCompression supported: 0
-2020/01/11-12:07:58.728071 7effc50af440 	kLZ4HCCompression supported: 0
-2020/01/11-12:07:58.728073 7effc50af440 	kLZ4Compression supported: 0
-2020/01/11-12:07:58.728074 7effc50af440 	kBZip2Compression supported: 1
-2020/01/11-12:07:58.728076 7effc50af440 	kZlibCompression supported: 1
-2020/01/11-12:07:58.728077 7effc50af440 	kSnappyCompression supported: 0
-2020/01/11-12:07:58.728079 7effc50af440 Fast CRC32 supported: Supported on x86
-2020/01/11-12:07:58.728282 7effc50af440 [/version_set.cc:4053] Recovering from manifest file: kvdb/MANIFEST-000005
-2020/01/11-12:07:58.728341 7effc50af440 [/column_family.cc:482] --------------- Options for column family [default]:
-2020/01/11-12:07:58.728344 7effc50af440               Options.comparator: leveldb.BytewiseComparator
-2020/01/11-12:07:58.728345 7effc50af440           Options.merge_operator: None
-2020/01/11-12:07:58.728347 7effc50af440        Options.compaction_filter: None
-2020/01/11-12:07:58.728348 7effc50af440        Options.compaction_filter_factory: None
-2020/01/11-12:07:58.728349 7effc50af440         Options.memtable_factory: SkipListFactory
-2020/01/11-12:07:58.728357 7effc50af440            Options.table_factory: BlockBasedTable
-2020/01/11-12:07:58.728388 7effc50af440            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x1eca3230)
-  cache_index_and_filter_blocks: 0
-  cache_index_and_filter_blocks_with_high_priority: 0
-  pin_l0_filter_and_index_blocks_in_cache: 0
-  pin_top_level_index_and_filter: 1
-  index_type: 0
-  data_block_index_type: 0
-  index_shortening: 1
-  data_block_hash_table_util_ratio: 0.750000
-  hash_index_allow_collision: 1
-  checksum: 1
-  no_block_cache: 0
-  block_cache: 0x20e93940
-  block_cache_name: LRUCache
-  block_cache_options:
-    capacity : 8388608
-    num_shard_bits : 4
-    strict_capacity_limit : 0
-    memory_allocator : None
-    high_pri_pool_ratio: 0.000
-  block_cache_compressed: (nil)
-  persistent_cache: (nil)
-  block_size: 4096
-  block_size_deviation: 10
-  block_restart_interval: 16
-  index_block_restart_interval: 1
-  metadata_block_size: 4096
-  partition_filters: 0
-  use_delta_encoding: 1
-  filter_policy: nullptr
-  whole_key_filtering: 1
-  verify_compression: 0
-  read_amp_bytes_per_bit: 0
-  format_version: 2
-  enable_index_compression: 1
-  block_align: 0
-2020/01/11-12:07:58.728392 7effc50af440        Options.write_buffer_size: 67108864
-2020/01/11-12:07:58.728393 7effc50af440  Options.max_write_buffer_number: 2
-2020/01/11-12:07:58.728395 7effc50af440          Options.compression: NoCompression
-2020/01/11-12:07:58.728396 7effc50af440                  Options.bottommost_compression: Disabled
-2020/01/11-12:07:58.728398 7effc50af440       Options.prefix_extractor: nullptr
-2020/01/11-12:07:58.728399 7effc50af440   Options.memtable_insert_with_hint_prefix_extractor: nullptr
-2020/01/11-12:07:58.728400 7effc50af440             Options.num_levels: 7
-2020/01/11-12:07:58.728401 7effc50af440        Options.min_write_buffer_number_to_merge: 1
-2020/01/11-12:07:58.728403 7effc50af440     Options.max_write_buffer_number_to_maintain: 0
-2020/01/11-12:07:58.728404 7effc50af440            Options.bottommost_compression_opts.window_bits: -14
-2020/01/11-12:07:58.728413 7effc50af440                  Options.bottommost_compression_opts.level: 32767
-2020/01/11-12:07:58.728415 7effc50af440               Options.bottommost_compression_opts.strategy: 0
-2020/01/11-12:07:58.728416 7effc50af440         Options.bottommost_compression_opts.max_dict_bytes: 0
-2020/01/11-12:07:58.728417 7effc50af440         Options.bottommost_compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:07:58.728418 7effc50af440                  Options.bottommost_compression_opts.enabled: false
-2020/01/11-12:07:58.728420 7effc50af440            Options.compression_opts.window_bits: -14
-2020/01/11-12:07:58.728421 7effc50af440                  Options.compression_opts.level: 32767
-2020/01/11-12:07:58.728422 7effc50af440               Options.compression_opts.strategy: 0
-2020/01/11-12:07:58.728423 7effc50af440         Options.compression_opts.max_dict_bytes: 0
-2020/01/11-12:07:58.728424 7effc50af440         Options.compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:07:58.728425 7effc50af440                  Options.compression_opts.enabled: false
-2020/01/11-12:07:58.728427 7effc50af440      Options.level0_file_num_compaction_trigger: 4
-2020/01/11-12:07:58.728428 7effc50af440          Options.level0_slowdown_writes_trigger: 20
-2020/01/11-12:07:58.728429 7effc50af440              Options.level0_stop_writes_trigger: 36
-2020/01/11-12:07:58.728430 7effc50af440                   Options.target_file_size_base: 67108864
-2020/01/11-12:07:58.728431 7effc50af440             Options.target_file_size_multiplier: 1
-2020/01/11-12:07:58.728432 7effc50af440                Options.max_bytes_for_level_base: 268435456
-2020/01/11-12:07:58.728434 7effc50af440                      Options.snap_refresh_nanos: 0
-2020/01/11-12:07:58.728435 7effc50af440 Options.level_compaction_dynamic_level_bytes: 0
-2020/01/11-12:07:58.728436 7effc50af440          Options.max_bytes_for_level_multiplier: 10.000000
-2020/01/11-12:07:58.728438 7effc50af440 Options.max_bytes_for_level_multiplier_addtl[0]: 1
-2020/01/11-12:07:58.728445 7effc50af440 Options.max_bytes_for_level_multiplier_addtl[1]: 1
-2020/01/11-12:07:58.728447 7effc50af440 Options.max_bytes_for_level_multiplier_addtl[2]: 1
-2020/01/11-12:07:58.728448 7effc50af440 Options.max_bytes_for_level_multiplier_addtl[3]: 1
-2020/01/11-12:07:58.728449 7effc50af440 Options.max_bytes_for_level_multiplier_addtl[4]: 1
-2020/01/11-12:07:58.728450 7effc50af440 Options.max_bytes_for_level_multiplier_addtl[5]: 1
-2020/01/11-12:07:58.728451 7effc50af440 Options.max_bytes_for_level_multiplier_addtl[6]: 1
-2020/01/11-12:07:58.728452 7effc50af440       Options.max_sequential_skip_in_iterations: 8
-2020/01/11-12:07:58.728453 7effc50af440                    Options.max_compaction_bytes: 1677721600
-2020/01/11-12:07:58.728455 7effc50af440                        Options.arena_block_size: 8388608
-2020/01/11-12:07:58.728456 7effc50af440   Options.soft_pending_compaction_bytes_limit: 68719476736
-2020/01/11-12:07:58.728457 7effc50af440   Options.hard_pending_compaction_bytes_limit: 274877906944
-2020/01/11-12:07:58.728458 7effc50af440       Options.rate_limit_delay_max_milliseconds: 100
-2020/01/11-12:07:58.728459 7effc50af440                Options.disable_auto_compactions: 0
-2020/01/11-12:07:58.728461 7effc50af440                        Options.compaction_style: kCompactionStyleLevel
-2020/01/11-12:07:58.728463 7effc50af440                          Options.compaction_pri: kMinOverlappingRatio
-2020/01/11-12:07:58.728464 7effc50af440 Options.compaction_options_universal.size_ratio: 1
-2020/01/11-12:07:58.728466 7effc50af440 Options.compaction_options_universal.min_merge_width: 2
-2020/01/11-12:07:58.728467 7effc50af440 Options.compaction_options_universal.max_merge_width: 4294967295
-2020/01/11-12:07:58.728468 7effc50af440 Options.compaction_options_universal.max_size_amplification_percent: 200
-2020/01/11-12:07:58.728469 7effc50af440 Options.compaction_options_universal.compression_size_percent: -1
-2020/01/11-12:07:58.728471 7effc50af440 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
-2020/01/11-12:07:58.728472 7effc50af440 Options.compaction_options_fifo.max_table_files_size: 1073741824
-2020/01/11-12:07:58.728473 7effc50af440 Options.compaction_options_fifo.allow_compaction: 0
-2020/01/11-12:07:58.728475 7effc50af440                   Options.table_properties_collectors: 
-2020/01/11-12:07:58.728476 7effc50af440                   Options.inplace_update_support: 0
-2020/01/11-12:07:58.728477 7effc50af440                 Options.inplace_update_num_locks: 10000
-2020/01/11-12:07:58.728478 7effc50af440               Options.memtable_prefix_bloom_size_ratio: 0.000000
-2020/01/11-12:07:58.728479 7effc50af440               Options.memtable_whole_key_filtering: 0
-2020/01/11-12:07:58.728481 7effc50af440   Options.memtable_huge_page_size: 0
-2020/01/11-12:07:58.728482 7effc50af440                           Options.bloom_locality: 0
-2020/01/11-12:07:58.728483 7effc50af440                    Options.max_successive_merges: 0
-2020/01/11-12:07:58.728484 7effc50af440                Options.optimize_filters_for_hits: 0
-2020/01/11-12:07:58.728485 7effc50af440                Options.paranoid_file_checks: 0
-2020/01/11-12:07:58.728486 7effc50af440                Options.force_consistency_checks: 0
-2020/01/11-12:07:58.728487 7effc50af440                Options.report_bg_io_stats: 0
-2020/01/11-12:07:58.728489 7effc50af440                               Options.ttl: 0
-2020/01/11-12:07:58.728490 7effc50af440          Options.periodic_compaction_seconds: 0
-2020/01/11-12:07:58.729011 7effc50af440 [/version_set.cc:4267] Recovered from manifest file:kvdb/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
-2020/01/11-12:07:58.729015 7effc50af440 [/version_set.cc:4276] Column family [default] (ID 0), log number is 4
-2020/01/11-12:07:58.729081 7effc50af440 EVENT_LOG_v1 {"time_micros": 1578715678729069, "job": 1, "event": "recovery_started", "log_files": [6]}
-2020/01/11-12:07:58.729092 7effc50af440 [/db_impl_open.cc:597] Recovering log #6 mode 2
-2020/01/11-12:07:58.729153 7effc50af440 [/version_set.cc:3546] Creating manifest 8
-2020/01/11-12:07:58.730101 7effc50af440 EVENT_LOG_v1 {"time_micros": 1578715678730097, "job": 1, "event": "recovery_finished"}
-2020/01/11-12:07:58.732193 7effc50af440 DB pointer 0x20e950b0
-2020/01/11-12:07:58.732503 7eff4a1fc700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-12:07:58.732538 7eff4a1fc700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 0.0 total, 0.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **

examples/demo-serving/kvdb/LOG.old.1578716645179721

-2020/01/11-12:14:13.197446 7fd2bb712440 RocksDB version: 6.2.4
-2020/01/11-12:14:13.197484 7fd2bb712440 Git sha rocksdb_build_git_sha:3513d4e93f8530ac44cc0c1efea04be4259c6938
-2020/01/11-12:14:13.197486 7fd2bb712440 Compile date Dec 23 2019
-2020/01/11-12:14:13.197489 7fd2bb712440 DB SUMMARY
-2020/01/11-12:14:13.197519 7fd2bb712440 CURRENT file:  CURRENT
-2020/01/11-12:14:13.197521 7fd2bb712440 IDENTITY file:  IDENTITY
-2020/01/11-12:14:13.197524 7fd2bb712440 MANIFEST file:  MANIFEST-000008 size: 59 Bytes
-2020/01/11-12:14:13.197527 7fd2bb712440 SST files in kvdb dir, Total Num: 0, files: 
-2020/01/11-12:14:13.197528 7fd2bb712440 Write Ahead Log file in kvdb: 000009.log size: 0 ; 
-2020/01/11-12:14:13.197531 7fd2bb712440                         Options.error_if_exists: 0
-2020/01/11-12:14:13.197532 7fd2bb712440                       Options.create_if_missing: 1
-2020/01/11-12:14:13.197533 7fd2bb712440                         Options.paranoid_checks: 1
-2020/01/11-12:14:13.197535 7fd2bb712440                                     Options.env: 0x1bed2ac0
-2020/01/11-12:14:13.197536 7fd2bb712440                                Options.info_log: 0x1fd86b00
-2020/01/11-12:14:13.197537 7fd2bb712440                Options.max_file_opening_threads: 16
-2020/01/11-12:14:13.197538 7fd2bb712440                              Options.statistics: (nil)
-2020/01/11-12:14:13.197540 7fd2bb712440                               Options.use_fsync: 0
-2020/01/11-12:14:13.197541 7fd2bb712440                       Options.max_log_file_size: 0
-2020/01/11-12:14:13.197542 7fd2bb712440                  Options.max_manifest_file_size: 1073741824
-2020/01/11-12:14:13.197543 7fd2bb712440                   Options.log_file_time_to_roll: 0
-2020/01/11-12:14:13.197545 7fd2bb712440                       Options.keep_log_file_num: 1000
-2020/01/11-12:14:13.197546 7fd2bb712440                    Options.recycle_log_file_num: 0
-2020/01/11-12:14:13.197547 7fd2bb712440                         Options.allow_fallocate: 1
-2020/01/11-12:14:13.197548 7fd2bb712440                        Options.allow_mmap_reads: 0
-2020/01/11-12:14:13.197549 7fd2bb712440                       Options.allow_mmap_writes: 0
-2020/01/11-12:14:13.197550 7fd2bb712440                        Options.use_direct_reads: 0
-2020/01/11-12:14:13.197551 7fd2bb712440                        Options.use_direct_io_for_flush_and_compaction: 0
-2020/01/11-12:14:13.197553 7fd2bb712440          Options.create_missing_column_families: 0
-2020/01/11-12:14:13.197555 7fd2bb712440                              Options.db_log_dir: 
-2020/01/11-12:14:13.197556 7fd2bb712440                                 Options.wal_dir: kvdb
-2020/01/11-12:14:13.197558 7fd2bb712440                Options.table_cache_numshardbits: 6
-2020/01/11-12:14:13.197559 7fd2bb712440                      Options.max_subcompactions: 1
-2020/01/11-12:14:13.197560 7fd2bb712440                  Options.max_background_flushes: -1
-2020/01/11-12:14:13.197561 7fd2bb712440                         Options.WAL_ttl_seconds: 0
-2020/01/11-12:14:13.197563 7fd2bb712440                       Options.WAL_size_limit_MB: 0
-2020/01/11-12:14:13.197564 7fd2bb712440             Options.manifest_preallocation_size: 4194304
-2020/01/11-12:14:13.197565 7fd2bb712440                     Options.is_fd_close_on_exec: 1
-2020/01/11-12:14:13.197566 7fd2bb712440                   Options.advise_random_on_open: 1
-2020/01/11-12:14:13.197567 7fd2bb712440                    Options.db_write_buffer_size: 0
-2020/01/11-12:14:13.197568 7fd2bb712440                    Options.write_buffer_manager: 0x1dd0b160
-2020/01/11-12:14:13.197570 7fd2bb712440         Options.access_hint_on_compaction_start: 1
-2020/01/11-12:14:13.197571 7fd2bb712440  Options.new_table_reader_for_compaction_inputs: 0
-2020/01/11-12:14:13.197572 7fd2bb712440           Options.random_access_max_buffer_size: 1048576
-2020/01/11-12:14:13.197573 7fd2bb712440                      Options.use_adaptive_mutex: 0
-2020/01/11-12:14:13.197574 7fd2bb712440                            Options.rate_limiter: (nil)
-2020/01/11-12:14:13.197576 7fd2bb712440     Options.sst_file_manager.rate_bytes_per_sec: 0
-2020/01/11-12:14:13.197577 7fd2bb712440                       Options.wal_recovery_mode: 2
-2020/01/11-12:14:13.197584 7fd2bb712440                  Options.enable_thread_tracking: 0
-2020/01/11-12:14:13.197585 7fd2bb712440                  Options.enable_pipelined_write: 0
-2020/01/11-12:14:13.197586 7fd2bb712440         Options.allow_concurrent_memtable_write: 1
-2020/01/11-12:14:13.197587 7fd2bb712440      Options.enable_write_thread_adaptive_yield: 1
-2020/01/11-12:14:13.197589 7fd2bb712440             Options.write_thread_max_yield_usec: 100
-2020/01/11-12:14:13.197590 7fd2bb712440            Options.write_thread_slow_yield_usec: 3
-2020/01/11-12:14:13.197591 7fd2bb712440                               Options.row_cache: None
-2020/01/11-12:14:13.197592 7fd2bb712440                              Options.wal_filter: None
-2020/01/11-12:14:13.197594 7fd2bb712440             Options.avoid_flush_during_recovery: 0
-2020/01/11-12:14:13.197595 7fd2bb712440             Options.allow_ingest_behind: 0
-2020/01/11-12:14:13.197596 7fd2bb712440             Options.preserve_deletes: 0
-2020/01/11-12:14:13.197597 7fd2bb712440             Options.two_write_queues: 0
-2020/01/11-12:14:13.197598 7fd2bb712440             Options.manual_wal_flush: 0
-2020/01/11-12:14:13.197599 7fd2bb712440             Options.atomic_flush: 0
-2020/01/11-12:14:13.197600 7fd2bb712440             Options.avoid_unnecessary_blocking_io: 0
-2020/01/11-12:14:13.197602 7fd2bb712440             Options.max_background_jobs: 2
-2020/01/11-12:14:13.197603 7fd2bb712440             Options.max_background_compactions: -1
-2020/01/11-12:14:13.197604 7fd2bb712440             Options.avoid_flush_during_shutdown: 0
-2020/01/11-12:14:13.197605 7fd2bb712440           Options.writable_file_max_buffer_size: 1048576
-2020/01/11-12:14:13.197606 7fd2bb712440             Options.delayed_write_rate : 16777216
-2020/01/11-12:14:13.197607 7fd2bb712440             Options.max_total_wal_size: 0
-2020/01/11-12:14:13.197609 7fd2bb712440             Options.delete_obsolete_files_period_micros: 21600000000
-2020/01/11-12:14:13.197610 7fd2bb712440                   Options.stats_dump_period_sec: 600
-2020/01/11-12:14:13.197611 7fd2bb712440                 Options.stats_persist_period_sec: 600
-2020/01/11-12:14:13.197612 7fd2bb712440                 Options.stats_history_buffer_size: 1048576
-2020/01/11-12:14:13.197614 7fd2bb712440                          Options.max_open_files: -1
-2020/01/11-12:14:13.197615 7fd2bb712440                          Options.bytes_per_sync: 0
-2020/01/11-12:14:13.197616 7fd2bb712440                      Options.wal_bytes_per_sync: 0
-2020/01/11-12:14:13.197617 7fd2bb712440                   Options.strict_bytes_per_sync: 0
-2020/01/11-12:14:13.197618 7fd2bb712440       Options.compaction_readahead_size: 0
-2020/01/11-12:14:13.197619 7fd2bb712440 Compression algorithms supported:
-2020/01/11-12:14:13.197632 7fd2bb712440 	kZSTDNotFinalCompression supported: 0
-2020/01/11-12:14:13.197642 7fd2bb712440 	kZSTD supported: 0
-2020/01/11-12:14:13.197644 7fd2bb712440 	kXpressCompression supported: 0
-2020/01/11-12:14:13.197645 7fd2bb712440 	kLZ4HCCompression supported: 0
-2020/01/11-12:14:13.197646 7fd2bb712440 	kLZ4Compression supported: 0
-2020/01/11-12:14:13.197647 7fd2bb712440 	kBZip2Compression supported: 1
-2020/01/11-12:14:13.197649 7fd2bb712440 	kZlibCompression supported: 1
-2020/01/11-12:14:13.197650 7fd2bb712440 	kSnappyCompression supported: 0
-2020/01/11-12:14:13.197652 7fd2bb712440 Fast CRC32 supported: Supported on x86
-2020/01/11-12:14:13.197844 7fd2bb712440 [/version_set.cc:4053] Recovering from manifest file: kvdb/MANIFEST-000008
-2020/01/11-12:14:13.197902 7fd2bb712440 [/column_family.cc:482] --------------- Options for column family [default]:
-2020/01/11-12:14:13.197906 7fd2bb712440               Options.comparator: leveldb.BytewiseComparator
-2020/01/11-12:14:13.197907 7fd2bb712440           Options.merge_operator: None
-2020/01/11-12:14:13.197909 7fd2bb712440        Options.compaction_filter: None
-2020/01/11-12:14:13.197910 7fd2bb712440        Options.compaction_filter_factory: None
-2020/01/11-12:14:13.197911 7fd2bb712440         Options.memtable_factory: SkipListFactory
-2020/01/11-12:14:13.197922 7fd2bb712440            Options.table_factory: BlockBasedTable
-2020/01/11-12:14:13.197955 7fd2bb712440            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x1e99cbf0)
-  cache_index_and_filter_blocks: 0
-  cache_index_and_filter_blocks_with_high_priority: 0
-  pin_l0_filter_and_index_blocks_in_cache: 0
-  pin_top_level_index_and_filter: 1
-  index_type: 0
-  data_block_index_type: 0
-  index_shortening: 1
-  data_block_hash_table_util_ratio: 0.750000
-  hash_index_allow_collision: 1
-  checksum: 1
-  no_block_cache: 0
-  block_cache: 0x1e9cf0f0
-  block_cache_name: LRUCache
-  block_cache_options:
-    capacity : 8388608
-    num_shard_bits : 4
-    strict_capacity_limit : 0
-    memory_allocator : None
-    high_pri_pool_ratio: 0.000
-  block_cache_compressed: (nil)
-  persistent_cache: (nil)
-  block_size: 4096
-  block_size_deviation: 10
-  block_restart_interval: 16
-  index_block_restart_interval: 1
-  metadata_block_size: 4096
-  partition_filters: 0
-  use_delta_encoding: 1
-  filter_policy: nullptr
-  whole_key_filtering: 1
-  verify_compression: 0
-  read_amp_bytes_per_bit: 0
-  format_version: 2
-  enable_index_compression: 1
-  block_align: 0
-2020/01/11-12:14:13.197959 7fd2bb712440        Options.write_buffer_size: 67108864
-2020/01/11-12:14:13.197960 7fd2bb712440  Options.max_write_buffer_number: 2
-2020/01/11-12:14:13.197962 7fd2bb712440          Options.compression: NoCompression
-2020/01/11-12:14:13.197963 7fd2bb712440                  Options.bottommost_compression: Disabled
-2020/01/11-12:14:13.197965 7fd2bb712440       Options.prefix_extractor: nullptr
-2020/01/11-12:14:13.197966 7fd2bb712440   Options.memtable_insert_with_hint_prefix_extractor: nullptr
-2020/01/11-12:14:13.197967 7fd2bb712440             Options.num_levels: 7
-2020/01/11-12:14:13.197968 7fd2bb712440        Options.min_write_buffer_number_to_merge: 1
-2020/01/11-12:14:13.197970 7fd2bb712440     Options.max_write_buffer_number_to_maintain: 0
-2020/01/11-12:14:13.197971 7fd2bb712440            Options.bottommost_compression_opts.window_bits: -14
-2020/01/11-12:14:13.197972 7fd2bb712440                  Options.bottommost_compression_opts.level: 32767
-2020/01/11-12:14:13.197973 7fd2bb712440               Options.bottommost_compression_opts.strategy: 0
-2020/01/11-12:14:13.197975 7fd2bb712440         Options.bottommost_compression_opts.max_dict_bytes: 0
-2020/01/11-12:14:13.197976 7fd2bb712440         Options.bottommost_compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:14:13.197977 7fd2bb712440                  Options.bottommost_compression_opts.enabled: false
-2020/01/11-12:14:13.197978 7fd2bb712440            Options.compression_opts.window_bits: -14
-2020/01/11-12:14:13.197980 7fd2bb712440                  Options.compression_opts.level: 32767
-2020/01/11-12:14:13.197981 7fd2bb712440               Options.compression_opts.strategy: 0
-2020/01/11-12:14:13.197982 7fd2bb712440         Options.compression_opts.max_dict_bytes: 0
-2020/01/11-12:14:13.197983 7fd2bb712440         Options.compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:14:13.197984 7fd2bb712440                  Options.compression_opts.enabled: false
-2020/01/11-12:14:13.197985 7fd2bb712440      Options.level0_file_num_compaction_trigger: 4
-2020/01/11-12:14:13.197987 7fd2bb712440          Options.level0_slowdown_writes_trigger: 20
-2020/01/11-12:14:13.197988 7fd2bb712440              Options.level0_stop_writes_trigger: 36
-2020/01/11-12:14:13.197989 7fd2bb712440                   Options.target_file_size_base: 67108864
-2020/01/11-12:14:13.197990 7fd2bb712440             Options.target_file_size_multiplier: 1
-2020/01/11-12:14:13.197991 7fd2bb712440                Options.max_bytes_for_level_base: 268435456
-2020/01/11-12:14:13.197992 7fd2bb712440                      Options.snap_refresh_nanos: 0
-2020/01/11-12:14:13.197994 7fd2bb712440 Options.level_compaction_dynamic_level_bytes: 0
-2020/01/11-12:14:13.197995 7fd2bb712440          Options.max_bytes_for_level_multiplier: 10.000000
-2020/01/11-12:14:13.197997 7fd2bb712440 Options.max_bytes_for_level_multiplier_addtl[0]: 1
-2020/01/11-12:14:13.198004 7fd2bb712440 Options.max_bytes_for_level_multiplier_addtl[1]: 1
-2020/01/11-12:14:13.198005 7fd2bb712440 Options.max_bytes_for_level_multiplier_addtl[2]: 1
-2020/01/11-12:14:13.198006 7fd2bb712440 Options.max_bytes_for_level_multiplier_addtl[3]: 1
-2020/01/11-12:14:13.198007 7fd2bb712440 Options.max_bytes_for_level_multiplier_addtl[4]: 1
-2020/01/11-12:14:13.198009 7fd2bb712440 Options.max_bytes_for_level_multiplier_addtl[5]: 1
-2020/01/11-12:14:13.198010 7fd2bb712440 Options.max_bytes_for_level_multiplier_addtl[6]: 1
-2020/01/11-12:14:13.198011 7fd2bb712440       Options.max_sequential_skip_in_iterations: 8
-2020/01/11-12:14:13.198012 7fd2bb712440                    Options.max_compaction_bytes: 1677721600
-2020/01/11-12:14:13.198013 7fd2bb712440                        Options.arena_block_size: 8388608
-2020/01/11-12:14:13.198014 7fd2bb712440   Options.soft_pending_compaction_bytes_limit: 68719476736
-2020/01/11-12:14:13.198016 7fd2bb712440   Options.hard_pending_compaction_bytes_limit: 274877906944
-2020/01/11-12:14:13.198017 7fd2bb712440       Options.rate_limit_delay_max_milliseconds: 100
-2020/01/11-12:14:13.198018 7fd2bb712440                Options.disable_auto_compactions: 0
-2020/01/11-12:14:13.198020 7fd2bb712440                        Options.compaction_style: kCompactionStyleLevel
-2020/01/11-12:14:13.198021 7fd2bb712440                          Options.compaction_pri: kMinOverlappingRatio
-2020/01/11-12:14:13.198022 7fd2bb712440 Options.compaction_options_universal.size_ratio: 1
-2020/01/11-12:14:13.198024 7fd2bb712440 Options.compaction_options_universal.min_merge_width: 2
-2020/01/11-12:14:13.198025 7fd2bb712440 Options.compaction_options_universal.max_merge_width: 4294967295
-2020/01/11-12:14:13.198026 7fd2bb712440 Options.compaction_options_universal.max_size_amplification_percent: 200
-2020/01/11-12:14:13.198027 7fd2bb712440 Options.compaction_options_universal.compression_size_percent: -1
-2020/01/11-12:14:13.198028 7fd2bb712440 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
-2020/01/11-12:14:13.198030 7fd2bb712440 Options.compaction_options_fifo.max_table_files_size: 1073741824
-2020/01/11-12:14:13.198031 7fd2bb712440 Options.compaction_options_fifo.allow_compaction: 0
-2020/01/11-12:14:13.198032 7fd2bb712440                   Options.table_properties_collectors: 
-2020/01/11-12:14:13.198033 7fd2bb712440                   Options.inplace_update_support: 0
-2020/01/11-12:14:13.198034 7fd2bb712440                 Options.inplace_update_num_locks: 10000
-2020/01/11-12:14:13.198035 7fd2bb712440               Options.memtable_prefix_bloom_size_ratio: 0.000000
-2020/01/11-12:14:13.198037 7fd2bb712440               Options.memtable_whole_key_filtering: 0
-2020/01/11-12:14:13.198038 7fd2bb712440   Options.memtable_huge_page_size: 0
-2020/01/11-12:14:13.198039 7fd2bb712440                           Options.bloom_locality: 0
-2020/01/11-12:14:13.198040 7fd2bb712440                    Options.max_successive_merges: 0
-2020/01/11-12:14:13.198042 7fd2bb712440                Options.optimize_filters_for_hits: 0
-2020/01/11-12:14:13.198043 7fd2bb712440                Options.paranoid_file_checks: 0
-2020/01/11-12:14:13.198044 7fd2bb712440                Options.force_consistency_checks: 0
-2020/01/11-12:14:13.198045 7fd2bb712440                Options.report_bg_io_stats: 0
-2020/01/11-12:14:13.198046 7fd2bb712440                               Options.ttl: 0
-2020/01/11-12:14:13.198047 7fd2bb712440          Options.periodic_compaction_seconds: 0
-2020/01/11-12:14:13.198592 7fd2bb712440 [/version_set.cc:4267] Recovered from manifest file:kvdb/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 10, last_sequence is 0, log_number is 7,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
-2020/01/11-12:14:13.198596 7fd2bb712440 [/version_set.cc:4276] Column family [default] (ID 0), log number is 7
-2020/01/11-12:14:13.198655 7fd2bb712440 EVENT_LOG_v1 {"time_micros": 1578716053198645, "job": 1, "event": "recovery_started", "log_files": [9]}
-2020/01/11-12:14:13.198669 7fd2bb712440 [/db_impl_open.cc:597] Recovering log #9 mode 2
-2020/01/11-12:14:13.198730 7fd2bb712440 [/version_set.cc:3546] Creating manifest 11
-2020/01/11-12:14:13.199526 7fd2bb712440 EVENT_LOG_v1 {"time_micros": 1578716053199520, "job": 1, "event": "recovery_finished"}
-2020/01/11-12:14:13.201585 7fd2bb712440 DB pointer 0x1fd857f0
-2020/01/11-12:14:13.201860 7fd23d1f4700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-12:14:13.201894 7fd23d1f4700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 0.0 total, 0.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **

examples/demo-serving/kvdb/LOG.old.1578717332758385

-2020/01/11-12:24:05.180155 7fcf91393440 RocksDB version: 6.2.4
-2020/01/11-12:24:05.180191 7fcf91393440 Git sha rocksdb_build_git_sha:3513d4e93f8530ac44cc0c1efea04be4259c6938
-2020/01/11-12:24:05.180194 7fcf91393440 Compile date Dec 23 2019
-2020/01/11-12:24:05.180197 7fcf91393440 DB SUMMARY
-2020/01/11-12:24:05.180227 7fcf91393440 CURRENT file:  CURRENT
-2020/01/11-12:24:05.180229 7fcf91393440 IDENTITY file:  IDENTITY
-2020/01/11-12:24:05.180232 7fcf91393440 MANIFEST file:  MANIFEST-000011 size: 59 Bytes
-2020/01/11-12:24:05.180235 7fcf91393440 SST files in kvdb dir, Total Num: 0, files: 
-2020/01/11-12:24:05.180237 7fcf91393440 Write Ahead Log file in kvdb: 000012.log size: 0 ; 
-2020/01/11-12:24:05.180239 7fcf91393440                         Options.error_if_exists: 0
-2020/01/11-12:24:05.180241 7fcf91393440                       Options.create_if_missing: 1
-2020/01/11-12:24:05.180242 7fcf91393440                         Options.paranoid_checks: 1
-2020/01/11-12:24:05.180243 7fcf91393440                                     Options.env: 0x1bed2ac0
-2020/01/11-12:24:05.180245 7fcf91393440                                Options.info_log: 0x1f6774a0
-2020/01/11-12:24:05.180246 7fcf91393440                Options.max_file_opening_threads: 16
-2020/01/11-12:24:05.180247 7fcf91393440                              Options.statistics: (nil)
-2020/01/11-12:24:05.180248 7fcf91393440                               Options.use_fsync: 0
-2020/01/11-12:24:05.180250 7fcf91393440                       Options.max_log_file_size: 0
-2020/01/11-12:24:05.180251 7fcf91393440                  Options.max_manifest_file_size: 1073741824
-2020/01/11-12:24:05.180252 7fcf91393440                   Options.log_file_time_to_roll: 0
-2020/01/11-12:24:05.180254 7fcf91393440                       Options.keep_log_file_num: 1000
-2020/01/11-12:24:05.180255 7fcf91393440                    Options.recycle_log_file_num: 0
-2020/01/11-12:24:05.180256 7fcf91393440                         Options.allow_fallocate: 1
-2020/01/11-12:24:05.180258 7fcf91393440                        Options.allow_mmap_reads: 0
-2020/01/11-12:24:05.180259 7fcf91393440                       Options.allow_mmap_writes: 0
-2020/01/11-12:24:05.180260 7fcf91393440                        Options.use_direct_reads: 0
-2020/01/11-12:24:05.180261 7fcf91393440                        Options.use_direct_io_for_flush_and_compaction: 0
-2020/01/11-12:24:05.180262 7fcf91393440          Options.create_missing_column_families: 0
-2020/01/11-12:24:05.180265 7fcf91393440                              Options.db_log_dir: 
-2020/01/11-12:24:05.180266 7fcf91393440                                 Options.wal_dir: kvdb
-2020/01/11-12:24:05.180268 7fcf91393440                Options.table_cache_numshardbits: 6
-2020/01/11-12:24:05.180269 7fcf91393440                      Options.max_subcompactions: 1
-2020/01/11-12:24:05.180270 7fcf91393440                  Options.max_background_flushes: -1
-2020/01/11-12:24:05.180272 7fcf91393440                         Options.WAL_ttl_seconds: 0
-2020/01/11-12:24:05.180273 7fcf91393440                       Options.WAL_size_limit_MB: 0
-2020/01/11-12:24:05.180274 7fcf91393440             Options.manifest_preallocation_size: 4194304
-2020/01/11-12:24:05.180275 7fcf91393440                     Options.is_fd_close_on_exec: 1
-2020/01/11-12:24:05.180277 7fcf91393440                   Options.advise_random_on_open: 1
-2020/01/11-12:24:05.180278 7fcf91393440                    Options.db_write_buffer_size: 0
-2020/01/11-12:24:05.180279 7fcf91393440                    Options.write_buffer_manager: 0x1f677500
-2020/01/11-12:24:05.180280 7fcf91393440         Options.access_hint_on_compaction_start: 1
-2020/01/11-12:24:05.180281 7fcf91393440  Options.new_table_reader_for_compaction_inputs: 0
-2020/01/11-12:24:05.180282 7fcf91393440           Options.random_access_max_buffer_size: 1048576
-2020/01/11-12:24:05.180283 7fcf91393440                      Options.use_adaptive_mutex: 0
-2020/01/11-12:24:05.180285 7fcf91393440                            Options.rate_limiter: (nil)
-2020/01/11-12:24:05.180286 7fcf91393440     Options.sst_file_manager.rate_bytes_per_sec: 0
-2020/01/11-12:24:05.180288 7fcf91393440                       Options.wal_recovery_mode: 2
-2020/01/11-12:24:05.180297 7fcf91393440                  Options.enable_thread_tracking: 0
-2020/01/11-12:24:05.180299 7fcf91393440                  Options.enable_pipelined_write: 0
-2020/01/11-12:24:05.180300 7fcf91393440         Options.allow_concurrent_memtable_write: 1
-2020/01/11-12:24:05.180301 7fcf91393440      Options.enable_write_thread_adaptive_yield: 1
-2020/01/11-12:24:05.180302 7fcf91393440             Options.write_thread_max_yield_usec: 100
-2020/01/11-12:24:05.180303 7fcf91393440            Options.write_thread_slow_yield_usec: 3
-2020/01/11-12:24:05.180304 7fcf91393440                               Options.row_cache: None
-2020/01/11-12:24:05.180306 7fcf91393440                              Options.wal_filter: None
-2020/01/11-12:24:05.180307 7fcf91393440             Options.avoid_flush_during_recovery: 0
-2020/01/11-12:24:05.180308 7fcf91393440             Options.allow_ingest_behind: 0
-2020/01/11-12:24:05.180309 7fcf91393440             Options.preserve_deletes: 0
-2020/01/11-12:24:05.180310 7fcf91393440             Options.two_write_queues: 0
-2020/01/11-12:24:05.180312 7fcf91393440             Options.manual_wal_flush: 0
-2020/01/11-12:24:05.180313 7fcf91393440             Options.atomic_flush: 0
-2020/01/11-12:24:05.180314 7fcf91393440             Options.avoid_unnecessary_blocking_io: 0
-2020/01/11-12:24:05.180315 7fcf91393440             Options.max_background_jobs: 2
-2020/01/11-12:24:05.180316 7fcf91393440             Options.max_background_compactions: -1
-2020/01/11-12:24:05.180318 7fcf91393440             Options.avoid_flush_during_shutdown: 0
-2020/01/11-12:24:05.180319 7fcf91393440           Options.writable_file_max_buffer_size: 1048576
-2020/01/11-12:24:05.180320 7fcf91393440             Options.delayed_write_rate : 16777216
-2020/01/11-12:24:05.180322 7fcf91393440             Options.max_total_wal_size: 0
-2020/01/11-12:24:05.180323 7fcf91393440             Options.delete_obsolete_files_period_micros: 21600000000
-2020/01/11-12:24:05.180324 7fcf91393440                   Options.stats_dump_period_sec: 600
-2020/01/11-12:24:05.180326 7fcf91393440                 Options.stats_persist_period_sec: 600
-2020/01/11-12:24:05.180327 7fcf91393440                 Options.stats_history_buffer_size: 1048576
-2020/01/11-12:24:05.180328 7fcf91393440                          Options.max_open_files: -1
-2020/01/11-12:24:05.180329 7fcf91393440                          Options.bytes_per_sync: 0
-2020/01/11-12:24:05.180330 7fcf91393440                      Options.wal_bytes_per_sync: 0
-2020/01/11-12:24:05.180331 7fcf91393440                   Options.strict_bytes_per_sync: 0
-2020/01/11-12:24:05.180333 7fcf91393440       Options.compaction_readahead_size: 0
-2020/01/11-12:24:05.180334 7fcf91393440 Compression algorithms supported:
-2020/01/11-12:24:05.180344 7fcf91393440 	kZSTDNotFinalCompression supported: 0
-2020/01/11-12:24:05.180350 7fcf91393440 	kZSTD supported: 0
-2020/01/11-12:24:05.180351 7fcf91393440 	kXpressCompression supported: 0
-2020/01/11-12:24:05.180352 7fcf91393440 	kLZ4HCCompression supported: 0
-2020/01/11-12:24:05.180354 7fcf91393440 	kLZ4Compression supported: 0
-2020/01/11-12:24:05.180355 7fcf91393440 	kBZip2Compression supported: 1
-2020/01/11-12:24:05.180356 7fcf91393440 	kZlibCompression supported: 1
-2020/01/11-12:24:05.180357 7fcf91393440 	kSnappyCompression supported: 0
-2020/01/11-12:24:05.180360 7fcf91393440 Fast CRC32 supported: Supported on x86
-2020/01/11-12:24:05.180560 7fcf91393440 [/version_set.cc:4053] Recovering from manifest file: kvdb/MANIFEST-000011
-2020/01/11-12:24:05.180623 7fcf91393440 [/column_family.cc:482] --------------- Options for column family [default]:
-2020/01/11-12:24:05.180627 7fcf91393440               Options.comparator: leveldb.BytewiseComparator
-2020/01/11-12:24:05.180629 7fcf91393440           Options.merge_operator: None
-2020/01/11-12:24:05.180630 7fcf91393440        Options.compaction_filter: None
-2020/01/11-12:24:05.180631 7fcf91393440        Options.compaction_filter_factory: None
-2020/01/11-12:24:05.180632 7fcf91393440         Options.memtable_factory: SkipListFactory
-2020/01/11-12:24:05.180645 7fcf91393440            Options.table_factory: BlockBasedTable
-2020/01/11-12:24:05.180680 7fcf91393440            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x1e2afd50)
-  cache_index_and_filter_blocks: 0
-  cache_index_and_filter_blocks_with_high_priority: 0
-  pin_l0_filter_and_index_blocks_in_cache: 0
-  pin_top_level_index_and_filter: 1
-  index_type: 0
-  data_block_index_type: 0
-  index_shortening: 1
-  data_block_hash_table_util_ratio: 0.750000
-  hash_index_allow_collision: 1
-  checksum: 1
-  no_block_cache: 0
-  block_cache: 0x1e2bdf80
-  block_cache_name: LRUCache
-  block_cache_options:
-    capacity : 8388608
-    num_shard_bits : 4
-    strict_capacity_limit : 0
-    memory_allocator : None
-    high_pri_pool_ratio: 0.000
-  block_cache_compressed: (nil)
-  persistent_cache: (nil)
-  block_size: 4096
-  block_size_deviation: 10
-  block_restart_interval: 16
-  index_block_restart_interval: 1
-  metadata_block_size: 4096
-  partition_filters: 0
-  use_delta_encoding: 1
-  filter_policy: nullptr
-  whole_key_filtering: 1
-  verify_compression: 0
-  read_amp_bytes_per_bit: 0
-  format_version: 2
-  enable_index_compression: 1
-  block_align: 0
-2020/01/11-12:24:05.180684 7fcf91393440        Options.write_buffer_size: 67108864
-2020/01/11-12:24:05.180686 7fcf91393440  Options.max_write_buffer_number: 2
-2020/01/11-12:24:05.180687 7fcf91393440          Options.compression: NoCompression
-2020/01/11-12:24:05.180689 7fcf91393440                  Options.bottommost_compression: Disabled
-2020/01/11-12:24:05.180690 7fcf91393440       Options.prefix_extractor: nullptr
-2020/01/11-12:24:05.180692 7fcf91393440   Options.memtable_insert_with_hint_prefix_extractor: nullptr
-2020/01/11-12:24:05.180693 7fcf91393440             Options.num_levels: 7
-2020/01/11-12:24:05.180694 7fcf91393440        Options.min_write_buffer_number_to_merge: 1
-2020/01/11-12:24:05.180696 7fcf91393440     Options.max_write_buffer_number_to_maintain: 0
-2020/01/11-12:24:05.180697 7fcf91393440            Options.bottommost_compression_opts.window_bits: -14
-2020/01/11-12:24:05.180698 7fcf91393440                  Options.bottommost_compression_opts.level: 32767
-2020/01/11-12:24:05.180699 7fcf91393440               Options.bottommost_compression_opts.strategy: 0
-2020/01/11-12:24:05.180701 7fcf91393440         Options.bottommost_compression_opts.max_dict_bytes: 0
-2020/01/11-12:24:05.180702 7fcf91393440         Options.bottommost_compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:24:05.180703 7fcf91393440                  Options.bottommost_compression_opts.enabled: false
-2020/01/11-12:24:05.180705 7fcf91393440            Options.compression_opts.window_bits: -14
-2020/01/11-12:24:05.180706 7fcf91393440                  Options.compression_opts.level: 32767
-2020/01/11-12:24:05.180707 7fcf91393440               Options.compression_opts.strategy: 0
-2020/01/11-12:24:05.180708 7fcf91393440         Options.compression_opts.max_dict_bytes: 0
-2020/01/11-12:24:05.180710 7fcf91393440         Options.compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:24:05.180711 7fcf91393440                  Options.compression_opts.enabled: false
-2020/01/11-12:24:05.180712 7fcf91393440      Options.level0_file_num_compaction_trigger: 4
-2020/01/11-12:24:05.180713 7fcf91393440          Options.level0_slowdown_writes_trigger: 20
-2020/01/11-12:24:05.180715 7fcf91393440              Options.level0_stop_writes_trigger: 36
-2020/01/11-12:24:05.180716 7fcf91393440                   Options.target_file_size_base: 67108864
-2020/01/11-12:24:05.180717 7fcf91393440             Options.target_file_size_multiplier: 1
-2020/01/11-12:24:05.180718 7fcf91393440                Options.max_bytes_for_level_base: 268435456
-2020/01/11-12:24:05.180720 7fcf91393440                      Options.snap_refresh_nanos: 0
-2020/01/11-12:24:05.180721 7fcf91393440 Options.level_compaction_dynamic_level_bytes: 0
-2020/01/11-12:24:05.180722 7fcf91393440          Options.max_bytes_for_level_multiplier: 10.000000
-2020/01/11-12:24:05.180724 7fcf91393440 Options.max_bytes_for_level_multiplier_addtl[0]: 1
-2020/01/11-12:24:05.180731 7fcf91393440 Options.max_bytes_for_level_multiplier_addtl[1]: 1
-2020/01/11-12:24:05.180732 7fcf91393440 Options.max_bytes_for_level_multiplier_addtl[2]: 1
-2020/01/11-12:24:05.180733 7fcf91393440 Options.max_bytes_for_level_multiplier_addtl[3]: 1
-2020/01/11-12:24:05.180734 7fcf91393440 Options.max_bytes_for_level_multiplier_addtl[4]: 1
-2020/01/11-12:24:05.180735 7fcf91393440 Options.max_bytes_for_level_multiplier_addtl[5]: 1
-2020/01/11-12:24:05.180737 7fcf91393440 Options.max_bytes_for_level_multiplier_addtl[6]: 1
-2020/01/11-12:24:05.180738 7fcf91393440       Options.max_sequential_skip_in_iterations: 8
-2020/01/11-12:24:05.180739 7fcf91393440                    Options.max_compaction_bytes: 1677721600
-2020/01/11-12:24:05.180740 7fcf91393440                        Options.arena_block_size: 8388608
-2020/01/11-12:24:05.180741 7fcf91393440   Options.soft_pending_compaction_bytes_limit: 68719476736
-2020/01/11-12:24:05.180743 7fcf91393440   Options.hard_pending_compaction_bytes_limit: 274877906944
-2020/01/11-12:24:05.180744 7fcf91393440       Options.rate_limit_delay_max_milliseconds: 100
-2020/01/11-12:24:05.180745 7fcf91393440                Options.disable_auto_compactions: 0
-2020/01/11-12:24:05.180747 7fcf91393440                        Options.compaction_style: kCompactionStyleLevel
-2020/01/11-12:24:05.180748 7fcf91393440                          Options.compaction_pri: kMinOverlappingRatio
-2020/01/11-12:24:05.180749 7fcf91393440 Options.compaction_options_universal.size_ratio: 1
-2020/01/11-12:24:05.180751 7fcf91393440 Options.compaction_options_universal.min_merge_width: 2
-2020/01/11-12:24:05.180752 7fcf91393440 Options.compaction_options_universal.max_merge_width: 4294967295
-2020/01/11-12:24:05.180753 7fcf91393440 Options.compaction_options_universal.max_size_amplification_percent: 200
-2020/01/11-12:24:05.180754 7fcf91393440 Options.compaction_options_universal.compression_size_percent: -1
-2020/01/11-12:24:05.180756 7fcf91393440 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
-2020/01/11-12:24:05.180757 7fcf91393440 Options.compaction_options_fifo.max_table_files_size: 1073741824
-2020/01/11-12:24:05.180758 7fcf91393440 Options.compaction_options_fifo.allow_compaction: 0
-2020/01/11-12:24:05.180759 7fcf91393440                   Options.table_properties_collectors: 
-2020/01/11-12:24:05.180761 7fcf91393440                   Options.inplace_update_support: 0
-2020/01/11-12:24:05.180762 7fcf91393440                 Options.inplace_update_num_locks: 10000
-2020/01/11-12:24:05.180763 7fcf91393440               Options.memtable_prefix_bloom_size_ratio: 0.000000
-2020/01/11-12:24:05.180765 7fcf91393440               Options.memtable_whole_key_filtering: 0
-2020/01/11-12:24:05.180766 7fcf91393440   Options.memtable_huge_page_size: 0
-2020/01/11-12:24:05.180767 7fcf91393440                           Options.bloom_locality: 0
-2020/01/11-12:24:05.180768 7fcf91393440                    Options.max_successive_merges: 0
-2020/01/11-12:24:05.180769 7fcf91393440                Options.optimize_filters_for_hits: 0
-2020/01/11-12:24:05.180770 7fcf91393440                Options.paranoid_file_checks: 0
-2020/01/11-12:24:05.180771 7fcf91393440                Options.force_consistency_checks: 0
-2020/01/11-12:24:05.180773 7fcf91393440                Options.report_bg_io_stats: 0
-2020/01/11-12:24:05.180774 7fcf91393440                               Options.ttl: 0
-2020/01/11-12:24:05.180775 7fcf91393440          Options.periodic_compaction_seconds: 0
-2020/01/11-12:24:05.181303 7fcf91393440 [/version_set.cc:4267] Recovered from manifest file:kvdb/MANIFEST-000011 succeeded,manifest_file_number is 11, next_file_number is 13, last_sequence is 0, log_number is 10,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
-2020/01/11-12:24:05.181308 7fcf91393440 [/version_set.cc:4276] Column family [default] (ID 0), log number is 10
-2020/01/11-12:24:05.181374 7fcf91393440 EVENT_LOG_v1 {"time_micros": 1578716645181364, "job": 1, "event": "recovery_started", "log_files": [12]}
-2020/01/11-12:24:05.181387 7fcf91393440 [/db_impl_open.cc:597] Recovering log #12 mode 2
-2020/01/11-12:24:05.181461 7fcf91393440 [/version_set.cc:3546] Creating manifest 14
-2020/01/11-12:24:05.182388 7fcf91393440 EVENT_LOG_v1 {"time_micros": 1578716645182384, "job": 1, "event": "recovery_finished"}
-2020/01/11-12:24:05.184452 7fcf91393440 DB pointer 0x1f676190
-2020/01/11-12:24:05.184787 7fcf12bfd700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-12:24:05.184823 7fcf12bfd700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 0.0 total, 0.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-2020/01/11-12:34:05.185022 7fcf12bfd700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-12:34:05.185072 7fcf12bfd700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 600.0 total, 600.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 600.0 total, 600.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 600.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **

examples/demo-serving/kvdb/LOG.old.1578717444163782

-2020/01/11-12:35:32.758840 7f43e73ec440 RocksDB version: 6.2.4
-2020/01/11-12:35:32.758875 7f43e73ec440 Git sha rocksdb_build_git_sha:3513d4e93f8530ac44cc0c1efea04be4259c6938
-2020/01/11-12:35:32.758878 7f43e73ec440 Compile date Dec 23 2019
-2020/01/11-12:35:32.758882 7f43e73ec440 DB SUMMARY
-2020/01/11-12:35:32.758911 7f43e73ec440 CURRENT file:  CURRENT
-2020/01/11-12:35:32.758913 7f43e73ec440 IDENTITY file:  IDENTITY
-2020/01/11-12:35:32.758916 7f43e73ec440 MANIFEST file:  MANIFEST-000014 size: 59 Bytes
-2020/01/11-12:35:32.758919 7f43e73ec440 SST files in kvdb dir, Total Num: 0, files: 
-2020/01/11-12:35:32.758921 7f43e73ec440 Write Ahead Log file in kvdb: 000015.log size: 0 ; 
-2020/01/11-12:35:32.758923 7f43e73ec440                         Options.error_if_exists: 0
-2020/01/11-12:35:32.758924 7f43e73ec440                       Options.create_if_missing: 1
-2020/01/11-12:35:32.758926 7f43e73ec440                         Options.paranoid_checks: 1
-2020/01/11-12:35:32.758927 7f43e73ec440                                     Options.env: 0x1bed2ac0
-2020/01/11-12:35:32.758928 7f43e73ec440                                Options.info_log: 0x207818b0
-2020/01/11-12:35:32.758930 7f43e73ec440                Options.max_file_opening_threads: 16
-2020/01/11-12:35:32.758931 7f43e73ec440                              Options.statistics: (nil)
-2020/01/11-12:35:32.758932 7f43e73ec440                               Options.use_fsync: 0
-2020/01/11-12:35:32.758934 7f43e73ec440                       Options.max_log_file_size: 0
-2020/01/11-12:35:32.758935 7f43e73ec440                  Options.max_manifest_file_size: 1073741824
-2020/01/11-12:35:32.758936 7f43e73ec440                   Options.log_file_time_to_roll: 0
-2020/01/11-12:35:32.758937 7f43e73ec440                       Options.keep_log_file_num: 1000
-2020/01/11-12:35:32.758938 7f43e73ec440                    Options.recycle_log_file_num: 0
-2020/01/11-12:35:32.758940 7f43e73ec440                         Options.allow_fallocate: 1
-2020/01/11-12:35:32.758941 7f43e73ec440                        Options.allow_mmap_reads: 0
-2020/01/11-12:35:32.758942 7f43e73ec440                       Options.allow_mmap_writes: 0
-2020/01/11-12:35:32.758943 7f43e73ec440                        Options.use_direct_reads: 0
-2020/01/11-12:35:32.758944 7f43e73ec440                        Options.use_direct_io_for_flush_and_compaction: 0
-2020/01/11-12:35:32.758945 7f43e73ec440          Options.create_missing_column_families: 0
-2020/01/11-12:35:32.758948 7f43e73ec440                              Options.db_log_dir: 
-2020/01/11-12:35:32.758949 7f43e73ec440                                 Options.wal_dir: kvdb
-2020/01/11-12:35:32.758951 7f43e73ec440                Options.table_cache_numshardbits: 6
-2020/01/11-12:35:32.758952 7f43e73ec440                      Options.max_subcompactions: 1
-2020/01/11-12:35:32.758953 7f43e73ec440                  Options.max_background_flushes: -1
-2020/01/11-12:35:32.758954 7f43e73ec440                         Options.WAL_ttl_seconds: 0
-2020/01/11-12:35:32.758955 7f43e73ec440                       Options.WAL_size_limit_MB: 0
-2020/01/11-12:35:32.758957 7f43e73ec440             Options.manifest_preallocation_size: 4194304
-2020/01/11-12:35:32.758958 7f43e73ec440                     Options.is_fd_close_on_exec: 1
-2020/01/11-12:35:32.758960 7f43e73ec440                   Options.advise_random_on_open: 1
-2020/01/11-12:35:32.758961 7f43e73ec440                    Options.db_write_buffer_size: 0
-2020/01/11-12:35:32.758962 7f43e73ec440                    Options.write_buffer_manager: 0x1e591160
-2020/01/11-12:35:32.758963 7f43e73ec440         Options.access_hint_on_compaction_start: 1
-2020/01/11-12:35:32.758964 7f43e73ec440  Options.new_table_reader_for_compaction_inputs: 0
-2020/01/11-12:35:32.758965 7f43e73ec440           Options.random_access_max_buffer_size: 1048576
-2020/01/11-12:35:32.758966 7f43e73ec440                      Options.use_adaptive_mutex: 0
-2020/01/11-12:35:32.758967 7f43e73ec440                            Options.rate_limiter: (nil)
-2020/01/11-12:35:32.758969 7f43e73ec440     Options.sst_file_manager.rate_bytes_per_sec: 0
-2020/01/11-12:35:32.758970 7f43e73ec440                       Options.wal_recovery_mode: 2
-2020/01/11-12:35:32.758977 7f43e73ec440                  Options.enable_thread_tracking: 0
-2020/01/11-12:35:32.758979 7f43e73ec440                  Options.enable_pipelined_write: 0
-2020/01/11-12:35:32.758980 7f43e73ec440         Options.allow_concurrent_memtable_write: 1
-2020/01/11-12:35:32.758981 7f43e73ec440      Options.enable_write_thread_adaptive_yield: 1
-2020/01/11-12:35:32.758982 7f43e73ec440             Options.write_thread_max_yield_usec: 100
-2020/01/11-12:35:32.758983 7f43e73ec440            Options.write_thread_slow_yield_usec: 3
-2020/01/11-12:35:32.758984 7f43e73ec440                               Options.row_cache: None
-2020/01/11-12:35:32.758985 7f43e73ec440                              Options.wal_filter: None
-2020/01/11-12:35:32.758987 7f43e73ec440             Options.avoid_flush_during_recovery: 0
-2020/01/11-12:35:32.758988 7f43e73ec440             Options.allow_ingest_behind: 0
-2020/01/11-12:35:32.758989 7f43e73ec440             Options.preserve_deletes: 0
-2020/01/11-12:35:32.758990 7f43e73ec440             Options.two_write_queues: 0
-2020/01/11-12:35:32.758991 7f43e73ec440             Options.manual_wal_flush: 0
-2020/01/11-12:35:32.758993 7f43e73ec440             Options.atomic_flush: 0
-2020/01/11-12:35:32.758994 7f43e73ec440             Options.avoid_unnecessary_blocking_io: 0
-2020/01/11-12:35:32.758995 7f43e73ec440             Options.max_background_jobs: 2
-2020/01/11-12:35:32.758996 7f43e73ec440             Options.max_background_compactions: -1
-2020/01/11-12:35:32.758997 7f43e73ec440             Options.avoid_flush_during_shutdown: 0
-2020/01/11-12:35:32.758998 7f43e73ec440           Options.writable_file_max_buffer_size: 1048576
-2020/01/11-12:35:32.758999 7f43e73ec440             Options.delayed_write_rate : 16777216
-2020/01/11-12:35:32.759001 7f43e73ec440             Options.max_total_wal_size: 0
-2020/01/11-12:35:32.759002 7f43e73ec440             Options.delete_obsolete_files_period_micros: 21600000000
-2020/01/11-12:35:32.759003 7f43e73ec440                   Options.stats_dump_period_sec: 600
-2020/01/11-12:35:32.759004 7f43e73ec440                 Options.stats_persist_period_sec: 600
-2020/01/11-12:35:32.759005 7f43e73ec440                 Options.stats_history_buffer_size: 1048576
-2020/01/11-12:35:32.759007 7f43e73ec440                          Options.max_open_files: -1
-2020/01/11-12:35:32.759008 7f43e73ec440                          Options.bytes_per_sync: 0
-2020/01/11-12:35:32.759009 7f43e73ec440                      Options.wal_bytes_per_sync: 0
-2020/01/11-12:35:32.759010 7f43e73ec440                   Options.strict_bytes_per_sync: 0
-2020/01/11-12:35:32.759011 7f43e73ec440       Options.compaction_readahead_size: 0
-2020/01/11-12:35:32.759012 7f43e73ec440 Compression algorithms supported:
-2020/01/11-12:35:32.759026 7f43e73ec440 	kZSTDNotFinalCompression supported: 0
-2020/01/11-12:35:32.759031 7f43e73ec440 	kZSTD supported: 0
-2020/01/11-12:35:32.759033 7f43e73ec440 	kXpressCompression supported: 0
-2020/01/11-12:35:32.759034 7f43e73ec440 	kLZ4HCCompression supported: 0
-2020/01/11-12:35:32.759035 7f43e73ec440 	kLZ4Compression supported: 0
-2020/01/11-12:35:32.759036 7f43e73ec440 	kBZip2Compression supported: 1
-2020/01/11-12:35:32.759038 7f43e73ec440 	kZlibCompression supported: 1
-2020/01/11-12:35:32.759039 7f43e73ec440 	kSnappyCompression supported: 0
-2020/01/11-12:35:32.759041 7f43e73ec440 Fast CRC32 supported: Supported on x86
-2020/01/11-12:35:32.759237 7f43e73ec440 [/version_set.cc:4053] Recovering from manifest file: kvdb/MANIFEST-000014
-2020/01/11-12:35:32.759300 7f43e73ec440 [/column_family.cc:482] --------------- Options for column family [default]:
-2020/01/11-12:35:32.759304 7f43e73ec440               Options.comparator: leveldb.BytewiseComparator
-2020/01/11-12:35:32.759305 7f43e73ec440           Options.merge_operator: None
-2020/01/11-12:35:32.759306 7f43e73ec440        Options.compaction_filter: None
-2020/01/11-12:35:32.759307 7f43e73ec440        Options.compaction_filter_factory: None
-2020/01/11-12:35:32.759309 7f43e73ec440         Options.memtable_factory: SkipListFactory
-2020/01/11-12:35:32.759320 7f43e73ec440            Options.table_factory: BlockBasedTable
-2020/01/11-12:35:32.759356 7f43e73ec440            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x1f223340)
-  cache_index_and_filter_blocks: 0
-  cache_index_and_filter_blocks_with_high_priority: 0
-  pin_l0_filter_and_index_blocks_in_cache: 0
-  pin_top_level_index_and_filter: 1
-  index_type: 0
-  data_block_index_type: 0
-  index_shortening: 1
-  data_block_hash_table_util_ratio: 0.750000
-  hash_index_allow_collision: 1
-  checksum: 1
-  no_block_cache: 0
-  block_cache: 0x1f24b760
-  block_cache_name: LRUCache
-  block_cache_options:
-    capacity : 8388608
-    num_shard_bits : 4
-    strict_capacity_limit : 0
-    memory_allocator : None
-    high_pri_pool_ratio: 0.000
-  block_cache_compressed: (nil)
-  persistent_cache: (nil)
-  block_size: 4096
-  block_size_deviation: 10
-  block_restart_interval: 16
-  index_block_restart_interval: 1
-  metadata_block_size: 4096
-  partition_filters: 0
-  use_delta_encoding: 1
-  filter_policy: nullptr
-  whole_key_filtering: 1
-  verify_compression: 0
-  read_amp_bytes_per_bit: 0
-  format_version: 2
-  enable_index_compression: 1
-  block_align: 0
-2020/01/11-12:35:32.759360 7f43e73ec440        Options.write_buffer_size: 67108864
-2020/01/11-12:35:32.759361 7f43e73ec440  Options.max_write_buffer_number: 2
-2020/01/11-12:35:32.759363 7f43e73ec440          Options.compression: NoCompression
-2020/01/11-12:35:32.759364 7f43e73ec440                  Options.bottommost_compression: Disabled
-2020/01/11-12:35:32.759366 7f43e73ec440       Options.prefix_extractor: nullptr
-2020/01/11-12:35:32.759367 7f43e73ec440   Options.memtable_insert_with_hint_prefix_extractor: nullptr
-2020/01/11-12:35:32.759368 7f43e73ec440             Options.num_levels: 7
-2020/01/11-12:35:32.759370 7f43e73ec440        Options.min_write_buffer_number_to_merge: 1
-2020/01/11-12:35:32.759371 7f43e73ec440     Options.max_write_buffer_number_to_maintain: 0
-2020/01/11-12:35:32.759372 7f43e73ec440            Options.bottommost_compression_opts.window_bits: -14
-2020/01/11-12:35:32.759373 7f43e73ec440                  Options.bottommost_compression_opts.level: 32767
-2020/01/11-12:35:32.759374 7f43e73ec440               Options.bottommost_compression_opts.strategy: 0
-2020/01/11-12:35:32.759376 7f43e73ec440         Options.bottommost_compression_opts.max_dict_bytes: 0
-2020/01/11-12:35:32.759377 7f43e73ec440         Options.bottommost_compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:35:32.759378 7f43e73ec440                  Options.bottommost_compression_opts.enabled: false
-2020/01/11-12:35:32.759379 7f43e73ec440            Options.compression_opts.window_bits: -14
-2020/01/11-12:35:32.759380 7f43e73ec440                  Options.compression_opts.level: 32767
-2020/01/11-12:35:32.759382 7f43e73ec440               Options.compression_opts.strategy: 0
-2020/01/11-12:35:32.759383 7f43e73ec440         Options.compression_opts.max_dict_bytes: 0
-2020/01/11-12:35:32.759384 7f43e73ec440         Options.compression_opts.zstd_max_train_bytes: 0
-2020/01/11-12:35:32.759386 7f43e73ec440                  Options.compression_opts.enabled: false
-2020/01/11-12:35:32.759387 7f43e73ec440      Options.level0_file_num_compaction_trigger: 4
-2020/01/11-12:35:32.759388 7f43e73ec440          Options.level0_slowdown_writes_trigger: 20
-2020/01/11-12:35:32.759389 7f43e73ec440              Options.level0_stop_writes_trigger: 36
-2020/01/11-12:35:32.759391 7f43e73ec440                   Options.target_file_size_base: 67108864
-2020/01/11-12:35:32.759392 7f43e73ec440             Options.target_file_size_multiplier: 1
-2020/01/11-12:35:32.759393 7f43e73ec440                Options.max_bytes_for_level_base: 268435456
-2020/01/11-12:35:32.759395 7f43e73ec440                      Options.snap_refresh_nanos: 0
-2020/01/11-12:35:32.759396 7f43e73ec440 Options.level_compaction_dynamic_level_bytes: 0
-2020/01/11-12:35:32.759397 7f43e73ec440          Options.max_bytes_for_level_multiplier: 10.000000
-2020/01/11-12:35:32.759399 7f43e73ec440 Options.max_bytes_for_level_multiplier_addtl[0]: 1
-2020/01/11-12:35:32.759415 7f43e73ec440 Options.max_bytes_for_level_multiplier_addtl[1]: 1
-2020/01/11-12:35:32.759416 7f43e73ec440 Options.max_bytes_for_level_multiplier_addtl[2]: 1
-2020/01/11-12:35:32.759417 7f43e73ec440 Options.max_bytes_for_level_multiplier_addtl[3]: 1
-2020/01/11-12:35:32.759418 7f43e73ec440 Options.max_bytes_for_level_multiplier_addtl[4]: 1
-2020/01/11-12:35:32.759420 7f43e73ec440 Options.max_bytes_for_level_multiplier_addtl[5]: 1
-2020/01/11-12:35:32.759421 7f43e73ec440 Options.max_bytes_for_level_multiplier_addtl[6]: 1
-2020/01/11-12:35:32.759422 7f43e73ec440       Options.max_sequential_skip_in_iterations: 8
-2020/01/11-12:35:32.759423 7f43e73ec440                    Options.max_compaction_bytes: 1677721600
-2020/01/11-12:35:32.759424 7f43e73ec440                        Options.arena_block_size: 8388608
-2020/01/11-12:35:32.759425 7f43e73ec440   Options.soft_pending_compaction_bytes_limit: 68719476736
-2020/01/11-12:35:32.759427 7f43e73ec440   Options.hard_pending_compaction_bytes_limit: 274877906944
-2020/01/11-12:35:32.759428 7f43e73ec440       Options.rate_limit_delay_max_milliseconds: 100
-2020/01/11-12:35:32.759429 7f43e73ec440                Options.disable_auto_compactions: 0
-2020/01/11-12:35:32.759431 7f43e73ec440                        Options.compaction_style: kCompactionStyleLevel
-2020/01/11-12:35:32.759432 7f43e73ec440                          Options.compaction_pri: kMinOverlappingRatio
-2020/01/11-12:35:32.759433 7f43e73ec440 Options.compaction_options_universal.size_ratio: 1
-2020/01/11-12:35:32.759435 7f43e73ec440 Options.compaction_options_universal.min_merge_width: 2
-2020/01/11-12:35:32.759436 7f43e73ec440 Options.compaction_options_universal.max_merge_width: 4294967295
-2020/01/11-12:35:32.759437 7f43e73ec440 Options.compaction_options_universal.max_size_amplification_percent: 200
-2020/01/11-12:35:32.759438 7f43e73ec440 Options.compaction_options_universal.compression_size_percent: -1
-2020/01/11-12:35:32.759440 7f43e73ec440 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize
-2020/01/11-12:35:32.759441 7f43e73ec440 Options.compaction_options_fifo.max_table_files_size: 1073741824
-2020/01/11-12:35:32.759442 7f43e73ec440 Options.compaction_options_fifo.allow_compaction: 0
-2020/01/11-12:35:32.759443 7f43e73ec440                   Options.table_properties_collectors: 
-2020/01/11-12:35:32.759445 7f43e73ec440                   Options.inplace_update_support: 0
-2020/01/11-12:35:32.759446 7f43e73ec440                 Options.inplace_update_num_locks: 10000
-2020/01/11-12:35:32.759447 7f43e73ec440               Options.memtable_prefix_bloom_size_ratio: 0.000000
-2020/01/11-12:35:32.759449 7f43e73ec440               Options.memtable_whole_key_filtering: 0
-2020/01/11-12:35:32.759450 7f43e73ec440   Options.memtable_huge_page_size: 0
-2020/01/11-12:35:32.759451 7f43e73ec440                           Options.bloom_locality: 0
-2020/01/11-12:35:32.759453 7f43e73ec440                    Options.max_successive_merges: 0
-2020/01/11-12:35:32.759454 7f43e73ec440                Options.optimize_filters_for_hits: 0
-2020/01/11-12:35:32.759455 7f43e73ec440                Options.paranoid_file_checks: 0
-2020/01/11-12:35:32.759456 7f43e73ec440                Options.force_consistency_checks: 0
-2020/01/11-12:35:32.759457 7f43e73ec440                Options.report_bg_io_stats: 0
-2020/01/11-12:35:32.759458 7f43e73ec440                               Options.ttl: 0
-2020/01/11-12:35:32.759459 7f43e73ec440          Options.periodic_compaction_seconds: 0
-2020/01/11-12:35:32.759987 7f43e73ec440 [/version_set.cc:4267] Recovered from manifest file:kvdb/MANIFEST-000014 succeeded,manifest_file_number is 14, next_file_number is 16, last_sequence is 0, log_number is 13,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0
-2020/01/11-12:35:32.759991 7f43e73ec440 [/version_set.cc:4276] Column family [default] (ID 0), log number is 13
-2020/01/11-12:35:32.760049 7f43e73ec440 EVENT_LOG_v1 {"time_micros": 1578717332760044, "job": 1, "event": "recovery_started", "log_files": [15]}
-2020/01/11-12:35:32.760062 7f43e73ec440 [/db_impl_open.cc:597] Recovering log #15 mode 2
-2020/01/11-12:35:32.760123 7f43e73ec440 [/version_set.cc:3546] Creating manifest 17
-2020/01/11-12:35:32.760790 7f43e73ec440 EVENT_LOG_v1 {"time_micros": 1578717332760786, "job": 1, "event": "recovery_finished"}
-2020/01/11-12:35:32.762750 7f43e73ec440 DB pointer 0x20780540
-2020/01/11-12:35:32.763037 7f43695ee700 [/db_impl.cc:779] ------- DUMPING STATS -------
-2020/01/11-12:35:32.763073 7f43695ee700 [/db_impl.cc:780] 
-** DB Stats **
-Uptime(secs): 0.0 total, 0.0 interval
-Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s
-Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s
-Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent
-Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s
-Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s
-Interval stall: 00:00:0.000 H:M:S, 0.0 percent
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **
-
-** Compaction Stats [default] **
-Level    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Sum      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
- Int      0/0    0.00 KB   0.0      0.0     0.0      0.0       0.0      0.0       0.0   0.0      0.0      0.0      0.00              0.00         0    0.000       0      0
-
-** Compaction Stats [default] **
-Priority    Files   Size     Score Read(GB)  Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-Uptime(secs): 0.0 total, 0.0 interval
-Flush(GB): cumulative 0.000, interval 0.000
-AddFile(GB): cumulative 0.000, interval 0.000
-AddFile(Total Files): cumulative 0, interval 0
-AddFile(L0 Files): cumulative 0, interval 0
-AddFile(Keys): cumulative 0, interval 0
-Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds
-Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count
-
-** File Read Latency Histogram By Level [default] **

examples/demo-serving/kvdb/OPTIONS-000020

-# This is a RocksDB option file.
-#
-# For detailed file format spec, please refer to the example file
-# in examples/rocksdb_option_file_example.ini
-#
-
-[Version]
-  rocksdb_version=6.2.4
-  options_file_version=1.1
-
-[DBOptions]
-  avoid_unnecessary_blocking_io=false
-  allow_mmap_reads=false
-  allow_fallocate=true
-  WAL_size_limit_MB=0
-  writable_file_max_buffer_size=1048576
-  allow_mmap_writes=false
-  allow_concurrent_memtable_write=true
-  use_direct_reads=false
-  max_open_files=-1
-  strict_bytes_per_sync=false
-  db_write_buffer_size=0
-  max_background_jobs=2
-  WAL_ttl_seconds=0
-  enable_thread_tracking=false
-  error_if_exists=false
-  is_fd_close_on_exec=true
-  recycle_log_file_num=0
-  max_manifest_file_size=1073741824
-  skip_log_error_on_recovery=false
-  skip_stats_update_on_db_open=false
-  max_total_wal_size=0
-  new_table_reader_for_compaction_inputs=false
-  manual_wal_flush=false
-  compaction_readahead_size=0
-  atomic_flush=false
-  random_access_max_buffer_size=1048576
-  create_missing_column_families=false
-  wal_bytes_per_sync=0
-  use_adaptive_mutex=false
-  use_direct_io_for_flush_and_compaction=false
-  max_background_compactions=-1
-  advise_random_on_open=true
-  base_background_compactions=-1
-  max_background_flushes=-1
-  two_write_queues=false
-  table_cache_numshardbits=6
-  keep_log_file_num=1000
-  write_thread_slow_yield_usec=3
-  stats_dump_period_sec=600
-  avoid_flush_during_recovery=false
-  log_file_time_to_roll=0
-  delayed_write_rate=16777216
-  manifest_preallocation_size=4194304
-  paranoid_checks=true
-  max_log_file_size=0
-  allow_2pc=false
-  wal_dir=kvdb
-  db_log_dir=
-  max_subcompactions=1
-  create_if_missing=true
-  enable_pipelined_write=false
-  bytes_per_sync=0
-  stats_persist_period_sec=600
-  stats_history_buffer_size=1048576
-  fail_if_options_file_error=false
-  use_fsync=false
-  wal_recovery_mode=kPointInTimeRecovery
-  delete_obsolete_files_period_micros=21600000000
-  enable_write_thread_adaptive_yield=true
-  avoid_flush_during_shutdown=false
-  write_thread_max_yield_usec=100
-  info_log_level=INFO_LEVEL
-  max_file_opening_threads=16
-  dump_malloc_stats=false
-  allow_ingest_behind=false
-  access_hint_on_compaction_start=NORMAL
-  preserve_deletes=false
-  
-
-[CFOptions "default"]
-  sample_for_compression=0
-  compaction_pri=kMinOverlappingRatio
-  merge_operator=nullptr
-  compaction_filter_factory=nullptr
-  memtable_factory=SkipListFactory
-  memtable_insert_with_hint_prefix_extractor=nullptr
-  comparator=leveldb.BytewiseComparator
-  target_file_size_base=67108864
-  max_sequential_skip_in_iterations=8
-  compaction_style=kCompactionStyleLevel
-  max_bytes_for_level_base=268435456
-  bloom_locality=0
-  write_buffer_size=67108864
-  compression_per_level=
-  memtable_huge_page_size=0
-  max_successive_merges=0
-  arena_block_size=8388608
-  memtable_whole_key_filtering=false
-  target_file_size_multiplier=1
-  max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
-  snap_refresh_nanos=0
-  num_levels=7
-  min_write_buffer_number_to_merge=1
-  max_write_buffer_number_to_maintain=0
-  max_write_buffer_number=2
-  compression=kNoCompression
-  level0_stop_writes_trigger=36
-  level0_slowdown_writes_trigger=20
-  compaction_filter=nullptr
-  level0_file_num_compaction_trigger=4
-  max_compaction_bytes=1677721600
-  compaction_options_universal={allow_trivial_move=false;size_ratio=1;min_merge_width=2;max_size_amplification_percent=200;max_merge_width=4294967295;compression_size_percent=-1;stop_style=kCompactionStopStyleTotalSize;}
-  memtable_prefix_bloom_size_ratio=0.000000
-  hard_pending_compaction_bytes_limit=274877906944
-  ttl=0
-  table_factory=BlockBasedTable
-  soft_pending_compaction_bytes_limit=68719476736
-  prefix_extractor=nullptr
-  bottommost_compression=kDisableCompressionOption
-  force_consistency_checks=false
-  paranoid_file_checks=false
-  compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
-  max_bytes_for_level_multiplier=10.000000
-  optimize_filters_for_hits=false
-  level_compaction_dynamic_level_bytes=false
-  inplace_update_num_locks=10000
-  inplace_update_support=false
-  periodic_compaction_seconds=0
-  disable_auto_compactions=false
-  report_bg_io_stats=false
-  
-[TableOptions/BlockBasedTable "default"]
-  pin_top_level_index_and_filter=true
-  enable_index_compression=true
-  read_amp_bytes_per_bit=8589934592
-  format_version=2
-  block_align=false
-  metadata_block_size=4096
-  block_size_deviation=10
-  partition_filters=false
-  block_size=4096
-  index_block_restart_interval=1
-  no_block_cache=false
-  checksum=kCRC32c
-  whole_key_filtering=true
-  index_shortening=kShortenSeparators
-  data_block_index_type=kDataBlockBinarySearch
-  index_type=kBinarySearch
-  verify_compression=false
-  filter_policy=nullptr
-  data_block_hash_table_util_ratio=0.750000
-  pin_l0_filter_and_index_blocks_in_cache=false
-  block_restart_interval=16
-  cache_index_and_filter_blocks_with_high_priority=false
-  cache_index_and_filter_blocks=false
-  hash_index_allow_collision=true
-  flush_block_policy_factory=FlushBlockBySizePolicyFactory
-  

examples/demo-serving/kvdb/OPTIONS-000023

-# This is a RocksDB option file.
-#
-# For detailed file format spec, please refer to the example file
-# in examples/rocksdb_option_file_example.ini
-#
-
-[Version]
-  rocksdb_version=6.2.4
-  options_file_version=1.1
-
-[DBOptions]
-  avoid_unnecessary_blocking_io=false
-  allow_mmap_reads=false
-  allow_fallocate=true
-  WAL_size_limit_MB=0
-  writable_file_max_buffer_size=1048576
-  allow_mmap_writes=false
-  allow_concurrent_memtable_write=true
-  use_direct_reads=false
-  max_open_files=-1
-  strict_bytes_per_sync=false
-  db_write_buffer_size=0
-  max_background_jobs=2
-  WAL_ttl_seconds=0
-  enable_thread_tracking=false
-  error_if_exists=false
-  is_fd_close_on_exec=true
-  recycle_log_file_num=0
-  max_manifest_file_size=1073741824
-  skip_log_error_on_recovery=false
-  skip_stats_update_on_db_open=false
-  max_total_wal_size=0
-  new_table_reader_for_compaction_inputs=false
-  manual_wal_flush=false
-  compaction_readahead_size=0
-  atomic_flush=false
-  random_access_max_buffer_size=1048576
-  create_missing_column_families=false
-  wal_bytes_per_sync=0
-  use_adaptive_mutex=false
-  use_direct_io_for_flush_and_compaction=false
-  max_background_compactions=-1
-  advise_random_on_open=true
-  base_background_compactions=-1
-  max_background_flushes=-1
-  two_write_queues=false
-  table_cache_numshardbits=6
-  keep_log_file_num=1000
-  write_thread_slow_yield_usec=3
-  stats_dump_period_sec=600
-  avoid_flush_during_recovery=false
-  log_file_time_to_roll=0
-  delayed_write_rate=16777216
-  manifest_preallocation_size=4194304
-  paranoid_checks=true
-  max_log_file_size=0
-  allow_2pc=false
-  wal_dir=kvdb
-  db_log_dir=
-  max_subcompactions=1
-  create_if_missing=true
-  enable_pipelined_write=false
-  bytes_per_sync=0
-  stats_persist_period_sec=600
-  stats_history_buffer_size=1048576
-  fail_if_options_file_error=false
-  use_fsync=false
-  wal_recovery_mode=kPointInTimeRecovery
-  delete_obsolete_files_period_micros=21600000000
-  enable_write_thread_adaptive_yield=true
-  avoid_flush_during_shutdown=false
-  write_thread_max_yield_usec=100
-  info_log_level=INFO_LEVEL
-  max_file_opening_threads=16
-  dump_malloc_stats=false
-  allow_ingest_behind=false
-  access_hint_on_compaction_start=NORMAL
-  preserve_deletes=false
-  
-
-[CFOptions "default"]
-  sample_for_compression=0
-  compaction_pri=kMinOverlappingRatio
-  merge_operator=nullptr
-  compaction_filter_factory=nullptr
-  memtable_factory=SkipListFactory
-  memtable_insert_with_hint_prefix_extractor=nullptr
-  comparator=leveldb.BytewiseComparator
-  target_file_size_base=67108864
-  max_sequential_skip_in_iterations=8
-  compaction_style=kCompactionStyleLevel
-  max_bytes_for_level_base=268435456
-  bloom_locality=0
-  write_buffer_size=67108864
-  compression_per_level=
-  memtable_huge_page_size=0
-  max_successive_merges=0
-  arena_block_size=8388608
-  memtable_whole_key_filtering=false
-  target_file_size_multiplier=1
-  max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
-  snap_refresh_nanos=0
-  num_levels=7
-  min_write_buffer_number_to_merge=1
-  max_write_buffer_number_to_maintain=0
-  max_write_buffer_number=2
-  compression=kNoCompression
-  level0_stop_writes_trigger=36
-  level0_slowdown_writes_trigger=20
-  compaction_filter=nullptr
-  level0_file_num_compaction_trigger=4
-  max_compaction_bytes=1677721600
-  compaction_options_universal={allow_trivial_move=false;size_ratio=1;min_merge_width=2;max_size_amplification_percent=200;max_merge_width=4294967295;compression_size_percent=-1;stop_style=kCompactionStopStyleTotalSize;}
-  memtable_prefix_bloom_size_ratio=0.000000
-  hard_pending_compaction_bytes_limit=274877906944
-  ttl=0
-  table_factory=BlockBasedTable
-  soft_pending_compaction_bytes_limit=68719476736
-  prefix_extractor=nullptr
-  bottommost_compression=kDisableCompressionOption
-  force_consistency_checks=false
-  paranoid_file_checks=false
-  compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;}
-  max_bytes_for_level_multiplier=10.000000
-  optimize_filters_for_hits=false
-  level_compaction_dynamic_level_bytes=false
-  inplace_update_num_locks=10000
-  inplace_update_support=false
-  periodic_compaction_seconds=0
-  disable_auto_compactions=false
-  report_bg_io_stats=false
-  
-[TableOptions/BlockBasedTable "default"]
-  pin_top_level_index_and_filter=true
-  enable_index_compression=true
-  read_amp_bytes_per_bit=8589934592
-  format_version=2
-  block_align=false
-  metadata_block_size=4096
-  block_size_deviation=10
-  partition_filters=false
-  block_size=4096
-  index_block_restart_interval=1
-  no_block_cache=false
-  checksum=kCRC32c
-  whole_key_filtering=true
-  index_shortening=kShortenSeparators
-  data_block_index_type=kDataBlockBinarySearch
-  index_type=kBinarySearch
-  verify_compression=false
-  filter_policy=nullptr
-  data_block_hash_table_util_ratio=0.750000
-  pin_l0_filter_and_index_blocks_in_cache=false
-  block_restart_interval=16
-  cache_index_and_filter_blocks_with_high_priority=false
-  cache_index_and_filter_blocks=false
-  hash_index_allow_collision=true
-  flush_block_policy_factory=FlushBlockBySizePolicyFactory
-  

examples/demo-serving/op/general_model_op.cpp

@@ -51,14 +51,14 @@ int GeneralModelOp::inference() {
   // infer
   if (batch_size > 0) {
     int var_num = req->insts(0).tensor_array_size();
-    VLOG(3) << "var num: " << var_num;
+    VLOG(2) << "var num: " << var_num;
     elem_type.resize(var_num);
     elem_size.resize(var_num);
     capacity.resize(var_num);
     paddle::PaddleTensor lod_tensor;
     for (int i = 0; i < var_num; ++i) {
       elem_type[i] = req->insts(0).tensor_array(i).elem_type();
-      VLOG(3) << "var[" << i << "] has elem type: " << elem_type[i];
+      VLOG(2) << "var[" << i << "] has elem type: " << elem_type[i];
       if (elem_type[i] == 0) {  // int64
         elem_size[i] = sizeof(int64_t);
         lod_tensor.dtype = paddle::PaddleDType::INT64;
@@ -70,17 +70,17 @@ int GeneralModelOp::inference() {
       if (req->insts(0).tensor_array(i).shape(0) == -1) {
         lod_tensor.lod.resize(1);
         lod_tensor.lod[0].push_back(0);
-        VLOG(3) << "var[" << i << "] is lod_tensor";
+        VLOG(2) << "var[" << i << "] is lod_tensor";
       } else {
         lod_tensor.shape.push_back(batch_size);
         capacity[i] = 1;
         for (int k = 0; k < req->insts(0).tensor_array(i).shape_size(); ++k) {
           int dim = req->insts(0).tensor_array(i).shape(k);
-          VLOG(3) << "shape for var[" << i << "]: " << dim;
+          VLOG(2) << "shape for var[" << i << "]: " << dim;
           capacity[i] *= dim;
           lod_tensor.shape.push_back(dim);
         }
-        VLOG(3) << "var[" << i << "] is tensor, capacity: " << capacity[i];
+        VLOG(2) << "var[" << i << "] is tensor, capacity: " << capacity[i];
       }
       if (i == 0) {
         lod_tensor.name = "words";
@@ -95,19 +95,19 @@ int GeneralModelOp::inference() {
         for (int j = 0; j < batch_size; ++j) {
           const Tensor &tensor = req->insts(j).tensor_array(i);
           int data_len = tensor.data_size();
-          VLOG(3) << "tensor size for var[" << i << "]: " << tensor.data_size();
+          VLOG(2) << "tensor size for var[" << i << "]: " << tensor.data_size();
           int cur_len = in->at(i).lod[0].back();
-          VLOG(3) << "current len: " << cur_len;
+          VLOG(2) << "current len: " << cur_len;
           in->at(i).lod[0].push_back(cur_len + data_len);
-          VLOG(3) << "new len: " << cur_len + data_len;
+          VLOG(2) << "new len: " << cur_len + data_len;
         }
         in->at(i).data.Resize(in->at(i).lod[0].back() * elem_size[i]);
         in->at(i).shape = {in->at(i).lod[0].back(), 1};
-        VLOG(3) << "var[" << i
+        VLOG(2) << "var[" << i
                 << "] is lod_tensor and len=" << in->at(i).lod[0].back();
       } else {
         in->at(i).data.Resize(batch_size * capacity[i] * elem_size[i]);
-        VLOG(3) << "var[" << i
+        VLOG(2) << "var[" << i
                 << "] is tensor and capacity=" << batch_size * capacity[i];
       }
     }
@@ -144,7 +144,7 @@ int GeneralModelOp::inference() {
       }
     }
 
-    VLOG(3) << "going to infer";
+    VLOG(2) << "going to infer";
     TensorVector *out = butil::get_object<TensorVector>();
     if (!out) {
       LOG(ERROR) << "Failed get tls output object";
@@ -157,7 +157,7 @@ int GeneralModelOp::inference() {
     for (uint32_t i = 0; i < 10; i++) {
       oss << *(example + i) << " ";
     }
-    VLOG(3) << "msg: " << oss.str();
+    VLOG(2) << "msg: " << oss.str();
 
     // infer
     if (predictor::InferManager::instance().infer(
@@ -167,7 +167,7 @@ int GeneralModelOp::inference() {
     }
     // print response
     float *example_1 = reinterpret_cast<float *>((*out)[0].data.data());
-    VLOG(3) << "result: " << *example_1;
+    VLOG(2) << "result: " << *example_1;
 
     Response *res = mutable_data<Response>();
 

paddle_inference/inferencer-fluid-cpu/CMakeLists.txt

@@ -3,7 +3,7 @@ add_library(fluid_cpu_engine ${fluid_cpu_engine_srcs})
 target_include_directories(fluid_cpu_engine PUBLIC
         ${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir/)
 add_dependencies(fluid_cpu_engine pdserving extern_paddle configure kvdb)
-target_link_libraries(fluid_cpu_engine pdserving paddle_fluid iomp5 mklml_intel -lpthread -lcrypto -lm -lrt -lssl -ldl -lz)
+target_link_libraries(fluid_cpu_engine pdserving paddle_fluid -lpthread -lcrypto -lm -lrt -lssl -ldl -lz)
 
 install(TARGETS fluid_cpu_engine 
         ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib

python/CMakeLists.txt

-file(GLOB_RECURSE SERVING_CLIENT_PY_FILES serving_client/*.py)
+if (CLIENT_ONLY)
+file(GLOB_RECURSE SERVING_CLIENT_PY_FILES paddle_serving_client/*.py)
 set(PY_FILES ${SERVING_CLIENT_PY_FILES})
 SET(PACKAGE_NAME "serving_client")
-set(SETUP_LOG_FILE "setup.py.log")
+set(SETUP_LOG_FILE "setup.py.client.log")
+endif()
+
+if (NOT CLIENT_ONLY)
+file(GLOB_RECURSE SERVING_SERVER_PY_FILES paddle_serving_server/*.py)
+set(PY_FILES ${SERVING_SERVER_PY_FILES})
+SET(PACKAGE_NAME "serving_server")
+set(SETUP_LOG_FILE "setup.py.server.log")
+endif()
 
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in
+if (CLIENT_ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.client.in
     ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
+endif()
+
+if (NOT CLIENT_ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.server.in
+    ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
+endif()
 
 set(SERVING_CLIENT_CORE ${PADDLE_SERVING_BINARY_DIR}/core/general-client/serving_client.so)
 
 message("python env: " ${py_env})
 
+if (CLIENT_ONLY)
 add_custom_command(
 	OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
-	COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving/ ${PADDLE_SERVING_BINARY_DIR}/python/
-	COMMAND ${CMAKE_COMMAND} -E copy ${PADDLE_SERVING_BINARY_DIR}/core/general-client/serving_client.so ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving/serving_client/
+	COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_client/ ${PADDLE_SERVING_BINARY_DIR}/python/
+	COMMAND ${CMAKE_COMMAND} -E copy ${PADDLE_SERVING_BINARY_DIR}/core/general-client/serving_client.so ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/
 	COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
 	DEPENDS ${SERVING_CLIENT_CORE} sdk_configure_py_proto ${PY_FILES})
-
 add_custom_target(paddle_python ALL DEPENDS serving_client ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
+endif()
+
+if (NOT CLIENT_ONLY)
+add_custom_command(
+   	OUTPUT ${PADDLE_SERVING_BINARY_DIR}/.timestamp
+	COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/paddle_serving_server/ ${PADDLE_SERVING_BINARY_DIR}/python/
+	COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
+	DEPENDS ${SERVING_SERVER_CORE} server_config_py_proto ${PY_FILES})
+add_custom_target(paddle_python ALL DEPENDS ${PADDLE_SERVING_BINARY_DIR}/.timestamp)
+endif()
 
 set(SERVING_CLIENT_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/)
+set(SERVING_SERVER_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/)
 
+if (CLIENT_ONLY)
 install(DIRECTORY ${SERVING_CLIENT_PYTHON_PACKAGE_DIR}
     DESTINATION opt/serving_client/share/wheels
 )
+endif()
+
+if (NOT CLIENT_ONLY)
+install(DIRECTORY ${SERVING_SERVER_PYTHON_PACKAGE_DIR}
+    DESTINATION opt/serving_server/share/wheels
+)
+endif()
 
 find_program(PATCHELF_EXECUTABLE patchelf)
 if(NOT PATCHELF_EXECUTABLE)

python/examples/imdb/clean_data.sh

+rm -rf imdb.vocab kvdb log *.pyc serving_client_conf serving_server_model test_data text_classification_data.tar.gz train_data work_dir1

python/examples/imdb/get_data.sh

-wget https://fleet.bj.bcebos.com/text_classification_data.tar.gz
+wget --no-check-certificate https://fleet.bj.bcebos.com/text_classification_data.tar.gz
 tar -zxvf text_classification_data.tar.gz
+#wget --no-check-certificate https://paddle-serving.bj.bcebos.com/imdb-demo%2Fimdb.tar.gz
+#tar -xzf imdb-demo%2Fimdb.tar.gz

python/examples/imdb/local_train.py

@@ -16,7 +16,6 @@ import sys
 import paddle
 import logging
 import paddle.fluid as fluid
-import paddle_serving as serving
 
 logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger("fluid")
@@ -42,7 +41,7 @@ if __name__ == "__main__":
     dataset = fluid.DatasetFactory().create_dataset()
     filelist = ["train_data/%s" % x for x in os.listdir("train_data")]
     dataset.set_use_var([data, label])
-    pipe_command = "python imdb_reader.py"
+    pipe_command = "/home/users/dongdaxiang/paddle_whls/custom_op/paddle_release_home/python/bin/python imdb_reader.py"
     dataset.set_pipe_command(pipe_command)
     dataset.set_batch_size(4)
     dataset.set_filelist(filelist)
@@ -54,15 +53,22 @@ if __name__ == "__main__":
 
     exe = fluid.Executor(fluid.CPUPlace())
     exe.run(fluid.default_startup_program())
-    epochs = 30
+    epochs = 6
     save_dirname = "cnn_model"
 
+    import paddle_serving_client.io as serving_io
+
     for i in range(epochs):
         exe.train_from_dataset(program=fluid.default_main_program(),
                                dataset=dataset, debug=False)
         logger.info("TRAIN --> pass: {}".format(i))
-        fluid.io.save_inference_model("%s/epoch%d.model" % (save_dirname, i),
-                                      [data.name, label.name], [acc], exe)
-        serving.save_model("%s/epoch%d.model" % (save_dirname, i), "client_config{}".format(i),
-                           {"words": data, "label": label},
-                           {"acc": acc, "cost": avg_cost, "prediction": prediction})
+        if i == 5:
+            serving_io.save_model("serving_server_model",
+                                  "serving_client_conf",
+                                  {"words": data, "label": label},
+                                  {"cost": avg_cost, "acc": acc,
+                                   "prediction": prediction},
+                                  fluid.default_main_program())
+
+
+

python/examples/imdb/test_client.py

-from paddle_serving import Client
+from paddle_serving_client import Client
 import sys
 
 client = Client()
@@ -7,7 +7,7 @@ client.connect(["127.0.0.1:9292"])
 
 for line in sys.stdin:
     group = line.strip().split()
-    words = [int(x) for x in group[1:int(group[0])]]
+    words = [int(x) for x in group[1:int(group[0]) + 1]]
     label = [int(group[-1])]
     feed = {"words": words, "label": label}
     fetch = ["acc", "cost", "prediction"]

python/examples/imdb/test_client_batch.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from paddle_serving import Client
+from paddle_serving_client import Client
 import sys
 import subprocess
 from multiprocessing import Pool

python/examples/imdb/test_client_multithread.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from paddle_serving import Client
+from paddle_serving_client import Client
 import sys
 import subprocess
 from multiprocessing import Pool

python/examples/imdb/test_server.py

+import os
+import sys
+from paddle_serving_server import OpMaker
+from paddle_serving_server import OpSeqMaker
+from paddle_serving_server import Server
+
+op_maker = OpMaker()
+read_op = op_maker.create('general_reader')
+general_infer_op = op_maker.create('general_infer')
+
+op_seq_maker = OpSeqMaker()
+op_seq_maker.add_op(read_op)
+op_seq_maker.add_op(general_infer_op)
+
+server = Server()
+server.set_op_sequence(op_seq_maker.get_op_sequence())
+server.load_model_config(sys.argv[1])
+server.prepare_server(workdir="work_dir1", port=9292, device="cpu")
+server.run_server()

python/paddle_serving/serving_client/__init__.py → python/paddle_serving_client/__init__.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 from .serving_client import PredictorClient
-from ..proto import sdk_configure_pb2 as sdk
+from .proto import sdk_configure_pb2 as sdk
+from .proto import general_model_config_pb2 as m_config
+import google.protobuf.text_format
 import time
 
 int_type = 0
@@ -74,34 +76,25 @@ class Client(object):
         self.feed_names_to_idx_ = {}
 
     def load_client_config(self, path):
+        model_conf = m_config.GeneralModelConfig()
+        f = open(path, 'r')
+        model_conf = google.protobuf.text_format.Merge(
+            str(f.read()), model_conf)
+
         # load configuraion here
         # get feed vars, fetch vars
         # get feed shapes, feed types
         # map feed names to index
         self.client_handle_ = PredictorClient()
         self.client_handle_.init(path)
-        self.feed_names_ = []
-        self.fetch_names_ = []
-        self.feed_shapes_ = []
-        self.feed_types_ = {}
+        self.feed_names_ = [var.alias_name for var in model_conf.feed_var]
+        self.fetch_names_ = [var.alias_name for var in model_conf.fetch_var]
+        self.feed_shapes_ = [var.shape for var in model_conf.feed_var]
         self.feed_names_to_idx_ = {}
+        for i, var in enumerate(model_conf.feed_var):
+            self.feed_names_to_idx_[var.alias_name] = i
+            self.feed_types_[var.alias_name] = var.feed_type
 
-        with open(path) as fin:
-            group = fin.readline().strip().split()
-            feed_num = int(group[0])
-            fetch_num = int(group[1])
-            for i in range(feed_num):
-                group = fin.readline().strip().split()
-                self.feed_names_.append(group[0])
-                tmp_shape = []
-                for s in group[2:-1]:
-                    tmp_shape.append(int(s))
-                self.feed_shapes_.append(tmp_shape)
-                self.feed_types_[group[0]] = int(group[-1])
-                self.feed_names_to_idx_[group[0]] = i
-            for i in range(fetch_num):
-                group = fin.readline().strip().split()
-                self.fetch_names_.append(group[0])
         return
 
     def connect(self, endpoints):
@@ -148,6 +141,9 @@ class Client(object):
         result = self.client_handle_.predict(
             float_slot, float_feed_names, int_slot, int_feed_names, fetch_names)
 
+        # TODO(guru4elephant): the order of fetch var name should be consistent with
+        #                      general_model_config, this is not friendly
+        #                      In the future, we need make the number of fetched variable changable
         result_map = {}
         for i, name in enumerate(fetch_names):
             result_map[name] = result[i]

python/paddle_serving/io/__init__.py → python/paddle_serving_client/io/__init__.py

@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from paddle.fluid import Executor
 from paddle.fluid.compiler import CompiledProgram
-from paddle.fluid.framework import Program
+from paddle.fluid.framework import core
 from paddle.fluid.framework import default_main_program
+from paddle.fluid.framework import Program
 from paddle.fluid import CPUPlace
-from paddle.fluid.io import save_persistables
+from paddle.fluid.io import save_inference_model
+from ..proto import general_model_config_pb2 as model_conf
 import os
 
 def save_model(server_model_folder,
@@ -26,39 +27,51 @@ def save_model(server_model_folder,
                feed_var_dict,
                fetch_var_dict,
                main_program=None):
-    if main_program is None:
-        main_program = default_main_program()
-    elif isinstance(main_program, CompiledProgram):
-        main_program = main_program._program
-        if main_program is None:
-            raise TypeError("program should be as Program type or None")
-    if not isinstance(main_program, Program):
-        raise TypeError("program should be as Program type or None")
-
     executor = Executor(place=CPUPlace())
 
-    save_persistables(executor, server_model_folder,
-                      main_program)
+    feed_var_names = [feed_var_dict[x].name for x in feed_var_dict]
+    target_vars = fetch_var_dict.values()
+
+    save_inference_model(server_model_folder, feed_var_names,
+                         target_vars, executor, main_program=main_program)
+
+    config = model_conf.GeneralModelConfig()
+
+    for key in feed_var_dict:
+        feed_var = model_conf.FeedVar()
+        feed_var.alias_name = key
+        feed_var.name = feed_var_dict[key].name
+        feed_var.is_lod_tensor = feed_var_dict[key].lod_level == 1
+        if feed_var_dict[key].dtype == core.VarDesc.VarType.INT32 or \
+           feed_var_dict[key].dtype == core.VarDesc.VarType.INT64:
+            feed_var.feed_type = 0
+        if feed_var_dict[key].dtype == core.VarDesc.VarType.FP32:
+            feed_var.feed_type = 1
+        if feed_var.is_lod_tensor:
+            feed_var.shape.extend([-1])
+        else:
+            tmp_shape = []
+            for v in feed_var_dict[key].shape:
+                if v >= 0:
+                    tmp_shape.append(v)
+            feed_var.shape.extend(tmp_shape)
+        config.feed_var.extend([feed_var])
+
+    for key in fetch_var_dict:
+        fetch_var = model_conf.FetchVar()
+        fetch_var.alias_name = key
+        fetch_var.name = fetch_var_dict[key].name
+        fetch_var.shape.extend(fetch_var_dict[key].shape)
+        config.fetch_var.extend([fetch_var])
 
     cmd = "mkdir -p {}".format(client_config_folder)
-    os.system(cmd)
-    with open("{}/client.conf".format(client_config_folder), "w") as fout:
-        fout.write("{} {}\n".format(len(feed_var_dict), len(fetch_var_dict)))
-        for key in feed_var_dict:
-            fout.write("{}".format(key))
-            if feed_var_dict[key].lod_level == 1:
-                fout.write(" 1 -1\n")
-            elif feed_var_dict[key].lod_level == 0:
-                fout.write(" {}".format(len(feed_var_dict[key].shape)))
-                for dim in feed_var_dict[key].shape:
-                    fout.write(" {}".format(dim))
-                fout.write("\n")
-        for key in fetch_var_dict:
-            fout.write("{} {}\n".format(key, fetch_var_dict[key].name))
 
-    cmd = "cp {}/client.conf {}/server.conf".format(
-        client_config_folder, server_model_folder)
     os.system(cmd)
+    with open("{}/serving_client_conf.prototxt".format(client_config_folder), "w") as fout:
+        fout.write(str(config))
+    with open("{}/serving_server_conf.prototxt".format(server_model_folder), "w") as fout:
+        fout.write(str(config))
+
 
     
 

python/paddle_serving/version.py → python/paddle_serving_client/version.py

@@ -13,4 +13,5 @@
 # limitations under the License.
 """ Paddle Serving Client version string """
 serving_client_version = "0.1.0"
+serving_server_version = "0.1.0"
 module_proto_version = "0.1.0"

python/paddle_serving_server/__init__.py

+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from .proto import server_configure_pb2 as server_sdk
+from .proto import general_model_config_pb2 as m_config
+import google.protobuf.text_format
+
+class OpMaker(object):
+    def __init__(self):
+        self.op_dict = {"general_infer":"GeneralInferOp",
+                        "general_reader":"GeneralReaderOp",
+                        "general_single_kv":"GeneralSingleKVOp",
+                        "general_dist_kv":"GeneralDistKVOp"}
+
+    # currently, inputs and outputs are not used
+    # when we have OpGraphMaker, inputs and outputs are necessary
+    def create(self, name, inputs=[], outputs=[]):
+        if name not in self.op_dict:
+            raise Exception("Op name {} is not supported right now".format(name))
+        node = server_sdk.DAGNode()
+        node.name = "{}_op".format(name)
+        node.type = self.op_dict[name]
+        return node
+
+class OpSeqMaker(object):
+    def __init__(self):
+        self.workflow = server_sdk.Workflow()
+        self.workflow.name = "workflow1"
+        self.workflow.workflow_type = "Sequence"
+
+    def add_op(self, node):
+        if len(self.workflow.nodes) >= 1:
+            dep = server_sdk.DAGNodeDependency()
+            dep.name = self.workflow.nodes[-1].name
+            dep.mode = "RO"
+            node.dependencies.extend([dep])
+        self.workflow.nodes.extend([node])
+
+    def get_op_sequence(self):
+        workflow_conf = server_sdk.WorkflowConf()
+        workflow_conf.workflows.extend([self.workflow])
+        return workflow_conf
+
+class Server(object):
+    def __init__(self):
+        self.server_handle_ = None
+        self.infer_service_conf = None
+        self.model_toolkit_conf = None
+        self.resource_conf = None
+        self.engine = None
+        self.memory_optimization = False
+        self.model_conf = None
+        self.workflow_fn = "workflow.prototxt"
+        self.resource_fn = "resource.prototxt"
+        self.infer_service_fn = "infer_service.prototxt"
+        self.model_toolkit_fn = "model_toolkit.prototxt"
+        self.general_model_config_fn = "general_model.prototxt"
+        self.workdir = ""
+        self.max_concurrency = 0
+        self.num_threads = 0
+        self.port = 8080
+        self.reload_interval_s = 10
+
+    def set_max_concurrency(self, concurrency):
+        self.max_concurrency = concurrency
+
+    def set_num_threads(self, threads):
+        self.num_threads = threads
+
+    def set_port(self, port):
+        self.port = port
+
+    def set_reload_interval(self, interval):
+        self.reload_interval_s = interval
+
+    def set_op_sequence(self, op_seq):
+        self.workflow_conf = op_seq
+
+    def set_memory_optimize(self, flag=False):
+        self.memory_optimization = flag
+
+    def _prepare_engine(self, model_config_path, device):
+        if self.model_toolkit_conf == None:
+            self.model_toolkit_conf = server_sdk.ModelToolkitConf()
+
+        if self.engine == None:
+            self.engine = server_sdk.EngineDesc()
+
+        self.model_config_path = model_config_path
+        self.engine.name = "general_model"
+        self.engine.reloadable_meta = model_config_path + "/fluid_time_file"
+        self.engine.reloadable_type = "timestamp_ne"
+        self.engine.runtime_thread_num = 0
+        self.engine.batch_infer_size = 0
+        self.engine.enable_batch_align = 0
+        self.engine.model_data_path = model_config_path
+        self.engine.enable_memory_optimization = self.memory_optimization
+        self.engine.static_optimization = False
+        self.engine.force_update_static_cache = False
+
+        if device == "cpu":
+            self.engine.type = "FLUID_CPU_ANALYSIS_DIR"
+        elif device == "gpu":
+            self.engine.type = "FLUID_GPU_ANALYSIS_DIR"
+
+        self.model_toolkit_conf.engines.extend([self.engine])
+
+    def _prepare_infer_service(self, port):
+        if self.infer_service_conf == None:
+            self.infer_service_conf = server_sdk.InferServiceConf()
+            self.infer_service_conf.port = port
+            infer_service = server_sdk.InferService()
+            infer_service.name = "GeneralModelService"
+            infer_service.workflows.extend(["workflow1"])
+            self.infer_service_conf.services.extend([infer_service])
+
+    def _prepare_resource(self, workdir):
+        if self.resource_conf == None:
+            with open("{}/{}".format(workdir, self.general_model_config_fn), "w") as fout:
+                fout.write(str(self.model_conf))
+            self.resource_conf = server_sdk.ResourceConf()
+            self.resource_conf.model_toolkit_path = workdir
+            self.resource_conf.model_toolkit_file = self.model_toolkit_fn
+            self.resource_conf.general_model_path = workdir
+            self.resource_conf.general_model_file = self.general_model_config_fn
+
+    def _write_pb_str(self, filepath, pb_obj):
+        with open(filepath, "w") as fout:
+            fout.write(str(pb_obj))
+
+    def load_model_config(self, path):
+        self.model_config_path = path
+        self.model_conf = m_config.GeneralModelConfig()
+        f = open("{}/serving_server_conf.prototxt".format(path), 'r')
+        self.model_conf = google.protobuf.text_format.Merge(
+            str(f.read()), self.model_conf)
+        # check config here
+        # print config here
+
+    def prepare_server(self, workdir=None, port=9292, device="cpu"):
+        if workdir == None:
+            workdir = "./tmp"
+            os.system("mkdir {}".format(workdir))
+        else:
+            os.system("mkdir {}".format(workdir))
+        os.system("touch {}/fluid_time_file".format(workdir))
+
+        self._prepare_resource(workdir)
+        self._prepare_engine(self.model_config_path, device)
+        self._prepare_infer_service(port)
+        self.workdir = workdir
+
+        infer_service_fn = "{}/{}".format(workdir, self.infer_service_fn)
+        workflow_fn = "{}/{}".format(workdir, self.workflow_fn)
+        resource_fn = "{}/{}".format(workdir, self.resource_fn)
+        model_toolkit_fn = "{}/{}".format(workdir, self.model_toolkit_fn)
+
+        self._write_pb_str(infer_service_fn, self.infer_service_conf)
+        self._write_pb_str(workflow_fn, self.workflow_conf)
+        self._write_pb_str(resource_fn, self.resource_conf)
+        self._write_pb_str(model_toolkit_fn, self.model_toolkit_conf)
+
+    def run_server(self):
+        # just run server with system command
+        # currently we do not load cube
+        command = "/home/users/dongdaxiang/github_develop/Serving/build_server/core/general-server/serving" \
+                  " -enable_model_toolkit " \
+                  "-inferservice_path {} " \
+                  "-inferservice_file {} " \
+                  "-max_concurrency {} " \
+                  "-num_threads {} " \
+                  "-port {} " \
+                  "-reload_interval_s {} " \
+                  "-resource_path {} " \
+                  "-resource_file {} " \
+                  "-workflow_path {} " \
+                  "-workflow_file {} ".format(
+                      self.workdir,
+                      self.infer_service_fn,
+                      self.max_concurrency,
+                      self.num_threads,
+                      self.port,
+                      self.reload_interval_s,
+                      self.workdir,
+                      self.resource_fn,
+                      self.workdir, 
+                      self.workflow_fn)
+        os.system(command)
+
+

python/paddle_serving/__init__.py → python/paddle_serving_server/version.py

-#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Licensed under the Apache License, Version 2.0 (the "License"
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -11,5 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from .serving_client import Client
-from .io import save_model
+""" Paddle Serving Client version string """
+serving_client_version = "0.1.0"
+serving_server_version = "0.1.0"
+module_proto_version = "0.1.0"

python/setup.py.client.in

+#   Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Setup for pip package."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import platform
+
+from setuptools import setup, Distribution, Extension
+from setuptools import find_packages
+from setuptools import setup
+from paddle_serving_client.version import serving_client_version
+
+def python_version():
+    return [int(v) for v in platform.python_version().split(".")]
+
+max_version, mid_version, min_version = python_version()
+
+REQUIRED_PACKAGES = [
+    'six >= 1.10.0', 'protobuf >= 3.1.0','paddlepaddle'
+]
+
+packages=['paddle_serving_client',
+          'paddle_serving_client.proto',
+          'paddle_serving_client.io']
+package_data={'paddle_serving_client': ['serving_client.so']}
+package_dir={'paddle_serving_client':
+             '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client',
+             'paddle_serving_client.proto':
+             '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto',
+             'paddle_serving_client.io':
+             '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/io'}
+
+setup(
+    name='paddle-serving-client',
+    version=serving_client_version.replace('-', ''),
+    description=
+    ('Paddle Serving Package for saved model with PaddlePaddle'),
+    url='https://github.com/PaddlePaddle/Serving',
+    author='PaddlePaddle Author',
+    author_email='guru4elephant@gmail.com',
+    install_requires=REQUIRED_PACKAGES,
+    packages=packages,
+    package_data=package_data,
+    package_dir=package_dir,
+    # PyPI package information.
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Education',
+        'Intended Audience :: Science/Research',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Topic :: Scientific/Engineering',
+        'Topic :: Scientific/Engineering :: Mathematics',
+        'Topic :: Scientific/Engineering :: Artificial Intelligence',
+        'Topic :: Software Development',
+        'Topic :: Software Development :: Libraries',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+    ],
+    license='Apache 2.0',
+    keywords=('paddle-serving serving-client deployment industrial easy-to-use'))
+

python/setup.py.server.in

+#   Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Setup for pip package."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import platform
+
+from setuptools import setup, Distribution, Extension
+from setuptools import find_packages
+from setuptools import setup
+from paddle_serving_server.version import serving_server_version
+
+def python_version():
+    return [int(v) for v in platform.python_version().split(".")]
+
+max_version, mid_version, min_version = python_version()
+
+REQUIRED_PACKAGES = [
+    'six >= 1.10.0', 'protobuf >= 3.1.0','paddlepaddle'
+]
+
+packages=['paddle_serving_server',
+          'paddle_serving_server.proto']
+
+package_dir={'paddle_serving_server':
+             '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server',
+             'paddle_serving_server.proto':
+             '${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto'}
+
+setup(
+    name='paddle-serving-server',
+    version=serving_server_version.replace('-', ''),
+    description=
+    ('Paddle Serving Package for saved model with PaddlePaddle'),
+    url='https://github.com/PaddlePaddle/Serving',
+    author='PaddlePaddle Author',
+    author_email='guru4elephant@gmail.com',
+    install_requires=REQUIRED_PACKAGES,
+    packages=packages,
+    package_dir=package_dir,
+    # PyPI package information.
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Education',
+        'Intended Audience :: Science/Research',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Topic :: Scientific/Engineering',
+        'Topic :: Scientific/Engineering :: Mathematics',
+        'Topic :: Scientific/Engineering :: Artificial Intelligence',
+        'Topic :: Software Development',
+        'Topic :: Software Development :: Libraries',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+    ],
+    license='Apache 2.0',
+    keywords=('paddle-serving serving-server deployment industrial easy-to-use'))
+