diff --git a/paddle_inference/paddle/include/paddle_engine.h b/paddle_inference/paddle/include/paddle_engine.h index 599d5e5e5477da72927f76c0189a82721db3c6b4..faec60a6147cfa9ae142aabe94b1340cb6b032d3 100644 --- a/paddle_inference/paddle/include/paddle_engine.h +++ b/paddle_inference/paddle/include/paddle_engine.h @@ -107,9 +107,9 @@ class PaddleInferenceEngine : public PaddleEngineBase { if (engine_conf.has_encrypted_model() && engine_conf.encrypted_model()) { // decrypt model std::string model_buffer, params_buffer, key_buffer; - predictor::ReadBinaryFile(model_path + "encrypt_model", &model_buffer); - predictor::ReadBinaryFile(model_path + "encrypt_params", ¶ms_buffer); - predictor::ReadBinaryFile(model_path + "key", &key_buffer); + predictor::ReadBinaryFile(model_path + "/encrypt_model", &model_buffer); + predictor::ReadBinaryFile(model_path + "/encrypt_params", ¶ms_buffer); + predictor::ReadBinaryFile(model_path + "/key", &key_buffer); auto cipher = paddle::MakeCipher(""); std::string real_model_buffer = cipher->Decrypt(model_buffer, key_buffer); diff --git a/python/examples/encryption/README.md b/python/examples/encryption/README.md index d8a04e29bf56439a24db7dadfdfe3ab5d9626e14..a08b8b84241fb699992d1a718f2bfbf986d8d180 100644 --- a/python/examples/encryption/README.md +++ b/python/examples/encryption/README.md @@ -31,6 +31,7 @@ dirname is the folder path where the model is located. If the parameter is discr The key is stored in the `key` file, and the encrypted model file and server-side configuration file are stored in the `encrypt_server` directory. client-side configuration file are stored in the `encrypt_client` directory. +**Notice：** When encryption prediction is used, the model configuration and parameter folder loaded by server and client should be encrypt_server/ and encrypt_client/ ## Start Encryption Service CPU Service ``` @@ -43,5 +44,5 @@ python -m paddle_serving_server.serve --model encrypt_server/ --port 9300 --use_ ## Prediction ``` -python test_client.py uci_housing_client/serving_client_conf.prototxt +python test_client.py encrypt_client/serving_client_conf.prototxt ``` diff --git a/python/examples/encryption/README_CN.md b/python/examples/encryption/README_CN.md index bb853ff37f914a5e2cfe1c6bbb097d17eb99a29a..f950796ec14dadfd7bf6744d94aba4959c838e7f 100644 --- a/python/examples/encryption/README_CN.md +++ b/python/examples/encryption/README_CN.md @@ -31,6 +31,8 @@ def serving_encryption(): 密钥保存在`key`文件中，加密模型文件以及server端配置文件保存在`encrypt_server`目录下，client端配置文件保存在`encrypt_client`目录下。 +**注意：** 当使用加密预测时，服务端和客户端启动加载的模型配置和参数文件夹是encrypt_server/和encrypt_client/ + ## 启动加密预测服务 CPU预测服务 ``` @@ -43,5 +45,5 @@ python -m paddle_serving_server.serve --model encrypt_server/ --port 9300 --use_ ## 预测 ``` -python test_client.py uci_housing_client/serving_client_conf.prototxt +python test_client.py encrypt_client/ ``` diff --git a/python/paddle_serving_server/server.py b/python/paddle_serving_server/server.py index d96253b592f70956591c345606eeb0d01e1e4b43..35b9bc65c17cb881845691666440413bc6d41c50 100755 --- a/python/paddle_serving_server/server.py +++ b/python/paddle_serving_server/server.py @@ -42,19 +42,30 @@ from concurrent import futures class Server(object): def __init__(self): + """ + self.model_toolkit_conf:'list'=[] # The quantity of self.model_toolkit_conf is equal to the InferOp quantity/Engine--OP + self.model_conf:'collections.OrderedDict()' # Save the serving_server_conf.prototxt content (feed and fetch information) this is a map for multi-model in a workflow + self.workflow_fn:'str'="workflow.prototxt" # Only one for one Service/Workflow + self.resource_fn:'str'="resource.prototxt" # Only one for one Service,model_toolkit_fn and general_model_config_fn is recorded in this file + self.infer_service_fn:'str'="infer_service.prototxt" # Only one for one Service,Service--Workflow + self.model_toolkit_fn:'list'=[] # ["general_infer_0/model_toolkit.prototxt"]The quantity is equal to the InferOp quantity,Engine--OP + self.general_model_config_fn:'list'=[] # ["general_infer_0/general_model.prototxt"]The quantity is equal to the InferOp quantity,Feed and Fetch --OP + self.subdirectory:'list'=[] # The quantity is equal to the InferOp quantity, and name = node.name = engine.name + self.model_config_paths:'collections.OrderedDict()' # Save the serving_server_conf.prototxt path (feed and fetch information) this is a map for multi-model in a workflow + """ self.server_handle_ = None self.infer_service_conf = None - self.model_toolkit_conf = []#The quantity is equal to the InferOp quantity,Engine--OP + self.model_toolkit_conf = [] self.resource_conf = None self.memory_optimization = False self.ir_optimization = False - self.model_conf = collections.OrderedDict()# save the serving_server_conf.prototxt content (feed and fetch information) this is a map for multi-model in a workflow - self.workflow_fn = "workflow.prototxt"#only one for one Service,Workflow--Op - self.resource_fn = "resource.prototxt"#only one for one Service,model_toolkit_fn and general_model_config_fn is recorded in this file - self.infer_service_fn = "infer_service.prototxt"#only one for one Service,Service--Workflow - self.model_toolkit_fn = []#["general_infer_0/model_toolkit.prototxt"]The quantity is equal to the InferOp quantity,Engine--OP - self.general_model_config_fn = []#["general_infer_0/general_model.prototxt"]The quantity is equal to the InferOp quantity,Feed and Fetch --OP - self.subdirectory = []#The quantity is equal to the InferOp quantity, and name = node.name = engine.name + self.model_conf = collections.OrderedDict() + self.workflow_fn = "workflow.prototxt" + self.resource_fn = "resource.prototxt" + self.infer_service_fn = "infer_service.prototxt" + self.model_toolkit_fn = [] + self.general_model_config_fn = [] + self.subdirectory = [] self.cube_config_fn = "cube.conf" self.workdir = "" self.max_concurrency = 0 @@ -71,12 +82,15 @@ class Server(object): self.use_trt = False self.use_lite = False self.use_xpu = False - self.model_config_paths = collections.OrderedDict() # save the serving_server_conf.prototxt path (feed and fetch information) this is a map for multi-model in a workflow + self.model_config_paths = collections.OrderedDict() self.product_name = None self.container_id = None - def get_fetch_list(self,infer_node_idx = -1 ): - fetch_names = [var.alias_name for var in list(self.model_conf.values())[infer_node_idx].fetch_var] + def get_fetch_list(self, infer_node_idx=-1): + fetch_names = [ + var.alias_name + for var in list(self.model_conf.values())[infer_node_idx].fetch_var + ] return fetch_names def set_max_concurrency(self, concurrency): @@ -195,9 +209,10 @@ class Server(object): self.workdir = workdir if self.resource_conf == None: self.resource_conf = server_sdk.ResourceConf() - for idx, op_general_model_config_fn in enumerate(self.general_model_config_fn): + for idx, op_general_model_config_fn in enumerate( + self.general_model_config_fn): with open("{}/{}".format(workdir, op_general_model_config_fn), - "w") as fout: + "w") as fout: fout.write(str(list(self.model_conf.values())[idx])) for workflow in self.workflow_conf.workflows: for node in workflow.nodes: @@ -212,9 +227,11 @@ class Server(object): if "quant" in node.name: self.resource_conf.cube_quant_bits = 8 self.resource_conf.model_toolkit_path.extend([workdir]) - self.resource_conf.model_toolkit_file.extend([self.model_toolkit_fn[idx]]) + self.resource_conf.model_toolkit_file.extend( + [self.model_toolkit_fn[idx]]) self.resource_conf.general_model_path.extend([workdir]) - self.resource_conf.general_model_file.extend([op_general_model_config_fn]) + self.resource_conf.general_model_file.extend( + [op_general_model_config_fn]) #TODO:figure out the meaning of product_name and container_id. if self.product_name != None: self.resource_conf.auth_product_name = self.product_name @@ -237,15 +254,18 @@ class Server(object): if os.path.isdir(single_model_config): pass elif os.path.isfile(single_model_config): - raise ValueError("The input of --model should be a dir not file.") - + raise ValueError( + "The input of --model should be a dir not file.") + if isinstance(model_config_paths_args, list): # If there is only one model path, use the default infer_op. # Because there are several infer_op type, we need to find # it from workflow_conf. default_engine_types = [ - 'GeneralInferOp', 'GeneralDistKVInferOp', - 'GeneralDistKVQuantInferOp','GeneralDetectionOp', + 'GeneralInferOp', + 'GeneralDistKVInferOp', + 'GeneralDistKVQuantInferOp', + 'GeneralDetectionOp', ] # now only support single-workflow. # TODO:support multi-workflow @@ -256,16 +276,24 @@ class Server(object): raise Exception( "You have set the engine_name of Op. Please use the form {op: model_path} to configure model path" ) - + f = open("{}/serving_server_conf.prototxt".format( - model_config_paths_args[model_config_paths_list_idx]), 'r') - self.model_conf[node.name] = google.protobuf.text_format.Merge(str(f.read()), m_config.GeneralModelConfig()) - self.model_config_paths[node.name] = model_config_paths_args[model_config_paths_list_idx] - self.general_model_config_fn.append(node.name+"/general_model.prototxt") - self.model_toolkit_fn.append(node.name+"/model_toolkit.prototxt") + model_config_paths_args[model_config_paths_list_idx]), + 'r') + self.model_conf[ + node.name] = google.protobuf.text_format.Merge( + str(f.read()), m_config.GeneralModelConfig()) + self.model_config_paths[ + node.name] = model_config_paths_args[ + model_config_paths_list_idx] + self.general_model_config_fn.append( + node.name + "/general_model.prototxt") + self.model_toolkit_fn.append(node.name + + "/model_toolkit.prototxt") self.subdirectory.append(node.name) model_config_paths_list_idx += 1 - if model_config_paths_list_idx == len(model_config_paths_args): + if model_config_paths_list_idx == len( + model_config_paths_args): break #Right now, this is not useful. elif isinstance(model_config_paths_args, dict): @@ -278,11 +306,12 @@ class Server(object): "that the input and output of multiple models are the same.") f = open("{}/serving_server_conf.prototxt".format(path), 'r') self.model_conf[node.name] = google.protobuf.text_format.Merge( - str(f.read()), m_config.GeneralModelConfig()) + str(f.read()), m_config.GeneralModelConfig()) else: - raise Exception("The type of model_config_paths must be str or list or " - "dict({op: model_path}), not {}.".format( - type(model_config_paths_args))) + raise Exception( + "The type of model_config_paths must be str or list or " + "dict({op: model_path}), not {}.".format( + type(model_config_paths_args))) # check config here # print config here @@ -409,7 +438,7 @@ class Server(object): resource_fn = "{}/{}".format(workdir, self.resource_fn) self._write_pb_str(resource_fn, self.resource_conf) - for idx,single_model_toolkit_fn in enumerate(self.model_toolkit_fn): + for idx, single_model_toolkit_fn in enumerate(self.model_toolkit_fn): model_toolkit_fn = "{}/{}".format(workdir, single_model_toolkit_fn) self._write_pb_str(model_toolkit_fn, self.model_toolkit_conf[idx]) @@ -498,6 +527,7 @@ class Server(object): os.system(command) + class MultiLangServer(object): def __init__(self): self.bserver_ = Server() @@ -553,22 +583,23 @@ class MultiLangServer(object): def set_gpuid(self, gpuid=0): self.bserver_.set_gpuid(gpuid) - def load_model_config(self, server_config_dir_paths, client_config_path=None): + def load_model_config(self, + server_config_dir_paths, + client_config_path=None): if isinstance(server_config_dir_paths, str): server_config_dir_paths = [server_config_dir_paths] elif isinstance(server_config_dir_paths, list): pass else: raise Exception("The type of model_config_paths must be str or list" - ", not {}.".format( - type(server_config_dir_paths))) - + ", not {}.".format(type(server_config_dir_paths))) for single_model_config in server_config_dir_paths: if os.path.isdir(single_model_config): pass elif os.path.isfile(single_model_config): - raise ValueError("The input of --model should be a dir not file.") + raise ValueError( + "The input of --model should be a dir not file.") self.bserver_.load_model_config(server_config_dir_paths) if client_config_path is None: @@ -576,27 +607,30 @@ class MultiLangServer(object): if isinstance(server_config_dir_paths, dict): self.is_multi_model_ = True client_config_path = [] - for server_config_path_items in list(server_config_dir_paths.items()): - client_config_path.append( server_config_path_items[1] ) + for server_config_path_items in list( + server_config_dir_paths.items()): + client_config_path.append(server_config_path_items[1]) elif isinstance(server_config_dir_paths, list): self.is_multi_model_ = False client_config_path = server_config_dir_paths else: - raise Exception("The type of model_config_paths must be str or list or " - "dict({op: model_path}), not {}.".format( - type(server_config_dir_paths))) + raise Exception( + "The type of model_config_paths must be str or list or " + "dict({op: model_path}), not {}.".format( + type(server_config_dir_paths))) if isinstance(client_config_path, str): client_config_path = [client_config_path] elif isinstance(client_config_path, list): pass - else:# dict is not support right now. - raise Exception("The type of client_config_path must be str or list or " - "dict({op: model_path}), not {}.".format( - type(client_config_path))) + else: # dict is not support right now. + raise Exception( + "The type of client_config_path must be str or list or " + "dict({op: model_path}), not {}.".format( + type(client_config_path))) if len(client_config_path) != len(server_config_dir_paths): - raise Warning("The len(client_config_path) is {}, != len(server_config_dir_paths) {}." - .format( len(client_config_path), len(server_config_dir_paths) ) - ) + raise Warning( + "The len(client_config_path) is {}, != len(server_config_dir_paths) {}." + .format(len(client_config_path), len(server_config_dir_paths))) self.bclient_config_path_list = client_config_path def prepare_server(self,