test int32 for java sdk && fix test_server_quant.py

java/examples/pom.xml

@@ -50,6 +50,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <nd4j.backend>nd4j-native</nd4j.backend>
         <nd4j.version>1.0.0-beta7</nd4j.version>
+        <datavec.version>1.0.0-beta7</datavec.version>
         <maven.compiler.source>1.7</maven.compiler.source>
         <maven.compiler.target>1.7</maven.compiler.target>
     </properties>
@@ -91,6 +92,11 @@
             <version>4.11</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.datavec</groupId>
+            <artifactId>datavec-data-image</artifactId>
+            <version>${datavec.version}</version>
+        </dependency>
     </dependencies>
 
 </project>

java/examples/src/main/java/PaddleServingClientExample.java

 import io.paddle.serving.client.*;
-import org.nd4j.linalg.api.ndarray.INDArray;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
 import org.nd4j.linalg.api.iter.NdIndexIterator;
+import org.nd4j.linalg.api.ndarray.INDArray;
+import org.datavec.image.loader.NativeImageLoader;
+import org.nd4j.linalg.api.ops.CustomOp;
+import org.nd4j.linalg.api.ops.DynamicCustomOp;
 import org.nd4j.linalg.factory.Nd4j;
 import java.util.*;
 
@@ -14,7 +20,63 @@ public class PaddleServingClientExample {
             = new HashMap<String, INDArray>() {{
                 put("x", npdata);
             }};
-        List<String> fetch = Arrays.asList("price");
+        List<String> fetch = Arrays.asList("save_infer_model/scale_0.tmp_0");
+        
+        Client client = new Client();
+        String target = "localhost:9393";
+        boolean succ = client.connect(target);
+        if (succ != true) {
+            System.out.println("connect failed.");
+            return false;
+        }
+
+        Map<String, INDArray> fetch_map = client.predict(feed_data, fetch);
+        if (fetch_map == null) {
+            return false;
+        }
+
+        for (Map.Entry<String, INDArray> e : fetch_map.entrySet()) {
+            System.out.println("Key = " + e.getKey() + ", Value = " + e.getValue());
+        }
+        return true;
+    }
+
+    boolean yolov4(String filename) {
+        // https://deeplearning4j.konduit.ai/
+        int height = 608;
+        int width = 608;
+        int channels = 3;
+        NativeImageLoader loader = new NativeImageLoader(height, width, channels);
+        INDArray BGRimage = null;
+        try {
+            BGRimage = loader.asMatrix(new File(filename));
+        } catch (java.io.IOException e) {
+            System.out.println("load image fail.");
+            return false;
+        }   
+
+        // shape: (channels, height, width)
+        BGRimage = BGRimage.reshape(channels, height, width);
+        INDArray RGBimage = Nd4j.create(BGRimage.shape());
+
+        // BGR2RGB
+        CustomOp op = DynamicCustomOp.builder("reverse")
+            .addInputs(BGRimage)
+            .addOutputs(RGBimage)
+            .addIntegerArguments(0)
+            .build();
+        Nd4j.getExecutioner().exec(op);
+        
+        // Div(255.0)
+        INDArray image = RGBimage.divi(255.0);
+        
+        INDArray im_size = Nd4j.createFromArray(new int[]{height, width});
+        HashMap<String, INDArray> feed_data
+            = new HashMap<String, INDArray>() {{
+                put("image", image);
+                put("im_size", im_size);
+            }};
+        List<String> fetch = Arrays.asList("save_infer_model/scale_0.tmp_0");
         
         Client client = new Client();
         String target = "localhost:9393";
@@ -255,26 +317,36 @@ public class PaddleServingClientExample {
         PaddleServingClientExample e = new PaddleServingClientExample();
         boolean succ = false;
         
-        for (String arg : args) {
-            System.out.format("[Example] %s\n", arg);
-            if ("fit_a_line".equals(arg)) {
+        if (args.length < 1) {
+            System.out.println("Usage: java -cp <jar> PaddleServingClientExample <test-type>.");
+            System.out.println("<test-type>: fit_a_line bert model_ensemble asyn_predict batch_predict cube_local cube_quant yolov4");
+            return;
+        }
+        String testType = args[0];
+        System.out.format("[Example] %s\n", testType);
+        if ("fit_a_line".equals(testType)) {
             succ = e.fit_a_line();
-            } else if ("bert".equals(arg)) {
+        } else if ("bert".equals(testType)) {
             succ = e.bert();
-            } else if ("model_ensemble".equals(arg)) {
+        } else if ("model_ensemble".equals(testType)) {
             succ = e.model_ensemble();
-            } else if ("asyn_predict".equals(arg)) {
+        } else if ("asyn_predict".equals(testType)) {
             succ = e.asyn_predict();
-            } else if ("batch_predict".equals(arg)) {
+        } else if ("batch_predict".equals(testType)) {
             succ = e.batch_predict();
-            } else if ("cube_local".equals(arg)) {
+        } else if ("cube_local".equals(testType)) {
             succ = e.cube_local();
-            } else if ("cube_quant".equals(arg)) {
+        } else if ("cube_quant".equals(testType)) {
             succ = e.cube_local();
-            } else {
-                System.out.format("%s not match: java -cp <jar> PaddleServingClientExample <exp>.\n", arg);
-                System.out.println("<exp>: fit_a_line bert model_ensemble asyn_predict batch_predict cube_local cube_quant.");
+        } else if ("yolov4".equals(testType)) {
+            if (args.length < 2) {
+                System.out.println("Usage: java -cp <jar> PaddleServingClientExample yolov4 <image-filepath>.");
+                return;
             }
+            succ = e.yolov4(args[1]);
+        } else {
+            System.out.format("test-type(%s) not match.\n", testType);
+            return;
         }
 
         if (succ == true) {

java/pom.xml

@@ -22,35 +22,21 @@
         </license>
     </licenses>
 
-    <!--
     <developers>
         <developer>
-            <name>Zhiru Zhu</name>
-            <email>zhiru.zhu@zilliz.com</email>
-            <organization>Milvus</organization>
-            <organizationUrl>http://www.milvus.io</organizationUrl>
-        </developer>
-        <developer>
-            <name>Xiaohai Xu</name>
-            <email>xiaohai.xu@zilliz.com</email>
-            <organization>Milvus</organization>
-            <organizationUrl>http://www.milvus.io</organizationUrl>
+            <name>PaddlePaddle Author</name>
+            <email>guru4elephant@gmail.com</email>
+            <organization>PaddlePaddle</organization>
+            <organizationUrl>https://github.com/PaddlePaddle/Serving</organizationUrl>
         </developer>
     </developers>
 
     <scm>
-        <connection>scm:git:https://github.com/milvus-io/milvus-sdk-java.git</connection>
-        <developerConnection>scm:git:https://github.com/milvus-io/milvus-sdk-java.git</developerConnection>
-        <url>https://github.com/milvus-io/milvus-sdk-java</url>
+        <connection>scm:git:https://github.com/PaddlePaddle/Serving.git</connection>
+        <developerConnection>scm:git:https://github.com/PaddlePaddle/Serving.git</developerConnection>
+        <url>https://github.com/PaddlePaddle/Serving</url>
     </scm>
 
-    <distributionManagement>
-        <snapshotRepository>
-            <id>ossrh</id>
-            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-        </snapshotRepository>
-    </distributionManagement>
-    -->
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <grpc.version>1.27.2</grpc.version>

java/src/main/java/io/paddle/serving/client/Client.java

@@ -324,7 +324,7 @@ public class Client {
             multi_result_map.put(engine_name, result_map);
         }
 
-        // TODO: tag
+        // TODO: tag(ABtest not support now)
         return multi_result_map;
     }
 
@@ -414,7 +414,7 @@ public class Client {
                 = new ArrayList<Map.Entry<String, HashMap<String, INDArray>>>(
                     ensemble_result.entrySet());
             if (list.size() != 1) {
-                System.out.format("grpc failed: please use ensemble_predict impl.\n");
+                System.out.format("predict failed: please use ensemble_predict impl.\n");
                 return null;
             }
             profiler_.record("java_postpro_1");
@@ -422,7 +422,7 @@ public class Client {
 
             return list.get(0).getValue();
         } catch (StatusRuntimeException e) {
-            System.out.format("grpc failed: %s\n", e.toString());
+            System.out.format("predict failed: %s\n", e.toString());
             return null;
         }
     }
@@ -448,7 +448,7 @@ public class Client {
 
             return ensemble_result;
         } catch (StatusRuntimeException e) {
-            System.out.format("grpc failed: %s\n", e.toString());
+            System.out.format("predict failed: %s\n", e.toString());
             return null;
         }
     }

python/examples/criteo_ctr_with_cube/test_server_quant.py

@@ -33,5 +33,9 @@ server = Server()
 server.set_op_sequence(op_seq_maker.get_op_sequence())
 server.set_num_threads(4)
 server.load_model_config(sys.argv[1])
-server.prepare_server(workdir="work_dir1", port=9292, device="cpu")
+server.prepare_server(
+    workdir="work_dir1",
+    port=9292,
+    device="cpu",
+    cube_conf="./cube/conf/cube.conf")
 server.run_server()

python/examples/grpc_impl_example/criteo_ctr_with_cube/test_server_quant.py

@@ -33,5 +33,9 @@ server = Server()
 server.set_op_sequence(op_seq_maker.get_op_sequence())
 server.set_num_threads(4)
 server.load_model_config(sys.argv[1], sys.argv[2])
-server.prepare_server(workdir="work_dir1", port=9292, device="cpu")
+server.prepare_server(
+    workdir="work_dir1",
+    port=9292,
+    device="cpu",
+    cube_conf="./cube/conf/cube.conf")
 server.run_server()

python/examples/grpc_impl_example/yolov4/README.md

+# Yolov4 Detection Service
+
+([简体中文](README_CN.md)|English)
+
+## Get Model
+
+```
+python -m paddle_serving_app.package --get_model yolov4
+tar -xzvf yolov4.tar.gz
+```
+
+## Start RPC Service
+
+```
+python -m paddle_serving_server_gpu.serve --model yolov4_model --port 9393 --gpu_ids 0 --use_multilang
+```
+
+## Prediction
+
+```
+python test_client.py 000000570688.jpg
+```
+After the prediction is completed, a json file to save the prediction result and a picture with the detection result box will be generated in the `./outpu folder.

python/examples/grpc_impl_example/yolov4/README_CN.md

+# Yolov4 检测服务
+
+(简体中文|[English](README.md))
+
+## 获取模型
+
+```
+python -m paddle_serving_app.package --get_model yolov4
+tar -xzvf yolov4.tar.gz
+```
+
+## 启动RPC服务
+
+```
+python -m paddle_serving_server_gpu.serve --model yolov4_model --port 9393 --gpu_ids 0 --use_multilang
+```
+
+## 预测
+
+```
+python test_client.py 000000570688.jpg
+```
+
+预测完成会在`./output`文件夹下生成保存预测结果的json文件以及标出检测结果框的图片。

python/examples/grpc_impl_example/yolov4/label_list.txt

+person
+bicycle
+car
+motorcycle
+airplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+couch
+potted plant
+bed
+dining table
+toilet
+tv
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush

python/examples/imdb/test_multilang_ensemble_client.py → python/examples/grpc_impl_example/yolov4/test_client.py

@@ -11,27 +11,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# pylint: disable=doc-string-missing
 
-from paddle_serving_client import MultiLangClient
-from imdb_reader import IMDBDataset
+import sys
+import numpy as np
+from paddle_serving_client import MultiLangClient as Client
+from paddle_serving_app.reader import *
+import cv2
 
-client = MultiLangClient()
-# If you have more than one model, make sure that the input
-# and output of more than one model are the same.
-client.connect(["127.0.0.1:9393"])
+preprocess = Sequential([
+    File2Image(), BGR2RGB(), Resize(
+        (608, 608), interpolation=cv2.INTER_LINEAR), Div(255.0), Transpose(
+            (2, 0, 1))
+])
 
-# you can define any english sentence or dataset here
-# This example reuses imdb reader in training, you
-# can define your own data preprocessing easily.
-imdb_dataset = IMDBDataset()
-imdb_dataset.load_resource('imdb.vocab')
+postprocess = RCNNPostprocess("label_list.txt", "output", [608, 608])
+client = Client()
+client.connect(['127.0.0.1:9393'])
+# client.set_rpc_timeout_ms(10000)
 
-for i in range(3):
-    line = 'i am very sad | 0'
-    word_ids, label = imdb_dataset.get_words_and_label(line)
-    feed = {"words": word_ids}
-    fetch = ["prediction"]
-    fetch_maps = client.predict(feed=feed, fetch=fetch)
-    for model, fetch_map in fetch_maps.items():
-        print("step: {}, model: {}, res: {}".format(i, model, fetch_map))
+im = preprocess(sys.argv[1])
+fetch_map = client.predict(
+    feed={
+        "image": im,
+        "im_size": np.array(list(im.shape[1:])),
+    },
+    fetch=["save_infer_model/scale_0.tmp_0"])
+fetch_map.pop("serving_status_code")
+fetch_map["image"] = sys.argv[1]
+postprocess(fetch_map)

python/examples/imdb/test_multilang_ensemble_server.py

-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# pylint: disable=doc-string-missing
-
-from paddle_serving_server import OpMaker
-from paddle_serving_server import OpGraphMaker
-from paddle_serving_server import MultiLangServer
-
-op_maker = OpMaker()
-read_op = op_maker.create('general_reader')
-cnn_infer_op = op_maker.create(
-    'general_infer', engine_name='cnn', inputs=[read_op])
-bow_infer_op = op_maker.create(
-    'general_infer', engine_name='bow', inputs=[read_op])
-response_op = op_maker.create(
-    'general_response', inputs=[cnn_infer_op, bow_infer_op])
-
-op_graph_maker = OpGraphMaker()
-op_graph_maker.add_op(read_op)
-op_graph_maker.add_op(cnn_infer_op)
-op_graph_maker.add_op(bow_infer_op)
-op_graph_maker.add_op(response_op)
-
-server = MultiLangServer()
-server.set_op_graph(op_graph_maker.get_op_graph())
-model_config = {cnn_infer_op: 'imdb_cnn_model', bow_infer_op: 'imdb_bow_model'}
-server.load_model_config(model_config)
-server.prepare_server(workdir="work_dir1", port=9393, device="cpu")
-server.run_server()

python/paddle_serving_client/__init__.py

@@ -522,7 +522,7 @@ class MultiLangClient(object):
                             tensor.float_data.extend(
                                 var.reshape(-1).astype('float32').tolist())
                         elif v_type == 2:
-                            tensor.int32_data.extend(
+                            tensor.int_data.extend(
                                 var.reshape(-1).astype('int32').tolist())
                         else:
                             raise Exception("error tensor value type.")
@@ -532,7 +532,7 @@ class MultiLangClient(object):
                         elif v_type == 1:
                             tensor.float_data.extend(self._flatten_list(var))
                         elif v_type == 2:
-                            tensor.int32_data.extend(self._flatten_list(var))
+                            tensor.int_data.extend(self._flatten_list(var))
                         else:
                             raise Exception("error tensor value type.")
                     else:

python/paddle_serving_server/__init__.py

@@ -524,7 +524,7 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc.
                     elif v_type == 1:  # float32
                         data = np.array(list(var.float_data), dtype="float32")
                     elif v_type == 2:  # int32
-                        data = np.array(list(var.int32_data), dtype="int32")
+                        data = np.array(list(var.int_data), dtype="int32")
                     else:
                         raise Exception("error type.")
                 data.shape = list(feed_inst.tensor_array[idx].shape)
@@ -559,7 +559,7 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc.
                         tensor.float_data.extend(model_result[name].reshape(-1)
                                                  .tolist())
                     elif v_type == 2:  # int32
-                        tensor.int32_data.extend(model_result[name].reshape(-1)
+                        tensor.int_data.extend(model_result[name].reshape(-1)
                                                .tolist())
                     else:
                         raise Exception("error type.")

python/paddle_serving_server_gpu/__init__.py

@@ -571,7 +571,7 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc.
                     elif v_type == 1:  # float32
                         data = np.array(list(var.float_data), dtype="float32")
                     elif v_type == 2:
-                        data = np.array(list(var.int32_data), dtype="int32")
+                        data = np.array(list(var.int_data), dtype="int32")
                     else:
                         raise Exception("error type.")
                 data.shape = list(feed_inst.tensor_array[idx].shape)
@@ -606,7 +606,7 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc.
                         tensor.float_data.extend(model_result[name].reshape(-1)
                                                  .tolist())
                     elif v_type == 2:  # int32
-                        tensor.int32_data.extend(model_result[name].reshape(-1)
+                        tensor.int_data.extend(model_result[name].reshape(-1)
                                                .tolist())
                     else:
                         raise Exception("error type.")