add detection demo

python/paddle_fl/paddle_fl/examples/detection_demo/download.sh

+wget --no-check-certificate https://paddlefl.bj.bcebos.com/datasets/fl_fruit.tar.gz
+tar -xf fl_fruit.tar.gz

python/paddle_fl/paddle_fl/examples/detection_demo/fl_master.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import paddle_fl.paddle_fl as fl
+import paddle.fluid as fluid
+from paddle_fl.paddle_fl.core.master.job_generator import JobGenerator
+from paddle_fl.paddle_fl.core.strategy.fl_strategy_base import FLStrategyFactory
+
+# Fedavg
+build_strategy = FLStrategyFactory()
+build_strategy.fed_avg = True
+build_strategy.inner_step = 5
+strategy = build_strategy.create_fl_strategy()
+
+endpoints = ["127.0.0.1:8181"]
+output = "fl_job_config"
+program_file = "faster_rcnn_program"
+job_generator = JobGenerator()
+job_generator.generate_fl_job_from_program(
+    strategy=strategy,
+    endpoints=endpoints,
+    worker_num=2,
+    program_input=program_file,
+    output=output)

python/paddle_fl/paddle_fl/examples/detection_demo/fl_scheduler.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_fl.paddle_fl.core.scheduler.agent_master import FLScheduler
+
+worker_num = 2
+server_num = 1
+#Define number of worker/server and the port for scheduler
+scheduler = FLScheduler(worker_num, server_num, port=9091)
+scheduler.set_sample_worker_num(2)
+scheduler.init_env()
+print("init env done.")
+scheduler.start_fl_training()

python/paddle_fl/paddle_fl/examples/detection_demo/fl_server.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#import paddle_fl.paddle_fl as fl
+import paddle_fl as fl
+import paddle.fluid as fluid
+from paddle_fl.paddle_fl.core.server.fl_server import FLServer
+from paddle_fl.paddle_fl.core.master.fl_job import FLRunTimeJob
+
+server = FLServer()
+server_id = 0
+job_path = "fl_job_config"
+job = FLRunTimeJob()
+job.load_server_job(job_path, server_id)
+job._scheduler_ep = "127.0.0.1:9091"  # IP address for scheduler
+server.set_server_job(job)
+server._current_ep = "127.0.0.1:8181"  # IP address for server
+server.start()

python/paddle_fl/paddle_fl/examples/detection_demo/fl_trainer.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_fl.paddle_fl.core.trainer.fl_trainer import FLTrainerFactory
+from paddle_fl.paddle_fl.core.master.fl_job import FLRunTimeJob
+import numpy as np
+import sys
+import paddle
+import pickle
+import paddle.fluid as fluid
+import logging
+import math
+import unittest
+import os
+from ppdet.data.source.voc import VOCDataSet
+from ppdet.data.reader import Reader
+from ppdet.utils.download import get_path
+from ppdet.utils.download import DATASET_HOME
+
+from ppdet.data.transform.operators import DecodeImage, RandomFlipImage, NormalizeImage, ResizeImage, Permute
+from ppdet.data.transform.batch_operators import PadBatch
+
+trainer_id = int(sys.argv[1])  # trainer id for each guest
+
+
+class DataReader():
+    def __init__(self):
+        """ setup
+        """
+        self.root_path = '/path/to/your/fl_fruit'
+        self.anno_path = '/path/to/your/fl_fruit/train' + str(
+            trainer_id) + '.txt'
+        self.image_dir = '/path/to/your/fl_fruit/JPEGImages'
+
+    def tearDownClass(self):
+        """ tearDownClass """
+        pass
+
+    def test_loader(self):
+        coco_loader = VOCDataSet(
+            dataset_dir=self.image_dir,
+            image_dir=self.root_path,
+            anno_path=self.anno_path,
+            sample_num=240,
+            use_default_label=False,
+            label_list='/path/to/your/fl_fruit/label_list.txt')
+        sample_trans = [
+            DecodeImage(to_rgb=True), RandomFlipImage(), NormalizeImage(
+                mean=[0.485, 0.456, 0.406],
+                std=[0.229, 0.224, 0.225],
+                is_scale=True,
+                is_channel_first=False), ResizeImage(
+                    target_size=800, max_size=1333, interp=1),
+            Permute(to_bgr=False)
+        ]
+        batch_trans = [PadBatch(pad_to_stride=32, use_padded_im_info=True), ]
+
+        inputs_def = {
+            'fields':
+            ['image', 'im_info', 'im_id', 'gt_bbox', 'gt_class', 'is_crowd'],
+        }
+        data_loader = Reader(
+            coco_loader,
+            sample_transforms=sample_trans,
+            batch_transforms=batch_trans,
+            batch_size=1,
+            shuffle=True,
+            drop_empty=True,
+            inputs_def=inputs_def)()
+
+        return data_loader
+
+
+job_path = "fl_job_config"
+job = FLRunTimeJob()
+job.load_trainer_job(job_path, trainer_id)
+job._scheduler_ep = "127.0.0.1:9091"  # Inform scheduler IP address to trainer
+trainer = FLTrainerFactory().create_fl_trainer(job)
+trainer._current_ep = "127.0.0.1:{}".format(9000 + trainer_id)
+trainer.start(fluid.CUDAPlace(trainer_id))
+
+test_program = trainer._main_program.clone(for_test=True)
+
+image = fluid.layers.data(
+    name='image', shape=[3, None, None], dtype='float32', lod_level=0)
+im_info = fluid.layers.data(
+    name='im_info', shape=[None, 3], dtype='float32', lod_level=0)
+im_id = fluid.layers.data(
+    name='im_id', shape=[None, 1], dtype='int64', lod_level=0)
+gt_bbox = fluid.layers.data(
+    name='gt_bbox', shape=[None, 4], dtype='float32', lod_level=1)
+gt_class = fluid.layers.data(
+    name='gt_class', shape=[None, 1], dtype='int32', lod_level=1)
+is_crowd = fluid.layers.data(
+    name='is_crowd', shape=[None, 1], dtype='int32', lod_level=1)
+place = fluid.CUDAPlace(trainer_id)
+feeder = fluid.DataFeeder(
+    feed_list=[image, im_info, im_id, gt_bbox, gt_class, is_crowd],
+    place=place)
+
+output_folder = "5_model_node%d" % trainer_id
+epoch_id = 0
+step = 0
+
+para_dir = "faster_rcnn_program"
+
+while not trainer.stop():
+    epoch_id += 1
+    if epoch_id > 120:
+        break
+    print("epoch %d start train" % (epoch_id))
+    test_class = DataReader()
+    data_loader = test_class.test_loader()
+    for step_id, data in enumerate(data_loader):
+        acc = trainer.run(feeder.feed(data), fetch=['sum_0.tmp_0'])
+        step += 1
+        print("step: {}, loss: {}".format(step, acc))
+
+    if trainer_id == 0:
+        save_dir = (output_folder + "/epoch_%d") % epoch_id
+        trainer.save(para_dir, save_dir)

python/paddle_fl/paddle_fl/examples/detection_demo/run.sh

+wget --no-check-certificate https://paddlefl.bj.bcebos.com/detection_programs/faster_rcnn_program.tar.gz
+tar -xf faster_rcnn_program.tar.gz
+
+unset http_proxy
+unset https_proxy
+export PYTHONPATH=/path/to/PaddleDetection
+CUDA_VISIBLE_DEVICES=0,1
+
+python fl_master.py
+sleep 2
+python -u fl_scheduler.py >scheduler.log &
+sleep 2 
+python -u fl_server.py >server0.log &
+sleep 2
+python -u fl_trainer.py 0 > 5_trainer0.log &
+sleep 2
+python -u fl_trainer.py 1 > 5_trainer1.log &
+sleep 2

python/paddle_fl/paddle_fl/examples/generate_job_from_program/fl_trainer.py

@@ -61,6 +61,7 @@ def train_test(train_test_program, train_test_feed, train_test_reader):
     return acc_val_mean
 
 
+para_dir = 'load_file'
 output_folder = "model_node%d" % trainer_id
 epoch_id = 0
 step = 0