Merge branch 'master' into secagg

docs/source/md/quick_start.md

@@ -67,7 +67,8 @@ from paddle_fl.core.scheduler.agent_master import FLScheduler
 
 worker_num = 2
 server_num = 1
-scheduler = FLScheduler(worker_num,server_num)
+# Define the number of worker/server and the port for scheduler
+scheduler = FLScheduler(worker_num,server_num,port=9091)
 scheduler.set_sample_worker_num(worker_num)
 scheduler.init_env()
 print("init env done.")
@@ -94,6 +95,7 @@ trainer_id = int(sys.argv[1]) # trainer id for each guest
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_trainer_job(job_path, trainer_id)
+job._scheduler_ep = "127.0.0.1:9091" # Inform the scheduler IP to trainer
 trainer = FLTrainerFactory().create_fl_trainer(job)
 trainer.start()
 
@@ -122,6 +124,8 @@ server_id = 0
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_server_job(job_path, server_id)
+job._scheduler_ep = "127.0.0.1:9091" # IP address for scheduler
 server.set_server_job(job)
+server._current_ep = "127.0.0.1:8181" # IP address for server
 server.start()
 ```

paddle_fl/core/scheduler/agent_master.py

@@ -140,4 +140,3 @@ class FLScheduler(object):
                 if len(finish_training_dict) == len(worker_dict):
                     all_finish_training = True
             time.sleep(5)
-            loop += 1

paddle_fl/core/trainer/diffiehellman/diffiehellman.py

@@ -44,7 +44,6 @@ try:
 except(AttributeError, ImportError):
     rng = os.urandom
 
-
 class DiffieHellman:
     """
     Implements the Diffie-Hellman key exchange protocol.

paddle_fl/examples/ctr_demo/fl_scheduler.py

@@ -2,7 +2,8 @@ from paddle_fl.core.scheduler.agent_master import FLScheduler
 
 worker_num = 2
 server_num = 1
-scheduler = FLScheduler(worker_num,server_num)
+# Define the number of worker/server and the port for scheduler
+scheduler = FLScheduler(worker_num,server_num,port=9091) 
 scheduler.set_sample_worker_num(worker_num)
 scheduler.init_env()
 print("init env done.")

paddle_fl/examples/ctr_demo/fl_server.py

@@ -21,8 +21,8 @@ server_id = 0
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_server_job(job_path, server_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # IP address for scheduler
 server.set_server_job(job)
-server._current_ep = "127.0.0.1:8181"
+server._current_ep = "127.0.0.1:8181" # IP address for server
 server.start()
 print("connect")

paddle_fl/examples/ctr_demo/fl_trainer.py

@@ -19,22 +19,22 @@ trainer_id = int(sys.argv[1]) # trainer id for each guest
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_trainer_job(job_path, trainer_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # Inform the scheduler IP to trainer
 trainer = FLTrainerFactory().create_fl_trainer(job)
 trainer._current_ep = "127.0.0.1:{}".format(9000+trainer_id)
 trainer.start()
 print(trainer._scheduler_ep, trainer._current_ep)
 output_folder = "fl_model"
-step_i = 0
+epoch_id = 0
 while not trainer.stop():
-    print("batch %d start train" % (step_i))
+    print("batch %d start train" % (epoch_id))
     train_step = 0
     for data in reader():
         trainer.run(feed=data, fetch=[])
         train_step += 1
         if train_step == trainer._step:
             break
-    step_i += 1
-    if step_i % 100 == 0:
+    epoch_id += 1
+    if epoch_id % 5 == 0:
         trainer.save_inference_program(output_folder)
     

paddle_fl/examples/dpsgd_demo/fl_scheduler.py

@@ -2,7 +2,8 @@ from paddle_fl.core.scheduler.agent_master import FLScheduler
 
 worker_num = 4
 server_num = 1
-scheduler = FLScheduler(worker_num,server_num)
+#Define number of worker/server and the port for scheduler
+scheduler = FLScheduler(worker_num,server_num,port=9091) 
 scheduler.set_sample_worker_num(4)
 scheduler.init_env()
 print("init env done.")

paddle_fl/examples/dpsgd_demo/fl_server.py

@@ -21,7 +21,7 @@ server_id = 0
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_server_job(job_path, server_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # IP address for scheduler
 server.set_server_job(job)
-server._current_ep = "127.0.0.1:8181"
+server._current_ep = "127.0.0.1:8181" # IP address for server
 server.start()

paddle_fl/examples/dpsgd_demo/fl_trainer.py

@@ -13,7 +13,7 @@ trainer_id = int(sys.argv[1]) # trainer id for each guest
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_trainer_job(job_path, trainer_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # Inform scheduler IP address to trainer
 trainer = FLTrainerFactory().create_fl_trainer(job)
 trainer._current_ep = "127.0.0.1:{}".format(9000+trainer_id)
 trainer.start()

paddle_fl/examples/dpsgd_demo/run.sh

+unset http_proxy
+unset https_proxy
 python fl_master.py
 sleep 2
 python -u fl_scheduler.py >scheduler.log &

paddle_fl/examples/femnist_demo/fl_scheduler.py

@@ -2,7 +2,8 @@ from paddle_fl.core.scheduler.agent_master import FLScheduler
 
 worker_num = 4
 server_num = 1
-scheduler = FLScheduler(worker_num,server_num)
+# Define the number of worker/server and the port for scheduler
+scheduler = FLScheduler(worker_num,server_num,port=9091)
 scheduler.set_sample_worker_num(4)
 scheduler.init_env()
 print("init env done.")

paddle_fl/examples/femnist_demo/fl_server.py

@@ -7,7 +7,7 @@ server_id = 0
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_server_job(job_path, server_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # IP address for scheduler
 server.set_server_job(job)
-server._current_ep = "127.0.0.1:8181"
+server._current_ep = "127.0.0.1:8181" # IP address for server
 server.start()

paddle_fl/examples/femnist_demo/fl_trainer.py

@@ -14,7 +14,7 @@ trainer_id = int(sys.argv[1]) # trainer id for each guest
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_trainer_job(job_path, trainer_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # Inform the scheduler IP to trainer
 print(job._target_names)
 trainer = FLTrainerFactory().create_fl_trainer(job)
 trainer._current_ep = "127.0.0.1:{}".format(9000+trainer_id)
@@ -40,7 +40,7 @@ def train_test(train_test_program, train_test_feed, train_test_reader):
 epoch_id = 0
 step = 0
 epoch = 3000
-count_by_step = True
+count_by_step = False
 if count_by_step:
 	output_folder = "model_node%d" % trainer_id
 else: 

paddle_fl/examples/femnist_demo/run.sh

+unset http_proxy
+unset https_proxy
 #killall python
 python fl_master.py
 sleep 2

paddle_fl/examples/gru4rec_demo/fl_scheduler.py

@@ -2,7 +2,8 @@ from paddle_fl.core.scheduler.agent_master import FLScheduler
 
 worker_num = 4
 server_num = 1
-scheduler = FLScheduler(worker_num,server_num)
+# Define the number of worker/server and the port for scheduler
+scheduler = FLScheduler(worker_num,server_num,port=9091)
 scheduler.set_sample_worker_num(4)
 scheduler.init_env()
 print("init env done.")

paddle_fl/examples/gru4rec_demo/fl_server.py

@@ -21,7 +21,7 @@ server_id = 0
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_server_job(job_path, server_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # IP address for scheduler
 server.set_server_job(job)
-server._current_ep = "127.0.0.1:8181"
+server._current_ep = "127.0.0.1:8181" # IP address for server
 server.start()

paddle_fl/examples/gru4rec_demo/fl_trainer.py

@@ -14,7 +14,7 @@ train_file_dir = "mid_data/node4/%d/" % trainer_id
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_trainer_job(job_path, trainer_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # Inform the scheduler IP to trainer
 trainer = FLTrainerFactory().create_fl_trainer(job)
 trainer._current_ep = "127.0.0.1:{}".format(9000+trainer_id)
 trainer.start()

paddle_fl/examples/secagg_demo/fl_scheduler.py

@@ -2,7 +2,8 @@ from paddle_fl.core.scheduler.agent_master import FLScheduler
 
 worker_num = 2
 server_num = 1
-scheduler = FLScheduler(worker_num,server_num)
+
+scheduler = FLScheduler(worker_num,server_num,port=9091)
 scheduler.set_sample_worker_num(worker_num)
 scheduler.init_env()
 print("init env done.")

paddle_fl/examples/secagg_demo/fl_server.py

@@ -21,8 +21,8 @@ server_id = 0
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_server_job(job_path, server_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # IP address for scheduler
 server.set_server_job(job)
-server._current_ep = "127.0.0.1:8181"
+server._current_ep = "127.0.0.1:8181" # IP address for server
 server.start()
 print("connect")

paddle_fl/examples/secagg_demo/fl_trainer.py

@@ -28,7 +28,7 @@ trainer_id = int(sys.argv[1]) # trainer id for each guest
 job_path = "fl_job_config"
 job = FLRunTimeJob()
 job.load_trainer_job(job_path, trainer_id)
-job._scheduler_ep = "127.0.0.1:9091"
+job._scheduler_ep = "127.0.0.1:9091" # Inform the scheduler IP to trainer
 trainer = FLTrainerFactory().create_fl_trainer(job)
 trainer.trainer_id = trainer_id
 trainer._current_ep = "127.0.0.1:{}".format(9000+trainer_id)
@@ -75,6 +75,7 @@ while not trainer.stop():
         if step_i % 100 == 0:
             print("Epoch: {0}, step: {1}, accuracy: {2}".format(epoch_id, step_i, accuracy[0]))
 
+    print(step_i)
     avg_loss_val, acc_val = train_test(train_test_program=test_program,
                                        train_test_reader=test_reader,
                                        train_test_feed=feeder)
@@ -82,5 +83,5 @@ while not trainer.stop():
 
     if epoch_id > 40:
         break
-    if step_i % 100 == 0:
+    if epoch_id % 5 == 0:
         trainer.save_inference_program(output_folder)

paddle_fl/examples/submitter_demo/conf.txt

@@ -2,7 +2,8 @@
 task_name=test_fl_job_submit_jingqinghe
 hdfs_output=/user/feed/mlarch/sequence_generator/dongdaxiang/job_44
 train_cmd=python dist_trainer.py
-monitor_cmd=python system_monitor_app.py 10 100
+#monitor_cmd=python system_monitor_app.py 10 100
+monitor_cmd=
 #train_cmd=python test_hadoop.py
 
 hdfs_path=afs://xingtian.afs.baidu.com:9902

paddle_fl/examples/submitter_demo/kill.sh

-/home/jingqinghe/mpi_feed4/smart_client/bin/qdel $1".yq01-hpc-lvliang01-smart-master.dmop.baidu.com"
+unset http_proxy
+unset https_proxy
+/home/jingqinghe/tools/mpi_feed4/smart_client/bin/qdel $1".yq01-hpc-lvliang01-smart-master.dmop.baidu.com"

paddle_fl/examples/submitter_demo/scheduler_client.py

@@ -18,7 +18,8 @@ print(random_port)
 current_ip = socket.gethostbyname(socket.gethostname())
 endpoints = "{}:{}".format(current_ip, random_port)
 #start a web server for remote endpoints to download their config 
-os.system("python -m SimpleHTTPServer 8080 &")
+#os.system("python -m SimpleHTTPServer 8080 &")
+os.system("python -m http.server 8080 &")
 if os.path.exists("job_config"):
     os.system("rm -rf job_config")
 if os.path.exists("package"):
@@ -120,10 +121,10 @@ print(ip_list)
 #allocate the role of each endpoint and their ids
 ip_role = {}
 for i in range(len(ip_list)):
-	if i < int(default_dict["server_nodes"]):
-		ip_role[ip_list[i]] = 'server%d' % i
+        if i < int(default_dict["server_nodes"]):
+                ip_role[ip_list[i]] = 'server%d' % i
         else:
-		ip_role[ip_list[i]] = 'trainer%d' % (i-int(default_dict["server_nodes"]))
+                ip_role[ip_list[i]] = 'trainer%d' % (i-int(default_dict["server_nodes"]))
 print(ip_role)
 
 def job_generate():
@@ -179,7 +180,7 @@ while not all_job_sent:
     message = zmq_socket.recv()
     group = message.split("\t")
     if group[0] == "GET_FL_JOB":
-	download_job.append(group[1])
+        download_job.append(group[1])
         zmq_socket.send(ip_role[group[1]])
     else:
         zmq_socket.send("WAIT\t0")

paddle_fl/examples/submitter_demo/train_program.py

@@ -89,15 +89,15 @@ else:
     trainer.start()
     print(trainer._scheduler_ep, trainer._current_ep)
     output_folder = "fl_model"
-    step_i = 0
+    epoch_id = 0
     while not trainer.stop():
         print("batch %d start train" % (step_i))
-        train_step = 0
+        step_i = 0
         for data in reader():
             trainer.run(feed=data, fetch=[])
-            train_step += 1
+            step_i += 1
             if train_step == trainer._step:
                 break
-        step_i += 1
-        if step_i % 100 == 0:
+        epoch_id += 1
+        if epoch_id % 5 == 0:
             trainer.save_inference_program(output_folder)