add qsub submit

fleet_rec/core/engine/cluster/cluster.py

@@ -28,7 +28,6 @@ class ClusterEngine(Engine):
     def __init_impl__(self):
         abs_dir = os.path.dirname(os.path.abspath(__file__))
         self.submit_script = os.path.join(abs_dir, "master.sh")
-        self.job_script = os.path.join(abs_dir, "worker.sh")
 
     def start_worker_procs(self):
         default_env = os.environ.copy()
@@ -58,10 +57,6 @@ class ClusterEngine(Engine):
         role = envs.get_runtime_environ("engine_role")
 
         if role == "MASTER":
-            worker_script = {}
-            worker_script["engine_worker"] = self.job_script
-            envs.set_runtime_environs(worker_script)
-
             self.start_master_procs()
 
         elif role == "WORKER":

models/rank/dnn/backend.yaml

@@ -27,8 +27,9 @@ engine:
     paddlerec: "/home/tangwei/fleet_rec_env/FleetRec"
 
   submit:
-    hpc: "/home/tangwei/submit-tieba/smart_client/"
-    qconf: "/home/tangwei/Plines/imageq/package/my_conf/para.conf"
+    hpc: "/home/tangwei/Plines/client/smart_client_khan/"
+    qconf: "/home/tangwei/Plines/imageq/qsub_f.conf"
     nodes: 10
 
-  scrpit: "{workspace}/submit.sh"
\ No newline at end of file
+  submit_scrpit: "{workspace}/submit.sh"
+  job_scrpit: "{workspace}/worker.sh"

models/rank/dnn/submit.sh

@@ -25,15 +25,16 @@ function package() {
   temp=${engine_temp_path}
   echo "package temp dir: " ${temp}
 
-  cp ${engine_worker} ${temp}
+  cp ${engine_job_scrpit} ${temp}
+  cp ${engine_submit_qconf} ${temp}
   echo "copy job.sh from " ${engine_worker} " to " ${temp}
 
-  mkdir ${temp}/python
-  cp -r ${engine_package_python}/* ${temp}/python/
+  mkdir -p ${temp}/package/python
+  cp -r ${engine_package_python}/* ${temp}/package/python/
   echo "copy python from " ${engine_package_python} " to " ${temp}
 
-  mkdir ${temp}/whl
-  cp ${engine_package_paddlerec}  ${temp}/whl/
+  mkdir ${temp}/package/whl
+  cp ${engine_package_paddlerec}  ${temp}/package/whl/
   echo "copy " ${engine_package_paddlerec} " to " ${temp}"/whl/"
 }
 
@@ -68,7 +69,11 @@ function submit() {
 
   g_job_entry="worker.sh"
 
-  ${$engine_submit_hpc}/bin/qsub_f \
+  engine_hdfs_output=${engine_hdfs_output}/`date +%Y%m%d%H%M%S`
+
+  cd ${engine_temp_path}
+
+  ${engine_submit_hpc}/bin/qsub_f \
     -N ${g_job_name} \
     --conf ${engine_submit_qconf} \
     --hdfs ${engine_hdfs_name} \

fleet_rec/core/engine/cluster/worker.sh → models/rank/dnn/worker.sh

@@ -49,7 +49,10 @@ function user_define_variables() {
 }
 
 function job() {
-  mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python -u ${g_job_entry}
+  echo "job"
+  g_run_stage="job"
+
+  #  mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python -u ${g_job_entry}
 }
 
 function main() {