提交 7c10b488 编写于 作者: T tangwei

add qsub submit

上级 16966dc5
...@@ -28,7 +28,6 @@ class ClusterEngine(Engine): ...@@ -28,7 +28,6 @@ class ClusterEngine(Engine):
def __init_impl__(self): def __init_impl__(self):
abs_dir = os.path.dirname(os.path.abspath(__file__)) abs_dir = os.path.dirname(os.path.abspath(__file__))
self.submit_script = os.path.join(abs_dir, "master.sh") self.submit_script = os.path.join(abs_dir, "master.sh")
self.job_script = os.path.join(abs_dir, "worker.sh")
def start_worker_procs(self): def start_worker_procs(self):
default_env = os.environ.copy() default_env = os.environ.copy()
...@@ -58,10 +57,6 @@ class ClusterEngine(Engine): ...@@ -58,10 +57,6 @@ class ClusterEngine(Engine):
role = envs.get_runtime_environ("engine_role") role = envs.get_runtime_environ("engine_role")
if role == "MASTER": if role == "MASTER":
worker_script = {}
worker_script["engine_worker"] = self.job_script
envs.set_runtime_environs(worker_script)
self.start_master_procs() self.start_master_procs()
elif role == "WORKER": elif role == "WORKER":
......
...@@ -27,8 +27,9 @@ engine: ...@@ -27,8 +27,9 @@ engine:
paddlerec: "/home/tangwei/fleet_rec_env/FleetRec" paddlerec: "/home/tangwei/fleet_rec_env/FleetRec"
submit: submit:
hpc: "/home/tangwei/submit-tieba/smart_client/" hpc: "/home/tangwei/Plines/client/smart_client_khan/"
qconf: "/home/tangwei/Plines/imageq/package/my_conf/para.conf" qconf: "/home/tangwei/Plines/imageq/qsub_f.conf"
nodes: 10 nodes: 10
scrpit: "{workspace}/submit.sh" submit_scrpit: "{workspace}/submit.sh"
\ No newline at end of file job_scrpit: "{workspace}/worker.sh"
...@@ -25,15 +25,16 @@ function package() { ...@@ -25,15 +25,16 @@ function package() {
temp=${engine_temp_path} temp=${engine_temp_path}
echo "package temp dir: " ${temp} echo "package temp dir: " ${temp}
cp ${engine_worker} ${temp} cp ${engine_job_scrpit} ${temp}
cp ${engine_submit_qconf} ${temp}
echo "copy job.sh from " ${engine_worker} " to " ${temp} echo "copy job.sh from " ${engine_worker} " to " ${temp}
mkdir ${temp}/python mkdir -p ${temp}/package/python
cp -r ${engine_package_python}/* ${temp}/python/ cp -r ${engine_package_python}/* ${temp}/package/python/
echo "copy python from " ${engine_package_python} " to " ${temp} echo "copy python from " ${engine_package_python} " to " ${temp}
mkdir ${temp}/whl mkdir ${temp}/package/whl
cp ${engine_package_paddlerec} ${temp}/whl/ cp ${engine_package_paddlerec} ${temp}/package/whl/
echo "copy " ${engine_package_paddlerec} " to " ${temp}"/whl/" echo "copy " ${engine_package_paddlerec} " to " ${temp}"/whl/"
} }
...@@ -68,7 +69,11 @@ function submit() { ...@@ -68,7 +69,11 @@ function submit() {
g_job_entry="worker.sh" g_job_entry="worker.sh"
${$engine_submit_hpc}/bin/qsub_f \ engine_hdfs_output=${engine_hdfs_output}/`date +%Y%m%d%H%M%S`
cd ${engine_temp_path}
${engine_submit_hpc}/bin/qsub_f \
-N ${g_job_name} \ -N ${g_job_name} \
--conf ${engine_submit_qconf} \ --conf ${engine_submit_qconf} \
--hdfs ${engine_hdfs_name} \ --hdfs ${engine_hdfs_name} \
......
...@@ -49,7 +49,10 @@ function user_define_variables() { ...@@ -49,7 +49,10 @@ function user_define_variables() {
} }
function job() { function job() {
mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python -u ${g_job_entry} echo "job"
g_run_stage="job"
# mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python -u ${g_job_entry}
} }
function main() { function main() {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册