From 8f4e37b95dbda321e01d0664dc855a30d8d5f746 Mon Sep 17 00:00:00 2001
From: wanghua <wanghua36@huawei.com>
Date: Mon, 25 May 2020 17:00:10 +0800
Subject: [PATCH] modify pretrain script

---
 example/bert_clue/dataset.py                 |  2 +-
 example/bert_clue/run_distribute_pretrain.sh | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/example/bert_clue/dataset.py b/example/bert_clue/dataset.py
index 9dbe7b8ce..f930b6733 100644
--- a/example/bert_clue/dataset.py
+++ b/example/bert_clue/dataset.py
@@ -52,7 +52,7 @@ def create_bert_dataset(epoch_size=1, device_num=1, rank=0, do_shuffle="true", e
     ds = ds.map(input_columns="input_ids", operations=type_cast_op)
     # apply batch operations
     ds = ds.batch(bert_net_cfg.batch_size, drop_remainder=True)
-    ds = ds.repeat(repeat_count)
+    ds = ds.repeat(new_repeat_count)
     logger.info("data size: {}".format(ds.get_dataset_size()))
     logger.info("repeatcount: {}".format(ds.get_repeat_count()))
     return ds, new_repeat_count
diff --git a/example/bert_clue/run_distribute_pretrain.sh b/example/bert_clue/run_distribute_pretrain.sh
index 6c726027d..58ae389a0 100644
--- a/example/bert_clue/run_distribute_pretrain.sh
+++ b/example/bert_clue/run_distribute_pretrain.sh
@@ -28,15 +28,20 @@ SCHEMA_DIR=$4
 export MINDSPORE_HCCL_CONFIG_PATH=$5
 export RANK_TABLE_FILE=$5
 export RANK_SIZE=$1
-
+cores=`cat /proc/cpuinfo|grep "processor" |wc -l`
+echo "the number of logical core" $cores
+avg_core_per_rank=`expr $cores \/ $RANK_SIZE`
+core_gap=`expr $avg_core_per_rank \- 1`
+echo "avg_core_per_rank" $avg_core_per_rank
+echo "core_gap" $core_gap
 for((i=0;i<RANK_SIZE;i++))
 do
-    start=`expr $i \* 12`
+    start=`expr $i \* $avg_core_per_rank`
     export DEVICE_ID=$i
     export RANK_ID=$i
     export DEPLOY_MODE=0
     export GE_USE_STATIC_MEMORY=1
-    end=`expr $start \+ 11`
+    end=`expr $start \+ $core_gap`
     cmdopt=$start"-"$end
 
     rm -rf LOG$i
-- 
GitLab