diff --git a/example/bert_clue/dataset.py b/example/bert_clue/dataset.py index 9dbe7b8ce41e583b5e654b85163208c80ad43191..f930b67330d4ab6123a10a7c0ebe2c415d215737 100644 --- a/example/bert_clue/dataset.py +++ b/example/bert_clue/dataset.py @@ -52,7 +52,7 @@ def create_bert_dataset(epoch_size=1, device_num=1, rank=0, do_shuffle="true", e ds = ds.map(input_columns="input_ids", operations=type_cast_op) # apply batch operations ds = ds.batch(bert_net_cfg.batch_size, drop_remainder=True) - ds = ds.repeat(repeat_count) + ds = ds.repeat(new_repeat_count) logger.info("data size: {}".format(ds.get_dataset_size())) logger.info("repeatcount: {}".format(ds.get_repeat_count())) return ds, new_repeat_count diff --git a/example/bert_clue/run_distribute_pretrain.sh b/example/bert_clue/run_distribute_pretrain.sh index 6c726027d7910f5de96d53586f4650e620ef22b7..58ae389a0ec3d21e4c75c9dcfe5a62a95f39f373 100644 --- a/example/bert_clue/run_distribute_pretrain.sh +++ b/example/bert_clue/run_distribute_pretrain.sh @@ -28,15 +28,20 @@ SCHEMA_DIR=$4 export MINDSPORE_HCCL_CONFIG_PATH=$5 export RANK_TABLE_FILE=$5 export RANK_SIZE=$1 - +cores=`cat /proc/cpuinfo|grep "processor" |wc -l` +echo "the number of logical core" $cores +avg_core_per_rank=`expr $cores \/ $RANK_SIZE` +core_gap=`expr $avg_core_per_rank \- 1` +echo "avg_core_per_rank" $avg_core_per_rank +echo "core_gap" $core_gap for((i=0;i