.github/workflows/cnn_e2e.yml

+name: 'resnet e2e test'
+on: 
+  pull_request:
+    types: [review_requested]
+    branches:
+      - "*"
+  workflow_dispatch:
+    inputs:
+      placeholder:
+        description: "placeholder, no effect"
+        required: false
+
+jobs:
+  build: 
+    name: 'Build and test this repo'
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."

Classification/cnns/args_train.sh

-rm -rf core.*
+#! /bin/bash
+# set -ex
 
-
-# bash args_train.sh ${NUM_NODES} ${NUM_GPUS_PER_NODE} ${BATCH_SIZE} ${USE_FP16} ${NUM_EPOCH} ${LOSS_PRINT_ITER} ${TRAIN_DATA_PATH} ${VAL_DATA_PATH} ${PYTHON_BIN} ${NODE_IPS} ${DEBUG_AND_NCCL} ${NSYS_BIN} ${ITER_NUM}
+# bash args_train.sh ${NUM_NODES} ${NUM_GPUS_PER_NODE} ${BATCH_SIZE} ${USE_FP16} ${NUM_EPOCH} ${LOSS_PRINT_ITER} ${TRAIN_DATA_PATH} ${VAL_DATA_PATH} ${PYTHON_BIN} ${NODE_IPS} ${DEBUG_AND_NCCL} ${NSYS_BIN} ${RUN_COMMIT}
 
 NUM_NODES=${1:-1}
 NUM_GPUS_PER_NODE=${2:-8}
@@ -29,7 +29,7 @@ LOG_FILENAME=$LOG_FOLDER/${TRAN_MODEL}_lazy_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_b
 
 export PYTHONUNBUFFERED=1
 echo PYTHONUNBUFFERED=$PYTHONUNBUFFERED
-export NCCL_LAUNCH_MODE=PARALLEL
+export NCCL_LAUNCH_MODE=GROUP
 echo NCCL_LAUNCH_MODE=$NCCL_LAUNCH_MODE
 echo DEBUG_AND_NCCL=$DEBUG_AND_NCCL
 if $DEBUG_AND_NCCL; then
@@ -39,6 +39,10 @@ if $DEBUG_AND_NCCL; then
     echo NCCL_DEBUG=$NCCL_DEBUG
 fi
 
+if [[ ${NUM_NODES} -gt 1 ]]; then
+    export ONEFLOW_COMM_NET_IB_ENABLE=1
+fi
+
 CMD=""
 
 if [[ ! -z "${NSYS_BIN}" ]]; then

Classification/cnns/config.py

@@ -92,12 +92,27 @@ def get_parser(parser=None):
     parser.add_argument(
         "--model_load_dir", type=str, default=None, help="model load directory if need"
     )
+    parser.add_argument(
+        "--save_epoch_interval",
+        type=int,
+        default=10,
+        help="Number of iterations between checkpoint saves.",
+    )
+    parser.add_argument(
+        "--save_last",
+        action="store_true",
+        default=False,
+        help="save model snapshot for last iteration",
+    )
+    parser.add_argument(
+        "--save_init",
+        action="store_true",
+        default=False,
+        help="save model snapshot for inited",
+    )
     parser.add_argument("--batch_size_per_device", type=int, default=64)
     parser.add_argument("--val_batch_size_per_device", type=int, default=8)
 
-    parser.add_argument(
-        "--use_rdma", type=str2bool, nargs="?", const=True, help="Use rdma.",
-    )
     parser.add_argument(
         "--nccl_fusion_threshold_mb",
         type=int,

Classification/cnns/of_cnn_train_val.py

@@ -64,8 +64,6 @@ if args.nccl_fusion_threshold_mb:
 if args.nccl_fusion_max_ops:
     flow.config.collective_boxing.nccl_fusion_max_ops(args.nccl_fusion_max_ops)
 
-if args.num_nodes > 1 and args.use_rdma:
-    flow.config.use_rdma(True)
 
 def label_smoothing(labels, classes, eta, dtype):
     assert classes > 0
@@ -132,11 +130,11 @@ def main():
     InitNodes(args)
     flow.env.log_dir(args.log_dir)
 
-    snapshot = Snapshot(args.model_save_dir, args.model_load_dir)
+    snapshot = Snapshot(args.model_save_dir, args.model_load_dir, args.save_init)
 
     print(" {} iter per epoch...".format(epoch_size))
 
-    for epoch in range(args.num_epochs):
+    for epoch in range(1, args.num_epochs + 1):
         metric = Metric(
             desc="train",
             calculate_batches=args.loss_print_every_n_iter,
@@ -154,7 +152,11 @@ def main():
             )
             for i in range(num_val_steps):
                 InferenceNet().async_get(metric.metric_cb(epoch, i))
-        #snapshot.save("epoch_{}".format(epoch))
+        #if epoch % args.save_epoch_interval == 0:
+        #    snapshot.save("epoch_{}".format(epoch))
+
+    if args.save_last:
+        snapshot.save("epoch_{}".format("last"))
 
 
 if __name__ == "__main__":

Classification/cnns/train_fp16.sh

@@ -24,6 +24,7 @@ export PYTHONUNBUFFERED=1
 echo PYTHONUNBUFFERED=$PYTHONUNBUFFERED
 export NCCL_LAUNCH_MODE=PARALLEL
 echo NCCL_LAUNCH_MODE=$NCCL_LAUNCH_MODE
+export ONEFLOW_COMM_NET_IB_ENABLE=1
 
 python3 of_cnn_train_val.py \
      --train_data_dir=$DATA_ROOT/train \

Classification/cnns/util.py

@@ -17,8 +17,6 @@ limitations under the License.
 import os
 import time
 import numpy as np
-import pandas as pd
-from datetime import datetime
 import oneflow.compatible.single_client as flow
 
 
@@ -36,14 +34,14 @@ def InitNodes(args):
 
 
 class Snapshot(object):
-    def __init__(self, model_save_dir, model_load_dir):
+    def __init__(self, model_save_dir, model_load_dir, save_init=False):
         self._model_save_dir = model_save_dir
         if model_load_dir:
             assert os.path.isdir(model_load_dir)
             print("Restoring model from {}.".format(model_load_dir))
             flow.load_variables(flow.checkpoint.get(model_load_dir))
-        else:
-            # flow.checkpoint.save("initial_model")
+        elif save_init:
+            flow.checkpoint.save("initial_model")
             print("Init model on demand.")
 
     def save(self, name):

LanguageModeling/BERT/args_run_pretraining.sh

-rm -rf core.*
-rm -rf ./output/logs/$HOSTNAME ./output/$HOSTNAME ./initial_model
+#! /bin/bash
+# set -ex
 
-# bash args_run_pretraining.sh ${NUM_NODES} ${NUM_GPUS_PER_NODE} ${BSZ_PER_DEVICE} ${USE_FP16} ${ITER_NUM} ${LOSS_PRINT_ITER} ${DATA_DIR} ${DATA_PART_NUM} ${SEQ_LENGHT} ${NUM_HIDDEN_LAYERS} ${NUM_ATTENTION_HEADS} ${PYTHON_BIN} ${NODE_IPS} ${NSYS_BIN}
+# bash args_run_pretraining.sh ${NUM_NODES} ${NUM_GPUS_PER_NODE} ${BSZ_PER_DEVICE} ${USE_FP16} ${ITER_NUM} ${LOSS_PRINT_ITER} ${DATA_DIR} ${DATA_PART_NUM} ${SEQ_LENGHT} ${NUM_HIDDEN_LAYERS} ${NUM_ATTENTION_HEADS} ${PYTHON_BIN} ${NODE_IPS} ${NSYS_BIN} ${RUN_COMMIT}
 
 NUM_NODES=${1:-1}
 NUM_GPUS_PER_NODE=${2:-8}
@@ -20,7 +20,7 @@ PYTHON_BIN=${12:-"python3"}
 NODE_IPS=${13:-"10.11.0.2,10.11.0.3,10.11.0.4,10.11.0.5"}
 DEBUG_AND_NCCL=${14:-false}
 NSYS_BIN=${15:-""}
-ITER_N=${16:-1}
+RUN_COMMIT=${16:-"master"}
 NUM_ACC_STEP=${17:-1}
 OPTIMIZER_TYPE=${18:-"adam"}
 
@@ -28,10 +28,11 @@ OPTIMIZER_TYPE=${18:-"adam"}
 RUN_TIME=$(date "+%Y%m%d_%H%M%S%N")
 LOG_FOLDER=./output/logs/$HOSTNAME/${NUM_NODES}n${NUM_GPUS_PER_NODE}g
 mkdir -p $LOG_FOLDER
-LOG_FILENAME=$LOG_FOLDER/bert_${RUN_TIME}_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_sq${SEQ_LENGHT}_nhl${NUM_HIDDEN_LAYERS}_nah${NUM_ATTENTION_HEADS}_bsz${BSZ_PER_DEVICE}_${OPTIMIZER_TYPE}_iter${ITER_N}.log
+LOG_FILENAME=$LOG_FOLDER/bert_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_sq${SEQ_LENGHT}_nhl${NUM_HIDDEN_LAYERS}_nah${NUM_ATTENTION_HEADS}_bsz${BSZ_PER_DEVICE}_${OPTIMIZER_TYPE}_${RUN_COMMIT}_${RUN_TIME}.log
 
 export PYTHONUNBUFFERED=1
 export GLOG_v=3
+export NCCL_LAUNCH_MODE=GROUP
 
 echo DEBUG_AND_NCCL=$DEBUG_AND_NCCL
 if $DEBUG_AND_NCCL; then
@@ -41,14 +42,14 @@ if $DEBUG_AND_NCCL; then
     echo NCCL_DEBUG=$NCCL_DEBUG
 fi
 
-# if [ $NUM_GPUS_PER_NODE -eq 1 ]; then
-#   export CUDA_VISIBLE_DEVICES=$(($ITER_N-1))
-# fi
+if [[ ${NUM_NODES} -gt 1 ]]; then
+    export ONEFLOW_COMM_NET_IB_ENABLE=1
+fi
 
 CMD=""
 
 if [[ ! -z "${NSYS_BIN}" ]]; then
-    CMD+="${NSYS_BIN} profile --stats true --output ${TRAN_MODEL}_v0.5.0_${NUM_NODES}_${NUM_GPUS_PER_NODE}_%h_%p "
+    CMD+="${NSYS_BIN} profile --stats true --output bert_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_sq${SEQ_LENGHT}_nhl${NUM_HIDDEN_LAYERS}_nah${NUM_ATTENTION_HEADS}_bsz${BSZ_PER_DEVICE}_${OPTIMIZER_TYPE}_${RUN_COMMIT}_%h_%p "
 fi
 
 CMD+="${PYTHON_BIN} run_pretraining.py "
@@ -88,8 +89,3 @@ echo "Rum cmd ${CMD}"
 $CMD 2>&1 | tee ${LOG_FILENAME}
 
 echo "Writting log to ${LOG_FILENAME}"
-
-if [ ! -d "./test_result" ]; then
-  mkdir ./test_result
-fi
-cp -r $LOG_FOLDER ./test_result/

LanguageModeling/BERT/config.py

@@ -121,6 +121,13 @@ def get_parser(parser=None):
         required=False,
         help="model save directory",
     )
+    parser.add_argument(
+        "--model_save_init",
+        action="store_true",
+        default=False,
+        help="save model snapshot for inited",
+    )
+    
     parser.add_argument(
         "--save_last_snapshot",
         type=str2bool,

LanguageModeling/BERT/run_pretraining.py

@@ -124,7 +124,7 @@ def main():
 
     InitNodes(args)
 
-    snapshot = Snapshot(args.model_save_dir, args.model_load_dir)
+    snapshot = Snapshot(args.model_save_dir, args.model_load_dir, args.model_save_init)
 
     print("num_accumulation_steps:", args.num_accumulation_steps)
     metric = Metric(

LanguageModeling/BERT/util.py

@@ -16,10 +16,7 @@ limitations under the License.
 
 import os
 import time
-import numpy as np
 from collections import OrderedDict
-import pandas as pd
-from datetime import datetime
 import oneflow.compatible.single_client as flow
 
 
@@ -37,13 +34,13 @@ def InitNodes(args):
 
 
 class Snapshot(object):
-    def __init__(self, model_save_dir, model_load_dir):
+    def __init__(self, model_save_dir, model_load_dir, model_save_init=False):
         self._model_save_dir = model_save_dir
         if model_load_dir:
             assert os.path.isdir(model_load_dir)
             print("Restoring model from {}.".format(model_load_dir))
             flow.load_variables(flow.checkpoint.get(model_load_dir))
-        else:
+        elif model_save_init:
             flow.checkpoint.save("initial_model")
             print("Init model on demand.")
 

LanguageModeling/GPT/README.md

@@ -173,7 +173,7 @@ python3 tools/launch_container.py \
 bash examples/pretrain_1n8d_2x4x1_16_1536x16.sh
 ```
 
-如果需要进一步扩充设备数量，由于单台机器适配的 GPU 设备数量有限，我们需要更多的物理机器 node。此时，我们需要配置 `--num-nodes` 和 `--node-ips` 参数，并且分别在每台机器上启动训练命令（`traning.py`）。同时在有 rdma 的环境中，可以开启 `--use-rdma` 来带来更佳的训练效率。以下是4机8卡下，各种并行方式混合的示例：
+如果需要进一步扩充设备数量，由于单台机器适配的 GPU 设备数量有限，我们需要更多的物理机器 node。此时，我们需要配置 `--num-nodes` 和 `--node-ips` 参数，并且分别在每台机器上启动训练命令（`traning.py`）。同时在有 rdma 的环境中，可以开启 `export ONEFLOW_COMM_NET_IB_ENABLE=1` 来带来更佳的训练效率。以下是4机8卡下，各种并行方式混合的示例：
 
 ```
 bash examples/distribute_pretrain_4n8d_2x4x4_512_2304x24.sh

LanguageModeling/GPT/examples/args_pretrain_gpt.sh

 #! /bin/bash
+# set -ex
 
 # Runs the "117M" parameter model
 
-# bash args_pretrain_gpt.sh $NUM_NODES $NUM_GPUS_PER_NODE $M_P $P_P $MICRO_BATCH_SIZE $GLOABAL_BATCH_SIZE $USE_FP16 $TRAIN_ITERS $LOG_INTERVAL $DATA_PATH $NUM_LAYERS $HIDDEN_SIZE $NUM_ATTENTION_HEADS $SEQ_LENGTH $PYTHON_BIN $NODE_IPS $DEBUG_AND_NCCL $NSYS_BIN $ITER_NUM
+# bash args_pretrain_gpt.sh $NUM_NODES $NUM_GPUS_PER_NODE $M_P $P_P $MICRO_BATCH_SIZE $GLOABAL_BATCH_SIZE $USE_FP16 $TRAIN_ITERS $LOG_INTERVAL $DATA_PATH $NUM_LAYERS $HIDDEN_SIZE $NUM_ATTENTION_HEADS $SEQ_LENGTH $PYTHON_BIN $NODE_IPS $DEBUG_AND_NCCL $NSYS_BIN $RUN_COMMIT
 
-rm -rf core.*
-rm -rf ./output/logs/$HOSTNAME ./model_save
 
 NUM_NODES=${1:-1}
 NUM_GPUS_PER_NODE=${2:-8}
@@ -25,18 +24,19 @@ PYTHON_BIN=${15:-"python3"}
 NODE_IPS=${16:-"127.0.0.1"}
 DEBUG_AND_NCCL=${17:-false}
 NSYS_BIN=${18:-""}
-ITER_NUM=${19:-1}
+RUN_COMMIT=${19:-1}
 
 WORLD_SIZE=$(($NUM_GPUS_PER_NODE*$NUM_NODES))
 D_P=$(($WORLD_SIZE/$M_P/$P_P))
 
-
+RUN_TIME=$(date "+%Y%m%d_%H%M%S%N")
 LOG_FOLDER=./output/logs/$HOSTNAME/${NUM_NODES}n${NUM_GPUS_PER_NODE}g
 mkdir -p $LOG_FOLDER
-LOG_FILENAME=$LOG_FOLDER/oneflow_gpt_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_dp${D_P}_mp${M_P}_pp${P_P}_mbs${MICRO_BATCH_SIZE}_gbs${GLOABAL_BATCH_SIZE}_sql${SEQ_LENGTH}_l${NUM_LAYERS}_hsz${HIDDEN_SIZE}_ahs${NUM_ATTENTION_HEADS}_pretrain_iter${ITER_NUM}.log
+LOG_FILENAME=$LOG_FOLDER/oneflow_gpt_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_dp${D_P}_mp${M_P}_pp${P_P}_mbs${MICRO_BATCH_SIZE}_gbs${GLOABAL_BATCH_SIZE}_sql${SEQ_LENGTH}_l${NUM_LAYERS}_hsz${HIDDEN_SIZE}_ahs${NUM_ATTENTION_HEADS}_${RUN_COMMIT}_${RUN_TIME}.log
 
-CHECKPOINT_PATH=./model_save
-mkdir -p $CHECKPOINT_PATH
+# save model
+# CHECKPOINT_PATH=./model_save
+# mkdir -p $CHECKPOINT_PATH
 
 
 export PYTHONUNBUFFERED=1
@@ -54,14 +54,14 @@ if $DEBUG_AND_NCCL; then
     echo NCCL_DEBUG=$NCCL_DEBUG
 fi
 
-if [ $NUM_GPUS_PER_NODE -eq 1 ]; then
-  export CUDA_VISIBLE_DEVICES=$(($ITER_NUM-1))
+if [[ ${NUM_NODES} -gt 1 ]]; then
+    export ONEFLOW_COMM_NET_IB_ENABLE=1
 fi
 
 CMD=""
 
 if [[ ! -z "${NSYS_BIN}" ]]; then
-    CMD+="${NSYS_BIN} profile --stats true --output oneflow_gpt_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_%h_%p "
+    CMD+="${NSYS_BIN} profile --stats true --output oneflow_gpt_${NUM_NODES}n${NUM_GPUS_PER_NODE}g_dp${D_P}_mp${M_P}_pp${P_P}_mbs${MICRO_BATCH_SIZE}_gbs${GLOABAL_BATCH_SIZE}_sql${SEQ_LENGTH}_l${NUM_LAYERS}_hsz${HIDDEN_SIZE}_ahs${NUM_ATTENTION_HEADS}_${RUN_COMMIT}_%h_%p "
 fi
 
 CMD+="${PYTHON_BIN} oneflow_gpt/training.py "
@@ -93,8 +93,8 @@ CMD+=" --lr-warmup-fraction 0.01"
 CMD+=" --optimizer adamw"
 CMD+=" --weight-decay 1e-2"
 CMD+=" --clip-grad 1.0"
-CMD+=" --save ${CHECKPOINT_PATH}"
-CMD+=" --save-interval 100000"
+# CMD+=" --save ${CHECKPOINT_PATH}"
+# CMD+=" --save-interval 100000"
 CMD+=" --log-interval ${LOG_INTERVAL}"
 CMD+=" --checkpoint-activations"
 CMD+=" --multihead-attention-fusion"
@@ -104,9 +104,6 @@ if $USE_FP16; then
     CMD+=" --fp16"
 fi
 
-if [[ ${NUM_NODES} -gt 1 ]]; then
-    CMD+=" --use-rdma"
-fi
 
 if [[ ! -z "${NSYS_BIN}" ]]; then
     CMD+=" --profile-transformer-layer"
@@ -118,9 +115,3 @@ echo "Rum cmd ${CMD}"
 $CMD 2>&1 | tee ${LOG_FILENAME}
 
 echo "Writting log to ${LOG_FILENAME}"
-
-if [ ! -d "./test_result" ]; then
-  mkdir ./test_result
-fi
-cp -r $LOG_FOLDER ./test_result/
-

LanguageModeling/GPT/examples/pretrain.sh

@@ -84,7 +84,7 @@ cmd+=" --multihead-attention-fusion"
 cmd+=" --fp16"
 
 if [[ ${num_nodes} -gt 1 ]]; then
-    cmd+=" --use-rdma"
+    export ONEFLOW_COMM_NET_IB_ENABLE=1
 fi
 
 if [[ ! -z "${ONEFLOW_GTP_PROFILE_FILE}" ]]; then

LanguageModeling/GPT/oneflow_gpt/config.py

@@ -635,11 +635,7 @@ def _add_misc_args(parser):
         action="store_true",
         help="open transformer layer profiler",
     )
-    group.add_argument(
-        "--use-rdma",
-        action="store_true",
-        help="Use rdma.",
-    )
+
     return parser
 
 

LanguageModeling/GPT/oneflow_gpt/training.py

@@ -47,8 +47,6 @@ def _init_config(args):
                 " please try other version."
             )
 
-    if args.use_rdma:
-        flow.config.use_rdma(True)
 
     flow.config.enable_legacy_model_io()
     flow.config.enable_model_io_v2(True)

requirements.txt

-numpy>=1.17.2
-pandas>=1.0.4
-pillow>=7.2.0
\ No newline at end of file