modify_cnn_training_param

Classification/cnns/job_function_util.py

@@ -30,6 +30,7 @@ def _default_config(args):
 def get_train_config(args):
     train_config = _default_config(args)
     train_config.train.primary_lr(args.learning_rate)
+    train_config.cudnn_conv_heuristic_search_algo(False)
 
 
     train_config.prune_parallel_cast_ops(True)

Classification/cnns/of_cnn_train_val.py

@@ -80,7 +80,7 @@ def TrainNet():
     else:
         loss = flow.nn.sparse_softmax_cross_entropy_with_logits(labels, logits, name="softmax_loss")
 
-    flow.losses.add_loss(loss)
+    flow.losses.add_loss(flow.math.reduce_mean(loss))
     predictions = flow.nn.softmax(logits)
     outputs = {"loss": loss, "predictions": predictions, "labels": labels}
     return outputs
@@ -124,7 +124,7 @@ def main():
                             save_summary_steps=num_val_steps, batch_size=val_batch_size)
             for i in range(num_val_steps):
                 InferenceNet().async_get(metric.metric_cb(epoch, i))
-        snapshot.save('epoch_{}'.format(epoch))
+        # snapshot.save('epoch_{}'.format(epoch))
 
 
 if __name__ == "__main__":

Classification/cnns/train.sh

+#!/bin/sh
 rm -rf core.* 
 rm -rf ./output/snapshots/*
 
-
+# training with synthetic data
 python3 of_cnn_train_val.py \
     --num_examples=50 \
     --num_val_examples=50 \
@@ -13,4 +14,30 @@ python3 of_cnn_train_val.py \
     --batch_size_per_device=16 \
     --val_batch_size_per_device=16 \
     --num_epoch=10 \
-    --model="resnet50"
\ No newline at end of file
+    --model="resnet50"
+
+
+
+
+# # training with imagenet
+# DATA_ROOT=/datasets/ImageNet/ofrecord
+# LOG_FOLDER=../logs
+# mkdir -p $LOG_FOLDER
+# LOGFILE=$LOG_FOLDER/resnet_training.log
+
+# python3 of_cnn_train_val.py \
+#     --train_data_dir=$DATA_ROOT/train \
+#     --train_data_part_num=256 \
+#     --val_data_dir=$DATA_ROOT/validation \
+#     --val_data_part_num=256 \
+#     --num_nodes=1 \
+#     --gpu_num_per_node=4 \
+#     --model_update="momentum" \
+#     --learning_rate=0.256 \
+#     --loss_print_every_n_iter=100 \
+#     --batch_size_per_device=64 \
+#     --val_batch_size_per_device=50 \
+#     --num_epoch=90 \
+#     --model="resnet50" 2>&1 | tee ${LOGFILE}
+
+# echo "Writting log to ${LOGFILE}"
\ No newline at end of file