disable_all_reduce_sequence = False

cnn_e2e/job_function_util.py

@@ -14,7 +14,7 @@ def _default_config(args):
 def get_train_config(args):
     train_config = _default_config(args)
     train_config.train.primary_lr(args.learning_rate)
-    train_config.disable_all_reduce_sequence(True)
+    train_config.disable_all_reduce_sequence(False)
     train_config.cudnn_conv_enable_pseudo_half(True)
     #train_config.all_reduce_group_min_mbyte(8)
     #train_config.all_reduce_group_num(128)

cnn_e2e/resnet_model.py

@@ -22,7 +22,7 @@ def _conv2d(
     trainable=True,
     weight_initializer=flow.variance_scaling_initializer(data_format="NCHW"),
     #weight_initializer=flow.variance_scaling_initializer(3, 'fan_in', 'random_normal', data_format="NCHW"),
-    weight_regularizer=flow.regularizers.l2(1e-4),
+    weight_regularizer=flow.regularizers.l2(1e-5),
 ):
     weight = flow.get_variable(
         name + "-weight",
@@ -143,7 +143,7 @@ def resnet50(images, trainable=True):
             #kernel_initializer=flow.variance_scaling_initializer(3, 'fan_in', 'random_normal'),
             kernel_initializer=flow.xavier_uniform_initializer(),
             bias_initializer=flow.zeros_initializer(),
-            kernel_regularizer=flow.regularizers.l2(1e-4),
+            kernel_regularizer=flow.regularizers.l2(1e-5),
             trainable=trainable,
             name="fc1001",
         )