add gpu_allocator_try_time config, test=develop (#19675)

paddle/fluid/memory/allocation/allocator_facade.cc

@@ -37,7 +37,7 @@
 #endif
 
 DEFINE_int64(
-    gpu_allocator_retry_time, 100,
+    gpu_allocator_retry_time, 10000,
     "The retry time (milliseconds) when allocator fails "
     "to allocate memory. No retry if this value is not greater than 0");
 

python/paddle/fluid/__init__.py

@@ -204,7 +204,7 @@ def __bootstrap__():
             'cudnn_exhaustive_search', 'selected_gpus', 'sync_nccl_allreduce',
             'limit_of_tmp_allocation',
             'times_excess_than_required_tmp_allocation',
-            'cudnn_batchnorm_spatial_persistent'
+            'cudnn_batchnorm_spatial_persistent', 'gpu_allocator_retry_time'
         ]
     core.init_gflags([sys.argv[0]] +
                      ["--tryfromenv=" + ",".join(read_env_flags)])