[ut] Update skip concept to ignore. (#37635)

paddle/fluid/framework/ir/graph_viz_pass.cc

@@ -19,6 +19,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/op_proto_maker.h"
 #include "paddle/fluid/framework/program_desc.h"
 #include "paddle/fluid/inference/analysis/dot.h"
+#include "paddle/fluid/inference/analysis/helper.h"
 
 namespace paddle {
 namespace framework {
@@ -38,6 +39,13 @@ std::string FormatName(const Node* node) {
 }  // namespace
 
 void GraphVizPass::ApplyImpl(ir::Graph* graph) const {
+  std::string optim_cache_dir;
+  if (Has("optim_cache_dir")) {
+    optim_cache_dir = Get<std::string>("optim_cache_dir");
+    if (!optim_cache_dir.empty()) {
+      paddle::inference::analysis::MakeDirIfNotExists(optim_cache_dir);
+    }
+  }
   const std::string& graph_viz_path = Get<std::string>(kGraphvizPath);
   VLOG(3) << "draw IR graph viz to " << graph_viz_path;
   std::unique_ptr<std::ostream> fout(new std::ofstream(graph_viz_path));
@@ -62,7 +70,6 @@ void GraphVizPass::ApplyImpl(ir::Graph* graph) const {
         }
       }
     }
-    const std::string& optim_cache_dir = Get<std::string>("optim_cache_dir");
     std::string program_bytes = program_desc.Proto()->SerializeAsString();
     // rename from "17_ir_fc_fuse_pass.dot" to "fc_fuse_pass.pdmodel"
     program_path =

paddle/fluid/inference/analysis/helper.h

@@ -196,16 +196,20 @@ static std::string GetDirRoot(const std::string &path) {
   return path;
 }
 
-static std::string GetOrCreateModelOptCacheDir(const std::string &model_root) {
-  std::string opt_cache_dir = model_root + "/_opt_cache/";
-  if (!PathExists(opt_cache_dir)) {
+static void MakeDirIfNotExists(const std::string &path) {
+  if (!PathExists(path)) {
     PADDLE_ENFORCE_NE(
-        MKDIR(opt_cache_dir.c_str()), -1,
+        MKDIR(path.c_str()), -1,
         platform::errors::PreconditionNotMet(
             "Can not create optimize cache directory: %s, Make sure you "
             "have permission to write",
-            opt_cache_dir));
+            path));
   }
+}
+
+static std::string GetOrCreateModelOptCacheDir(const std::string &model_root) {
+  std::string opt_cache_dir = model_root + "/_opt_cache/";
+  MakeDirIfNotExists(opt_cache_dir);
   return opt_cache_dir;
 }
 

paddle/fluid/inference/api/analysis_config.cc

@@ -257,18 +257,10 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
     // Update() will reset all the passes, when some tensorRT pass is deleted in
     // other.pass_builder(), it will set again, so we just remove the
     // deleted_pass.
-    auto all_passes = kTRTSubgraphPasses;
+    pass_builder_->ClearPasses();
     auto other_passes = other.pass_builder()->AllPasses();
-    // We should sort them, because the user may call the SwitchIrDebug
-    // interface, which will change the pass.
-    std::sort(all_passes.begin(), all_passes.end());
-    std::sort(other_passes.begin(), other_passes.end());
-    std::vector<std::string> deleted_passes;
-    std::set_difference(all_passes.begin(), all_passes.end(),
-                        other_passes.begin(), other_passes.end(),
-                        std::inserter(deleted_passes, deleted_passes.begin()));
-    for (auto ps : deleted_passes) {
-      pass_builder_->DeletePass(ps);
+    for (auto pass : other_passes) {
+      pass_builder_->AppendPass(pass);
     }
   }
   if (use_dlnne_) {
@@ -479,6 +471,7 @@ void AnalysisConfig::Update() {
       pass_builder()->AppendPass(pass);
     }
   }
+
   if (use_dlnne_) {
     pass_builder()->ClearPasses();
     for (const auto &pass : kDlnneSubgraphPasses) {
@@ -688,8 +681,6 @@ void AnalysisConfig::SetModelBuffer(const char *prog_buffer,
   prog_file_ = std::string(prog_buffer, prog_buffer + prog_buffer_size);
   params_file_ = std::string(param_buffer, param_buffer + param_buffer_size);
   model_from_memory_ = true;
-
-  Update();
 }
 
 NativeConfig AnalysisConfig::ToNativeConfig() const {

python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py

@@ -59,7 +59,7 @@ else:
     settings.load_profile("dev")
 
 
-class SkipReasons(enum.Enum):
+class IgnoreReasons(enum.Enum):
     # Paddle not support, but trt support, we need to add the feature.
     TRT_NOT_IMPLEMENTED = 0
     # TRT not support.
@@ -70,19 +70,23 @@ class SkipReasons(enum.Enum):
     MKLDNN_ACCURACY_ERROR = 3
 
 
+# TODO(wilber): just for backward compatible
+SkipReasons = IgnoreReasons
+
+
 class AutoScanTest(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         np.random.seed(1024)
         paddle.enable_static()
         super(AutoScanTest, self).__init__(*args, **kwargs)
-        self.skip_cases = []
+        self.ignore_cases = []
         abs_dir = os.path.abspath(os.path.dirname(__file__))
         self.cache_dir = os.path.join(abs_dir,
                                       str(self.__module__) + '_cache_dir')
         self.available_passes_in_framework = set()
         self.num_ran_programs = 0
         self.num_invalid_programs = 0
-        self.num_skipped_tests = 0
+        self.num_ignore_tests = 0
         self.num_predictor_kinds = 0
 
     @abc.abstractmethod
@@ -98,12 +102,12 @@ class AutoScanTest(unittest.TestCase):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def add_skip_case(
+    def add_ignore_check_case(
             self,
             teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]],
-            reason: SkipReasons,
+            reason: IgnoreReasons,
             note: str):
-        self.skip_cases.append((teller, reason, note))
+        self.ignore_cases.append((teller, reason, note))
 
     def is_program_valid(self, program_config: ProgramConfig) -> bool:
         return True
@@ -117,7 +121,6 @@ class AutoScanTest(unittest.TestCase):
         predictor = paddle_infer.create_predictor(pred_config)
         self.available_passes_in_framework = self.available_passes_in_framework | set(
             pred_config.pass_builder().all_passes())
-
         for name, _ in prog_config.inputs.items():
             input_tensor = predictor.get_input_handle(name)
             input_tensor.copy_from_cpu(feed_data[name]['data'])
@@ -164,12 +167,12 @@ class AutoScanTest(unittest.TestCase):
         return ops
 
     @abc.abstractmethod
-    def skip_log(self, msg: str):
+    def ignore_log(self, msg: str):
         logging.warning("SKIP: " + msg)
 
     @abc.abstractmethod
     def fail_log(self, msg: str):
-        logging.error("FAILE: " + msg)
+        logging.error("FAIL: " + msg)
 
     @abc.abstractmethod
     def success_log(self, msg: str):
@@ -232,14 +235,16 @@ class MkldnnAutoScanTest(AutoScanTest):
             for pred_config, (
                     atol, rtol) in self.sample_predictor_configs(prog_config):
                 # skip info
-                skip_flag = False
-                for skip_info in self.skip_cases:
-                    if skip_info[0](prog_config, pred_config):
-                        skip_flag = True
-                        if skip_info[1] == SkipReasons.MKLDNN_ACCURACY_ERROR:
-                            self.skip_log("[MKLDNN_ACCURACY_ERROR] " +
-                                          skip_info[2] + ' ' + ' vs ' + self.
-                                          inference_config_str(pred_config))
+                ignore_flag = False
+                for ignore_info in self.ignore_cases:
+                    if ignore_info[0](prog_config, pred_config):
+                        ignore_flag = True
+                        if ignore_info[
+                                1] == IgnoreReasons.MKLDNN_ACCURACY_ERROR:
+                            self.ignore_log("[MKLDNN_ACCURACY_ERROR] " +
+                                            ignore_info[2] + ' ' + ' vs ' +
+                                            self.inference_config_str(
+                                                pred_config))
                         else:
                             raise NotImplementedError
                         break
@@ -259,7 +264,7 @@ class MkldnnAutoScanTest(AutoScanTest):
                     self.fail_log(
                         self.inference_config_str(pred_config) +
                         '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)))
-                    if not skip_flag:
+                    if not ignore_flag:
                         status = False
                     continue
                 self.success_log('RUN predictor_config ' + self.
@@ -291,7 +296,7 @@ class PassAutoScanTest(AutoScanTest):
                 status = False
         return status
 
-    def add_skip_pass_case(self):
+    def add_ignore_pass_case(self):
         return
 
     def assert_op_list(self, op_list_after_fusion):
@@ -343,7 +348,7 @@ class PassAutoScanTest(AutoScanTest):
         assert passes is not None, "Parameter of passes must be defined in function run_and_statis."
         self.passes = passes
 
-        self.add_skip_pass_case()
+        self.add_ignore_pass_case()
 
         def program_generator(draw):
             return self.sample_program_config(draw)
@@ -364,17 +369,16 @@ class PassAutoScanTest(AutoScanTest):
         logging.info("Number of Invalid Programs: {}".format(
             self.num_invalid_programs))
         logging.info("Number of Ran Programs: {}".format(self.num_ran_programs))
-        logging.info("Number of Skipped Tests: {}".format(
-            self.num_skipped_tests))
+        logging.info("Number of Ignore Tests: {}".format(self.num_ignore_tests))
         successful_ran_programs = int(self.num_ran_programs -
-                                      self.num_skipped_tests /
+                                      self.num_ignore_tests /
                                       self.num_predictor_kinds)
         logging.info(
             "Number of successfully ran programs approximately equal to {}".
             format(successful_ran_programs))
         if successful_ran_programs < min_success_num:
             logging.warning(
-                "satisfied_programs = ran_programs - num_skipped_tests / num_predictor_kinds"
+                "satisfied_programs = ran_programs - num_ignore_tests / num_predictor_kinds"
             )
             logging.error(
                 "At least {} programs need to ran successfully, but now only about {} programs satisfied.".
@@ -421,15 +425,16 @@ class PassAutoScanTest(AutoScanTest):
                     atol, rtol) in self.sample_predictor_configs(prog_config):
                 self.num_predictor_kinds += 1
                 # skip info
-                skip_flag = False
-                for skip_info in self.skip_cases:
-                    if skip_info[0](prog_config, pred_config):
-                        skip_flag = True
-                        self.num_skipped_tests += 1
-                        if skip_info[1] == SkipReasons.PASS_ACCURACY_ERROR:
-                            self.skip_log("[PASS_ACCURACY_ERROR] " + skip_info[
-                                2] + ' ' + ' vs ' + self.inference_config_str(
-                                    pred_config))
+                ignore_flag = False
+                for ignore_info in self.ignore_cases:
+                    if ignore_info[0](prog_config, pred_config):
+                        ignore_flag = True
+                        self.num_ignore_tests += 1
+                        if ignore_info[1] == IgnoreReasons.PASS_ACCURACY_ERROR:
+                            self.ignore_log("[PASS_ACCURACY_ERROR] " +
+                                            ignore_info[2] + ' ' + ' vs ' +
+                                            self.inference_config_str(
+                                                pred_config))
                         else:
                             raise NotImplementedError
                         break
@@ -445,14 +450,14 @@ class PassAutoScanTest(AutoScanTest):
                                              pred_config, feed_data))
                     self.assert_tensors_near(atol, rtol, results[-1],
                                              results[0])
-                    if not skip_flag:
+                    if not ignore_flag:
                         self.assert_op_list(op_list)
 
                 except Exception as e:
                     self.fail_log(
                         self.inference_config_str(pred_config) +
                         '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)))
-                    if not skip_flag:
+                    if not ignore_flag:
                         status = False
                     continue
                 self.success_log('RUN predictor_config ' + self.
@@ -487,8 +492,6 @@ class PassAutoScanTest(AutoScanTest):
         config.enable_use_gpu(100, 0)
         config.set_optim_cache_dir(self.cache_dir)
         config.switch_ir_debug()
-        # for assert_op_size.
-        self.passes = ['transpose_flatten_concat_fuse_pass']
         return config
 
 
@@ -656,16 +659,17 @@ class TrtLayerAutoScanTest(AutoScanTest):
                 ) == paddle_infer.PrecisionType.Int8 and not quant:
                     continue
 
-                skip_flag = False
-                for skip_info in self.skip_cases:
-                    if skip_info[0](prog_config, pred_config):
-                        skip_flag = True
-                        if skip_info[1] == SkipReasons.TRT_NOT_IMPLEMENTED:
-                            self.skip_log("[TRT_NOT_IMPLEMENTED] " + skip_info[
-                                2] + ' ' + ' vs ' + self.inference_config_str(
-                                    pred_config))
-                        elif skip_info[1] == SkipReasons.TRT_NOT_SUPPORT:
-                            self.skip_log("[TRT_NOT_SUPPORT] " + skip_info[
+                ignore_flag = False
+                for ignore_info in self.ignore_cases:
+                    if ignore_info[0](prog_config, pred_config):
+                        ignore_flag = True
+                        if ignore_info[1] == IgnoreReasons.TRT_NOT_IMPLEMENTED:
+                            self.ignore_log("[TRT_NOT_IMPLEMENTED] " +
+                                            ignore_info[2] + ' ' + ' vs ' +
+                                            self.inference_config_str(
+                                                pred_config))
+                        elif ignore_info[1] == IgnoreReasons.TRT_NOT_SUPPORT:
+                            self.ignore_log("[TRT_NOT_SUPPORT] " + ignore_info[
                                 2] + ' ' + ' vs ' + self.inference_config_str(
                                     pred_config))
                         else:
@@ -679,7 +683,7 @@ class TrtLayerAutoScanTest(AutoScanTest):
                                              pred_config, feed_data))
                     self.assert_tensors_near(atol, rtol, results[-1],
                                              results[0])
-                    if not skip_flag:
+                    if not ignore_flag:
                         self.assert_op_size(nodes_num[0], nodes_num[1])
                     # deserialize test
                     if nodes_num[0] > 0:
@@ -690,10 +694,18 @@ class TrtLayerAutoScanTest(AutoScanTest):
                         str(prog_config) + ' vs ' + self.inference_config_str(
                             pred_config) +
                         '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)))
-                    if not skip_flag:
+                    if not ignore_flag:
                         status = False
                     continue
                 self.success_log('RUN predictor_config ' + self.
                                  inference_config_str(pred_config) + ' done')
 
         self.assertTrue(status)
+
+    # TODO(wilber): just for backward compatible
+    def add_skip_case(
+            self,
+            teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]],
+            reason: IgnoreReasons,
+            note: str):
+        self.ignore_cases.append((teller, reason, note))

python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from auto_scan_test import PassAutoScanTest, SkipReasons
+from auto_scan_test import PassAutoScanTest, IgnoreReasons
 from program_config import TensorConfig, ProgramConfig, OpConfig
 import numpy as np
 import paddle.inference as paddle_infer
@@ -270,7 +270,7 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest):
             })
         yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5)
 
-    def add_skip_pass_case(self):
+    def add_ignore_pass_case(self):
         def teller1(program_config, predictor_config):
             if program_config.ops[3].attrs['axis'] in [
                     -1, 2
@@ -280,8 +280,10 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest):
                 return True
             return False
 
-        self.add_skip_case(teller1, SkipReasons.PASS_ACCURACY_ERROR,
-                           "The pass output has diff in a specific case.")
+        self.add_ignore_check_case(
+            teller1, IgnoreReasons.PASS_ACCURACY_ERROR,
+            "The pass output has diff in a specific case. We need to fix it as soon as possible."
+        )
 
     def test(self):
         # this fuse need to fix, now there's no program can ran successfully

python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from auto_scan_test import PassAutoScanTest, SkipReasons
+from auto_scan_test import PassAutoScanTest, IgnoreReasons
 from program_config import TensorConfig, ProgramConfig, OpConfig
 import numpy as np
 import paddle.inference as paddle_infer
@@ -46,7 +46,7 @@ class TestFcFusePass(PassAutoScanTest):
         config = self.create_inference_config(use_gpu=True)
         yield config, ["fc"], (1e-5, 1e-5)
 
-    def add_skip_pass_case(self):
+    def add_ignore_pass_case(self):
         # Here we put some skip rules to avoid known bugs
         def teller1(program_config, predictor_config):
             # shape of bias should be [1, mul_y_shape[-1]] or [mul_y_shape[-1]]
@@ -64,14 +64,14 @@ class TestFcFusePass(PassAutoScanTest):
                 return True
             return False
 
-        self.add_skip_case(
+        self.add_ignore_check_case(
             teller1,
-            SkipReasons.PASS_ACCURACY_ERROR,
+            IgnoreReasons.PASS_ACCURACY_ERROR,
             "The pass output has diff while shape of bias is not [out_size] or [1, out_size].",
         )
-        self.add_skip_case(
+        self.add_ignore_check_case(
             teller2,
-            SkipReasons.PASS_ACCURACY_ERROR,
+            IgnoreReasons.PASS_ACCURACY_ERROR,
             "The pass output has diff while axis of elementwise_add is not -1.",
         )