From ae544242ac5e566b968b3f4cd7b9e54c9c18f435 Mon Sep 17 00:00:00 2001 From: Wilber Date: Mon, 29 Nov 2021 10:30:05 +0800 Subject: [PATCH] [ut] Update skip concept to ignore. (#37635) --- paddle/fluid/framework/ir/graph_viz_pass.cc | 9 +- paddle/fluid/inference/analysis/helper.h | 14 ++- paddle/fluid/inference/api/analysis_config.cc | 17 +-- .../unittests/ir/inference/auto_scan_test.py | 110 ++++++++++-------- .../test_emb_eltwise_layernorm_fuse_pass.py | 10 +- .../ir/inference/test_fc_fuse_pass.py | 12 +- 6 files changed, 94 insertions(+), 78 deletions(-) diff --git a/paddle/fluid/framework/ir/graph_viz_pass.cc b/paddle/fluid/framework/ir/graph_viz_pass.cc index 735b433b6c..7311eb4b91 100644 --- a/paddle/fluid/framework/ir/graph_viz_pass.cc +++ b/paddle/fluid/framework/ir/graph_viz_pass.cc @@ -19,6 +19,7 @@ limitations under the License. */ #include "paddle/fluid/framework/op_proto_maker.h" #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/inference/analysis/dot.h" +#include "paddle/fluid/inference/analysis/helper.h" namespace paddle { namespace framework { @@ -38,6 +39,13 @@ std::string FormatName(const Node* node) { } // namespace void GraphVizPass::ApplyImpl(ir::Graph* graph) const { + std::string optim_cache_dir; + if (Has("optim_cache_dir")) { + optim_cache_dir = Get("optim_cache_dir"); + if (!optim_cache_dir.empty()) { + paddle::inference::analysis::MakeDirIfNotExists(optim_cache_dir); + } + } const std::string& graph_viz_path = Get(kGraphvizPath); VLOG(3) << "draw IR graph viz to " << graph_viz_path; std::unique_ptr fout(new std::ofstream(graph_viz_path)); @@ -62,7 +70,6 @@ void GraphVizPass::ApplyImpl(ir::Graph* graph) const { } } } - const std::string& optim_cache_dir = Get("optim_cache_dir"); std::string program_bytes = program_desc.Proto()->SerializeAsString(); // rename from "17_ir_fc_fuse_pass.dot" to "fc_fuse_pass.pdmodel" program_path = diff --git a/paddle/fluid/inference/analysis/helper.h b/paddle/fluid/inference/analysis/helper.h index ebea4d0386..61c5d8d0e4 100644 --- a/paddle/fluid/inference/analysis/helper.h +++ b/paddle/fluid/inference/analysis/helper.h @@ -196,16 +196,20 @@ static std::string GetDirRoot(const std::string &path) { return path; } -static std::string GetOrCreateModelOptCacheDir(const std::string &model_root) { - std::string opt_cache_dir = model_root + "/_opt_cache/"; - if (!PathExists(opt_cache_dir)) { +static void MakeDirIfNotExists(const std::string &path) { + if (!PathExists(path)) { PADDLE_ENFORCE_NE( - MKDIR(opt_cache_dir.c_str()), -1, + MKDIR(path.c_str()), -1, platform::errors::PreconditionNotMet( "Can not create optimize cache directory: %s, Make sure you " "have permission to write", - opt_cache_dir)); + path)); } +} + +static std::string GetOrCreateModelOptCacheDir(const std::string &model_root) { + std::string opt_cache_dir = model_root + "/_opt_cache/"; + MakeDirIfNotExists(opt_cache_dir); return opt_cache_dir; } diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 0440801cfc..ceca7e8146 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -257,18 +257,10 @@ AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) { // Update() will reset all the passes, when some tensorRT pass is deleted in // other.pass_builder(), it will set again, so we just remove the // deleted_pass. - auto all_passes = kTRTSubgraphPasses; + pass_builder_->ClearPasses(); auto other_passes = other.pass_builder()->AllPasses(); - // We should sort them, because the user may call the SwitchIrDebug - // interface, which will change the pass. - std::sort(all_passes.begin(), all_passes.end()); - std::sort(other_passes.begin(), other_passes.end()); - std::vector deleted_passes; - std::set_difference(all_passes.begin(), all_passes.end(), - other_passes.begin(), other_passes.end(), - std::inserter(deleted_passes, deleted_passes.begin())); - for (auto ps : deleted_passes) { - pass_builder_->DeletePass(ps); + for (auto pass : other_passes) { + pass_builder_->AppendPass(pass); } } if (use_dlnne_) { @@ -479,6 +471,7 @@ void AnalysisConfig::Update() { pass_builder()->AppendPass(pass); } } + if (use_dlnne_) { pass_builder()->ClearPasses(); for (const auto &pass : kDlnneSubgraphPasses) { @@ -688,8 +681,6 @@ void AnalysisConfig::SetModelBuffer(const char *prog_buffer, prog_file_ = std::string(prog_buffer, prog_buffer + prog_buffer_size); params_file_ = std::string(param_buffer, param_buffer + param_buffer_size); model_from_memory_ = true; - - Update(); } NativeConfig AnalysisConfig::ToNativeConfig() const { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py b/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py index f244b875dd..33754fac12 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py @@ -59,7 +59,7 @@ else: settings.load_profile("dev") -class SkipReasons(enum.Enum): +class IgnoreReasons(enum.Enum): # Paddle not support, but trt support, we need to add the feature. TRT_NOT_IMPLEMENTED = 0 # TRT not support. @@ -70,19 +70,23 @@ class SkipReasons(enum.Enum): MKLDNN_ACCURACY_ERROR = 3 +# TODO(wilber): just for backward compatible +SkipReasons = IgnoreReasons + + class AutoScanTest(unittest.TestCase): def __init__(self, *args, **kwargs): np.random.seed(1024) paddle.enable_static() super(AutoScanTest, self).__init__(*args, **kwargs) - self.skip_cases = [] + self.ignore_cases = [] abs_dir = os.path.abspath(os.path.dirname(__file__)) self.cache_dir = os.path.join(abs_dir, str(self.__module__) + '_cache_dir') self.available_passes_in_framework = set() self.num_ran_programs = 0 self.num_invalid_programs = 0 - self.num_skipped_tests = 0 + self.num_ignore_tests = 0 self.num_predictor_kinds = 0 @abc.abstractmethod @@ -98,12 +102,12 @@ class AutoScanTest(unittest.TestCase): raise NotImplementedError @abc.abstractmethod - def add_skip_case( + def add_ignore_check_case( self, teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]], - reason: SkipReasons, + reason: IgnoreReasons, note: str): - self.skip_cases.append((teller, reason, note)) + self.ignore_cases.append((teller, reason, note)) def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -117,7 +121,6 @@ class AutoScanTest(unittest.TestCase): predictor = paddle_infer.create_predictor(pred_config) self.available_passes_in_framework = self.available_passes_in_framework | set( pred_config.pass_builder().all_passes()) - for name, _ in prog_config.inputs.items(): input_tensor = predictor.get_input_handle(name) input_tensor.copy_from_cpu(feed_data[name]['data']) @@ -164,12 +167,12 @@ class AutoScanTest(unittest.TestCase): return ops @abc.abstractmethod - def skip_log(self, msg: str): + def ignore_log(self, msg: str): logging.warning("SKIP: " + msg) @abc.abstractmethod def fail_log(self, msg: str): - logging.error("FAILE: " + msg) + logging.error("FAIL: " + msg) @abc.abstractmethod def success_log(self, msg: str): @@ -232,14 +235,16 @@ class MkldnnAutoScanTest(AutoScanTest): for pred_config, ( atol, rtol) in self.sample_predictor_configs(prog_config): # skip info - skip_flag = False - for skip_info in self.skip_cases: - if skip_info[0](prog_config, pred_config): - skip_flag = True - if skip_info[1] == SkipReasons.MKLDNN_ACCURACY_ERROR: - self.skip_log("[MKLDNN_ACCURACY_ERROR] " + - skip_info[2] + ' ' + ' vs ' + self. - inference_config_str(pred_config)) + ignore_flag = False + for ignore_info in self.ignore_cases: + if ignore_info[0](prog_config, pred_config): + ignore_flag = True + if ignore_info[ + 1] == IgnoreReasons.MKLDNN_ACCURACY_ERROR: + self.ignore_log("[MKLDNN_ACCURACY_ERROR] " + + ignore_info[2] + ' ' + ' vs ' + + self.inference_config_str( + pred_config)) else: raise NotImplementedError break @@ -259,7 +264,7 @@ class MkldnnAutoScanTest(AutoScanTest): self.fail_log( self.inference_config_str(pred_config) + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) - if not skip_flag: + if not ignore_flag: status = False continue self.success_log('RUN predictor_config ' + self. @@ -291,7 +296,7 @@ class PassAutoScanTest(AutoScanTest): status = False return status - def add_skip_pass_case(self): + def add_ignore_pass_case(self): return def assert_op_list(self, op_list_after_fusion): @@ -343,7 +348,7 @@ class PassAutoScanTest(AutoScanTest): assert passes is not None, "Parameter of passes must be defined in function run_and_statis." self.passes = passes - self.add_skip_pass_case() + self.add_ignore_pass_case() def program_generator(draw): return self.sample_program_config(draw) @@ -364,17 +369,16 @@ class PassAutoScanTest(AutoScanTest): logging.info("Number of Invalid Programs: {}".format( self.num_invalid_programs)) logging.info("Number of Ran Programs: {}".format(self.num_ran_programs)) - logging.info("Number of Skipped Tests: {}".format( - self.num_skipped_tests)) + logging.info("Number of Ignore Tests: {}".format(self.num_ignore_tests)) successful_ran_programs = int(self.num_ran_programs - - self.num_skipped_tests / + self.num_ignore_tests / self.num_predictor_kinds) logging.info( "Number of successfully ran programs approximately equal to {}". format(successful_ran_programs)) if successful_ran_programs < min_success_num: logging.warning( - "satisfied_programs = ran_programs - num_skipped_tests / num_predictor_kinds" + "satisfied_programs = ran_programs - num_ignore_tests / num_predictor_kinds" ) logging.error( "At least {} programs need to ran successfully, but now only about {} programs satisfied.". @@ -421,15 +425,16 @@ class PassAutoScanTest(AutoScanTest): atol, rtol) in self.sample_predictor_configs(prog_config): self.num_predictor_kinds += 1 # skip info - skip_flag = False - for skip_info in self.skip_cases: - if skip_info[0](prog_config, pred_config): - skip_flag = True - self.num_skipped_tests += 1 - if skip_info[1] == SkipReasons.PASS_ACCURACY_ERROR: - self.skip_log("[PASS_ACCURACY_ERROR] " + skip_info[ - 2] + ' ' + ' vs ' + self.inference_config_str( - pred_config)) + ignore_flag = False + for ignore_info in self.ignore_cases: + if ignore_info[0](prog_config, pred_config): + ignore_flag = True + self.num_ignore_tests += 1 + if ignore_info[1] == IgnoreReasons.PASS_ACCURACY_ERROR: + self.ignore_log("[PASS_ACCURACY_ERROR] " + + ignore_info[2] + ' ' + ' vs ' + + self.inference_config_str( + pred_config)) else: raise NotImplementedError break @@ -445,14 +450,14 @@ class PassAutoScanTest(AutoScanTest): pred_config, feed_data)) self.assert_tensors_near(atol, rtol, results[-1], results[0]) - if not skip_flag: + if not ignore_flag: self.assert_op_list(op_list) except Exception as e: self.fail_log( self.inference_config_str(pred_config) + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) - if not skip_flag: + if not ignore_flag: status = False continue self.success_log('RUN predictor_config ' + self. @@ -487,8 +492,6 @@ class PassAutoScanTest(AutoScanTest): config.enable_use_gpu(100, 0) config.set_optim_cache_dir(self.cache_dir) config.switch_ir_debug() - # for assert_op_size. - self.passes = ['transpose_flatten_concat_fuse_pass'] return config @@ -656,16 +659,17 @@ class TrtLayerAutoScanTest(AutoScanTest): ) == paddle_infer.PrecisionType.Int8 and not quant: continue - skip_flag = False - for skip_info in self.skip_cases: - if skip_info[0](prog_config, pred_config): - skip_flag = True - if skip_info[1] == SkipReasons.TRT_NOT_IMPLEMENTED: - self.skip_log("[TRT_NOT_IMPLEMENTED] " + skip_info[ - 2] + ' ' + ' vs ' + self.inference_config_str( - pred_config)) - elif skip_info[1] == SkipReasons.TRT_NOT_SUPPORT: - self.skip_log("[TRT_NOT_SUPPORT] " + skip_info[ + ignore_flag = False + for ignore_info in self.ignore_cases: + if ignore_info[0](prog_config, pred_config): + ignore_flag = True + if ignore_info[1] == IgnoreReasons.TRT_NOT_IMPLEMENTED: + self.ignore_log("[TRT_NOT_IMPLEMENTED] " + + ignore_info[2] + ' ' + ' vs ' + + self.inference_config_str( + pred_config)) + elif ignore_info[1] == IgnoreReasons.TRT_NOT_SUPPORT: + self.ignore_log("[TRT_NOT_SUPPORT] " + ignore_info[ 2] + ' ' + ' vs ' + self.inference_config_str( pred_config)) else: @@ -679,7 +683,7 @@ class TrtLayerAutoScanTest(AutoScanTest): pred_config, feed_data)) self.assert_tensors_near(atol, rtol, results[-1], results[0]) - if not skip_flag: + if not ignore_flag: self.assert_op_size(nodes_num[0], nodes_num[1]) # deserialize test if nodes_num[0] > 0: @@ -690,10 +694,18 @@ class TrtLayerAutoScanTest(AutoScanTest): str(prog_config) + ' vs ' + self.inference_config_str( pred_config) + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) - if not skip_flag: + if not ignore_flag: status = False continue self.success_log('RUN predictor_config ' + self. inference_config_str(pred_config) + ' done') self.assertTrue(status) + + # TODO(wilber): just for backward compatible + def add_skip_case( + self, + teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]], + reason: IgnoreReasons, + note: str): + self.ignore_cases.append((teller, reason, note)) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py index 7bbf850295..7379a8d333 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from auto_scan_test import PassAutoScanTest, SkipReasons +from auto_scan_test import PassAutoScanTest, IgnoreReasons from program_config import TensorConfig, ProgramConfig, OpConfig import numpy as np import paddle.inference as paddle_infer @@ -270,7 +270,7 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): }) yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5) - def add_skip_pass_case(self): + def add_ignore_pass_case(self): def teller1(program_config, predictor_config): if program_config.ops[3].attrs['axis'] in [ -1, 2 @@ -280,8 +280,10 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): return True return False - self.add_skip_case(teller1, SkipReasons.PASS_ACCURACY_ERROR, - "The pass output has diff in a specific case.") + self.add_ignore_check_case( + teller1, IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass output has diff in a specific case. We need to fix it as soon as possible." + ) def test(self): # this fuse need to fix, now there's no program can ran successfully diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py index 34f611e0bb..1db3a00713 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from auto_scan_test import PassAutoScanTest, SkipReasons +from auto_scan_test import PassAutoScanTest, IgnoreReasons from program_config import TensorConfig, ProgramConfig, OpConfig import numpy as np import paddle.inference as paddle_infer @@ -46,7 +46,7 @@ class TestFcFusePass(PassAutoScanTest): config = self.create_inference_config(use_gpu=True) yield config, ["fc"], (1e-5, 1e-5) - def add_skip_pass_case(self): + def add_ignore_pass_case(self): # Here we put some skip rules to avoid known bugs def teller1(program_config, predictor_config): # shape of bias should be [1, mul_y_shape[-1]] or [mul_y_shape[-1]] @@ -64,14 +64,14 @@ class TestFcFusePass(PassAutoScanTest): return True return False - self.add_skip_case( + self.add_ignore_check_case( teller1, - SkipReasons.PASS_ACCURACY_ERROR, + IgnoreReasons.PASS_ACCURACY_ERROR, "The pass output has diff while shape of bias is not [out_size] or [1, out_size].", ) - self.add_skip_case( + self.add_ignore_check_case( teller2, - SkipReasons.PASS_ACCURACY_ERROR, + IgnoreReasons.PASS_ACCURACY_ERROR, "The pass output has diff while axis of elementwise_add is not -1.", ) -- GitLab