Enable fc on bfloat16 (#43154)

* Enable fc on bfloat16

* Add pass for residual connection

* Dissable Residual connection pass for now

* Ban ResidualData from DQ

* style

* WO for python tests

paddle/fluid/framework/ir/mkldnn/cpu_bfloat16_pass.cc

@@ -188,7 +188,8 @@ class DeQuantizer final : public Quanter {
   bool IsNotPermittedName(const std::string& output_name) const override {
     std::unordered_map<std::string, std::vector<std::string>> block_list{
         {"layer_norm",
-         {"Mean", "Variance"}}};  // not used in inference in MKLDNN
+         {"Mean", "Variance"}},     // not used in inference in MKLDNN
+        {"fc", {"ResidualData"}}};  // artifical output, already dequantized
 
     std::vector<std::string> blocked_outputs{"XShape"};  // blocklist for any op
     auto op_name = op->Name();

paddle/fluid/inference/api/paddle_pass_builder.cc

@@ -348,6 +348,10 @@ void CpuPassStrategy::EnableMkldnnQuantizer() {
 void CpuPassStrategy::EnableMkldnnBfloat16() {
 #ifdef PADDLE_WITH_MKLDNN
   if (!use_mkldnn_bfloat16_) {
+    passes_.push_back("fc_mkldnn_pass");
+    passes_.push_back("fc_act_mkldnn_fuse_pass");
+    passes_.push_back("fc_elementwise_add_mkldnn_fuse_pass");
+
     passes_.push_back("cpu_bfloat16_placement_pass");
     passes_.push_back("cpu_bfloat16_pass");
     passes_.push_back("cpu_quantize_squash_pass");