graph_pattern_detector.h 65.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

#ifdef PADDLE_WITH_TESTING
#include <gtest/gtest_prod.h>
#endif

21
#include <map>
22
#include <memory>
23
#include <numeric>
24
#include <set>
25
#include <string>
26 27
#include <unordered_map>
#include <unordered_set>
28 29
#include <utility>
#include <vector>
W
wanghuancoder 已提交
30

31 32
#include "paddle/fluid/framework/ir/graph.h"
#include "paddle/fluid/framework/ir/node.h"
33
#include "paddle/fluid/inference/analysis/dot.h"
34

W
wanghuancoder 已提交
35 36 37 38 39 40 41 42 43
namespace paddle {
namespace framework {
namespace ir {
class Graph;
class Node;
}  // namespace ir
}  // namespace framework
}  // namespace paddle

44 45 46
namespace paddle {
namespace framework {
namespace ir {
47
class PDPattern;
48

49
// Some basic terminologies:
50 51 52 53 54 55 56 57 58
//   - PDPattern: a pattern defined as a data flow graph.
//   - PDNode: the node in the pattern, each PDNode represents an `ir::Node`
//     that meets some conditions defined in `PDNode.teller`.
//   - A pattern is defined with PDNodes with edges.

// Pattern detector node. This node helps to build a pattern.
struct PDNode {
  // tell whether an ir::Node* is a candidation for a PDNode.
  using teller_t = std::function<bool(Node*)>;
59
  enum class Type { kOp, kVar };
Y
Yan Chunwei 已提交
60 61 62 63 64 65
  enum class Role {
    kUnknown,      // No role,
    kInput,        // an input and will be retained,
    kOutput,       // an output and will be retained,
    kIntermediate  // will be removed after handler.
  };
66

67 68 69
  // this link to others
  PDNode& LinksTo(const std::vector<PDNode*>& others);
  PDNode& LinksFrom(const std::vector<PDNode*>& others);
70 71

  bool Tell(Node* node) const {
Y
Yan Chunwei 已提交
72 73 74 75 76 77
    if (teller_) return teller_(node);

    for (auto& asrt : asserts_) {
      if (!asrt(node)) return false;
    }
    return true;
78 79
  }

80 81 82
  bool IsOp() const { return type_ == Type::kOp; }
  bool IsVar() const { return type_ == Type::kVar; }

83
  const std::string& name() const { return name_; }
J
JingZhuangzhuang 已提交
84
  const PDPattern* pdpattern() const { return pattern_; }
85 86

  PDNode& operator=(const PDNode&) = delete;
Y
Yan Chunwei 已提交
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
  PDNode(const PDNode&) = delete;

  // Mark this node is an Input of a subgraph and will be retained.
  PDNode* AsInput() {
    role_ = Role::kInput;
    return this;
  }
  // Mark this node is an Output of a subgraph and will be retained.
  PDNode* AsOutput() {
    role_ = Role::kOutput;
    return this;
  }
  // Mark this node will be removed, so all the links should be inside a matched
  // sub-graph.
  PDNode* AsIntermediate() {
    role_ = Role::kIntermediate;
    return this;
  }

  bool IsIntermediate() const { return role_ == Role::kIntermediate; }
  bool IsInput() const { return role_ == Role::kInput; }
  bool IsOutput() const { return role_ == Role::kOutput; }

  // Assertions, helper functions to simplify the pattern definition.
  PDNode* assert_is_op();
  PDNode* assert_is_op(const std::string& op_type);
S
shentanyue 已提交
113
  PDNode* assert_is_not_op_type(const std::string& op_type);
Y
Yan Chunwei 已提交
114
  PDNode* assert_is_var();
Z
Zhen Wang 已提交
115
  PDNode* assert_var_dtype(proto::VarType::Type dtype);
C
chengduo 已提交
116
  PDNode* assert_is_not_ctrl_var();
Y
Yan Chunwei 已提交
117 118 119
  PDNode* assert_var_not_persistable();
  PDNode* assert_is_persistable_var();
  PDNode* assert_is_op_output(const std::string& op_type);
120 121
  PDNode* assert_is_op_output(const std::string& op_type,
                              const std::string& argument);
Y
Yan Chunwei 已提交
122
  PDNode* assert_is_op_input(const std::string& op_type);
123 124
  PDNode* assert_is_op_input(const std::string& op_type,
                             const std::string& argument);
Y
Yan Chunwei 已提交
125
  PDNode* assert_is_op_nth_input(const std::string& op_type,
126 127
                                 const std::string& argument,
                                 int nth);
Z
Zhen Wang 已提交
128
  PDNode* assert_is_not_op_input(const std::string& argument);
Y
Yan Chunwei 已提交
129
  PDNode* assert_is_op_nth_output(const std::string& op_type,
130 131
                                  const std::string& argument,
                                  int nth);
Y
Yan Chunwei 已提交
132 133 134 135 136
  PDNode* assert_is_only_input_of_op(const std::string& op_type);
  PDNode* assert_is_only_output_of_op(const std::string& op_type);
  PDNode* assert_op_has_n_inputs(const std::string& op_type, size_t n);
  PDNode* assert_op_has_n_outputs(const std::string& op_type, size_t n);
  PDNode* assert_more(teller_t&& teller);
137

C
chengduo 已提交
138 139 140 141 142 143
  PDNode* assert_is_ops_output(const std::unordered_set<std::string>& op_types);
  PDNode* assert_is_ops(const std::unordered_set<std::string>& op_types);
  PDNode* assert_is_ops_output(const std::unordered_set<std::string>& op_types,
                               const std::string& argument);
  PDNode* assert_is_ops_nth_input(
      const std::unordered_set<std::string>& op_types,
144 145
      const std::string& argument,
      int nth);
C
chengduo 已提交
146 147 148 149 150
  PDNode* assert_is_ops_input(const std::unordered_set<std::string>& op_types);
  PDNode* assert_is_ops_input(const std::unordered_set<std::string>& op_types,
                              const std::string& argument);
  PDNode* assert_is_ops_nth_output(
      const std::unordered_set<std::string>& op_types,
151 152
      const std::string& argument,
      int nth);
C
chengduo 已提交
153

154 155 156 157 158
  PDNode* assert_is_only_input_of_ops(
      const std::unordered_set<std::string>& op_types);
  PDNode* assert_is_only_output_of_ops(
      const std::unordered_set<std::string>& op_types);

159 160 161
  PDNode* assert_has_n_inputs(size_t n);
  PDNode* assert_has_n_outputs(size_t n);

T
tensor-tang 已提交
162 163 164 165
  template <typename T>
  PDNode* assert_op_attr(const std::string& attr_name, const T& attr) {
    asserts_.emplace_back([=](Node* x) {
      return x && x->IsOp() && x->Op()->HasAttr(attr_name) &&
R
Ruibiao Chen 已提交
166
             PADDLE_GET_CONST(T, x->Op()->GetAttr(attr_name)) == attr;
T
tensor-tang 已提交
167 168 169 170
    });
    return this;
  }

171
 private:
172 173
  PDNode(PDPattern* pattern,
         const std::string& name = "",
Y
Yan Chunwei 已提交
174 175
         Type type = Type::kVar)
      : pattern_(pattern), name_(name), type_(type) {}
176 177 178
  PDNode(teller_t&& teller,
         PDPattern* pattern,
         const std::string& name = "",
179 180 181 182 183
         Type type = Type::kVar)
      : teller_(std::move(teller)),
        pattern_(pattern),
        name_(name),
        type_(type) {
184 185 186
    PADDLE_ENFORCE_NOT_NULL(
        teller_,
        platform::errors::NotFound("invalid teller is set, teller is null"));
187 188 189 190 191 192
  }

  PDNode(PDNode&& other) = default;

  friend class PDPattern;

Y
Yan Chunwei 已提交
193
  // Will removed latter.
194
  teller_t teller_;
Y
Yan Chunwei 已提交
195
  std::vector<teller_t> asserts_;
196
  PDPattern* pattern_;
197
  std::string name_;
198
  Type type_;
Y
Yan Chunwei 已提交
199
  Role role_{Role::kUnknown};
200 201 202 203 204 205 206 207 208 209 210 211
};

/*
 * A pattern in a graph, which defined with PDNode and edges. Most graph
 * patterns can be divided into PDNodes and link relations between them.
 *
 * For example, the FC fusion need to filter the MUL and ELEMENTWISE_ADD
 * operators from the computation graph, the MUL's output should have only one
 * consumer which is the ELEMENTWISE_ADD.
 * This pattern can be defined as with the following pseudo codes
 *
 *     // Create two operator PDNodes.
Y
Yan Chunwei 已提交
212 213
 *     MUL = PDPattern.NewNode().assert_is_op("mul");
 *     ELE = PDPattern.NewNode().assert_is_op("elementwise_add");
214
 *     // Create the variable PDNodes.
Y
Yan Chunwei 已提交
215 216 217 218 219 220
 *     MUL_out = PDPattern.NewNode().assert_is_op_output("mul") \
 *                                  .assert_is_op_input("elementwise_add") \
 *                                  .AsIntermediate();
 *     // Add relations.
 *     MUL->LinksTo({MUL_out});
 *     MUL_out->LinksTo({ELE});
221
 *
Y
Yan Chunwei 已提交
222 223
 * One can add more specific asserts for PDNodes or edges, both the Operator
 * and Variable Nodes can be ruled in PDNode.assert_more(...).
224 225 226 227 228 229 230 231 232 233 234
 *
 * PDPattern can record the general patterns, such as the pattern represents
 *   - Op in CPU -> Op in GPU -> Op in CPU, to findout the IO abnormal place.
 *   - Ops whose inputs and outputs share the same variables
 */
class PDPattern {
 public:
  using edge_t = std::pair<PDNode*, PDNode*>;

  void AddEdge(PDNode* a, PDNode* b);

235
  PDNode* NewNode(PDNode::teller_t&& teller, const std::string& name = NewID());
Y
Yan Chunwei 已提交
236
  PDNode* NewNode(const std::string& name = NewID());
237 238 239
  PDNode* NewNode(const std::string& prefix, const std::string& name) {
    return NewNode(prefix + "/" + name);
  }
Y
Yan Chunwei 已提交
240
  PDNode* RetrieveNode(const std::string& id) const;
241 242 243 244

  const std::vector<std::unique_ptr<PDNode>>& nodes() const { return nodes_; }
  const std::vector<edge_t>& edges() const { return edges_; }

245 246
  std::string DotString() const;

247 248 249 250 251 252
 private:
#ifdef PADDLE_WITH_TESTING
  FRIEND_TEST(PDPattern, AddEdge);
  FRIEND_TEST(PDPattern, NewNode);
#endif

253 254
  static std::string NewID() { return "pdnode-" + std::to_string(id_++); }

255 256
  std::vector<std::unique_ptr<PDNode>> nodes_;
  std::vector<edge_t> edges_;
257
  std::map<std::string, PDNode*> node_map_;
258
  static size_t id_;
259 260 261
};

/*
262
 * GraphPatternDetector helps to detect the specific patterns in the graph.
263 264 265 266 267 268 269 270 271 272 273
 * Input a pattern, output a list of the matched subgraphs/nodes.
 * This helper can be used to support fuse(conv+batchnorm => batchnorm e.g.).
 *
 * The algorithm has three phases:
 *   1. Mark the nodes that match the defined PDNodes in a PDPattern,
 *   2. Extend a PDNode to subgraphs by deducing the connection relation defined
 *      in PAPattern(the edges),
 *   3. Get the filtered subgraphs and treat them with a pre-defined handler.
 *
 * Usage:
 *    // Create a detector
274
 *    GraphPatternDetector detector;
275 276 277 278 279 280 281 282
 *    // Define the detector's pattern, by adding PDNode and define the edges.
 *    auto* node0 = detector.mutable_pattern().AddNode(...)
 *    auto* node1 = detector.mutable_pattern().AddNode(...)
 *    node0->teller = some lambda.
 *    node1->teller = some lambda.
 *    detector.mutable_pattern().AddEdge(node0, node1);
 *    // Create an handler, to define the behavior of treating the filtered
 *    // subgraphs that comply with the patterns.
283
 *    GraphPatternDetector::handle_t handler = some labmda
284 285 286
 *    // Execute the detector.
 *    detector(&graph, handler);
 */
287
class GraphPatternDetector {
288
 public:
J
JingZhuangzhuang 已提交
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
  struct NodeIdCompare {
    bool operator()(Node* node1, Node* node2) const {
      return node1->id() < node2->id();
    }
  };

  struct PDNodeCompare {
    bool operator()(const PDNode* node1, const PDNode* node2) const {
      auto& nodes1 = node1->pdpattern()->nodes();
      auto& nodes2 = node2->pdpattern()->nodes();
      if (nodes1.size() != nodes2.size()) {
        return nodes1.size() < nodes2.size();
      } else {
        std::string pdnode_hash_key1 = "";
        std::string pdnode_hash_key2 = "";
        for (auto& node : nodes1) {
          pdnode_hash_key1 += node.get()->name();
          pdnode_hash_key1 += "#";
        }
        pdnode_hash_key1 += node1->name();
        for (auto& node : nodes2) {
          pdnode_hash_key2 += node.get()->name();
          pdnode_hash_key2 += "#";
        }
        pdnode_hash_key2 += node2->name();

        auto pdnode_key1 =
            std::to_string(std::hash<std::string>()(pdnode_hash_key1));
        auto pdnode_key2 =
            std::to_string(std::hash<std::string>()(pdnode_hash_key2));

        return pdnode_key1 < pdnode_key2;
      }
      return false;
    }
  };

  using subgraph_t = std::map<PDNode*, Node*, PDNodeCompare>;
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346

  // Operate on the detected pattern.
  using handle_t =
      std::function<void(const subgraph_t& /*hitted pattern*/, Graph*)>;

  void operator()(Graph* graph, handle_t handler);

  const PDPattern& pattern() const { return pattern_; }
  PDPattern* mutable_pattern() { return &pattern_; }

 private:
  // Mark the nodes that fits the pattern.
  bool MarkPDNodesInGraph(const ir::Graph& graph);

  // Detect all the pattern and output the hit records.
  std::vector<subgraph_t> DetectPatterns();

  // Remove duplicate patterns.
  void UniquePatterns(std::vector<subgraph_t>* subgraphs);

Z
Zhang Ting 已提交
347 348 349 350 351 352
  // Sort subgraphs, sort subgraphs by the specified node so that
  // the removed forward and backward subgraphs are corresponding
  // when two subgraphs are overlapped. Note: this function is
  // currently only used for bn_add_act, refer to PR28196 for details.
  void SortSubgraphs(std::vector<subgraph_t>* subgraphs);

353
  // Remove overlapped match subgraphs, when overlapped, keep the previous one.
Y
Yan Chunwei 已提交
354 355
  // The intermediate PDNodes will be removed, so can't shared by multiple
  // patterns.
356 357
  void RemoveOverlappedMatch(std::vector<subgraph_t>* subgraphs);

Y
Yan Chunwei 已提交
358 359 360
  // Validate whether the intermediate nodes are linked by external nodes.
  void ValidateByNodeRole(std::vector<subgraph_t>* subgraphs);

361 362 363 364 365 366 367 368 369
#ifdef PADDLE_WITH_TESTING
  FRIEND_TEST(GraphPatternDetecter, MarkPDNodesInGraph);
  FRIEND_TEST(GraphPatternDetecter, DetectPatterns);
#endif

 private:
  using hit_rcd_t =
      std::pair<Node* /*node in graph*/, PDNode* /*node in pattern*/>;
  PDPattern pattern_;
J
JingZhuangzhuang 已提交
370 371
  std::map<const PDNode*, std::set<Node*, NodeIdCompare>, PDNodeCompare>
      pdnodes2nodes_;
372 373
};

374 375
// some helper methods.

376 377 378 379 380
// Tell if a var links to an Op
bool VarLinksToOp(Node* node, const std::string& op_type);

// Tell if an op links to a var
bool VarLinksFromOp(Node* node, const std::string& op_type);
381 382

// Check whether a var node is a op node's nth input.
383
bool IsNthInput(Node* var, Node* op, const std::string& argument, size_t nth);
384

385 386 387
// Check whether the op node has input of given name.
bool HasInput(Node* op, const std::string& argument);

388 389 390
// Check whether the op node has output of given name.
bool HasOutput(Node* op, const std::string& argument);

391 392 393 394
// Tell whether a var node is a op node's nth output.
bool IsNthOutput(Node* var, Node* op, const std::string& argument, size_t nth);

// Graph safely remove some nodes, will automatically clean up the edges.
395 396 397 398
void GraphSafeRemoveNodes(
    Graph* graph,
    const std::unordered_set<const Node*>& nodes,
    std::unordered_set<std::shared_ptr<Node>>* saved_nodes = nullptr);
399 400

// Some pre-defined patterns those can be reused in multiple passes.
401 402
// The related Fluid Layer or Op should be one pattern here for better re-usage
// across different fusion.
403 404
namespace patterns {

Y
Yan Chunwei 已提交
405 406 407 408 409 410
struct KeyCounter {
  static KeyCounter& Instance() {
    static KeyCounter x;
    return x;
  }

411 412 413 414 415 416 417
#ifdef PADDLE_WITH_TENSORRT
  static int IncCounter(const std::string& key) { return dic_[key]++; }
  static void CleanCounter() { dic_.clear(); }

 private:
  static thread_local std::unordered_map<std::string, size_t> dic_;
#else
Y
Yan Chunwei 已提交
418 419 420 421
  int IncCounter(const std::string& key) { return dic_[key]++; }

 private:
  std::unordered_map<std::string, size_t> dic_;
422
#endif
Y
Yan Chunwei 已提交
423 424 425 426 427
};

// Generate a unique PDNode's name with name_scope and id.
// The format is {name_scope}/{repr}/{id}/{name}
static std::string PDNodeName(const std::string& name_scope,
428 429
                              const std::string& repr,
                              size_t id,
Y
Yan Chunwei 已提交
430 431 432 433 434 435 436
                              const std::string& name) {
  return string::Sprintf("%s/%s/%d/%s", name_scope, repr, id, name);
}
// Generate a unique PDNode's name.
// The format is {name_scope}/{repr}/{id}
static std::string PDNodeName(const std::string& name_scope,
                              const std::string& repr) {
437 438
  return string::Sprintf(
      "%s/%s/%d", name_scope, repr, KeyCounter::Instance().IncCounter(repr));
Y
Yan Chunwei 已提交
439 440 441 442 443
}
// Generate a unique key. It can be used for a universally unique temporary
// name.
// The format is {repr}/{id}
static std::string UniqueKey(const std::string& repr) {
444 445
  return string::Sprintf(
      "%s/%d", repr, KeyCounter::Instance().IncCounter(repr));
Y
Yan Chunwei 已提交
446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
}

// Declare a PDNode in a pattern, will create two methods:
// std::string xxx_repr(); return this PDNode's string id.
// PDNode* xxx_n(); return the corresponding PDNode.
#define PATTERN_DECL_NODE(name__)                        \
  std::string name__##_repr() const {                    \
    return PDNodeName(name_scope_, repr_, id_, #name__); \
  }                                                      \
  PDNode* name__##_n() const { return pattern->RetrieveNode(name__##_repr()); }

// Get an ir::Node* from the matched subgraph.
// var: variable.
// arg: the argument declared by PATTERN_DECL_NODE in a pattern definition.
// pat: the pattern object.
461
#define GET_IR_NODE_FROM_SUBGRAPH(var, arg, pat)                               \
462 463
  PADDLE_ENFORCE_NE(subgraph.count(pat.arg##_n()),                             \
                    0UL,                                                       \
464 465 466
                    platform::errors::NotFound("Node not found for PDNode %s", \
                                               pat.arg##_repr()));             \
  Node* var = subgraph.at(pat.arg##_n());                                      \
467 468 469
  PADDLE_ENFORCE_NOT_NULL(var,                                                 \
                          platform::errors::NotFound(                          \
                              "node %s not exists in the sub-graph", #arg));
Y
Yan Chunwei 已提交
470 471 472

// The base class of all the patterns.
struct PatternBase {
473 474
  PatternBase(PDPattern* pattern,
              const std::string& name_scope,
Y
Yan Chunwei 已提交
475 476 477 478 479 480 481 482 483 484 485 486 487 488
              const std::string& repr)
      : pattern(pattern),
        name_scope_(name_scope),
        repr_(repr),
        id_(KeyCounter::Instance().IncCounter(repr)) {}

  PDPattern* pattern;

 protected:
  std::string name_scope_;
  std::string repr_;
  size_t id_;
};

S
Sylwester Fraczek 已提交
489 490 491 492 493 494 495 496 497 498 499
// Conv with batch norm
// op: conv + (elementwise_add +) batch_norm
// named nodes:
// conv_weight, conv_out, conv,
// bn_x, bn_scale, bn_bias, bn_mean,  bn_variance,
// bn_batch_norm, bn_y, bn_mean_out, bn_variance_out,
// bn_saved_mean, bn_saved_variance
struct ConvBN : public PatternBase {
  ConvBN(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "conv_bn") {}

500 501
  PDNode* operator()(PDNode* conv_input,
                     const std::string& conv_type,
502
                     bool with_eltwise_add);
S
Sylwester Fraczek 已提交
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528

  // declare operator node's name
  PATTERN_DECL_NODE(conv);
  PATTERN_DECL_NODE(batch_norm);
  PATTERN_DECL_NODE(eltwise);  // ELEMENTWISE_ADD
  // CONV inputs
  PATTERN_DECL_NODE(conv_weight);  // Filter
  // CONV outputs
  PATTERN_DECL_NODE(conv_out);  // tmp
  // ELTWISE inputs
  PATTERN_DECL_NODE(eltwise_y_in);
  // ELTWISE outputs
  PATTERN_DECL_NODE(eltwise_out);  // tmp
  // BN inputs
  PATTERN_DECL_NODE(bn_scale);
  PATTERN_DECL_NODE(bn_bias);
  PATTERN_DECL_NODE(bn_mean);
  PATTERN_DECL_NODE(bn_variance);
  // BN outputs
  PATTERN_DECL_NODE(bn_out);  // Out
  PATTERN_DECL_NODE(bn_mean_out);
  PATTERN_DECL_NODE(bn_variance_out);
  PATTERN_DECL_NODE(bn_saved_mean);
  PATTERN_DECL_NODE(bn_saved_variance);
};

529 530 531 532 533 534 535 536 537 538 539 540 541
struct LayerNormShiftScale : public PatternBase {
  LayerNormShiftScale(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "layer_norm_shift_scale") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(layer_norm_in);
  PATTERN_DECL_NODE(layer_norm_op);
  PATTERN_DECL_NODE(layer_norm_bias);
  PATTERN_DECL_NODE(layer_norm_scale);
  PATTERN_DECL_NODE(layer_norm_out);
};

542 543 544
struct OperatorActivation : public PatternBase {
  OperatorActivation(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "operator_activation") {}
545

546
  PDNode* operator()(const std::string& operator_type,
547 548
                     const std::string& activation_type);

549 550
  PATTERN_DECL_NODE(preceding_op);
  PATTERN_DECL_NODE(preceding_op_out);
551 552 553 554
  PATTERN_DECL_NODE(activation);
  PATTERN_DECL_NODE(activation_out);
};

555 556 557 558 559 560 561 562 563 564 565 566
struct Squeeze2Transpose2 : public PatternBase {
  Squeeze2Transpose2(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "squeeze2_transpose2") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(squeeze2_op_in);
  PATTERN_DECL_NODE(squeeze2_op);
  PATTERN_DECL_NODE(squeeze2_op_out);
  PATTERN_DECL_NODE(transpose2_op);
};

567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
struct OperatorUnsqueeze2 : public PatternBase {
  OperatorUnsqueeze2(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "operator_unsqueeze2") {}

  PDNode* operator()(const std::string& operator_type,
                     const int num_of_outputs);

  PATTERN_DECL_NODE(preceding_op);
  PATTERN_DECL_NODE(preceding_op_out);
  PATTERN_DECL_NODE(unsqueeze2_op);
  PATTERN_DECL_NODE(unsqueeze2_out);
};

struct OperatorReshape2 : public PatternBase {
  OperatorReshape2(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "operator_reshape2") {}

  PDNode* operator()(const std::string& operator_type,
                     const int num_of_outputs);

  PATTERN_DECL_NODE(preceding_op);
  PATTERN_DECL_NODE(preceding_op_out);
  PATTERN_DECL_NODE(reshape2_op);
  PATTERN_DECL_NODE(reshape2_out);
};

T
tensor-tang 已提交
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
// SEQCONV with Elementwise_Add ReLU
// op: seqconv + elementwise_add + relu
// named nodes:
// seqconv_input, seqconv_weight,
// seqconv_out, seqconv,
// elementwise_add_bias, elementwise_add_out, elementwise_add
// relu_out, relu
struct SeqConvEltAddRelu : public PatternBase {
  SeqConvEltAddRelu(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "seqconv_eltadd_relu") {}

  PDNode* operator()(PDNode* seqconv_input);

  // declare operator node's name
  PATTERN_DECL_NODE(seqconv);
  PATTERN_DECL_NODE(eltadd);
  PATTERN_DECL_NODE(relu);
  // declare variable node's name
  PATTERN_DECL_NODE(seqconv_weight);
  PATTERN_DECL_NODE(seqconv_out);
  PATTERN_DECL_NODE(eltadd_bias);
  PATTERN_DECL_NODE(eltadd_out);
  PATTERN_DECL_NODE(relu_out);
};

618 619 620 621 622
// FC with bias
// op: mul + elementwise_add
// named nodes:
// mul, elementwise_add
// w, mul_out, bias, fc_out
Y
Yan Chunwei 已提交
623 624 625 626
struct FC : public PatternBase {
  FC(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "fc") {}

627
  PDNode* operator()(PDNode* x, bool with_bias, bool with_relu);
Y
Yan Chunwei 已提交
628 629 630 631 632

  // declare operator node's name
  PATTERN_DECL_NODE(fc);
  PATTERN_DECL_NODE(mul);
  PATTERN_DECL_NODE(elementwise_add);
633
  PATTERN_DECL_NODE(relu);
Y
Yan Chunwei 已提交
634 635 636 637
  // declare variable node's name
  PATTERN_DECL_NODE(w);
  PATTERN_DECL_NODE(mul_out);  // (x,w) -> mul_out
  PATTERN_DECL_NODE(bias);
638 639
  PATTERN_DECL_NODE(elementwise_add_out);
  PATTERN_DECL_NODE(relu_out);
Y
Yan Chunwei 已提交
640 641
};

642 643 644 645
// MKL-DNN's FC with bias
// op: fc
// named node:
// fc
646
// w, bias, output, residual_data
647 648 649 650
struct FCMKLDNN : public PatternBase {
  FCMKLDNN(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "fc_mkldnn") {}

651
  PDNode* operator()(bool with_residual_data);
652 653 654 655

  // declare operator node's name
  PATTERN_DECL_NODE(fc);
  // declare variable node's name
M
Michał Gallus 已提交
656
  PATTERN_DECL_NODE(input);
657 658 659
  PATTERN_DECL_NODE(weights);
  PATTERN_DECL_NODE(bias);
  PATTERN_DECL_NODE(output);
660
  PATTERN_DECL_NODE(residual_data);
661 662
};

663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
// Embedding
struct Embedding : public PatternBase {
  Embedding(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "embedding") {}

  PDNode* operator()(PDNode* x);

  // declare operator node's name
  PATTERN_DECL_NODE(lookup_table);
  // Inputs
  //
  PATTERN_DECL_NODE(Ids);
  PATTERN_DECL_NODE(W);  // embeddings
  // Outputs
  PATTERN_DECL_NODE(Out);
};

Y
Yan Chunwei 已提交
680 681 682
struct LSTM : public PatternBase {
  LSTM(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "lstm") {}
683

Y
Yan Chunwei 已提交
684
  PDNode* operator()(PDNode* x);
685

Y
Yan Chunwei 已提交
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
  // Operators
  PATTERN_DECL_NODE(lstm);

  // Inputs
  PATTERN_DECL_NODE(Input);
  PATTERN_DECL_NODE(H0);
  PATTERN_DECL_NODE(C0);
  PATTERN_DECL_NODE(Weight);
  PATTERN_DECL_NODE(Bias);

  // Outputs
  PATTERN_DECL_NODE(Hidden);
  PATTERN_DECL_NODE(Cell);
  PATTERN_DECL_NODE(BatchGate);
  PATTERN_DECL_NODE(BatchCellPreAct);
};

struct GRU : public PatternBase {
  GRU(PDPattern* pattern, const std::string& name_scope)
S
superjomn 已提交
705
      : PatternBase(pattern, name_scope, "gru") {}
Y
Yan Chunwei 已提交
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721

  PDNode* operator()(PDNode* x);

  // Operators
  PATTERN_DECL_NODE(gru);

  // Inputs
  PATTERN_DECL_NODE(Bias);
  PATTERN_DECL_NODE(Weight);

  // Outputs
  PATTERN_DECL_NODE(BatchGate);
  PATTERN_DECL_NODE(BatchResetHiddenPrev);
  PATTERN_DECL_NODE(BatchHidden);
  PATTERN_DECL_NODE(Hidden);
};
T
tensor-tang 已提交
722

Z
Zhen Wang 已提交
723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
// The following pattern is used to fuse batch_norm and act
// formula: act(bn(x))
// op: batch_norm + act
struct BatchNormAct : public PatternBase {
  BatchNormAct(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "bn_act") {}

  PDNode* operator()(PDNode* x, std::unordered_set<std::string> acts);

  // declare operator node's name
  PATTERN_DECL_NODE(batch_norm);
  PATTERN_DECL_NODE(act);
  // declare variable node's name
  // BN inputs
  PATTERN_DECL_NODE(bn_scale);
  PATTERN_DECL_NODE(bn_bias);
  PATTERN_DECL_NODE(bn_variance);
  PATTERN_DECL_NODE(bn_mean);
  // BN outputs
  PATTERN_DECL_NODE(bn_mean_out);
  PATTERN_DECL_NODE(bn_variance_out);
  PATTERN_DECL_NODE(bn_saved_variance);
  PATTERN_DECL_NODE(bn_saved_mean);
  PATTERN_DECL_NODE(bn_reserve_space);
  PATTERN_DECL_NODE(bn_out);
  // ACT output
  PATTERN_DECL_NODE(act_out);
};

// the backward of act(bn(x))
// op: batch_norm_grad + act_grad
struct BatchNormActGrad : public PatternBase {
  BatchNormActGrad(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "bn_act_grad") {}

  // act_grad: in["Out", "Out@GRAD"], out["X@GRAD"]
  // bn_grad: in["X", "Y@GRAD", "Scale", "Bias", "SavedMean", "SavedVariance",
  // "ReserveSpace"],
  // out["X@GRAD", "Scale@GRAD", "Bias@GRAD"]
  PDNode* operator()(PDNode* x, std::unordered_set<std::string> act_grad_types);

  // declare operator node's name
  PATTERN_DECL_NODE(act_grad);
  PATTERN_DECL_NODE(batch_norm_grad);
  // declare variable node's name
  PATTERN_DECL_NODE(act_out);
  PATTERN_DECL_NODE(d_itermediate_out);
  PATTERN_DECL_NODE(bn_x);
  PATTERN_DECL_NODE(bn_scale);
  PATTERN_DECL_NODE(bn_bias);
  PATTERN_DECL_NODE(bn_saved_mean);
  PATTERN_DECL_NODE(bn_saved_variance);
  PATTERN_DECL_NODE(bn_reserve_space);
  PATTERN_DECL_NODE(d_bn_x);
  PATTERN_DECL_NODE(d_bn_scale);
  PATTERN_DECL_NODE(d_bn_bias);
};

781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
//
// \brief   Pattern looking for batch_norm and a directly following activation
// operator.
//
// \note    Currently only ReLU is supported as an activation function.
//          Formula: act(bn(x))
//          Op: batch_norm + act
struct BatchNormActOneDNN : public PatternBase {
  BatchNormActOneDNN(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "bn_act_onednn") {}

  PDNode* operator()(const std::string& act_type);

  // declare operator node's name
  PATTERN_DECL_NODE(bn_in);
  PATTERN_DECL_NODE(batch_norm);
  PATTERN_DECL_NODE(act);
  PATTERN_DECL_NODE(bn_out);
  PATTERN_DECL_NODE(act_out);
};

Z
Zhang Ting 已提交
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
// The following pattern is used to fuse batch_norm, elewise_add, and act
// formula: act(bn(x) + z)
// op: batch_norm + elewise_add + act
struct BatchNormAddAct : public PatternBase {
  BatchNormAddAct(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "bn_add_act") {}

  PDNode* operator()(PDNode* x, std::unordered_set<std::string> acts);

  // declare operator node's name
  PATTERN_DECL_NODE(batch_norm);
  PATTERN_DECL_NODE(elewise_add);
  PATTERN_DECL_NODE(act);
  // declare variable node's name
  // BN inputs
  PATTERN_DECL_NODE(bn_scale);
  PATTERN_DECL_NODE(bn_bias);
  // BN outputs
  PATTERN_DECL_NODE(bn_mean_out);
  PATTERN_DECL_NODE(bn_variance_out);
  PATTERN_DECL_NODE(bn_saved_variance);
  PATTERN_DECL_NODE(bn_saved_mean);
  PATTERN_DECL_NODE(bn_reserve_space);
  PATTERN_DECL_NODE(bn_out);
  // Elewise_Add input
  PATTERN_DECL_NODE(elewise_add_in);
  // Elewise_Add output
  PATTERN_DECL_NODE(elewise_add_out);
  // ACT output
  PATTERN_DECL_NODE(act_out);
};

// the backward of act(bn(x) + z)
// op: batch_norm_grad + elewise_add_grad + act_grad
struct BatchNormAddActGrad : public PatternBase {
  BatchNormAddActGrad(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "bn_add_act_grad") {}

  // act_grad: in["Out", "Out@GRAD"], out["X@GRAD"]
  // elewise_add_grad: in["Out@GRAD"], out["X@GRAD", "Y@GRAD"]
  // bn_grad: in["X", "Z", "Y@GRAD", "Scale", "Bias", "SavedMean",
  // "SavedVariance",
  // "ReserveSpace"],
  // out["X@GRAD", "Z@GRAD", "Scale@GRAD", "Bias@GRAD"]
  PDNode* operator()(PDNode* x, std::unordered_set<std::string> act_grad_types);

  // declare operator node's name
  PATTERN_DECL_NODE(act_grad);
  PATTERN_DECL_NODE(elewise_add_grad);
  PATTERN_DECL_NODE(batch_norm_grad);
  // declare variable node's name
  PATTERN_DECL_NODE(act_out);
  PATTERN_DECL_NODE(d_act_x);
  PATTERN_DECL_NODE(d_elewise_add_in);
  PATTERN_DECL_NODE(d_bn_out);
  PATTERN_DECL_NODE(bn_x);
  PATTERN_DECL_NODE(bn_scale);
  PATTERN_DECL_NODE(bn_bias);
  PATTERN_DECL_NODE(bn_saved_mean);
  PATTERN_DECL_NODE(bn_saved_variance);
  PATTERN_DECL_NODE(bn_reserve_space);
  PATTERN_DECL_NODE(d_bn_x);
  PATTERN_DECL_NODE(d_bn_scale);
  PATTERN_DECL_NODE(d_bn_bias);
};

C
chengduo 已提交
868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929
// The following patterns are used to fuse elewise_add and act
// formula: act(ele_add(x, y))
// op: elementwise_add + act
// named nodes: elementwise_add, act
//              ele_x, ele_y, elewise_add_out, act_out
struct ElewiseAddAct : public PatternBase {
  ElewiseAddAct(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "elewise_add_act") {}

  PDNode* operator()(PDNode* x, std::unordered_set<std::string> acts);

  // declare operator node's name
  PATTERN_DECL_NODE(ele_add);
  PATTERN_DECL_NODE(act);
  // declare variable node's name
  PATTERN_DECL_NODE(elewise_add_out);
  PATTERN_DECL_NODE(ele_y);
  PATTERN_DECL_NODE(act_out);
};

// formula: ele_add(x, act(y))
// op: elementwise_add + act
// named nodes: elementwise_add, act
//              act_in, act_out, ele_x, elewise_add_out
struct ActElewiseAdd : public PatternBase {
  ActElewiseAdd(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "act_elewise_add") {}

  PDNode* operator()(PDNode* x, std::unordered_set<std::string> acts);

  // declare operator node's name
  PATTERN_DECL_NODE(act);
  PATTERN_DECL_NODE(ele_add);
  // declare variable node's name
  PATTERN_DECL_NODE(act_out);
  PATTERN_DECL_NODE(ele_x);
  PATTERN_DECL_NODE(elewise_add_out);
};

// the backward of act(ele_add(x, y))
// the act is inplace.
// op: elementwise_add_grad + act_grad
// named nodes: elementwise_add_grad, act_grad
//              act_out, act_out_g, ele_y, d_itermediate_out, d_ele_x, d_ele_y
struct ElewiseAddActInplaceGrad : public PatternBase {
  ElewiseAddActInplaceGrad(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "elewise_add_act_grad1") {}

  // act_grad: in["Out", "Out@GRAD"], out["X@GRAD"]
  // ele_add_grad: in["Y", "Out@GRAD"], out["X@GRAD", "Y@GRAD"]
  PDNode* operator()(PDNode* x, std::unordered_set<std::string> acts);

  // declare operator node's name
  PATTERN_DECL_NODE(act_grad);
  PATTERN_DECL_NODE(ele_add_grad);
  // declare variable node's name
  PATTERN_DECL_NODE(act_out);
  PATTERN_DECL_NODE(d_itermediate_out);
  PATTERN_DECL_NODE(d_ele_x);
  PATTERN_DECL_NODE(d_ele_y);
  PATTERN_DECL_NODE(ele_y);
};
M
Michal Gallus 已提交
930

931 932 933 934 935 936 937 938 939 940 941 942
// The following patterns are used to fuse linear and act (ReLu or GeLU)
// formula: act(F.linear(x))
// op: matmul_v2 + elementwise_add + act
// named nodes: matmul, elementwise_add, act
//              matmul_w, matmul_out
//              ele_bias, elewise_add_out, act_out
struct LinearAct : public PatternBase {
  LinearAct(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "linear_act") {}

  PDNode* operator()(PDNode* x,
                     const std::unordered_set<std::string>& act_types,
943 944
                     bool with_grad_link,
                     bool is_act_grad_x_from_act);
945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972

  // declare operator node's name
  PATTERN_DECL_NODE(matmul);
  PATTERN_DECL_NODE(ele_add);
  PATTERN_DECL_NODE(act);
  PATTERN_DECL_NODE(act_grad);
  // declare variable node's name
  PATTERN_DECL_NODE(matmul_w);
  PATTERN_DECL_NODE(matmul_out);
  PATTERN_DECL_NODE(elewise_add_out);
  PATTERN_DECL_NODE(ele_bias);
  PATTERN_DECL_NODE(act_out);
};

// The following patterns are used to fuse linear_grad and act_grad (ReLu or
// GeLU)
// formula: the backward of F.linear( act(x) )
// op: elementwise_add_grad + matmul_v2_grad + act_grad
// named nodes: ele_add_grad, matmul_grad, act_grad
//              ele_grad_bias, ele_grad_dx, ele_grad_dbias
//              matmul_grad_x, matmul_grad_dx, matmul_grad_dx
//              matmul_grad_dw, act_grad_dx
struct ElewiseAddMatmulAct : public PatternBase {
  ElewiseAddMatmulAct(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "elewiseadd_matmul_act") {}

  PDNode* operator()(PDNode* x,
                     const std::unordered_set<std::string>& act_grad_types,
973 974
                     bool without_x_gradient,
                     bool is_act_grad_x_from_act);
975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991

  // declare operator node's name
  PATTERN_DECL_NODE(ele_add_grad);
  PATTERN_DECL_NODE(matmul_grad);
  PATTERN_DECL_NODE(act_grad);
  // declare variable node's name
  PATTERN_DECL_NODE(ele_out);
  PATTERN_DECL_NODE(ele_grad_bias);
  PATTERN_DECL_NODE(ele_grad_dx);
  PATTERN_DECL_NODE(ele_grad_dbias);
  PATTERN_DECL_NODE(matmul_grad_x);
  PATTERN_DECL_NODE(matmul_grad_w);
  PATTERN_DECL_NODE(matmul_grad_dx);
  PATTERN_DECL_NODE(matmul_grad_dw);
  PATTERN_DECL_NODE(act_grad_dx);
};

M
Michal Gallus 已提交
992 993 994 995 996 997 998 999 1000 1001
// Conv with Elementwise_add as bias
// op: conv + elementwise_add
// named nodes:
// conv_input, conv_weight,
// conv_out, conv,
// eltwise_bias, eltwise_out,
// elementwise_add
struct ConvBias : public PatternBase {
  ConvBias(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "conv_bias") {}
1002
  PDNode* operator()(PDNode* conv_input, std::string conv_type = "conv2d");
M
Michal Gallus 已提交
1003 1004 1005 1006 1007 1008 1009 1010 1011
  // declare operator node's name
  PATTERN_DECL_NODE(conv);
  PATTERN_DECL_NODE(eltwise);
  // declare variable node's name
  PATTERN_DECL_NODE(conv_weight);
  PATTERN_DECL_NODE(conv_out);
  PATTERN_DECL_NODE(eltwise_bias);
  PATTERN_DECL_NODE(eltwise_out);
};
1012

1013 1014 1015 1016 1017 1018 1019 1020 1021
// Convolution op
// Forward pass for convolution.
// conv_input, conv_bias and conv_filter are inputs.
// conv_output is a result of the operator.
// residual_data is data used by skip connection.
// If residual connection fusion is on, the formula is:
// conv_output = conv_op(conv_filter, conv_input, conv_bias)
//             + conv_residual_data
// If the fusion is off, conv_residual_data is not added.
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
struct Conv : public PatternBase {
  Conv(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "convolution") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(conv_op);
  PATTERN_DECL_NODE(conv_input);
  PATTERN_DECL_NODE(conv_filter);
  PATTERN_DECL_NODE(conv_output);
};

1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
// Convolution op with residual data
struct ConvResidual : public PatternBase {
  ConvResidual(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "conv_residual") {}

  PDNode* operator()(bool with_residual_data);

  PATTERN_DECL_NODE(conv_op);
  PATTERN_DECL_NODE(conv_input);
  PATTERN_DECL_NODE(conv_filter);
  PATTERN_DECL_NODE(conv_residual_data);
  PATTERN_DECL_NODE(conv_output);
};

// Pool op
// Forward pass for pooling.
// pool_input is the input.
// pool_output is a result of the operator.
struct Pool : public PatternBase {
  Pool(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "pooling") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(pool_op);
  PATTERN_DECL_NODE(pool_input);
  PATTERN_DECL_NODE(pool_output);
};

Z
Zuza 已提交
1063
// Elementwise ops
1064
// Forward pass for element-wise operators
1065
// elementwise_out is the result of the operator
Z
Zuza 已提交
1066 1067 1068 1069
struct Elementwise : public PatternBase {
  Elementwise(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "elementwise") {}

1070 1071
  PDNode* operator()(PDNode* x_var,
                     PDNode* y_var,
1072
                     const std::string& elementwise_type);
Z
Zuza 已提交
1073 1074 1075 1076 1077

  PATTERN_DECL_NODE(elementwise_op);
  PATTERN_DECL_NODE(elementwise_x);
  PATTERN_DECL_NODE(elementwise_y);
  PATTERN_DECL_NODE(elementwise_out);
1078
};
1079

1080 1081 1082 1083 1084 1085 1086
// Elementwise ops
// Forward pass for element-wise operators
// elementwise_out is the result of the operator
struct ElementwiseOp : public PatternBase {
  ElementwiseOp(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "elementwise") {}

1087
  PDNode* operator()(const std::string& elementwise_type);
1088 1089 1090 1091 1092

  PATTERN_DECL_NODE(elementwise_op);
  PATTERN_DECL_NODE(elementwise_out);
};

1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
struct MatmulElementwiseAdd : public PatternBase {
  MatmulElementwiseAdd(PDPattern* pattern,
                       const std::string& name_scope,
                       const std::string& matmul_type,
                       bool as_x)
      : PatternBase(pattern, name_scope, "matmul_elementwise_add") {}

  PDNode* operator()(const std::string& matmul_type, bool as_x);
  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(matmul_out);
  PATTERN_DECL_NODE(elementwise_addend);
  PATTERN_DECL_NODE(elementwise_add_op);
  PATTERN_DECL_NODE(elementwise_add_out);
};

1108 1109 1110 1111
// Residual Elementwise ops
// This pattern allows operator output to be X or Y
// and residual data Y or X, based on as_x flag
struct ResidualElementwise : public PatternBase {
1112 1113
  ResidualElementwise(PDPattern* pattern,
                      const std::string& name_scope,
1114 1115
                      bool as_x)
      : PatternBase(pattern, name_scope, "residual_elementwise") {}
1116 1117
  PDNode* operator()(PDNode* op_var,
                     PDNode* residual_var,
1118
                     const std::string& elementwise_type,
1119
                     bool as_x);
1120 1121 1122 1123 1124 1125 1126

  PATTERN_DECL_NODE(operator_output);
  PATTERN_DECL_NODE(residual_data);
  PATTERN_DECL_NODE(elementwise_op);
  PATTERN_DECL_NODE(elementwise_out);
};

1127
// General struct for immutable ops:
P
Paulina Gacek 已提交
1128
// reshape, transpose, slice, shape, nearest-interp, split
1129 1130 1131 1132 1133
// Forward pass for no weights-op.
// immutable_out is a result of the operator.
struct Immutable : public PatternBase {
  Immutable(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "immutable") {}
1134

1135 1136
  PDNode* operator()(const std::string& immutable_type,
                     const std::string& input_name);
1137
  PATTERN_DECL_NODE(prev_op);
1138 1139 1140
  PATTERN_DECL_NODE(immutable_in);
  PATTERN_DECL_NODE(immutable_op);
  PATTERN_DECL_NODE(immutable_out);
1141 1142
};

1143 1144 1145 1146
// Matmul op
// Forward pass for matmul.
struct Matmul : public PatternBase {
  Matmul(PDPattern* pattern, const std::string& name_scope)
1147 1148 1149 1150 1151 1152 1153 1154 1155
      : PatternBase(pattern, name_scope, "matmul") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(matmul_in_x);
  PATTERN_DECL_NODE(matmul_in_y);
  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(matmul_out);
};

1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
// MatmulV2: tensor * weight
struct MatmulV2Weight : public PatternBase {
  MatmulV2Weight(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "matmul_v2_weight") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(matmul_v2_in_x);
  PATTERN_DECL_NODE(matmul_v2_in_y);
  PATTERN_DECL_NODE(matmul_v2_op);
  PATTERN_DECL_NODE(matmul_v2_out);
};

// MatmulV2: tensor * tensor or tensor * weight
1169 1170 1171 1172 1173
struct MatmulV2 : public PatternBase {
  MatmulV2(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "matmul_v2") {}

  PDNode* operator()();
1174 1175 1176 1177
  PATTERN_DECL_NODE(matmul_v2_in_x);
  PATTERN_DECL_NODE(matmul_v2_in_y);
  PATTERN_DECL_NODE(matmul_v2_op);
  PATTERN_DECL_NODE(matmul_v2_out);
1178 1179
};

H
heliqi 已提交
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
// Matmul + scale
// Forward pass.
struct MatmulScale : public PatternBase {
  MatmulScale(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "matmul_scale") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(matmul_in_x);
  PATTERN_DECL_NODE(matmul_in_y);
  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(scale_in_x);
  PATTERN_DECL_NODE(scale_op);
  PATTERN_DECL_NODE(scale_out);
};

// Matmul_v2 + scale
// Forward pass.
struct MatmulV2Scale : public PatternBase {
  MatmulV2Scale(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "matmul_v2_scale") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(matmul_v2_in_x);
  PATTERN_DECL_NODE(matmul_v2_in_y);
  PATTERN_DECL_NODE(matmul_v2_op);
  PATTERN_DECL_NODE(scale_in_x);
  PATTERN_DECL_NODE(scale_op);
  PATTERN_DECL_NODE(scale_out);
};

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
// Squeeze2 + Matmul
// Forward pass.
struct Squeeze2Matmul : public PatternBase {
  Squeeze2Matmul(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "squeeze2_matmul") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(squeeze2_in_x);
  PATTERN_DECL_NODE(squeeze2_op);
  PATTERN_DECL_NODE(matmul_in_x);
  PATTERN_DECL_NODE(matmul_in_y);
  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(matmul_out);
};

// Reshape2 + Matmul
// Forward pass.
struct Reshape2Matmul : public PatternBase {
  Reshape2Matmul(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "reshape2_matmul") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(reshape2_in_x);
  PATTERN_DECL_NODE(reshape2_op);
  PATTERN_DECL_NODE(matmul_in_x);
  PATTERN_DECL_NODE(matmul_in_y);
  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(matmul_out);
};

// Forward pass for two input ops and matmul op.
// matmul_out is a result of the operator.
struct MatmulWithInputOps : public PatternBase {
  MatmulWithInputOps(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "matmul_with_input_ops") {}
1245

1246
  PDNode* operator()(bool with_residual);
1247 1248 1249 1250 1251
  PATTERN_DECL_NODE(prev_op_x);
  PATTERN_DECL_NODE(prev_op_y);
  PATTERN_DECL_NODE(matmul_in_x);
  PATTERN_DECL_NODE(matmul_in_y);
  PATTERN_DECL_NODE(matmul_op);
1252
  PATTERN_DECL_NODE(matmul_residual_data);
1253 1254 1255
  PATTERN_DECL_NODE(matmul_out);
};

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
// Flatten2 + Matmul
// Forward pass.
struct Flatten2Matmul : public PatternBase {
  Flatten2Matmul(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "flatten2_matmul") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(flatten2_in_x);
  PATTERN_DECL_NODE(flatten2_op);
  PATTERN_DECL_NODE(matmul_in_x);
  PATTERN_DECL_NODE(matmul_in_y);
  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(matmul_out);
};

1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
// Concat op
// Forward pass for concat.
// concat_out is a result of the operator.
struct Concat : public PatternBase {
  Concat(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "concat") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(concat_op);
  PATTERN_DECL_NODE(concat_out);
};

J
joanna.wozna.intel 已提交
1284
// Op + Requant
1285
// named nodes:
J
joanna.wozna.intel 已提交
1286
// any_op, any_out
1287
// requant_op, requant_out
J
joanna.wozna.intel 已提交
1288 1289 1290
struct OpRequant : public PatternBase {
  OpRequant(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "op_requant") {}
1291 1292 1293

  PDNode* operator()();

J
joanna.wozna.intel 已提交
1294 1295
  PATTERN_DECL_NODE(any_op);
  PATTERN_DECL_NODE(requant_in);
1296 1297 1298 1299
  PATTERN_DECL_NODE(requant_op);
  PATTERN_DECL_NODE(requant_out);
};

1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315
// Requant + Op
// named nodes:
// requant_in, requant_op,
// requant_out, any_op
struct RequantOp : public PatternBase {
  RequantOp(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "requant_op") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(any_op);
  PATTERN_DECL_NODE(requant_in);
  PATTERN_DECL_NODE(requant_op);
  PATTERN_DECL_NODE(requant_out);
};

1316
// Op + Dequant
1317
// named nodes:
1318
// any_op, dequant_in
1319
// dequant_op, dequant_out
1320 1321 1322
struct OpDequant : public PatternBase {
  OpDequant(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "op_dequant") {}
1323 1324 1325

  PDNode* operator()();

1326 1327
  PATTERN_DECL_NODE(any_op);
  PATTERN_DECL_NODE(dequant_in);
1328 1329 1330 1331
  PATTERN_DECL_NODE(dequant_op);
  PATTERN_DECL_NODE(dequant_out);
};

1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
// Dequantize + Scale
struct DequantScale : public PatternBase {
  DequantScale(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "dequant_scale") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(dequant_op);
  PATTERN_DECL_NODE(dequant_out);
  PATTERN_DECL_NODE(scale_op);
  PATTERN_DECL_NODE(scale_out);
};

1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
// Scale + Quantize
struct ScaleQuant : public PatternBase {
  ScaleQuant(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "scale_quant") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(scale_in);
  PATTERN_DECL_NODE(scale_op);
  PATTERN_DECL_NODE(quant_in);
  PATTERN_DECL_NODE(quant_op);
};

// Quantize + Conv2d
struct QuantConv : public PatternBase {
  QuantConv(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "quant_conv") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(quant_in);
  PATTERN_DECL_NODE(quant_op);
  PATTERN_DECL_NODE(conv_in);
  PATTERN_DECL_NODE(conv_op);
};

1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
// Scale + Matmul
struct ScaleMatmul : public PatternBase {
  ScaleMatmul(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "scale_matmul") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(scale_in);
  PATTERN_DECL_NODE(scale_op);
  PATTERN_DECL_NODE(scale_out);
  PATTERN_DECL_NODE(matmul_op);
};

1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
// PriorBox operator
// operator: prior_box_op
// inputs: prior_box_input, prior_box_image
// outputs: prior_box_boxes, prior_box_variances
struct PriorBox : public PatternBase {
  PriorBox(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "PriorBox") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(prior_box_op);
  PATTERN_DECL_NODE(prior_box_input);
  PATTERN_DECL_NODE(prior_box_image);
  PATTERN_DECL_NODE(prior_box_boxes);
  PATTERN_DECL_NODE(prior_box_variances);
};

F
feng_shuai 已提交
1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
// vit_attention
struct VitAttention : public PatternBase {
  VitAttention(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "vit_attention") {}

  PDNode* operator()(PDNode* in);

  PATTERN_DECL_NODE(matmul0_op);
  PATTERN_DECL_NODE(matmul0_in_y);
  PATTERN_DECL_NODE(matmul0_out);

  PATTERN_DECL_NODE(elementwise0_op);
  PATTERN_DECL_NODE(elementwise0_in_y);
  PATTERN_DECL_NODE(elementwise0_out);

  PATTERN_DECL_NODE(reshape1_op);
  PATTERN_DECL_NODE(reshape1_out);

  PATTERN_DECL_NODE(transpose1_op);
  PATTERN_DECL_NODE(transpose1_out);

  PATTERN_DECL_NODE(slice1_op);
  PATTERN_DECL_NODE(slice1_out);

  PATTERN_DECL_NODE(slice2_op);
  PATTERN_DECL_NODE(slice2_out);

  PATTERN_DECL_NODE(slice3_op);
  PATTERN_DECL_NODE(slice3_out);

  PATTERN_DECL_NODE(matmul2_op);
  PATTERN_DECL_NODE(matmul2_out);

  PATTERN_DECL_NODE(matmul1_op);
  PATTERN_DECL_NODE(matmul1_out);

  PATTERN_DECL_NODE(transpose2_op);
  PATTERN_DECL_NODE(transpose2_out);

  PATTERN_DECL_NODE(scale1_op);
  PATTERN_DECL_NODE(scale1_out);

  PATTERN_DECL_NODE(softmax1_op);
  PATTERN_DECL_NODE(softmax1_out);

  PATTERN_DECL_NODE(transpose3_op);
  PATTERN_DECL_NODE(transpose3_out);

  PATTERN_DECL_NODE(reshape2_op);
  PATTERN_DECL_NODE(reshape2_out);
};

1452
// Conv + ElementwiseAdd + an activation
1453
// This pattern can further fuse the conv related ops after the conv+bn fusion.
1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
struct ConvElementwiseaddAct : public PatternBase {
  ConvElementwiseaddAct(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "conv_elementwiseadd_act") {}

  PDNode* operator()(PDNode* conv_in);

  PATTERN_DECL_NODE(conv_op);
  PATTERN_DECL_NODE(conv_out);
  PATTERN_DECL_NODE(conv_filter);

  PATTERN_DECL_NODE(elementwise_add_op);
  PATTERN_DECL_NODE(elementwise_add_in_y);  // input
  PATTERN_DECL_NODE(elementwise_add_out);

  PATTERN_DECL_NODE(act_op);
  PATTERN_DECL_NODE(act_out);
};

// Conv + ElementwiseAdd + ElementwiseAdd + Activation
struct ConvElementwiseadd2Act : public PatternBase {
  ConvElementwiseadd2Act(PDPattern* pattern, const std::string& name_scope)
1475 1476
      : PatternBase(
            pattern, name_scope, "conv_elementwiseadd2_elementwiseadd_act") {}
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495

  PDNode* operator()(PDNode* conv_in);

  PATTERN_DECL_NODE(conv_op);
  PATTERN_DECL_NODE(conv_filter);
  PATTERN_DECL_NODE(conv_out);

  PATTERN_DECL_NODE(elementwise_add_op);
  PATTERN_DECL_NODE(elementwise_add_in_y);  // input
  PATTERN_DECL_NODE(elementwise_add_out);

  PATTERN_DECL_NODE(elementwise_add_op_1);
  PATTERN_DECL_NODE(elementwise_add_in_y_1);  // input
  PATTERN_DECL_NODE(elementwise_add_out_1);

  PATTERN_DECL_NODE(act_op);
  PATTERN_DECL_NODE(act_out);
};

N
nhzlx 已提交
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
// Conv + ElementwiseAdd
// This pattern should be used after ConvElementwiseadd2Act or
// ConvElementwiseadd pass
struct ConvElementwiseadd : public PatternBase {
  ConvElementwiseadd(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "conv_elementwiseadd") {}

  PDNode* operator()(PDNode* conv_in);

  PATTERN_DECL_NODE(conv_op);
  PATTERN_DECL_NODE(conv_out);
  PATTERN_DECL_NODE(conv_filter);

  PATTERN_DECL_NODE(elementwise_add_op);
  PATTERN_DECL_NODE(elementwise_add_in_y);
  PATTERN_DECL_NODE(elementwise_add_out);
};

N
nhzlx 已提交
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545
// Conv with affine_channel
// op: conv + (elementwise_add +) affine_channel
// named nodes:
// conv_weight, conv_out, conv,
// ac_x, ac_scale, ac_bias
// affine_channel, ac_out
struct ConvAffineChannel : public PatternBase {
  ConvAffineChannel(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "conv_affine_channel") {}

  PDNode* operator()(PDNode* conv_input, bool with_eltwise_add);

  // declare operator node's name
  PATTERN_DECL_NODE(conv);
  PATTERN_DECL_NODE(affine_channel);
  PATTERN_DECL_NODE(eltwise);  // ELEMENTWISE_ADD
  // CONV inputs
  PATTERN_DECL_NODE(conv_weight);  // Filter
  // CONV outputs
  PATTERN_DECL_NODE(conv_out);  // tmp
  // ELTWISE inputs
  PATTERN_DECL_NODE(eltwise_y_in);
  // ELTWISE outputs
  PATTERN_DECL_NODE(eltwise_out);  // tmp

  // AC(Affine_Channel) inputs
  PATTERN_DECL_NODE(ac_scale);
  PATTERN_DECL_NODE(ac_bias);
  // AC outputs
  PATTERN_DECL_NODE(ac_out);  // Out
};

1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
// Dequantize + Quantize + anyOP
// This pattern is used for squashing the dequantize-quantize pairs.
struct DequantQuantAny : public PatternBase {
  DequantQuantAny(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "dequant_quant_any") {}
  PDNode* operator()();

  PATTERN_DECL_NODE(dequant_in);
  PATTERN_DECL_NODE(dequant_op);
  PATTERN_DECL_NODE(dequant_out);
  PATTERN_DECL_NODE(quant_op);
  PATTERN_DECL_NODE(quant_out);
  PATTERN_DECL_NODE(next_op);
};

// Dequantize + anyOP
// This quantize is used for getting number of ops the Dequantize's
// output is an input to.
struct DequantAny : public PatternBase {
  DequantAny(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "dequant_any") {}
  PDNode* operator()();

  PATTERN_DECL_NODE(dequant_op);
  PATTERN_DECL_NODE(dequant_out);
  PATTERN_DECL_NODE(next_op);
};

1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
// anyOp + more then one quantize op
// This pattern is used for squashing multiple quantize with the same scale.
struct MultipleQuantize : public PatternBase {
  MultipleQuantize(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "multiple_quantize") {}
  PDNode* operator()();

  PATTERN_DECL_NODE(prev_out);
};

1584 1585 1586 1587 1588 1589 1590 1591 1592
struct QuantizePlacement : public PatternBase {
  QuantizePlacement(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "quantize_placement") {}
  PDNode* operator()(
      const std::unordered_set<std::string>& quantize_enabled_op_types);

  PATTERN_DECL_NODE(op);
};

1593 1594 1595 1596 1597 1598
struct Bfloat16Placement : public PatternBase {
  Bfloat16Placement(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "bfloat16_placement") {}
  PDNode* operator()(
      const std::unordered_set<std::string>& bfloat16_enabled_op_types);

1599
  PATTERN_DECL_NODE(op_in);
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
  PATTERN_DECL_NODE(op);
};

struct OrphanedBfloat16 : public PatternBase {
  OrphanedBfloat16(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "orphaned_bfloat16") {}
  PDNode* operator()();

  PATTERN_DECL_NODE(prev_op);
  PATTERN_DECL_NODE(prev_out);
  PATTERN_DECL_NODE(op);
  PATTERN_DECL_NODE(op_out);
  PATTERN_DECL_NODE(next_op);
};

W
wenbin 已提交
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
struct UnsupportedBfloat16 : public PatternBase {
  UnsupportedBfloat16(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "unsupported_bfloat16") {}
  PDNode* operator()();

  PATTERN_DECL_NODE(prev_op);
  PATTERN_DECL_NODE(prev_out);
  PATTERN_DECL_NODE(op);
};

T
Tomasz Socha 已提交
1625 1626 1627
struct Bloat16Ops : public PatternBase {
  Bloat16Ops(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "many_bfloat16_ops") {}
1628 1629 1630 1631 1632 1633

  PDNode* operator()();

  PATTERN_DECL_NODE(op);
};

1634 1635 1636 1637 1638 1639 1640
// Pattern used for enforcing inplace computation for in-place computation
// supporting DNNL ops. softmax, batch_norm and layer_norm
struct MKLDNNInPlace : public PatternBase {
  MKLDNNInPlace(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "mkldnn_inplace") {}
  PDNode* operator()();

1641
  // MKL-DNN's in-place ops: BatchNorm, Softmax, Elementwise_add
1642 1643 1644 1645
  PATTERN_DECL_NODE(inplace_to_be_op);
  PATTERN_DECL_NODE(inplace_to_be_op_in);
  PATTERN_DECL_NODE(inplace_to_be_op_out);
  PATTERN_DECL_NODE(next_op);
1646
  PATTERN_DECL_NODE(next_op_out);
1647 1648
};

1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
struct TransposeFlattenConcat : public PatternBase {
  TransposeFlattenConcat(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "transpose_flatten_concat") {}

  PDNode* operator()(std::vector<PDNode*> conv_inputs, int times);

  std::string GetNodeName(const std::string& op_type) {
    return PDNodeName(name_scope_, repr_, id_, op_type);
  }

  PDNode* GetPDNode(const std::string& op_type) {
    return pattern->RetrieveNode(GetNodeName(op_type));
  }
};

1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
struct DeleteQuantOpFuse : public PatternBase {
  DeleteQuantOpFuse(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "delete_quant_fuse") {}

  void operator()(PDNode* input_act_node, const std::string& quant_type);

  std::string GetNodeName(const std::string& op_type) {
    return PDNodeName(name_scope_, repr_, id_, op_type);
  }

  PDNode* GetPDNode(const std::string& op_type) {
    return pattern->RetrieveNode(GetNodeName(op_type));
  }
};

struct DequantOpFuse : public PatternBase {
  DequantOpFuse(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "dequant_fuse") {}

1683 1684
  void operator()(PDNode* quant_op_input,
                  const std::string& quantized_op_type,
1685 1686
                  const std::string& dequant_type,
                  const std::string& weight_name);
N
nhzlx 已提交
1687 1688 1689 1690 1691 1692 1693 1694 1695 1696

  std::string GetNodeName(const std::string& op_type) {
    return PDNodeName(name_scope_, repr_, id_, op_type);
  }

  PDNode* GetPDNode(const std::string& op_type) {
    return pattern->RetrieveNode(GetNodeName(op_type));
  }
};

1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
struct ShuffleChannelPattern : public PatternBase {
  ShuffleChannelPattern(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "shufflechannel_pattern") {}

  void operator()(PDNode* reshape1_in);

  PATTERN_DECL_NODE(reshape1_op);
  PATTERN_DECL_NODE(reshape1_out);

  PATTERN_DECL_NODE(transpose_op);
  PATTERN_DECL_NODE(transpose_out);
  PATTERN_DECL_NODE(reshape2_op);
  PATTERN_DECL_NODE(reshape2_out);
};

D
denglin-github 已提交
1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724
struct DeleteDropoutOpPattern : public PatternBase {
  DeleteDropoutOpPattern(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "delete_dropout_op_pattern") {}

  void operator()();

  PATTERN_DECL_NODE(any_op_out);
  PATTERN_DECL_NODE(dropout_op);
  PATTERN_DECL_NODE(dropout_op_out);
  PATTERN_DECL_NODE(dropout_op_outmask);
  PATTERN_DECL_NODE(any_op2);
};

1725 1726 1727 1728
struct DeleteQuantDequantOpPattern : public PatternBase {
  DeleteQuantDequantOpPattern(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "delete_quantdequant_op_pattern") {}

1729
  void operator()(PDNode* input_node, const std::string& quantdequant_types);
1730 1731 1732 1733 1734 1735 1736

  PATTERN_DECL_NODE(quant_dequant_op_inscale);
  PATTERN_DECL_NODE(quant_dequant_op);
  PATTERN_DECL_NODE(quant_dequant_op_outscale);
  PATTERN_DECL_NODE(quant_dequant_op_out);
};

1737 1738 1739
struct DeleteQuantDequantFilterOpPattern : public PatternBase {
  DeleteQuantDequantFilterOpPattern(PDPattern* pattern,
                                    const std::string& name_scope)
1740 1741
      : PatternBase(
            pattern, name_scope, "delete_quantdequant_filter_op_pattern") {}
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751

  void operator()();

  PATTERN_DECL_NODE(quant_dequant_op_x);
  PATTERN_DECL_NODE(quant_dequant_op);
  PATTERN_DECL_NODE(quant_dequant_op_outscale);
  PATTERN_DECL_NODE(quant_dequant_op_out);
  PATTERN_DECL_NODE(any_op2);
};

1752 1753 1754
struct DeleteWeightQuantDequantLinearOpPattern : public PatternBase {
  DeleteWeightQuantDequantLinearOpPattern(PDPattern* pattern,
                                          const std::string& name_scope)
1755 1756
      : PatternBase(pattern,
                    name_scope,
1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
                    "delete_weight_quant_dequant_linear_op_pattern") {}

  void operator()();

  PATTERN_DECL_NODE(weight_dequantize_linear_op_x);
  PATTERN_DECL_NODE(weight_dequantize_linear_op_scale);
  PATTERN_DECL_NODE(weight_dequantize_linear_op);
  PATTERN_DECL_NODE(weight_dequantize_linear_op_out);
  PATTERN_DECL_NODE(any_op2);
};

struct DeleteQuantDequantLinearOpPattern : public PatternBase {
  DeleteQuantDequantLinearOpPattern(PDPattern* pattern,
                                    const std::string& name_scope)
1771 1772
      : PatternBase(
            pattern, name_scope, "delete_quant_dequant_linear_op_pattern") {}
1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785

  void operator()();

  PATTERN_DECL_NODE(quantize_linear_op_x);
  PATTERN_DECL_NODE(quantize_linear_op_scale);
  PATTERN_DECL_NODE(quantize_linear_op);
  PATTERN_DECL_NODE(quantize_linear_op_out);
  PATTERN_DECL_NODE(dequantize_linear_op);
  // PATTERN_DECL_NODE(dequantize_linear_op_scale);  // Can not add this node.
  // Todo: Wangzheee
  PATTERN_DECL_NODE(dequantize_linear_op_out);
};

1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
// Reshape + Transpose + Matmul
// named nodes:
// reshape_op, reshape_out, reshape_xshape,
// transpose_op, transpose_out, transpose_xshape,
// matmul_op, matmul_out
struct ReshapeTransposeMatmulPattern : public PatternBase {
  ReshapeTransposeMatmulPattern(PDPattern* pattern,
                                const std::string& name_scope)
      : PatternBase(pattern, name_scope, "reshape_transpose_matmul") {}

1796 1797
  PDNode* operator()(const std::string& op_name,
                     bool with_reshape_xshape,
1798
                     bool with_transpose_xshape);
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810

  PATTERN_DECL_NODE(reshape_in);
  PATTERN_DECL_NODE(reshape_op);
  PATTERN_DECL_NODE(reshape_out);
  PATTERN_DECL_NODE(reshape_xshape);
  PATTERN_DECL_NODE(transpose_op);
  PATTERN_DECL_NODE(transpose_out);
  PATTERN_DECL_NODE(transpose_xshape);
  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(matmul_out);
};

1811 1812 1813 1814 1815 1816
// Matmul + Transpose + Reshape
struct MatmulTransposeReshapePattern : public PatternBase {
  MatmulTransposeReshapePattern(PDPattern* pattern,
                                const std::string& name_scope)
      : PatternBase(pattern, name_scope, "matmul_transpose_reshape") {}

1817
  PDNode* operator()(const std::string& op_name);
1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828

  PATTERN_DECL_NODE(matmul_op);
  PATTERN_DECL_NODE(matmul_out);
  PATTERN_DECL_NODE(transpose_op);
  PATTERN_DECL_NODE(transpose_out);
  PATTERN_DECL_NODE(transpose_out_xshape);
  PATTERN_DECL_NODE(reshape_op);
  PATTERN_DECL_NODE(reshape_out);
  PATTERN_DECL_NODE(reshape_out_xshape);
};

1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843
// fusion_gru op
// Forward pass for fusion_gru.
// fusion_gru out is a result of the operator.
struct FusionGru : public PatternBase {
  FusionGru(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "fusion_gru") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(op);
  PATTERN_DECL_NODE(x);
  PATTERN_DECL_NODE(weight_h);
  PATTERN_DECL_NODE(weight_x);
  PATTERN_DECL_NODE(out);
};

1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865
// fusion_lstm op
// Forward pass for fusion_lstm.
// fusion_lstm out is a result of the operator.
struct FusionLSTM : public PatternBase {
  FusionLSTM(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "fusion_lstm") {}
  // TODO(lidanqing): Is it enough to detect fusion_lstm with these things
  PDNode* operator()();

  // declare op
  PATTERN_DECL_NODE(op);

  // declate inputs
  PATTERN_DECL_NODE(x);
  PATTERN_DECL_NODE(weight_h);
  PATTERN_DECL_NODE(weight_x);

  // decalre outputs
  PATTERN_DECL_NODE(hidden);
  PATTERN_DECL_NODE(cell);
};

1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
// two concatenated fusion_gru ops
// Forward pass for fusion of two concatenated fusion_gru ops.
// concat_out is a result of the operator().
struct TwoFusionGruConcat : public PatternBase {
  TwoFusionGruConcat(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "bi_fusion_gru") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(x);
  PATTERN_DECL_NODE(gru1);
  PATTERN_DECL_NODE(gru2);
  PATTERN_DECL_NODE(wh1);
  PATTERN_DECL_NODE(wh2);
  PATTERN_DECL_NODE(wx1);
  PATTERN_DECL_NODE(wx2);
  PATTERN_DECL_NODE(b1);
  PATTERN_DECL_NODE(b2);
  PATTERN_DECL_NODE(h1);
  PATTERN_DECL_NODE(h2);
  PATTERN_DECL_NODE(concat);
  PATTERN_DECL_NODE(out);
};

1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915
// two subsequent bi_fusion_gru ops
// Forward pass for fusion of two subsequent fusion_gru ops.
// Hidden of the last fusion_gru op is a result of the operator().
struct MultiGruSeq : public PatternBase {
  MultiGruSeq(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "multi_gru_seq") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(x);
  PATTERN_DECL_NODE(gru1);
  PATTERN_DECL_NODE(wx11);
  PATTERN_DECL_NODE(wx12);
  PATTERN_DECL_NODE(wh11);
  PATTERN_DECL_NODE(wh12);
  PATTERN_DECL_NODE(b11);
  PATTERN_DECL_NODE(b12);
  PATTERN_DECL_NODE(h1);
  PATTERN_DECL_NODE(gru2);
  PATTERN_DECL_NODE(wx21);
  PATTERN_DECL_NODE(wx22);
  PATTERN_DECL_NODE(wh21);
  PATTERN_DECL_NODE(wh22);
  PATTERN_DECL_NODE(b21);
  PATTERN_DECL_NODE(b22);
  PATTERN_DECL_NODE(h2);
};

1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
// multi_gru op
// Quantization pass for multi_gru op.
// Hidden of the multi_gru op is a result of the operator().
struct MultiGru : public PatternBase {
  MultiGru(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "multi_gru") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(x);
  PATTERN_DECL_NODE(gru);
  PATTERN_DECL_NODE(wx);
  PATTERN_DECL_NODE(wh);
  PATTERN_DECL_NODE(h);
};

1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965
//
// \brief   Pattern looking for subgraph representing layer normalization
//          operation.
//
struct LayerNorm : public PatternBase {
  LayerNorm(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "layer_norm") {}

  PDNode* operator()();

  PATTERN_DECL_NODE(x);
  PATTERN_DECL_NODE(x_mean);
  PATTERN_DECL_NODE(x_mean_out);
  PATTERN_DECL_NODE(x_sub_mean);
  PATTERN_DECL_NODE(x_sub_mean_out);
  PATTERN_DECL_NODE(sqr_pow);
  PATTERN_DECL_NODE(x_sub_mean_sqr);
  PATTERN_DECL_NODE(x_sub_mean_sqr_out);
  PATTERN_DECL_NODE(std_dev);
  PATTERN_DECL_NODE(std_dev_out);
  PATTERN_DECL_NODE(eps);
  PATTERN_DECL_NODE(std_dev_eps);
  PATTERN_DECL_NODE(std_dev_eps_out);
  PATTERN_DECL_NODE(std_dev_eps_sqrt);
  PATTERN_DECL_NODE(std_dev_eps_sqrt_out);
  PATTERN_DECL_NODE(division);
  PATTERN_DECL_NODE(division_out);
  PATTERN_DECL_NODE(gamma);
  PATTERN_DECL_NODE(scale);
  PATTERN_DECL_NODE(scale_out);
  PATTERN_DECL_NODE(beta);
  PATTERN_DECL_NODE(shift);
  PATTERN_DECL_NODE(shift_out);
};

W
wenbin 已提交
1966 1967
//
// \brief   Pattern looking for subgraph representing layernorm_shift_partition
1968
//          operation with shift_size = 0.
W
wenbin 已提交
1969 1970 1971
//
struct LayernormShiftPartitionPattern : public PatternBase {
  LayernormShiftPartitionPattern(PDPattern* pattern,
1972 1973 1974 1975
                                 const std::string& name_scope,
                                 bool with_roll)
      : PatternBase(pattern, name_scope, "layernorm_shift_partition"),
        with_roll_(with_roll) {}
W
wenbin 已提交
1976 1977

  PDNode* operator()();
1978
  bool with_roll_;
W
wenbin 已提交
1979 1980 1981 1982 1983 1984 1985
  PATTERN_DECL_NODE(layer_norm_in);
  PATTERN_DECL_NODE(layer_norm_op);
  PATTERN_DECL_NODE(layer_norm_bias);
  PATTERN_DECL_NODE(layer_norm_scale);
  PATTERN_DECL_NODE(layer_norm_out);
  PATTERN_DECL_NODE(reshape1_op);
  PATTERN_DECL_NODE(reshape1_out);
1986 1987 1988 1989
  // optional op roll
  PATTERN_DECL_NODE(roll1_op);
  PATTERN_DECL_NODE(roll1_out);

W
wenbin 已提交
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
  PATTERN_DECL_NODE(reshape2_op);
  PATTERN_DECL_NODE(reshape2_out);
  PATTERN_DECL_NODE(transpose_op);
  PATTERN_DECL_NODE(transpose_out);
  PATTERN_DECL_NODE(reshape3_op);
  PATTERN_DECL_NODE(reshape3_out);
  PATTERN_DECL_NODE(reshape4_op);
  PATTERN_DECL_NODE(reshape4_out);
};

W
Wang Bojun 已提交
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
// pattern for merge_layernorm
struct MergeLayernormPattern : public PatternBase {
  MergeLayernormPattern(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "merge_layernorm") {}

  PDNode* operator()(PDNode* reshape2_in);

  PATTERN_DECL_NODE(reshape2_00_op);
  PATTERN_DECL_NODE(reshape2_00_out);
  PATTERN_DECL_NODE(strided_slice_10_op);
  PATTERN_DECL_NODE(strided_slice_10_out);
  PATTERN_DECL_NODE(strided_slice_11_op);
  PATTERN_DECL_NODE(strided_slice_11_out);
  PATTERN_DECL_NODE(strided_slice_12_op);
  PATTERN_DECL_NODE(strided_slice_12_out);
  PATTERN_DECL_NODE(strided_slice_13_op);
  PATTERN_DECL_NODE(strided_slice_13_out);
  PATTERN_DECL_NODE(concat_20_op);
  PATTERN_DECL_NODE(concat_20_out);
  PATTERN_DECL_NODE(reshape2_30_op);
  PATTERN_DECL_NODE(reshape2_30_out);
  PATTERN_DECL_NODE(layernorm_40_op);
  PATTERN_DECL_NODE(layernorm_40_in_bias);
  PATTERN_DECL_NODE(layernorm_40_in_scale);
  PATTERN_DECL_NODE(layernorm_40_out);
};

2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
// Add support int8 flag
struct AddSupportInt8 : public PatternBase {
  AddSupportInt8(PDPattern* pattern, const std::string& name_scope)
      : PatternBase(pattern, name_scope, "Add_support_int8") {}

  PDNode* operator()();
  PATTERN_DECL_NODE(quant_op);
  PATTERN_DECL_NODE(quant_out);
};

2037
}  // namespace patterns
2038

Y
Yan Chunwei 已提交
2039
// Link two ir::Nodes from each other.
2040 2041 2042 2043
#define IR_NODE_LINK_TO(a, b) \
  a->outputs.push_back(b);    \
  b->inputs.push_back(a);

2044 2045 2046 2047 2048 2049 2050 2051
// UnLink 2 ir::Nodes from each other.
#define IR_NODE_UNLINK(a, b)                                                  \
  a->outputs.erase(                                                           \
      std::remove(std::begin(a->outputs), std::end(a->outputs), b),           \
      std::end(a->outputs));                                                  \
  b->inputs.erase(std::remove(std::begin(b->inputs), std::end(b->inputs), a), \
                  std::end(b->inputs));

C
chengduo 已提交
2052 2053 2054 2055 2056 2057
// Set the out_var as the output of the op
#define IR_OP_VAR_LINK(op, out_var) \
  op->outputs.push_back(out_var);   \
  out_var->inputs.clear();          \
  out_var->inputs.push_back(op);

2058 2059 2060 2061 2062 2063
// Set the in_var as the input of the op
#define IR_VAR_OP_LINK(in_var, op) \
  in_var->outputs.clear();         \
  in_var->outputs.push_back(op);   \
  op->inputs.push_back(in_var);

2064 2065 2066
}  // namespace ir
}  // namespace framework
}  // namespace paddle