diff --git a/mace/python/tools/converter_tool/base_converter.py b/mace/python/tools/converter_tool/base_converter.py
index cfd8409abfaad1de1b066583665afd9f47d4af41..9a5440f440b9307f268956361d5f31e2eb3505c1 100644
--- a/mace/python/tools/converter_tool/base_converter.py
+++ b/mace/python/tools/converter_tool/base_converter.py
@@ -195,7 +195,6 @@ class TransformerRule(Enum):
     QUANTIZE_NODES = 23
     ADD_QUANTIZE_TENSOR_RANGE = 24
     QUANTIZE_WEIGHTS = 25
-    TRANSPOSE_MATMUL_WEIGHT = 26
 
 
 class ConverterInterface(object):
@@ -346,8 +345,6 @@ class ConverterOption(object):
                 TransformerRule.FOLD_ACTIVATION,
                 TransformerRule.TRANSFORM_GLOBAL_CONV_TO_FC,
                 TransformerRule.RESHAPE_FC_WEIGHT,
-                # Transpose the weight of matmul if necessary
-                TransformerRule.TRANSPOSE_MATMUL_WEIGHT,
                 # Model data format related transformation
                 TransformerRule.TRANSPOSE_FILTERS,
                 TransformerRule.TRANSPOSE_DATA_FORMAT,
diff --git a/mace/python/tools/converter_tool/transformer.py b/mace/python/tools/converter_tool/transformer.py
index 7908c31581f21fb0631fd539cadc44c5dc94b96e..b9dabae2c0158400c6e5a1daa813ffa179fb8beb 100644
--- a/mace/python/tools/converter_tool/transformer.py
+++ b/mace/python/tools/converter_tool/transformer.py
@@ -64,9 +64,6 @@ class Transformer(base_converter.ConverterInterface):
             TransformerRule.FOLD_BIASADD: self.fold_biasadd,
             TransformerRule.FLATTEN_ATROUS_CONV: self.flatten_atrous_conv,
             TransformerRule.FOLD_ACTIVATION: self.fold_activation,
-            # TODO(liuqi): should move to transpose_filter
-            TransformerRule.TRANSPOSE_MATMUL_WEIGHT:
-                self.transpose_matmul_weight,
             TransformerRule.TRANSPOSE_FILTERS: self.transpose_filters,
             TransformerRule.TRANSPOSE_DATA_FORMAT: self.transpose_data_format,
             TransformerRule.ADD_IN_OUT_TENSOR_INFO:
@@ -1057,6 +1054,8 @@ class Transformer(base_converter.ConverterInterface):
                     filter.dims[:] = filter_data.shape
                     transposed_filter.add(op.input[1])
 
+            self.transpose_matmul_weight()
+
         return False
 
     def buffer_to_image(self, op, input_idx, input_type):