test_op_nn.py

#!/usr/bin/env python3

# Copyright (c) 2021 CINN Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import paddle as paddle
import numpy as np
import unittest
import math
import cinn
from cinn import frontend
from cinn import runtime
from cinn import lang
from cinn import framework
from cinn import ir
from cinn import common
from cinn.poly import create_stages
import logging
from test_utils import SingleOpTester
import pool_utils
import conv2d_utils


class OpTest_relu(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        return np.maximum(X, np.zeros(X.shape).astype("float32"))

    def test_op(self):
        attrs = framework.NodeAttr()
        self.to_test_op([[32]], [[32]], "relu", attrs)


class OpTest_relu6(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        return np.minimum(
            np.maximum(X, np.zeros(np.array(X).shape).astype("float32")), 6
        )

    def test_op(self):
        attrs = framework.NodeAttr()
        self.to_test_op([[32, 32]], [[32, 32]], "relu6", attrs)


class OpTest_conv2d_nchw(SingleOpTester):
    def init_testcase(self):
        self.input_size = [1, 3, 10, 10]
        self.groups = 1
        assert np.mod(self.input_size[1], self.groups) == 0
        f_c = self.input_size[1] // self.groups
        self.filter_size = [2, f_c, 2, 2]
        assert np.mod(self.filter_size[0], self.groups) == 0
        self.data_format = "NCHW"
        self.attrs = framework.NodeAttr()
        self.padding = [1, 1]
        self.stride = [2, 2]
        self.dilation = [2, 2]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, False
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "conv2d",
            self.attrs,
            0,
            True,
        )


class OpTest_conv2d_nchw_1(SingleOpTester):
    def init_testcase(self):
        self.input_size = [1, 3, 224, 224]
        self.groups = 1
        assert np.mod(self.input_size[1], self.groups) == 0
        f_c = self.input_size[1] // self.groups
        self.filter_size = [64, f_c, 7, 7]
        self.data_format = "NCHW"
        self.attrs = framework.NodeAttr()
        self.padding = [3, 3]
        self.stride = [2, 2]
        self.dilation = [1, 1]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, False
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "conv2d",
            self.attrs,
            0,
            True,
        )


class OpTest_conv2d_nchw_group(SingleOpTester):
    def init_testcase(self):
        self.input_size = [2, 8, 10, 10]
        self.groups = 4
        assert np.mod(self.input_size[1], self.groups) == 0
        f_c = self.input_size[1] // self.groups
        self.filter_size = [16, f_c, 7, 7]
        self.data_format = "NCHW"
        self.attrs = framework.NodeAttr()
        self.padding = [1, 1]
        self.stride = [2, 2]
        self.dilation = [1, 1]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, False
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "conv2d",
            self.attrs,
            0,
            True,
        )


class OpTest_conv2d_nchw_depthwise(SingleOpTester):
    def init_testcase(self):
        self.input_size = [2, 8, 10, 10]
        self.groups = 8
        assert np.mod(self.input_size[1], self.groups) == 0
        f_c = self.input_size[1] // self.groups
        self.filter_size = [16, f_c, 7, 7]
        self.data_format = "NCHW"
        self.attrs = framework.NodeAttr()
        self.padding = [1, 1]
        self.stride = [2, 2]
        self.dilation = [1, 1]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, False
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "conv2d",
            self.attrs,
            0,
            True,
        )


class OpTest_conv2d_nhwc_group(SingleOpTester):
    def init_testcase(self):
        self.input_size = [2, 10, 10, 8]
        self.groups = 4
        assert np.mod(self.input_size[3], self.groups) == 0
        f_c = self.input_size[3] // self.groups
        self.filter_size = [16, f_c, 7, 7]
        self.data_format = "NHWC"
        self.attrs = framework.NodeAttr()
        self.padding = [2, 2]
        self.stride = [2, 2]
        self.dilation = [2, 2]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, False
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "conv2d",
            self.attrs,
            0,
            True,
        )


class OpTest_conv2d_nhwc_depthwise(SingleOpTester):
    def init_testcase(self):
        self.input_size = [2, 10, 10, 8]
        self.groups = 8
        assert np.mod(self.input_size[3], self.groups) == 0
        f_c = self.input_size[3] // self.groups
        self.filter_size = [16, f_c, 7, 7]
        self.data_format = "NHWC"
        self.attrs = framework.NodeAttr()
        self.padding = [1, 1]
        self.stride = [2, 2]
        self.dilation = [1, 1]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, False
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "conv2d",
            self.attrs,
            0,
            True,
        )


# test channel multiplier format
class OpTest_depthwise_conv2d_nchw(SingleOpTester):
    def init_testcase(self):
        self.input_size = [2, 8, 10, 10]
        self.groups = self.input_size[1]
        assert np.mod(self.input_size[1], self.groups) == 0
        channel_multiplier = 1
        self.filter_size = [self.input_size[1], channel_multiplier, 7, 7]
        self.data_format = "NCHW"
        self.attrs = framework.NodeAttr()
        self.padding = [1, 1]
        self.stride = [2, 2]
        self.dilation = [1, 1]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, True
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "depthwise_conv2d",
            self.attrs,
            0,
            True,
        )


# test channel multiplier format
class OpTest_depthwise_conv2d_nhwc(SingleOpTester):
    def init_testcase(self):
        self.input_size = [2, 10, 10, 8]
        self.groups = self.input_size[3]
        assert np.mod(self.input_size[3], self.groups) == 0
        channel_multiplier = 4
        self.filter_size = [self.input_size[3], channel_multiplier, 7, 7]
        self.data_format = "NHWC"
        self.attrs = framework.NodeAttr()
        self.padding = [1, 1]
        self.stride = [2, 2]
        self.dilation = [1, 1]
        self.attrs.set_attr("stride", self.stride)
        self.attrs.set_attr("padding", self.padding)
        self.attrs.set_attr("dilation", self.dilation)
        self.attrs.set_attr("groups", self.groups)
        self.attrs.set_attr("data_format", self.data_format)

    def create_target_data(self, inputs_data, attrs):
        return conv2d_utils.conv2d_native(
            inputs_data, self.input_size, self.filter_size, self.attrs, True
        )

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [self.input_size, self.filter_size],
            None,
            "depthwise_conv2d",
            self.attrs,
            0,
            True,
        )


class OpTest_pool1d(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2])
    attrs.set_attr("stride_size", [2])
    attrs.set_attr("padding_size", [1, 1])
    attrs.set_attr("pool_type", "max")
    attrs.set_attr("ceil_mode", False)
    attrs.set_attr("exclusive", True)
    attrs.set_attr("data_format", "NCW")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool1d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 3, 8]
        self.to_test_op([input_shape], None, "pool1d", self.attrs)


class OpTest_pool1d_1(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2])
    attrs.set_attr("stride_size", [2])
    attrs.set_attr("padding_size", [2, 3])
    attrs.set_attr("pool_type", "avg")
    attrs.set_attr("ceil_mode", False)
    attrs.set_attr("exclusive", True)
    attrs.set_attr("data_format", "NCW")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool1d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 3, 8]
        self.to_test_op([input_shape], None, "pool1d", self.attrs)


class OpTest_pool1d_2(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2])
    attrs.set_attr("stride_size", [3])
    attrs.set_attr("padding_size", [4, 5])
    attrs.set_attr("pool_type", "avg")
    attrs.set_attr("ceil_mode", True)
    attrs.set_attr("exclusive", False)
    attrs.set_attr("data_format", "NWC")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool1d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 8, 3]
        self.to_test_op([input_shape], None, "pool1d", self.attrs)


class OpTest_pool2d(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2, 2])
    attrs.set_attr("stride_size", [2, 2])
    attrs.set_attr("padding_size", [1, 1, 1, 1])
    attrs.set_attr("pool_type", "max")
    attrs.set_attr("ceil_mode", False)
    attrs.set_attr("exclusive", True)
    attrs.set_attr("data_format", "NCHW")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool2d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 3, 8, 8]
        self.to_test_op([input_shape], None, "pool2d", self.attrs)


class OpTest_pool2d_1(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2, 2])
    attrs.set_attr("stride_size", [2, 2])
    attrs.set_attr("padding_size", [2, 3, 4, 5])
    attrs.set_attr("pool_type", "avg")
    attrs.set_attr("ceil_mode", False)
    attrs.set_attr("exclusive", True)
    attrs.set_attr("data_format", "NCHW")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool2d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 3, 8, 8]
        self.to_test_op([input_shape], None, "pool2d", self.attrs)


class OpTest_pool2d_2(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2, 2])
    attrs.set_attr("stride_size", [3, 3])
    attrs.set_attr("padding_size", [2, 3, 4, 5])
    attrs.set_attr("pool_type", "avg")
    attrs.set_attr("ceil_mode", True)
    attrs.set_attr("exclusive", False)
    attrs.set_attr("data_format", "NHWC")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool2d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 8, 8, 3]
        self.to_test_op([input_shape], None, "pool2d", self.attrs)


# The following test is temporarily broken

# class OpTest_pool3d(SingleOpTester):
#     attrs = framework.NodeAttr()
#     attrs.attr_store = {
#         "kernel_size": [2, 2, 2],
#         "stride_size": [2, 2, 2],
#         "padding_size": [1, 2, 3, 4, 5, 6],
#         "pool_type": "max",
#         "ceil_mode": False,
#         "exclusive": True,
#         "data_format": "NCDHW"
#     }

#     def create_target_data(self, inputs_data, attrs):
#         return pool_utils.pool3d(inputs_data[0], self.attrs)

#     def test_op(self):
#         input_shape = [2, 3, 8, 8, 8]
#         self.to_test_op([input_shape], None, "pool3d", self.attrs)


class OpTest_pool3d_1(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2, 2, 2])
    attrs.set_attr("stride_size", [2, 2, 2])
    attrs.set_attr("padding_size", [1, 1, 1, 1, 1, 1])
    attrs.set_attr("pool_type", "avg")
    attrs.set_attr("ceil_mode", False)
    attrs.set_attr("exclusive", True)
    attrs.set_attr("data_format", "NCDHW")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool3d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 3, 8, 8, 8]
        self.to_test_op([input_shape], None, "pool3d", self.attrs)


class OpTest_pool3d_2(SingleOpTester):
    attrs = framework.NodeAttr()
    attrs.set_attr("kernel_size", [2, 2, 2])
    attrs.set_attr("stride_size", [2, 2, 2])
    attrs.set_attr("padding_size", [1, 2, 3, 4, 5, 6])
    attrs.set_attr("pool_type", "avg")
    attrs.set_attr("ceil_mode", True)
    attrs.set_attr("exclusive", False)
    attrs.set_attr("data_format", "NDHWC")

    def create_target_data(self, inputs_data, attrs):
        return pool_utils.pool3d(inputs_data[0], self.attrs)

    def test_op(self):
        input_shape = [1, 8, 8, 8, 3]
        self.to_test_op([input_shape], None, "pool3d", self.attrs)


class OpTest_batchnorm(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X, Scale, Bias, Mean, Variance] = inputs_data
        c = X.shape[1]
        for i in range(0, c):
            X[:, i, :, :] = (X[:, i, :, :] - Mean[i]) / math.sqrt(
                Variance[i] + 0.00001
            ) * Scale[i] + Bias[i]
        return X

    def test_op(self):
        attrs = framework.NodeAttr()
        self.to_test_op(
            [[1, 64, 112, 112], [64], [64], [64], [64]],
            [[1, 64, 112, 112]],
            "batch_norm",
            attrs,
        )


class OpTest_softmax_0(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        Y = np.zeros(X.shape).astype("float32")
        for i in range(0, Y.shape[1]):
            Y[:, i, :] = (
                np.exp(X[:, i, :])
                / np.sum(np.exp(X), axis=1, keepdims=True)[:, 0, :]
            )
        return Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axis", 1)
        self.to_test_op(
            [[12, 224, 224]],
            [[12, 224, 224], [12, 224, 224]],
            "softmax",
            attrs,
            0,
        )


class OpTest_softmax_1(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        Y = np.zeros(X.shape).astype("float32")
        for i in range(0, Y.shape[2]):
            Y[:, :, i] = (
                np.exp(X[:, :, i])
                / np.sum(np.exp(X), axis=2, keepdims=True)[:, :, 0]
            )
        return Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axis", -1)
        self.to_test_op(
            [[12, 224, 224]],
            [[12, 224, 224], [12, 224, 224]],
            "softmax",
            attrs,
            0,
        )


class OpTest_softmax_2(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        Y = np.zeros(X.shape).astype("float32")
        for i in range(0, Y.shape[0]):
            Y[i, :, :] = (
                np.exp(X[i, :, :])
                / np.sum(np.exp(X), axis=0, keepdims=True)[0, :, :]
            )
        return Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axis", 0)
        self.to_test_op(
            [[12, 224, 224]],
            [[12, 224, 224], [12, 224, 224]],
            "softmax",
            attrs,
            0,
        )


class OpTest_sigmoid(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        x = np.array(inputs_data[0])
        y = 1 / (1 + np.exp(-x))
        return y

    def test_op(self):
        attrs = framework.NodeAttr()
        self.to_test_op([[3, 224, 224]], [[3, 224, 224]], "sigmoid", attrs)


class OpTest_slice_0(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        Y = X[:, 0:2, 2:4, :]
        return Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axes", [0, 1, 2])
        attrs.set_attr("starts", [-3, 0, 2])
        attrs.set_attr("ends", [3, 2, 4])
        self.to_test_op([[3, 4, 5, 6]], [[3, 2, 2, 6]], "slice", attrs)


class OpTest_slice_1(SingleOpTester):
    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        Y = X[:, 0:3, 1:2, 2:4]
        return Y

    def test_op(self):
        attrs = framework.NodeAttr()
        attrs.set_attr("axes", [1, 2, 3])
        attrs.set_attr("starts", [0, 1, 2])
        attrs.set_attr("ends", [3, 2, 4])
        self.to_test_op([[3, 4, 5, 6]], [[3, 3, 1, 2]], "slice", attrs)


class OpTest_dropout_infer_0(SingleOpTester):
    def init_testcase(self):
        self.attrs = framework.NodeAttr()
        self.attrs.set_attr("dropout_prob", 0.2)
        self.attrs.set_attr("dropout_implementation", "downgrade_in_infer")

    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        assert "dropout_implementation" in self.attrs.attr_store
        if (
            self.attrs.attr_store["dropout_implementation"]
            == "downgrade_in_infer"
        ):
            return X * (1 - self.attrs.attr_store["dropout_prob"])
        else:
            return X

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [[2, 1280, 2, 2]], [[2, 1280, 2, 2]], "dropout_infer", self.attrs
        )


class OpTest_dropout_infer_1(SingleOpTester):
    def init_testcase(self):
        self.attrs = framework.NodeAttr()
        self.attrs.set_attr("dropout_prob", 0.2)
        self.attrs.set_attr("dropout_implementation", "upscale_in_train")

    def create_target_data(self, inputs_data, attrs):
        [X] = inputs_data
        assert "dropout_implementation" in self.attrs.attr_store
        if (
            self.attrs.attr_store["dropout_implementation"]
            == "downgrade_in_infer"
        ):
            return X * (1 - self.attrs.attr_store["dropout_prob"])
        else:
            return X

    def test_op(self):
        self.init_testcase()
        self.to_test_op(
            [[2, 1280, 2, 2]], [[2, 1280, 2, 2]], "dropout_infer", self.attrs
        )


if __name__ == "__main__":
    unittest.main()