conda_build.py

#!/bin/python

# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
import platform
import argparse
import os


def parse_args():
    parser = argparse.ArgumentParser("conda build for paddlepaddle version")
    parser.add_argument(
        "--paddle_version",
        type=str,
        required=True,
        help="paddle version for conda build.",
    )
    args = parser.parse_args()

    return args


class ConstantVar:
    def __init__(self):
        self.build = r"""
build:
  number: '0'
  string: """

        self.requirement_build = r"""
requirements:
  build:
    - numpy>=1.13
    - cython
    - setuptools
"""

        self.requirement_run = r"""
  run:
    - requests>=2.20.0
    - numpy>=1.13
    - protobuf>=3.1.0
    - gast==0.3.3
    - Pillow
    - decorator
    - astor
"""

        self.requirement_run_windows = r"""
  run:
    - requests>=2.20.0
    - numpy>=1.13
    - protobuf>=3.1.0
    - gast==0.3.3
    - Pillow
    - decorator
    - astor
"""
        self.test = r"""
test:
  import:
    paddle
"""

        self.about = r"""
about:
  home: http://www.paddlepaddle.org/
  license: APACHE 2.0
  license_family: APACHE
  summary: an easy-to-use, efficient, flexible and scalable deep learning platform
"""

        self.build_const = r"""
"""

        self.blt_const = r"""
"""

        self.python36 = r"    - python>=3.6, <3.7"
        self.python37 = r"    - python>=3.7, <3.8"
        self.python38 = r"    - python>=3.8, <3.9"
        self.python39 = r"    - python>=3.9, <3.10"

        self.python_version = [
            self.python36,
            self.python37,
            self.python38,
            self.python39,
        ]

        self.cuda101 = r"""
    - cudatoolkit>=10.1, <10.2
    - cudnn>=7.6, <7.7
    """
        self.cuda102 = r"""
    - cudatoolkit>=10.2, <10.3
    - cudnn>=7.6, <7.7
    """
        self.cuda112 = r"""
    - cudatoolkit>=11.2, <11.3
    - cudnn>=8.1, <8.2
    """

        self.cuda_info = [
            (self.cuda101, "cuda10.1", ".post101"),
            (self.cuda102, "cuda10.2", ""),
            (self.cuda112, "cuda11.2", ".post112"),
        ]
        self.py_str = ["py36", "py37", "py38", "py39"]
        self.pip_end = ".whl --no-deps"
        self.pip_prefix_linux = "pip install /package/paddlepaddle"
        self.pip_prefix_windows = r"pip install C:\package\paddlepaddle"
        self.pip_gpu = "_gpu-"
        self.pip_cpu = "-"
        self.mac_pip = [
            "-cp36-cp36m-macosx_10_6_intel",
            "-cp37-cp37m-macosx_10_6_intel",
            "-cp38-cp38-macosx_10_14_x86_64",
            "-cp39-cp39-macosx_10_14_x86_64",
        ]
        self.linux_pip = [
            "-cp36-cp36m-linux_x86_64",
            "-cp37-cp37m-linux_x86_64",
            "-cp38-cp38-linux_x86_64",
            "-cp39-cp39-linux_x86_64",
        ]
        self.windows_pip = [
            "-cp36-cp36m-win_amd64",
            "-cp37-cp37m-win_amd64",
            "-cp38-cp38-win_amd64",
            "-cp39-cp39-win_amd64",
        ]


def meta_build_mac(var, python_str, paddle_version, build_var, build_name_str):
    package_str = (
        """
package:
  name: paddlepaddle
  version: """
        + paddle_version
    )
    requirement = (
        var.requirement_build + python_str + var.requirement_run + python_str
    )
    meta_build = var.build + build_name_str
    meta_str = package_str + meta_build + requirement + var.test + var.about
    build_str = var.build_const + build_var

    meta_filename = "meta.yaml"
    build_filename = "build.sh"
    with open(meta_filename, 'w') as f:
        f.write(meta_str)
    with open(build_filename, 'w') as f:
        f.write(build_str)


def meta_build_linux(
    var, python_str, paddle_version, build_var, build_name_str, cuda_str=None
):
    if cuda_str is None:
        package_str = (
            """
package:
  name: paddlepaddle
  version: """
            + paddle_version
        )
    else:
        package_str = (
            """
package:
  name: paddlepaddle-gpu
  version: """
            + paddle_version
        )
    requirement = (
        var.requirement_build + python_str + var.requirement_run + python_str
    )
    meta_build = var.build + build_name_str
    meta_str = package_str + meta_build + requirement
    if not (cuda_str is None):
        meta_str = meta_str + cuda_str
    meta_str = meta_str + var.test + var.about

    build_str = var.build_const + build_var

    meta_filename = "meta.yaml"
    build_filename = "build.sh"
    with open(meta_filename, 'w') as f:
        f.write(meta_str)
    with open(build_filename, 'w') as f:
        f.write(build_str)


def meta_build_windows(
    var, python_str, paddle_version, blt_var, build_name_str, cuda_str=None
):
    if cuda_str is None:
        package_str = (
            """
package:
  name: paddlepaddle
  version: """
            + paddle_version
        )
    else:
        package_str = (
            """
package:
  name: paddlepaddle-gpu
  version: """
            + paddle_version
        )

    requirement = (
        var.requirement_build
        + python_str
        + var.requirement_run_windows
        + python_str
    )
    meta_build = var.build + build_name_str
    meta_str = package_str + meta_build + requirement

    if not (cuda_str is None):
        meta_str = meta_str + cuda_str

    blt_str = var.blt_const + blt_var

    meta_str = meta_str + var.test + var.about
    meta_filename = "meta.yaml"
    build_filename = "bld.bat"
    with open(meta_filename, 'w') as f:
        f.write(meta_str)
    with open(build_filename, 'w') as f:
        f.write(blt_str)


def conda_build(paddle_version, var):
    sysstr = platform.system()
    if sysstr == "Windows":
        os.system("mkdir paddle")
        os.chdir(r"./paddle")
        for i in range(len(var.python_version)):
            blt_var = (
                var.pip_prefix_windows
                + var.pip_cpu
                + paddle_version
                + var.windows_pip[i]
                + var.pip_end
            )
            name = var.py_str[i] + "_cpu_windows"
            python_str = var.python_version[i]
            meta_build_windows(var, python_str, paddle_version, blt_var, name)
            os.system("conda build .")

        for i in range(len(var.python_version)):
            for cuda_str in var.cuda_info:
                post = cuda_str[2]
                blt_var = (
                    var.pip_prefix_windows
                    + var.pip_gpu
                    + paddle_version
                    + post
                    + var.windows_pip[i]
                    + var.pip_end
                )
                name = var.py_str[i] + "_gpu_" + cuda_str[1] + "_windows"
                cuda_cudnn_str = cuda_str[0]
                python_str = var.python_version[i]
                meta_build_windows(
                    var,
                    python_str,
                    paddle_version,
                    blt_var,
                    name,
                    cuda_cudnn_str,
                )
                os.system("conda build .")

    elif sysstr == "Linux":
        os.system("mkdir paddle")
        os.chdir(r"./paddle")
        for i in range(len(var.python_version)):
            build_var = (
                var.pip_prefix_linux
                + var.pip_cpu
                + paddle_version
                + var.linux_pip[i]
                + var.pip_end
            )
            name = var.py_str[i] + "_cpu_many_linux"
            python_str = var.python_version[i]
            meta_build_linux(var, python_str, paddle_version, build_var, name)
            os.system("conda build .")

        for i in range(len(var.python_version)):
            for cuda_str in var.cuda_info:
                post = cuda_str[2]
                build_var = (
                    var.pip_prefix_linux
                    + var.pip_gpu
                    + paddle_version
                    + post
                    + var.linux_pip[i]
                    + var.pip_end
                )
                name = var.py_str[i] + "_gpu_" + cuda_str[1] + "_many_linux"
                cuda_cudnn_str = cuda_str[0]
                python_str = var.python_version[i]
                meta_build_linux(
                    var,
                    python_str,
                    paddle_version,
                    build_var,
                    name,
                    cuda_cudnn_str,
                )
                os.system("conda build .")

        os.system("cd ..")

    elif sysstr == "Darwin":
        os.system("mkdir paddle")
        os.chdir(r"./paddle")
        for i in range(len(var.python_version)):
            build_var = (
                var.pip_prefix_linux
                + var.pip_cpu
                + paddle_version
                + var.mac_pip[i]
                + var.pip_end
            )
            name = var.py_str[i] + "_mac"
            python_str = var.python_version[i]
            meta_build_mac(var, python_str, paddle_version, build_var, name)
            os.system("conda build .")

        os.system("cd ..")


if __name__ == "__main__":
    args = parse_args()
    paddle_version = args.paddle_version
    var = ConstantVar()
    conda_build(paddle_version, var)