envs.py 6.3 KB
Newer Older
T
tangwei 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

T
tangwei 已提交
15
from contextlib import closing
T
tangwei12 已提交
16
import copy
T
tangwei 已提交
17
import os
C
chengmo 已提交
18
import socket
T
tangwei 已提交
19
import sys
T
tangwei 已提交
20

T
tangwei12 已提交
21
global_envs = {}
T
tangwei 已提交
22

X
fix  
xjqbest 已提交
23

T
tangwei 已提交
24
def flatten_environs(envs, separator="."):
T
tangwei 已提交
25
    flatten_dict = {}
T
tangwei 已提交
26 27
    assert isinstance(envs, dict)

T
fix bug  
tangwei 已提交
28
    def fatten_env_namespace(namespace_nests, local_envs):
T
fix bug  
tangwei 已提交
29
        if not isinstance(local_envs, dict):
T
tangwei 已提交
30
            global_k = separator.join(namespace_nests)
T
tangwei 已提交
31
            flatten_dict[global_k] = str(local_envs)
T
fix bug  
tangwei 已提交
32 33 34 35 36 37 38
        else:
            for k, v in local_envs.items():
                if isinstance(v, dict):
                    nests = copy.deepcopy(namespace_nests)
                    nests.append(k)
                    fatten_env_namespace(nests, v)
                else:
T
tangwei 已提交
39
                    global_k = separator.join(namespace_nests + [k])
T
tangwei 已提交
40
                    flatten_dict[global_k] = str(v)
T
fix bug  
tangwei 已提交
41

T
tangwei 已提交
42
    for k, v in envs.items():
T
fix bug  
tangwei 已提交
43
        fatten_env_namespace([k], v)
T
tangwei 已提交
44

T
tangwei 已提交
45
    return flatten_dict
T
tangwei 已提交
46

T
tangwei 已提交
47 48 49

def set_runtime_environs(environs):
    for k, v in environs.items():
T
tangwei 已提交
50
        os.environ[k] = str(v)
T
tangwei 已提交
51

T
tangwei 已提交
52

T
tangwei 已提交
53
def get_runtime_environ(key):
T
tangwei 已提交
54 55
    return os.getenv(key, None)

T
tangwei 已提交
56

T
tangwei 已提交
57
def get_trainer():
T
tangwei 已提交
58
    train_mode = get_runtime_environ("train.trainer.trainer")
T
tangwei 已提交
59 60 61
    return train_mode


T
tangwei12 已提交
62 63
def set_global_envs(envs):
    assert isinstance(envs, dict)
T
tangwei 已提交
64

T
tangwei12 已提交
65 66 67 68 69 70
    def fatten_env_namespace(namespace_nests, local_envs):
        for k, v in local_envs.items():
            if isinstance(v, dict):
                nests = copy.deepcopy(namespace_nests)
                nests.append(k)
                fatten_env_namespace(nests, v)
X
fix  
xjqbest 已提交
71 72
            elif (k == "dataset" or k == "phase" or
                  k == "runner") and isinstance(v, list):
X
fix  
xjqbest 已提交
73 74 75 76 77 78 79
                for i in v:
                    if i.get("name") is None:
                        raise ValueError("name must be in dataset list ", v)
                    nests = copy.deepcopy(namespace_nests)
                    nests.append(k)
                    nests.append(i["name"])
                    fatten_env_namespace(nests, i)
T
tangwei12 已提交
80 81 82
            else:
                global_k = ".".join(namespace_nests + [k])
                global_envs[global_k] = v
T
tangwei 已提交
83

X
fix  
xjqbest 已提交
84
    fatten_env_namespace([], envs)
T
tangwei 已提交
85

X
fix  
xjqbest 已提交
86

T
tangwei12 已提交
87
def get_global_env(env_name, default_value=None, namespace=None):
T
tangwei 已提交
88 89 90
    """
    get os environment value
    """
C
chengmo 已提交
91 92
    _env_name = env_name if namespace is None else ".".join(
        [namespace, env_name])
T
tangwei12 已提交
93 94 95
    return global_envs.get(_env_name, default_value)


T
tangwei 已提交
96 97 98 99
def get_global_envs():
    return global_envs


T
tangwei 已提交
100
def path_adapter(path):
T
tangwei 已提交
101 102
    if path.startswith("paddlerec."):
        package = get_runtime_environ("PACKAGE_BASE")
F
frankwhzhang 已提交
103 104
        l_p = path.split("paddlerec.")[1].replace(".", "/")
        return os.path.join(package, l_p)
T
tangwei 已提交
105
    else:
T
tangwei 已提交
106
        return path
T
tangwei 已提交
107 108


T
tangwei 已提交
109 110 111 112 113 114 115
def windows_path_converter(path):
    if get_platform() == "WINDOWS":
        return path.replace("/", "\\")
    else:
        return path.replace("\\", "/")


T
tangwei 已提交
116
def update_workspace():
X
fix  
xjqbest 已提交
117
    workspace = global_envs.get("workspace")
T
tangwei 已提交
118 119
    if not workspace:
        return
T
tangwei 已提交
120
    workspace = path_adapter(workspace)
T
tangwei 已提交
121 122 123

    for name, value in global_envs.items():
        if isinstance(value, str):
T
tangwei 已提交
124
            value = value.replace("{workspace}", workspace)
T
tangwei 已提交
125
            value = windows_path_converter(value)
T
tangwei 已提交
126 127
            global_envs[name] = value

T
tangwei 已提交
128

T
tangwei12 已提交
129
def pretty_print_envs(envs, header=None):
T
tangwei12 已提交
130 131
    spacing = 5
    max_k = 45
T
tangwei 已提交
132
    max_v = 50
T
tangwei12 已提交
133

T
tangwei 已提交
134
    for k, v in envs.items():
T
tangwei12 已提交
135 136
        max_k = max(max_k, len(k))

T
tangwei12 已提交
137
    h_format = "{{:^{}s}}{}{{:<{}s}}\n".format(max_k, " " * spacing, max_v)
T
tangwei12 已提交
138 139 140 141 142 143 144 145
    l_format = "{{:<{}s}}{{}}{{:<{}s}}\n".format(max_k, max_v)
    length = max_k + max_v + spacing

    border = "".join(["="] * length)
    line = "".join(["-"] * length)

    draws = ""
    draws += border + "\n"
T
tangwei 已提交
146 147 148 149

    if header:
        draws += h_format.format(header[0], header[1])
    else:
150
        draws += h_format.format("paddlerec Global Envs", "Value")
T
tangwei 已提交
151

T
tangwei12 已提交
152 153
    draws += line + "\n"

T
tangwei 已提交
154
    for k, v in envs.items():
T
tangwei 已提交
155 156 157 158 159 160
        if isinstance(v, str) and len(v) >= max_v:
            str_v = "... " + v[-46:]
        else:
            str_v = v

        draws += l_format.format(k, " " * spacing, str(str_v))
T
tangwei12 已提交
161 162 163 164 165

    draws += border

    _str = "\n{}\n".format(draws)
    return _str
T
tangwei 已提交
166 167


T
tangwei 已提交
168
def lazy_instance_by_package(package, class_name):
T
tangwei 已提交
169
    models = get_global_env("train.model.models")
T
tangwei 已提交
170 171
    model_package = __import__(package,
                               globals(), locals(), package.split("."))
T
tangwei 已提交
172 173
    instance = getattr(model_package, class_name)
    return instance
T
tangwei 已提交
174 175


T
tangwei 已提交
176 177
def lazy_instance_by_fliename(abs, class_name):
    dirname = os.path.dirname(abs)
T
tangwei 已提交
178
    sys.path.append(dirname)
T
tangwei 已提交
179
    package = os.path.splitext(os.path.basename(abs))[0]
T
tangwei 已提交
180

T
tangwei 已提交
181 182
    model_package = __import__(package,
                               globals(), locals(), package.split("."))
T
tangwei 已提交
183 184
    instance = getattr(model_package, class_name)
    return instance
T
tangwei 已提交
185 186


T
tangwei 已提交
187 188 189 190 191 192 193 194 195
def get_platform():
    import platform
    plats = platform.platform()
    if 'Linux' in plats:
        return "LINUX"
    if 'Darwin' in plats:
        return "DARWIN"
    if 'Windows' in plats:
        return "WINDOWS"
C
chengmo 已提交
196 197 198 199


def find_free_port():
    def __free_port():
T
tangwei 已提交
200
        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
C
chengmo 已提交
201 202
            s.bind(('', 0))
            return s.getsockname()[1]
T
tangwei 已提交
203

C
chengmo 已提交
204 205
    new_port = __free_port()
    return new_port
X
test  
xjqbest 已提交
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228


def load_yaml(config):
    vs = [int(i) for i in yaml.__version__.split(".")]
    if vs[0] < 5:
        use_full_loader = False
    elif vs[0] > 5:
        use_full_loader = True
    else:
        if vs[1] >= 1:
            use_full_loader = True
        else:
            use_full_loader = False

    if os.path.isfile(config):
        with open(config, 'r') as rb:
            if use_full_loader:
                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
            else:
                _config = yaml.load(rb.read())
            return _config
    else:
        raise ValueError("config {} can not be supported".format(config))