envs.py 6.6 KB
Newer Older
T
tangwei 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

T
tangwei 已提交
15
from contextlib import closing
Y
yaoxuefeng 已提交
16
import yaml
T
tangwei12 已提交
17
import copy
T
tangwei 已提交
18
import os
C
chengmo 已提交
19
import socket
T
tangwei 已提交
20
import sys
X
xionghang 已提交
21
import traceback
T
tangwei 已提交
22

T
tangwei12 已提交
23
global_envs = {}
T
tangwei 已提交
24

X
fix  
xjqbest 已提交
25

T
tangwei 已提交
26
def flatten_environs(envs, separator="."):
T
tangwei 已提交
27
    flatten_dict = {}
T
tangwei 已提交
28 29
    assert isinstance(envs, dict)

T
fix bug  
tangwei 已提交
30
    def fatten_env_namespace(namespace_nests, local_envs):
T
fix bug  
tangwei 已提交
31
        if not isinstance(local_envs, dict):
T
tangwei 已提交
32
            global_k = separator.join(namespace_nests)
T
tangwei 已提交
33
            flatten_dict[global_k] = str(local_envs)
T
fix bug  
tangwei 已提交
34 35 36 37 38 39 40
        else:
            for k, v in local_envs.items():
                if isinstance(v, dict):
                    nests = copy.deepcopy(namespace_nests)
                    nests.append(k)
                    fatten_env_namespace(nests, v)
                else:
T
tangwei 已提交
41
                    global_k = separator.join(namespace_nests + [k])
T
tangwei 已提交
42
                    flatten_dict[global_k] = str(v)
T
fix bug  
tangwei 已提交
43

T
tangwei 已提交
44
    for k, v in envs.items():
T
fix bug  
tangwei 已提交
45
        fatten_env_namespace([k], v)
T
tangwei 已提交
46

T
tangwei 已提交
47
    return flatten_dict
T
tangwei 已提交
48

T
tangwei 已提交
49 50 51

def set_runtime_environs(environs):
    for k, v in environs.items():
T
tangwei 已提交
52
        os.environ[k] = str(v)
T
tangwei 已提交
53

T
tangwei 已提交
54

T
tangwei 已提交
55
def get_runtime_environ(key):
T
tangwei 已提交
56 57
    return os.getenv(key, None)

T
tangwei 已提交
58

T
tangwei 已提交
59
def get_trainer():
T
tangwei 已提交
60
    train_mode = get_runtime_environ("train.trainer.trainer")
T
tangwei 已提交
61 62 63
    return train_mode


T
tangwei12 已提交
64 65
def set_global_envs(envs):
    assert isinstance(envs, dict)
T
tangwei 已提交
66

T
tangwei12 已提交
67 68 69 70 71 72
    def fatten_env_namespace(namespace_nests, local_envs):
        for k, v in local_envs.items():
            if isinstance(v, dict):
                nests = copy.deepcopy(namespace_nests)
                nests.append(k)
                fatten_env_namespace(nests, v)
X
fix  
xjqbest 已提交
73 74
            elif (k == "dataset" or k == "phase" or
                  k == "runner") and isinstance(v, list):
X
fix  
xjqbest 已提交
75 76 77 78 79 80 81
                for i in v:
                    if i.get("name") is None:
                        raise ValueError("name must be in dataset list ", v)
                    nests = copy.deepcopy(namespace_nests)
                    nests.append(k)
                    nests.append(i["name"])
                    fatten_env_namespace(nests, i)
T
tangwei12 已提交
82 83 84
            else:
                global_k = ".".join(namespace_nests + [k])
                global_envs[global_k] = v
T
tangwei 已提交
85

X
fix  
xjqbest 已提交
86
    fatten_env_namespace([], envs)
T
tangwei 已提交
87

X
fix  
xjqbest 已提交
88

T
tangwei12 已提交
89
def get_global_env(env_name, default_value=None, namespace=None):
T
tangwei 已提交
90 91 92
    """
    get os environment value
    """
C
chengmo 已提交
93 94
    _env_name = env_name if namespace is None else ".".join(
        [namespace, env_name])
T
tangwei12 已提交
95 96 97
    return global_envs.get(_env_name, default_value)


T
tangwei 已提交
98 99 100 101
def get_global_envs():
    return global_envs


T
tangwei 已提交
102
def path_adapter(path):
T
tangwei 已提交
103 104
    if path.startswith("paddlerec."):
        package = get_runtime_environ("PACKAGE_BASE")
F
frankwhzhang 已提交
105 106
        l_p = path.split("paddlerec.")[1].replace(".", "/")
        return os.path.join(package, l_p)
T
tangwei 已提交
107
    else:
T
tangwei 已提交
108
        return path
T
tangwei 已提交
109 110


T
tangwei 已提交
111 112 113 114 115 116 117
def windows_path_converter(path):
    if get_platform() == "WINDOWS":
        return path.replace("/", "\\")
    else:
        return path.replace("\\", "/")


T
tangwei 已提交
118
def update_workspace():
X
fix  
xjqbest 已提交
119
    workspace = global_envs.get("workspace")
T
tangwei 已提交
120 121
    if not workspace:
        return
T
tangwei 已提交
122
    workspace = path_adapter(workspace)
T
tangwei 已提交
123 124 125

    for name, value in global_envs.items():
        if isinstance(value, str):
T
tangwei 已提交
126
            value = value.replace("{workspace}", workspace)
T
tangwei 已提交
127
            value = windows_path_converter(value)
T
tangwei 已提交
128 129
            global_envs[name] = value

T
tangwei 已提交
130

T
tangwei12 已提交
131
def pretty_print_envs(envs, header=None):
T
tangwei12 已提交
132 133
    spacing = 5
    max_k = 45
T
tangwei 已提交
134
    max_v = 50
T
tangwei12 已提交
135

T
tangwei 已提交
136
    for k, v in envs.items():
T
tangwei12 已提交
137 138
        max_k = max(max_k, len(k))

T
tangwei12 已提交
139
    h_format = "{{:^{}s}}{}{{:<{}s}}\n".format(max_k, " " * spacing, max_v)
T
tangwei12 已提交
140 141 142 143 144 145 146 147
    l_format = "{{:<{}s}}{{}}{{:<{}s}}\n".format(max_k, max_v)
    length = max_k + max_v + spacing

    border = "".join(["="] * length)
    line = "".join(["-"] * length)

    draws = ""
    draws += border + "\n"
T
tangwei 已提交
148 149 150 151

    if header:
        draws += h_format.format(header[0], header[1])
    else:
152
        draws += h_format.format("paddlerec Global Envs", "Value")
T
tangwei 已提交
153

T
tangwei12 已提交
154 155
    draws += line + "\n"

T
tangwei 已提交
156
    for k, v in envs.items():
T
tangwei 已提交
157 158 159 160 161 162
        if isinstance(v, str) and len(v) >= max_v:
            str_v = "... " + v[-46:]
        else:
            str_v = v

        draws += l_format.format(k, " " * spacing, str(str_v))
T
tangwei12 已提交
163 164 165 166 167

    draws += border

    _str = "\n{}\n".format(draws)
    return _str
T
tangwei 已提交
168 169


T
tangwei 已提交
170
def lazy_instance_by_package(package, class_name):
X
xionghang 已提交
171 172 173 174 175 176 177 178 179
    try:
        model_package = __import__(package,
                                   globals(), locals(), package.split("."))
        instance = getattr(model_package, class_name)
        return instance
    except Exception, err:
        traceback.print_exc()
        print('Catch Exception:%s' % str(err))
        return None
T
tangwei 已提交
180 181


T
tangwei 已提交
182
def lazy_instance_by_fliename(abs, class_name):
X
xionghang 已提交
183 184 185 186 187 188 189 190 191 192 193 194 195
    try:
        dirname = os.path.dirname(abs)
        sys.path.append(dirname)
        package = os.path.splitext(os.path.basename(abs))[0]

        model_package = __import__(package,
                                   globals(), locals(), package.split("."))
        instance = getattr(model_package, class_name)
        return instance
    except Exception, err:
        traceback.print_exc()
        print('Catch Exception:%s' % str(err))
        return None
T
tangwei 已提交
196 197


T
tangwei 已提交
198 199 200 201 202 203 204 205 206
def get_platform():
    import platform
    plats = platform.platform()
    if 'Linux' in plats:
        return "LINUX"
    if 'Darwin' in plats:
        return "DARWIN"
    if 'Windows' in plats:
        return "WINDOWS"
C
chengmo 已提交
207 208 209 210


def find_free_port():
    def __free_port():
T
tangwei 已提交
211
        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
C
chengmo 已提交
212 213
            s.bind(('', 0))
            return s.getsockname()[1]
T
tangwei 已提交
214

C
chengmo 已提交
215 216
    new_port = __free_port()
    return new_port
X
test  
xjqbest 已提交
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239


def load_yaml(config):
    vs = [int(i) for i in yaml.__version__.split(".")]
    if vs[0] < 5:
        use_full_loader = False
    elif vs[0] > 5:
        use_full_loader = True
    else:
        if vs[1] >= 1:
            use_full_loader = True
        else:
            use_full_loader = False

    if os.path.isfile(config):
        with open(config, 'r') as rb:
            if use_full_loader:
                _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
            else:
                _config = yaml.load(rb.read())
            return _config
    else:
        raise ValueError("config {} can not be supported".format(config))