pod.py 5.2 KB
Newer Older
K
kuizhiqing 已提交
1
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2
#
K
kuizhiqing 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
6
#
K
kuizhiqing 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
K
kuizhiqing 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import OrderedDict
from .container import Container

from .status import Status

import random
import time


class PodSepc(object):
25

K
kuizhiqing 已提交
26 27 28 29 30 31 32 33 34 35 36 37
    def __init__(self):
        self._name = ''.join(
            random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6))

        # by controller
        self._init_containers: List[Container] = []
        self._containers: List[Container] = []

        #self.resource: Resource = None
        #self.status: Status = None

        self._rank = -1
38
        self._init_timeout = None
K
kuizhiqing 已提交
39 40 41 42 43 44
        self._restart = -1
        self._replicas = 0  # number of containers
        self._exit_code = 0


class Pod(PodSepc):
45

K
kuizhiqing 已提交
46 47 48 49
    def __init__(self):
        super().__init__()

    def __str__(self):
50 51 52
        return "Pod: {}, replicas {}, status {}".format(self.name,
                                                        self.replicas,
                                                        self.status)
K
kuizhiqing 已提交
53 54

    def failed_container(self):
55
        cs = []
K
kuizhiqing 已提交
56
        for c in self._containers:
57 58 59
            if c.status == Status.FAILED:
                cs.append(c)
        return cs
K
kuizhiqing 已提交
60 61 62 63 64 65 66 67 68 69 70

    @property
    def name(self):
        return self._name

    @property
    def replicas(self):
        return self._replicas

    @replicas.setter
    def replicas(self, r):
71
        self._replicas = max(r, 1)
K
kuizhiqing 已提交
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103

    @property
    def rank(self):
        return self._rank

    @rank.setter
    def rank(self, r):
        self._rank = r

    @property
    def restart(self):
        return self._restart

    @property
    def containers(self):
        return self._containers

    def add_container(self, c):
        c.rank = len(self._containers)
        self._containers.append(c)

    @property
    def init_containers(self):
        return self._init_containers

    def add_init_container(self, c):
        c.rank = len(self._init_containers)
        self._init_containers.append(c)

    @property
    def exit_code(self):
        for c in self._containers:
104 105
            if c.exit_code != 0:
                return c.exit_code
K
kuizhiqing 已提交
106 107 108
        return 0

    def deploy(self):
109
        # init container should stop before run containers
K
kuizhiqing 已提交
110
        for i in self._init_containers:
111 112
            i.start()
            i.wait(self._init_timeout)
K
kuizhiqing 已提交
113 114 115 116 117 118

        for c in self._containers:
            c.start()

        self._restart += 1

K
kuizhiqing 已提交
119
    def stop(self, sigint=15, timeout=None):
K
kuizhiqing 已提交
120
        for c in self._containers:
K
kuizhiqing 已提交
121 122 123 124 125 126 127 128 129 130 131 132
            if isinstance(sigint, int) and timeout is None:
                c.send_signal(sigint)
            else:
                c.terminate()

        if isinstance(timeout, int):
            if not self.join(timeout):
                for c in self._containers:
                    c.terminate(force=True)
                return False
            else:
                return True
K
kuizhiqing 已提交
133

K
kuizhiqing 已提交
134
    def join(self, timeout=None):
K
kuizhiqing 已提交
135
        for c in self._containers:
K
kuizhiqing 已提交
136 137 138
            if not c.wait(timeout):
                return False
        return True
K
kuizhiqing 已提交
139

140
    @property
K
kuizhiqing 已提交
141 142 143 144 145 146 147
    def status(self):
        if self.is_failed():
            return Status.FAILED

        if self.is_completed():
            return Status.COMPLETED

148 149 150
        if self.is_running():
            return Status.RUNNING

K
kuizhiqing 已提交
151 152 153 154 155 156 157 158
        return Status.READY

    def reset(self):
        self._init_containers = []
        self._containers = []

    def is_failed(self):
        for c in self._containers:
159
            if c.status == Status.FAILED:
K
kuizhiqing 已提交
160 161 162 163 164
                return True
        return False

    def is_completed(self):
        for c in self._containers:
165 166 167 168 169 170 171
            if c.status != Status.COMPLETED:
                return False
        return True

    def is_running(self):
        for c in self._containers:
            if c.status != Status.RUNNING:
K
kuizhiqing 已提交
172 173 174 175 176
                return False
        return True

    def logs(self, idx=None):
        if idx is None:
177
            self._containers[0].logs()
K
kuizhiqing 已提交
178 179 180 181 182
        else:
            self._containers[idx].logs()

    def tail(self, idx=None):
        if idx is None:
183
            self._containers[0].tail()
K
kuizhiqing 已提交
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
        else:
            self._containers[idx].tail()

    def watch(self,
              all_list=[Status.COMPLETED],
              any_list=[Status.FAILED],
              interval=1,
              timeout=-1):
        '''
        watch return if any container status in any_list
        or all container status in all_list
        '''
        end = time.time() + timeout
        while timeout < 0 or time.time() < end:
            for c in self._containers:
199 200
                if c.status in any_list:
                    return c.status
K
kuizhiqing 已提交
201

202
            s = [c.status for c in self._containers]
K
kuizhiqing 已提交
203 204 205 206
            if len(set(s)) == 1 and s[0] in all_list:
                return s[0]

            time.sleep(interval)