CspFileReader.py

# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import time
import json
import glob
import logging
import pandas as pd
from multiprocessing import Process, Lock
""" Some terms to clarify the code
    in most case, one or more paremeters may be set as input args for a class or a function
    in form of single variable or k-v dict

    1.  trainerId
    2.  gpuId
    3.  rankId
    4.  gpuPerTrainer
    5.  groupSize
    6.  groupId
    7.  groupNum
    8.  displaySize
    9.  dataPath
    10. resultPath
    11. fileOrganizeForm -- "byRank" OR "byTrainer" or "other"

"""

PIPELINEINFO_TRACE_NUM = 1

dcgmMetricParameterMap = {
    "02_gpuUtility": [("GPUTL", "GPUTL"), ("GRACT", "GRACT")],
    "03_smUtility": [("SMACT", "SMACT"), ("SMOCC", "SMOCC")],
    "04_memUtility": [("FB_USED_RATIO", "FB_USED_RATIO"), ("DRAMA", "DRAMA")],
    "05_txUtility": [("NVLTX", "NVLTX"), ("NVLRX", "NVLRX"), ("PCITX", "PCITX"),
                     ("PCIRX", "PCIRX")],
    "06_calUtility": [("FP32A", "FP32A"), ("FP16A", "FP16A"),
                      ("TENSO", "TENSO")]
}
DCGMINFO_TRACE_NUM = len(dcgmMetricParameterMap.keys())
NETINFO_TRACE_NUM = 2

DCGM_PATH = "dcgm"
NET_PATH = "net"
TIME_PATH = "time"
PROFILE_PATH = "profile"

FILEORGANIZEFORM_BYRANK = "byRank"
FILEORGANIZEFORM_BYTRAINER = "byTrainer"
FILEORGANIZEFORM_BYOTHER = "other"
FILEORGANIZEFORM = [
    FILEORGANIZEFORM_BYRANK, FILEORGANIZEFORM_BYTRAINER,
    FILEORGANIZEFORM_BYOTHER
]


class FileReader(object):

    def __init__(self, logger, args):
        self._logger = logger
        self._args = args

        self._fileList = []
        self._fileNum = 0

        self._dataPath = ""
        self._groupSize = 0
        self._displaySize = 0
        self._organizeForm = FILEORGANIZEFORM_BYOTHER
        self._gpuPerTrainer = 0

        self._checkArgs()
        self._getFileList()

        self._lock = Lock()

    def printArgs(self):
        self._logger.info("dataPath:")
        self._logger.info(self._dataPath)
        self._logger.info("groupSize:")
        self._logger.info(self._groupSize)
        self._logger.info("displaySize:")
        self._logger.info(self._displaySize)
        self._logger.info("organizeForm:")
        self._logger.info(self._organizeForm)
        self._logger.info("gpuPerTrainer:")
        self._logger.info(self._gpuPerTrainer)
        self._logger.info("minTimeStamp:")
        self._logger.info(self._minTimeStamp)

    def _checkArgsKey(self, key, type):
        if not self._args.has_key(key):
            raise KeyError("args should has key [%s]!" % key)

        if not isinstance(self._args[key], type):
            raise TypeError(
                "Invalid type of key [%s] in args dict, it should be a %s!" %
                (key, type))

        exec("self._%s = self._args[\"%s\"]" % (key, key))

    def _align_ts(self, ts):
        return ts - self._minTimeStamp

    def _checkArgs(self):
        if not isinstance(self._args, dict):
            raise TypeError("Invalid type of args, it should be a dict!")

        self._checkArgsKey("organizeForm", str)
        if self._organizeForm not in FILEORGANIZEFORM or \
            self._organizeForm == FILEORGANIZEFORM_BYOTHER:
            raise NotImplementedError(
                "we have not known how to process this form of file [%s]!" %
                self._organizeForm)

        self._checkArgsKey("gpuPerTrainer", int)

        self._checkArgsKey("dataPath", str)
        if not os.path.exists(self._dataPath):
            raise IOError("input data path [%s] not existed!" %
                          (self._dataPath))

        self._checkArgsKey("groupSize", int)
        self._checkArgsKey("displaySize", int)
        self._checkArgsKey("minTimeStamp", int)

    def getFileListByGroup(self, groupId):
        lIndext = 0
        rIndext = 0

        if self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
            lIndext = groupId * self._groupSize
            rIndext = (groupId + 1) * self._groupSize
        elif self._organizeForm == FILEORGANIZEFORM_BYRANK:
            lIndext = groupId * self._groupSize * self._gpuPerTrainer
            rIndext = (groupId + 1) * self._groupSize * self._gpuPerTrainer

        try:
            return self._fileList[lIndext:rIndext]
        except IndexError:
            raise IndexError("invalid index of file list")

    def getFileList(self):
        return self._getFileList

    def _cmp(self, x, y):
        return self._getId(x, self._organizeForm) - self._getId(
            y, self._organizeForm)

    def _getFileList(self):
        self._fileList = glob.glob(os.path.join(self._dataPath, "*.*"))

        # check unique
        idList = []
        newFileList = []
        for file in self._fileList:
            id = self._getId(file, self._organizeForm)
            if id not in idList:
                idList.append(id)
                newFileList.append(file)
            else:
                raise NotImplementedError(
                    "[%s] is repeated by id, we don not how to process it!" %
                    file)

        if not self._fileList:
            if (self._getId(self._fileList[-1]) -
                    self._getId(self._fileList[0])) != len(self._fileList) - 1:
                raise Exception("The file id should be countious!")
        # sort
        def _sortBySuffix(elem):
            return int(elem.split(".")[-1])

        self._fileList.sort(key=_sortBySuffix)

        if not self._fileList:
            self._logger.warning("we can not find any file in dir [%s]!" %
                                 self._dataPath)
        else:
            self._logger.info("file list in dir [%s] is : %s !" %
                              (self._dataPath, ',  '.join(self._fileList)))

        return self._fileList

    def _getId(self, fileName, organizeForm, sed="."):
        if self._organizeForm != organizeForm:
            raise TypeError(
                "Can not get rank id when organizer form is not %s!" %
                organizeForm)

        if not os.path.isfile(fileName):
            raise IOError("[%s] is not a valid file!" % (fileName))

        try:
            prefix_str = fileName.split(sed)[-1]
            try:
                return int(prefix_str)
            except ValueError, Argument:
                print(Argument)
                raise TypeError("invalid fileName [%s]" % fileName)

        except IndexError, Argument:
            print(Argument)
            raise TypeError(
                "invalid fileName [%s], the prefix should be a number!" %
                fileName)

    def getRankId(self, fileName, sed="."):
        return self._getId(fileName, FILEORGANIZEFORM_BYRANK, sed)

    def getRankNum(self):
        if self._organizeForm == FILEORGANIZEFORM_BYRANK:
            return len(self._fileList)

        elif self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
            return len(self._fileList) * self._gpuPerTrainer

    def getTrainerNum(self):
        if self._organizeForm == FILEORGANIZEFORM_BYRANK:
            return len(self._fileList) / self._gpuPerTrainer

        elif self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
            return len(self._fileList)

    def getTrainerId(self, fileName, sed="."):
        return self._getId(fileName, FILEORGANIZEFORM_BYTRAINER, sed)

    def _splitTaskListForMultiProcess(self, ls, n):
        if not isinstance(ls, list) or not isinstance(n, int):
            return []
        ls_len = len(ls)
        if n <= 0 or 0 == ls_len:
            return []
        if n >= ls_len:
            return [[i] for i in ls]
        else:
            j = int((ls_len + n - 1) / n)
            k = ls_len % n
            ls_return = []
            end = 0
            for i in range(0, (n) * j, j):
                if i < len(ls) and (i + j) < len(ls):
                    ls_return.append(ls[i:i + j])
                    end = i + j
            ls_return.append(ls[end:])
            return ls_return

    def getOpInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
        return self.getFileName("opinfo", groupId, gpuId, tmpPath)

    def getPipeLineInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
        return self.getFileName("pipilineinfo", groupId, gpuId, tmpPath)

    def getDCGMInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
        return self.getFileName("dcgm", groupId, gpuId, tmpPath)

    def getFileName(self, name, groupId, gpuId, tmpPath="./tmp"):
        return os.path.join(tmpPath, "%s_%d_%d.json" % (name, groupId, gpuId))

    def getOpInfoDict(self, groupId, gpuId, tmpPath="./tmp"):
        return self.getDict("opinfo", groupId, gpuId, tmpPath)

    def getDcgmInfoDict(self, groupId, gpuId, tmpPath="./tmp"):
        return self.getDict("dcgm", groupId, gpuId, tmpPath)

    def getDict(self, name, groupId, gpuId, tmpPath="./tmp"):
        fileName = self.getFileName(name, groupId, gpuId, tmpPath)
        if not os.path.isfile(fileName):
            raise IOError("[%s] is not existed!" % fileName)

        data = {}
        with open(fileName, "r") as rf:
            try:
                data = json.load(rf)
            except Exception:
                self._logger.error("read [%s] error. not a json file!" %
                                   (fileName))
                raise TypeError("read [%s] error. not a json file!" %
                                (fileName))
        return data

    def dumpOpInfoDict(self,
                       data,
                       groupId,
                       gpuId,
                       pretty=False,
                       tmpPath="./tmp"):
        return self.dumpDict(data,
                             "opinfo",
                             groupId,
                             gpuId,
                             pretty=False,
                             tmpPath="./tmp")

    def dumpDCGMDict(self, data, groupId, gpuId, pretty=False, tmpPath="./tmp"):
        return self.dumpDict(data,
                             "dcgm",
                             groupId,
                             gpuId,
                             pretty=False,
                             tmpPath="./tmp")

    def dumpDict(self,
                 data,
                 name,
                 groupId,
                 gpuId,
                 pretty=False,
                 tmpPath="./tmp"):
        self._lock.acquire()
        if not os.path.exists(tmpPath):
            os.makedirs(tmpPath)
        self._lock.release()
        if pretty:
            jsObj = json.dumps(data, indent=4, separators=(',', ': '))
        else:
            jsObj = json.dumps(data, separators=(',', ':'))

        fileName = self.getFileName(name, groupId, gpuId, tmpPath)
        if os.path.isfile(fileName):
            os.remove(fileName)

        fileObject = open(fileName, 'w')
        fileObject.write(jsObj)
        fileObject.close()
        self._logger.info("dump [%s] sucessfully!" % fileName)


def getLogger():
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    rq = time.strftime('%Y%m%d%H%M.%s', time.localtime(time.time()))
    log_path = os.path.dirname(os.getcwd()) + '/Logs/'
    if not os.path.exists(log_path):
        os.makedirs(log_path)

    log_name = log_path + rq + '.log'
    logfile = log_name
    fh = logging.FileHandler(logfile, mode='w')
    fh.setLevel(logging.DEBUG)

    formatter = logging.Formatter(
        "%(asctime)s - %(filename)s[line:%(lineno)d] - %(process)d - %(levelname)s: %(message)s"
    )
    fh.setFormatter(formatter)

    logger.addHandler(fh)
    return logger


def test_FileReader(args):
    try:
        testReader = FileReader(None, args)
    except Exception, Argument:
        print(Argument)
    else:
        testReader.printArgs()


if __name__ == "__main__":
    args = 0
    test_FileReader(args)

    args = {
        "dataPath": ".",
        "groupSize": 1,
        "displaySize": 1,
        "gpuPerTrainer": 8,
        "organizeForm": FILEORGANIZEFORM_BYOTHER,
    }
    test_FileReader(args)

    args = {
        "dataPath": ".",
        "groupSize": 1,
        "displaySize": 1,
        "gpuPerTrainer": 8,
        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
    }
    test_FileReader(args)

    args = {
        "dataPath": "./res",
        "groupSize": 1,
        "displaySize": 1,
        "gpuPerTrainer": 8,
        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
    }
    test_FileReader(args)

    args = {
        "dataPath": ".",
        "groupSize": "",
        "displaySize": 1,
        "gpuPerTrainer": 8,
        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
    }
    test_FileReader(args)