From 0023ce5590ff68eb37d6c5c89d96824228b7d883 Mon Sep 17 00:00:00 2001 From: Wang Feng Date: Tue, 7 Jul 2020 19:23:32 +0800 Subject: [PATCH] feat(detection): support time and isort (#42) --- .isort.cfg | 13 ++++++++++ .../detection/layers/basic/functional.py | 4 ++-- .../vision/detection/layers/basic/norm.py | 2 +- .../vision/detection/layers/det/anchor.py | 4 ++-- .../vision/detection/layers/det/box_utils.py | 1 - official/vision/detection/layers/det/loss.py | 1 - .../vision/detection/layers/det/pooler.py | 1 + .../vision/detection/layers/det/retinanet.py | 1 - official/vision/detection/layers/det/rpn.py | 3 ++- .../detection/models/faster_rcnn_fpn.py | 1 + official/vision/detection/models/retinanet.py | 3 ++- official/vision/detection/tools/gpu_nms.py | 1 + official/vision/detection/tools/inference.py | 3 ++- official/vision/detection/tools/test.py | 5 ++-- official/vision/detection/tools/train.py | 24 +++++++++++++++---- 15 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..68da32b --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,13 @@ +[isort] +line_length = 100 +multi_line_output = 3 +balanced_wrapping = True +# skip = tools/test_net.py, tools/train_net.py +known_standard_library = setuptools +known_myself = official +known_data_processing = cv2,numpy,scipy,PIL,matplotlib +known_datasets = pycocotools +known_deeplearning = megengine +sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,data_processing,datasets,deeplearning,myself,LOCALFOLDER +no_lines_before=STDLIB,THIRDPARTY,datasets +default_section = FIRSTPARTY diff --git a/official/vision/detection/layers/basic/functional.py b/official/vision/detection/layers/basic/functional.py index 8fdff3f..36bcf7f 100644 --- a/official/vision/detection/layers/basic/functional.py +++ b/official/vision/detection/layers/basic/functional.py @@ -6,10 +6,10 @@ # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -import megengine as mge -import megengine.functional as F import numpy as np +import megengine as mge +import megengine.functional as F from megengine.core import Tensor diff --git a/official/vision/detection/layers/basic/norm.py b/official/vision/detection/layers/basic/norm.py index 43a0917..073d758 100644 --- a/official/vision/detection/layers/basic/norm.py +++ b/official/vision/detection/layers/basic/norm.py @@ -24,9 +24,9 @@ # This file has been modified by Megvii ("Megvii Modifications"). # All Megvii Modifications are Copyright (C) 2014-2020 Megvii Inc. All rights reserved. # --------------------------------------------------------------------- -import megengine.module as M import numpy as np +import megengine.module as M from megengine.core import Buffer diff --git a/official/vision/detection/layers/det/anchor.py b/official/vision/detection/layers/det/anchor.py index 505553f..83be5ac 100644 --- a/official/vision/detection/layers/det/anchor.py +++ b/official/vision/detection/layers/det/anchor.py @@ -8,10 +8,10 @@ # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. from abc import ABCMeta, abstractmethod -import megengine.functional as F import numpy as np -from megengine.core import tensor, Tensor +import megengine.functional as F +from megengine.core import Tensor, tensor class BaseAnchorGenerator(metaclass=ABCMeta): diff --git a/official/vision/detection/layers/det/box_utils.py b/official/vision/detection/layers/det/box_utils.py index 4d03f5e..23bbba6 100644 --- a/official/vision/detection/layers/det/box_utils.py +++ b/official/vision/detection/layers/det/box_utils.py @@ -9,7 +9,6 @@ from abc import ABCMeta, abstractmethod import megengine.functional as F - from megengine.core import Tensor diff --git a/official/vision/detection/layers/det/loss.py b/official/vision/detection/layers/det/loss.py index b0fa4bf..3e75f29 100644 --- a/official/vision/detection/layers/det/loss.py +++ b/official/vision/detection/layers/det/loss.py @@ -7,7 +7,6 @@ # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. import megengine.functional as F - from megengine.core import Tensor diff --git a/official/vision/detection/layers/det/pooler.py b/official/vision/detection/layers/det/pooler.py index d415b52..a3456ec 100644 --- a/official/vision/detection/layers/det/pooler.py +++ b/official/vision/detection/layers/det/pooler.py @@ -9,6 +9,7 @@ import math import numpy as np + import megengine as mge import megengine.functional as F diff --git a/official/vision/detection/layers/det/retinanet.py b/official/vision/detection/layers/det/retinanet.py index df6d195..88c8c6e 100644 --- a/official/vision/detection/layers/det/retinanet.py +++ b/official/vision/detection/layers/det/retinanet.py @@ -10,7 +10,6 @@ import math from typing import List import megengine.module as M - from megengine.core import Tensor from official.vision.detection.layers import basic diff --git a/official/vision/detection/layers/det/rpn.py b/official/vision/detection/layers/det/rpn.py index dfce347..3c0aebe 100644 --- a/official/vision/detection/layers/det/rpn.py +++ b/official/vision/detection/layers/det/rpn.py @@ -7,9 +7,10 @@ # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. import megengine as mge -import megengine.random as rand import megengine.functional as F import megengine.module as M +import megengine.random as rand + from official.vision.detection import layers from official.vision.detection.tools.gpu_nms import batched_nms diff --git a/official/vision/detection/models/faster_rcnn_fpn.py b/official/vision/detection/models/faster_rcnn_fpn.py index eaaff9a..e2aa411 100644 --- a/official/vision/detection/models/faster_rcnn_fpn.py +++ b/official/vision/detection/models/faster_rcnn_fpn.py @@ -7,6 +7,7 @@ # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. import numpy as np + import megengine as mge import megengine.functional as F import megengine.module as M diff --git a/official/vision/detection/models/retinanet.py b/official/vision/detection/models/retinanet.py index f576c4b..e509239 100644 --- a/official/vision/detection/models/retinanet.py +++ b/official/vision/detection/models/retinanet.py @@ -6,10 +6,11 @@ # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +import numpy as np + import megengine as mge import megengine.functional as F import megengine.module as M -import numpy as np from official.vision.classification.resnet.model import resnet50 from official.vision.detection import layers diff --git a/official/vision/detection/tools/gpu_nms.py b/official/vision/detection/tools/gpu_nms.py index cf84229..4a3ee3f 100644 --- a/official/vision/detection/tools/gpu_nms.py +++ b/official/vision/detection/tools/gpu_nms.py @@ -5,6 +5,7 @@ import os import struct import numpy as np + import megengine as mge import megengine.functional as F from megengine._internal.craniotome import CraniotomeBase diff --git a/official/vision/detection/tools/inference.py b/official/vision/detection/tools/inference.py index c1a3556..3bc53f3 100644 --- a/official/vision/detection/tools/inference.py +++ b/official/vision/detection/tools/inference.py @@ -12,8 +12,9 @@ import os import sys import cv2 -import megengine as mge import numpy as np + +import megengine as mge from megengine import jit from megengine.data.dataset import COCO diff --git a/official/vision/detection/tools/test.py b/official/vision/detection/tools/test.py index 4fe87a5..5cf46d0 100644 --- a/official/vision/detection/tools/test.py +++ b/official/vision/detection/tools/test.py @@ -13,13 +13,14 @@ import os import random import sys from multiprocessing import Process, Queue +from tqdm import tqdm import cv2 -import megengine as mge import numpy as np + +import megengine as mge from megengine import jit from megengine.data import DataLoader, SequentialSampler -from tqdm import tqdm from official.vision.detection.tools.data_mapper import data_mapper from official.vision.detection.tools.nms import py_cpu_nms diff --git a/official/vision/detection/tools/train.py b/official/vision/detection/tools/train.py index fb280c5..d842e61 100644 --- a/official/vision/detection/tools/train.py +++ b/official/vision/detection/tools/train.py @@ -14,16 +14,18 @@ import importlib import multiprocessing as mp import os import sys +import time from collections import defaultdict +from tabulate import tabulate -import megengine as mge import numpy as np + +import megengine as mge from megengine import distributed as dist from megengine import jit from megengine import optimizer as optim from megengine.data import Collator, DataLoader, Infinite, RandomSampler from megengine.data import transform as T -from tabulate import tabulate from official.vision.detection.tools.data_mapper import data_mapper @@ -81,7 +83,8 @@ def worker(rank, world_size, args): weights = mge.load(args.weight_file) model.backbone.bottom_up.load_state_dict(weights) - logger.info("Prepare dataset") + if rank == 0: + logger.info("Prepare dataset") loader = build_dataloader(model.batch_size, args.dataset_dir, model.cfg) train_loader = iter(loader["train"]) @@ -155,21 +158,32 @@ def train_one_epoch( return losses meter = AverageMeter(record_len=model.cfg.num_losses) + time_meter = AverageMeter(record_len=2) log_interval = model.cfg.log_interval for step in range(tot_steps): adjust_learning_rate(opt, epoch_id, step, model, world_size) + + data_tik = time.time() mini_batch = next(data_queue) + data_tok = time.time() + model.inputs["image"].set_value(mini_batch["data"]) model.inputs["gt_boxes"].set_value(mini_batch["gt_boxes"]) model.inputs["im_info"].set_value(mini_batch["im_info"]) + tik = time.time() opt.zero_grad() loss_list = propagate() opt.step() + tok = time.time() + + time_meter.update([tok - tik, data_tok - data_tik]) if rank == 0: + info_str = "e%d, %d/%d, lr:%f, " loss_str = ", ".join(["{}:%f".format(loss) for loss in model.cfg.losses_keys]) - log_info_str = "e%d, %d/%d, lr:%f, " + loss_str + time_str = ", train_time:%.3fs, data_time:%.3fs" + log_info_str = info_str + loss_str + time_str meter.update([loss.numpy() for loss in loss_list]) if step % log_interval == 0: average_loss = meter.average() @@ -180,8 +194,10 @@ def train_one_epoch( tot_steps, opt.param_groups[0]["lr"], *average_loss, + *time_meter.average() ) meter.reset() + time_meter.reset() def make_parser(): -- GitLab