eval.py

#  Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
#
#Licensed under the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.

import os
import sys
import argparse
import ast
import logging
import numpy as np
import paddle.fluid as fluid
from paddle.fluid.dygraph.base import to_variable
from paddle.io import DataLoader, Dataset, DistributedBatchSampler
from paddle.hapi.model import _all_gather
from paddle.fluid.dygraph.parallel import ParallelEnv

from model import *
from config_utils import *
from kinetics_dataset import KineticsDataset

logging.root.handlers = []
FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s'
logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout)
logger = logging.getLogger(__name__)


def parse_args():
    parser = argparse.ArgumentParser(
        "SLOWFAST test for performance evaluation.")
    parser.add_argument(
        '--config_file',
        type=str,
        default='slowfast.yaml',
        help='path to config file of model')
    parser.add_argument(
        '--batch_size',
        type=int,
        default=None,
        help='total eval batch size of all gpus.')
    parser.add_argument(
        '--use_gpu',
        type=ast.literal_eval,
        default=True,
        help='default use gpu.')
    parser.add_argument(
        '--use_data_parallel',
        type=ast.literal_eval,
        default=True,
        help='default use data parallel.')
    parser.add_argument(
        '--weights',
        type=str,
        default=None,
        help='Weight path, None to use config setting.')
    parser.add_argument(
        '--log_interval',
        type=int,
        default=1,
        help='mini-batch interval to log.')
    args = parser.parse_args()
    return args


# Performance Evaluation
def test_slowfast(args):
    config = parse_config(args.config_file)
    test_config = merge_configs(config, 'test', vars(args))
    print_configs(test_config, "Test")

    if not args.use_gpu:
        place = fluid.CPUPlace()
    elif not args.use_data_parallel:
        place = fluid.CUDAPlace(0)
    else:
        place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id)

    _nranks = ParallelEnv().nranks  # num gpu
    bs_single = int(test_config.TEST.batch_size /
                    _nranks)  # batch_size of each gpu

    with fluid.dygraph.guard(place):
        #build model
        slowfast = SlowFast(cfg=test_config, num_classes=400)
        if args.weights:
            assert os.path.exists(args.weights + '.pdparams'),\
                "Given weight dir {} not exist.".format(args.weights)

        logger.info('load test weights from {}'.format(args.weights))
        model_dict, _ = fluid.load_dygraph(args.weights)
        slowfast.set_dict(model_dict)

        if args.use_data_parallel:
            strategy = fluid.dygraph.parallel.prepare_context()
            slowfast = fluid.dygraph.parallel.DataParallel(
                slowfast, strategy, find_unused_parameters=False)

        #create reader
        test_data = KineticsDataset(mode="test", cfg=test_config)
        test_sampler = DistributedBatchSampler(
            test_data, batch_size=bs_single, shuffle=False, drop_last=False)
        test_loader = DataLoader(
            test_data,
            batch_sampler=test_sampler,
            places=place,
            feed_list=None,
            num_workers=8,
            return_list=True)

        # start eval
        num_ensemble_views = test_config.TEST.num_ensemble_views
        num_spatial_crops = test_config.TEST.num_spatial_crops
        num_cls = test_config.MODEL.num_classes
        num_clips = num_ensemble_views * num_spatial_crops
        num_videos = len(test_data) // num_clips
        video_preds = np.zeros((num_videos, num_cls))
        video_labels = np.zeros((num_videos, 1), dtype="int64")
        clip_count = {}

        print(
            "[EVAL] eval start, number of videos {}, total number of clips {}".
            format(num_videos, num_clips * num_videos))
        slowfast.eval()
        for batch_id, data in enumerate(test_loader):
            # call net
            model_inputs = [data[0], data[1]]
            preds = slowfast(model_inputs, training=False)
            labels = data[2]
            clip_ids = data[3]

            # gather mulit card, results of following process in each card is the same.
            if _nranks > 1:
                preds = _all_gather(preds, _nranks)
                labels = _all_gather(labels, _nranks)
                clip_ids = _all_gather(clip_ids, _nranks)

            # to numpy
            preds = preds.numpy()
            labels = labels.numpy().astype("int64")
            clip_ids = clip_ids.numpy()

            # preds ensemble
            for ind in range(preds.shape[0]):
                vid_id = int(clip_ids[ind]) // num_clips
                ts_idx = int(clip_ids[ind]) % num_clips
                if vid_id not in clip_count:
                    clip_count[vid_id] = []
                if ts_idx in clip_count[vid_id]:
                    print(
                        "[EVAL] Passed!! read video {} clip index {} / {} repeatedly.".
                        format(vid_id, ts_idx, clip_ids[ind]))
                else:
                    clip_count[vid_id].append(ts_idx)
                    video_preds[vid_id] += preds[ind]  # ensemble method: sum
                    if video_labels[vid_id].sum() > 0:
                        assert video_labels[vid_id] == labels[ind]
                    video_labels[vid_id] = labels[ind]
            if batch_id % args.log_interval == 0:
                print("[EVAL] Processing batch {}/{} ...".format(
                    batch_id, len(test_data) // test_config.TEST.batch_size))

        # check clip index of each video
        for key in clip_count.keys():
            if len(clip_count[key]) != num_clips or sum(clip_count[
                    key]) != num_clips * (num_clips - 1) / 2:
                print(
                    "[EVAL] Warning!! video [{}] clip count [{}] not match number clips {}".
                    format(key, clip_count[key], num_clips))

        video_preds = to_variable(video_preds)
        video_labels = to_variable(video_labels)
        acc_top1 = fluid.layers.accuracy(
            input=video_preds, label=video_labels, k=1)
        acc_top5 = fluid.layers.accuracy(
            input=video_preds, label=video_labels, k=5)
        print('[EVAL] eval finished, avg_acc1= {}, avg_acc5= {} '.format(
            acc_top1.numpy(), acc_top5.numpy()))


if __name__ == "__main__":
    args = parse_args()
    logger.info(args)
    test_slowfast(args)