diff --git a/PaddleCV/PaddleVideo/eval.py b/PaddleCV/PaddleVideo/eval.py index 3c2641d03ebe9c8e2df53281a48f4296c01014c1..5be1408d157c580a7614b7f92e6d5bd7864a6d38 100644 --- a/PaddleCV/PaddleVideo/eval.py +++ b/PaddleCV/PaddleVideo/eval.py @@ -108,19 +108,35 @@ def test(args): test_feeder = fluid.DataFeeder(place=place, feed_list=test_feeds) epoch_period = [] + for test_iter, data in enumerate(test_reader()): - cur_time = time.time() - test_outs = exe.run(fetch_list=test_fetch_list, + if args.model_name!="TALL": + cur_time = time.time() + test_outs = exe.run(fetch_list=test_fetch_list, feed=test_feeder.feed(data)) - period = time.time() - cur_time - epoch_period.append(period) - test_metrics.accumulate(test_outs) + period = time.time() - cur_time + epoch_period.append(period) + test_metrics.accumulate(test_outs) # metric here - if args.log_interval > 0 and test_iter % args.log_interval == 0: - info_str = '[EVAL] Batch {}'.format(test_iter) - test_metrics.calculate_and_log_out(test_outs, info_str) - + if args.log_interval > 0 and test_iter % args.log_interval == 0: + info_str = '[EVAL] Batch {}'.format(test_iter) + test_metrics.calculate_and_log_out(test_outs, info_str) + else: + data_feed_in = [items[:2] for items in data] + + cur_time = time.time() + test_outs = exe.run(fetch_list=test_fetch_list, + feed=test_feeder(data_feed_in)) + period = time.time() - cur_time + epoch_period.append(period) + + calc_info = [items[2:] for items in data] + test_result_list = [item for item in test_outs] + calc_info + + test_metrics.accumulate(test_result_list) + + #shipping back to develop metrics if not os.path.isdir(args.save_dir): os.makedirs(args.save_dir) test_metrics.finalize_and_log_out("[EVAL] eval finished. ", args.save_dir) diff --git a/PaddleCV/PaddleVideo/metrics/metrics_util.py b/PaddleCV/PaddleVideo/metrics/metrics_util.py index 1228b77a4c68e73ac42f212afe6052723185b62e..91ea3b4ab64731ba86d2d2606389726609918c0b 100644 --- a/PaddleCV/PaddleVideo/metrics/metrics_util.py +++ b/PaddleCV/PaddleVideo/metrics/metrics_util.py @@ -430,12 +430,24 @@ class TallMetrics(Metrics): self.name = name self.mode =mode self.calculator = tall_metrics.MetricsCalculator(cfg=cfg, name=self.name, mode=self.mode) + self.IoU_thresh = [0.1, 0.3, 0.5, 0.7] + self.all_correct_num_10 = [0.0] * 5 + self.all_correct_num_5 = [0.0] * 5 + self.all_correct_num_1 = [0.0] * 5 + self.all_retrievd = 0.0 def calculator_and_log_out(self, fetch_list, info=""): if self.mode == "train": loss = np.array(fetch_list[0]) logger.info(info +'\tLoss = {}'.format('%.6f' % np.mean(loss))) - elif self.mode == "valid": + + + elif self.mode == "test": + + pass + + def accumalate(): + if self.mode == "test": outs = fetch_list[0] outputs = np.squeeze(outs) start = fetch_list[1] @@ -459,27 +471,34 @@ class TallMetrics(Metrics): clips = [b[0] for b in movie_clip_featmaps] sclips = [b[0] for b in movie_clip_sentences] - for i in range(len(IoU_thresh)): - IoU = IoU_thresh[i] - correct_num_10 = compute_IoU_recall_top_n_forreg(10, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips) - correct_num_5 = compute_IoU_recall_top_n_forreg(5, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips) - correct_num_1 = compute_IoU_recall_top_n_forreg(1, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips) - logger.info(info + " IoU=" + str(IoU) + ", R@10: " + str(correct_num_10 / len(sclips)) + "; IoU=" + str(IoU) + ", R@5: " + str(correct_num_5 / len(sclips)) + "; IoU=" + str(IoU) + ", R@1: " + str(correct_num_1 / len(sclips))) + for i in range(len(sel.IoU_thresh)): + IoU = self.IoU_thresh[i] + self.current_correct_num_10 = compute_IoU_recall_top_n_forreg(10, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips) + self_current_correct_num_5 = compute_IoU_recall_top_n_forreg(5, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips) + self.current_correct_num_1 = compute_IoU_recall_top_n_forreg(1, IoU, sentence_image_mat, sentence_image_reg_mat, sclips, iclips) + + #logger.info(info + " IoU=" + str(IoU) + ", R@10: " + str(correct_num_10 / len(sclips)) + "; IoU=" + str(IoU) + ", R@5: " + str(correct_num_5 / len(sclips)) + "; IoU=" + str(IoU) + ", R@1: " + str(correct_num_1 / len(sclips))) - all_correct_num_10[i] += correct_num_10 - all_correct_num_5[i] += correct_num_5 - all_correct_num_1[i] += correct_num_1 - all_retrievd += len(sclips) + self.all_correct_num_10[i] += correct_num_10 + self.all_correct_num_5[i] += correct_num_5 + self.all_correct_num_1[i] += correct_num_1 + + self.all_retrievd += len(sclips) - else: - pass - - def accumalate(): def finalize_and_log_out(self, info="", savedir="/"): + all_retrievd = self.all_retrievd + for k in range(len(self.IoU_thresh)): + print(" IoU=" + str(self.IoU_thresh[k]) + ", R@10: " + str(all_correct_num_10[k] / all_retrievd) + "; IoU=" + str(self.IoU_thresh[k]) + ", R@5: " + str(all_correct_num_5[k] / all_retrievd) + "; IoU=" + str(self.IoU_thresh[k]) + ", R@1: " + str(all_correct_num_1[k] / all_retrievd)) + + R1_IOU5 = self all_correct_num_1[2] / all_retrievd + R5_IOU5 = self.all_correct_num_5[2] / all_retrievd + + print "{}\n".format("best_R1_IOU5: %0.3f" % R1_IOU5) + print "{}\n".format("best_R5_IOU5: %0.3f" % R5_IOU5) def reset(self): - self.calculator.clear() + self.calculator.reset() class MetricsZoo(object): diff --git a/PaddleCV/PaddleVideo/metrics/tall/accuracy_metrics.py b/PaddleCV/PaddleVideo/metrics/tall/accuracy_metrics.py index 8a97b4121e066be01800cf92f474f9c5a7dc291c..7302325ac78a2fa05e1592ac328741ac4603f7ac 100644 --- a/PaddleCV/PaddleVideo/metrics/tall/accuracy_metrics.py +++ b/PaddleCV/PaddleVideo/metrics/tall/accuracy_metrics.py @@ -31,8 +31,14 @@ class MetricsCalculator(): def reset(self): logger.info("Resetting {} metrics...".format(self.mode)) - + self.IoU_thresh = [0.1, 0.3, 0.5, 0.7] + self.all_correct_num_10 = [0.0] * 5 + self.all_correct_num_5 = [0.0] * 5 + self.all_correct_num_1 = [0.0] * 5 + self.all_retrievd = 0.0 + def finalize_metrics(self): + return def calculate_metrics(self,): return diff --git a/PaddleCV/PaddleVideo/reader/tacos_reader.py b/PaddleCV/PaddleVideo/reader/tacos_reader.py deleted file mode 100644 index 51f99ac2136e7a8444b459bf0b29285eb133b02b..0000000000000000000000000000000000000000 --- a/PaddleCV/PaddleVideo/reader/tacos_reader.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import os -import sys -import cv2 -import math -import random -import functools -try: - import cPickle as pickle - from cStringIO import StringIO -except ImportError: - import pickle - from io import BytesIO -import numpy as np -import paddle -from PIL import Image, ImageEnhance -import logging - -from .reader_utils import DataReader - -class TacosReader(DataReader): - - def __init__(self, name, mode, cfg): - self.name = name - self.mode = mode - self.cfg = cfg - def create_reader(self): - cfg = self.cfg - mode = self.mode - num_reader_threads = cfg[mode.upper()]['num_reader_threads'] - assert num_reader_threads >=1, \ - "number of reader threads({}) should be a positive integer".format(num_reader_threads) - if num_reader_threads == 1: - reader_func = make_reader - else: - reader_func = make_multi_reader - - filelist = cfg[mode.upper()][''] - if self.mode == 'train': - return reader_func() - elif self.mode == 'valid': - return reader_func() - else: - logger.info("Not implemented") - raise NotImplementedError - -def make_reader(cfg): - def reader(): - cs = cPickle.load(open(cfg.TRAIN.train_clip_sentvec)) - movie_length_info = cPickle.load(open(cfg.TRAIN.movie_length_info)) - - #put train() in here - - diff --git a/PaddleCV/PaddleVideo/reader/tall_reader.py b/PaddleCV/PaddleVideo/reader/tall_reader.py index d3d60cf48f9647088e0c641c44105f9857ecab4b..e144facbdcaa480605893281412ac6814e36440b 100644 --- a/PaddleCV/PaddleVideo/reader/tall_reader.py +++ b/PaddleCV/PaddleVideo/reader/tall_reader.py @@ -22,7 +22,6 @@ import paddle import paddle.fluid as fluid import functools -import pdb random.seed(0) @@ -35,17 +34,18 @@ class TallReader(DataReader): def __init__(self, name, mode, cfg): self.name = name self.mode = mode - self.cfg = cfg + self.cfg = cfg def create_reader(self): cfg = self.cfg mode = self.mode - + if self.mode == 'train': train_batch_size = cfg.TRAIN.batch_size return paddle.batch(train(cfg),batch_size = train_batch_size, drop_last=True) elif self.mode == 'valid': - return test(cfg) + test_batch_size = cfg.VALID.batch_size + return paddle.batch(test(cfg),batch_size = test_batch_size, drop_last=True) else: logger.info("Not implemented") raise NotImplementedError @@ -201,13 +201,13 @@ def train(cfg): clip_sentence_pairs_iou.append((clip_sentence[0], clip_sentence[1], clip_name, start_offset, end_offset)) # count += 1 # if count > 200: - # break + # /yield break num_samples_iou = len(clip_sentence_pairs_iou) print "TRAIN: " + str(len(clip_sentence_pairs_iou))+" iou clip-sentence pairs are readed" return make_train_reader(cfg, clip_sentence_pairs_iou, shuffle=True, is_train=True) -class test(cfg): +class TACoS_Test_dataset(cfg): ''' ''' def __init__(self, cfg): @@ -304,3 +304,35 @@ class test(cfg): comb_feat = np.hstack((left_context_feat,feature_data,right_context_feat)) movie_clip_featmap.append((self.sliding_clip_names[k], comb_feat)) return movie_clip_featmap, movie_clip_sentences + + +def test(cfg): + test_dataset = TACoS_Test_dataset(cfg) + + idx = 0 + for movie_name in test_dataset.movie_names: + idx += 1 + print("%d/%d" % (idx, all_number)) + + movie_clip_featmaps, movie_clip_sentences = test_dataset.load_movie_slidingclip(movie_name, 16) + print("sentences: " + str(len(movie_clip_sentences))) + print("clips: " + str(len(movie_clip_featmaps))) # candidate clips) + + sentence_image_mat = np.zeros([len(movie_clip_sentences), len(movie_clip_featmaps)]) + sentence_image_reg_mat = np.zeros([len(movie_clip_sentences), len(movie_clip_featmaps), 2]) + + for k in range(len(movie_clip_sentences)): + sent_vec = movie_clip_sentences[k][1] + sent_vec = np.reshape(sent_vec, [1, sent_vec.shape[0]]) # 1,4800 + #sent_vec = torch.from_numpy(sent_vec).cuda() + + for t in range(len(movie_clip_featmaps)): + featmap = movie_clip_featmaps[t][1] + visual_clip_name = movie_clip_featmaps[t][0] + + start = float(visual_clip_name.split("_")[1]) + end = float(visual_clip_name.split("_")[2].split("_")[0]) + + featmap = np.reshape(featmap, [1, featmap.shape[0]]) + feed_data = [[featmap, sent_vec, start, end, k, t, movie_clip_sentences, movie_clip_featmaps]] + yield feed_data