Created by: xinsirBUPT2016

• 标题：运行测试程序的时候总出现内存溢出的错误，batch占用的内存没有被回收
• 版本、环境信息：    1）PaddlePaddle版本：1.7.1    2）CPU：Intel(R) Xeon(R) CPU E5-2650 v4 @ 2.20GHz    3）GPU：P40    4）系统环境：CentOS 6.3
• 模型信息    1）resnet50(any model) 2）aliproduct val 3）无 4）
• 复现信息：建议登录本人的开发机进行操作，直接测试即可

    from __future__ import absolute_import
    from __future__ import division
    from __future__ import print_function
    
    import os
    import sys
    import math
    import time
    import argparse
    import functools
    import numpy as np
    import paddle
    import paddle.fluid as fluid
    import models
    import reader
    from utility import add_arguments, print_arguments, check_cuda
    from losses import ArcMarginLoss
    
    import gc
    
    parser = argparse.ArgumentParser(description=__doc__)
    add_arg = functools.partial(add_arguments, argparser=parser)
    # yapf: disable
    add_arg('model', str, "ResNet50", "Set the network to use.")
    add_arg('embedding_size', int, 512, "Embedding size.")
    add_arg('batch_size', int, 2048, "Minibatch size.")
    add_arg('image_shape', str, "3,224,224", "Input image size.")
    add_arg('use_gpu', bool, True, "Whether to use GPU or not.")
    add_arg('class_dim', int, 50030, "Class number.")
    add_arg('pretrained_model', str, "../checkpoint/balance/440000/", "Whether to use pretrained model.")
    #add_arg('pretrained_model', str, "./online_models/136000/", "Whether to use pretrained model.")
    #add_arg('pretrained_model', str, "./Ali_output/ResNet50/1000//", "Whether to use pretrained model.")
    add_arg('data_dir', str, "/home/work/workspace/", '')
    add_arg('train_list', str, "./dataset/train_all.txt", '')
    #add_arg('val_list', str, "./dataset/val_all.txt", '')
    add_arg('val_list', str, "./val_all.list", '')
    # yapf: enable
    model_list = [m for m in dir(models) if "__" not in m]
    
    
    def infer(args):
        # parameters from arguments
        model_name = args.model
        pretrained_model = args.pretrained_model
        image_shape = [int(m) for m in args.image_shape.split(",")]
    
        assert model_name in model_list, "{} is not in lists: {}".format(args.model,
                                                                         model_list)
    
        # image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
        def train_program():
            image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
    
            # model definition
            model = models.__dict__[model_name]()
            out = model.net(input=image, embedding_size=args.embedding_size)
            metricloss = ArcMarginLoss(class_dim = args.class_dim)
            logit = metricloss.get_arc_margin_product(out, args.class_dim, 80)
            softmax_out = fluid.layers.softmax(logit, use_cudnn=False)#B class_num
            return softmax_out
        
        softmax_out = train_program()
    
        test_program = fluid.default_main_program().clone(for_test=True)
    
        place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
        exe = fluid.Executor(place)
        exe.run(fluid.default_startup_program())
    
        if pretrained_model:
    
            def if_exist(var):
                print (var.name, os.path.exists(os.path.join(pretrained_model, var.name)))
                return os.path.exists(os.path.join(pretrained_model, var.name))
    
            fluid.io.load_vars(exe, pretrained_model, predicate=if_exist)
    
        infer_reader = paddle.batch(reader.infer(args), batch_size=args.batch_size, drop_last=False)
        feed_var_list = [
            test_program.global_block().var(var_name) for var_name in ['image']
        ]
        feeder = fluid.DataFeeder(place=place, feed_list=feed_var_list)
    
        #fetch_list = [out.name]
        fetch_list = [softmax_out.name]
    
        test_exe = fluid.ParallelExecutor(
            use_cuda=args.use_gpu,
            main_program=test_program)#,
            #share_vars_from=exe)
    
        t1 = time.time()
        fp = open('result_4k.txt', 'w+')
        for batch_id, data in enumerate(infer_reader()):
    
            img_names = []
            gt_labels = []
            datas = []
            for sample in data:
                img_names.append(sample[1])
                gt_labels.append(sample[1].split('/')[-2])
                datas.append([sample[0]])
            print ('{} begin process'.format(batch_id))
    
            #data = [(np.array(data[0][0]),)]
            #print (data)
            #_data = data[0][0][:,np.newaxis,:,:]
            #result = exe.run(test_program, fetch_list=fetch_list, feed=feeder.feed(_data))
    
            result, = test_exe.run(feed=feeder.feed(datas),
                              fetch_list=fetch_list)   # batch_size x cls_num
    
    
            pred_index = np.argmax(result, 1)
            pred_value = np.max(result, 1)
            gt_labels = np.array(gt_labels)
            #print("Test-{0}-feature: {1}".format(batch_id, result))
            #print (len(result))
            for i in range(gt_labels.shape[0]):
                #print ('Pred: gt: {} top1 index: {} prob: {}\n'.format(gt_labels[i], pred_index[i], pred_value[i]))
                fp.write(img_names[i] + ' ' + str(gt_labels[i]) + ' ' + str(pred_index[i]) + ' ' + str(pred_value[i]) + '\n')
            sys.stdout.flush()
            gc.collect()
        fp.close()
        t2 = time.time()
        print ('total time', t2 - t1)
    
    
    def main():
        args = parser.parse_args()
        print_arguments(args)
        check_cuda(args.use_gpu)
        infer(args)
    
    
    if __name__ == '__main__':
        main()

• 问题描述： 启动测试程序以后，内存占用会随着batch的增加而无限增加，最终超过内存限制，导致开发机卡死