Created by: CaiQiuYu

问题类型：模型训练
问题描述

====================
请在这里描述您在使用过程中的问题，如模型训练出错，建议贴上模型训练代码，以便开发人员分析，并快速响应 错误日志 2020-09-12 22:12:52 [INFO] Connecting PaddleHub server to get pretrain weights... 2020-09-12 22:12:56 [INFO] Load pretrain weights from output/HRNet/pretrain/HRNet_W64. 2020-09-12 22:12:56,709-WARNING: output/HRNet/pretrain/HRNet_W64.pdparams not found, try to load model file saved with [ save_params, save_persistables, save_vars ] 2020-09-12 22:12:59 [INFO] There are 1525 varaibles in output/HRNet/pretrain/HRNet_W64 are loaded. 2020-09-12 22:14:24 [INFO] [TRAIN] Epoch=1/150, Step=50/82, loss=0.383345, lr=0.009964, time_each_step=1.37s, eta=5:44:47 2020-09-12 22:15:13 [INFO] [TRAIN] Epoch 1 finished, loss=0.399562, lr=0.00997 . 2020-09-12 22:15:13 [INFO] Start to evaluating(total_samples=146, total_steps=19)... 0%| | 0/19 [00:00<?, ?it/s]Process Process-9: Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() Process Process-10: File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) Process Process-11: File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) ValueError: too many values to unpack (expected 2) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) Process Process-12: File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape ValueError: too many values to unpack (expected 2) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) Process Process-13: ValueError: too many values to unpack (expected 2) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() ValueError: too many values to unpack (expected 2) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) Process Process-14: File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) ValueError: too many values to unpack (expected 2) Process Process-15: File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() ValueError: too many values to unpack (expected 2) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape ValueError: too many values to unpack (expected 2) Process Process-16: Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 166, in _read_into_queue six.reraise(*sys.exc_info()) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/six.py", line 703, in reraise raise value File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py", line 158, in _read_into_queue result = mapper(sample[0], sample[1], sample[2]) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 137, in call im, label = self.decode_image(im, label) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/transforms/seg_transforms.py", line 118, in decode_image label_height, label_width = label.shape ValueError: too many values to unpack (expected 2) ---------------------------------------------------------------------------ValueError Traceback (most recent call last) in 15 log_interval_steps=50, 16 save_interval_epochs=1, ---> 17 save_dir='output/HRNet') /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/hrnet.py in train(self, num_epochs, train_dataset, train_batch_size, eval_dataset, save_interval_epochs, log_interval_steps, save_dir, pretrain_weights, optimizer, learning_rate, lr_decay_power, use_vdl, sensitivities_file, eval_metric_loss, early_stop, early_stop_patience, resume_checkpoint) 176 pretrain_weights, optimizer, learning_rate, lr_decay_power, 177 use_vdl, sensitivities_file, eval_metric_loss, early_stop, --> 178 early_stop_patience, resume_checkpoint) /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/deeplabv3p.py in train(self, num_epochs, train_dataset, train_batch_size, eval_dataset, save_interval_epochs, log_interval_steps, save_dir, pretrain_weights, optimizer, learning_rate, lr_decay_power, use_vdl, sensitivities_file, eval_metric_loss, early_stop, early_stop_patience, resume_checkpoint) 356 use_vdl=use_vdl, 357 early_stop=early_stop, --> 358 early_stop_patience=early_stop_patience) 359 360 def evaluate(self, /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/base.py in train_loop(self, num_epochs, train_dataset, train_batch_size, eval_dataset, save_interval_epochs, log_interval_steps, save_dir, use_vdl, early_stop, early_stop_patience) 557 batch_size=eval_batch_size, 558 epoch_id=i + 1, --> 559 return_details=True) 560 logging.info('[EVAL] Finished, Epoch={}, {} .'.format( 561 i + 1, dict2str(self.eval_metrics))) /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/deeplabv3p.py in evaluate(self, eval_dataset, batch_size, epoch_id, return_details) 395 eval_dataset.num_samples, total_steps)) 396 for step, data in tqdm.tqdm( --> 397 enumerate(data_generator()), total=total_steps): 398 images = np.array([d[0] for d in data]) 399 im_info = [d[1] for d in data] /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/tqdm/std.py in iter(self) 1079 """), fp_write=getattr(self.fp, 'write', sys.stderr.write)) 1080 -> 1081 for obj in iterable: 1082 yield obj 1083 # Update and possibly print the progressbar. /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/datasets/dataset.py in queue_reader() 185 finish_num += 1 186 elif sample == "": --> 187 raise ValueError("multiprocess reader raises an exception") 188 else: 189 batch_data.append(sample) ValueError: multiprocess reader raises an exception