Created by: sagunkayastha

Hi, I am trying to train ppyolo on custom dataset. But i am getting following error

Traceback (most recent call last): File "tools/train.py", line 370, in main() File "tools/train.py", line 243, in main outs = exe.run(compiled_train_prog, fetch_list=train_values) File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/paddle/fluid/executor.py", line 1071, in run six.reraise(*sys.exc_info()) File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/six.py", line 703, in reraise raise value File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/paddle/fluid/executor.py", line 1066, in run return_merged=return_merged) File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/paddle/fluid/executor.py", line 1167, in _run_impl return_merged=return_merged) File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/paddle/fluid/executor.py", line 879, in _run_parallel tensors = exe.run(fetch_var_names, return_merged)._move_to_list() paddle.fluid.core_avx.EnforceNotMet:

C++ Call Stacks (More useful to developers):

0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int) 1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) 2 paddle::framework::OperatorWithKernel::ParseInputDataType(paddle::framework::ExecutionContext const&, std::string const&, paddle::framework::proto::VarType_Type*) const 3 paddle::framework::OperatorWithKernel::IndicateVarDataType(paddle::framework::ExecutionContext const&, std::string const&) const 4 paddle::operators::CropOp::GetExpectedKernelType(paddle::framework::ExecutionContext const&) const 5 paddle::framework::OperatorWithKernel::ChooseKernel(paddle::framework::RuntimeContext const&, paddle::framework::Scope const&, paddle::platform::Place const&) const 6 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&, paddle::framework::RuntimeContext*) const 7 paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, paddle::platform::Place const&) const 8 paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, paddle::platform::Place const&) 9 paddle::framework::details::ComputationOpHandle::RunImpl() 10 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) 11 paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr<paddle::framework::BlockingQueue > const&, unsigned long*) 12 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) 13 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) 14 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const

Python Call Stacks (More useful to users):

File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/paddle/fluid/framework.py", line 2610, in append_op attrs=kwargs.get("attrs", None)) File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op return self.main_program.current_block().append_op(*args, **kwargs) File "/home/ubuntu/sagun/env_pollen/lib/python3.7/site-packages/paddle/fluid/layers/nn.py", line 9949, in crop attrs=None if len(attrs) == 0 else attrs) File "/home/ubuntu/sagun/PaddleDetection/ppdet/modeling/losses/iou_loss.py", line 169, in _bbox_transform gj = fluid.layers.crop(x=gj_max, shape=dcx) File "/home/ubuntu/sagun/PaddleDetection/ppdet/modeling/losses/iou_aware_loss.py", line 73, in call batch_size, True, scale_x_y, eps) File "/home/ubuntu/sagun/PaddleDetection/ppdet/modeling/losses/yolo_loss.py", line 183, in _get_fine_grained_loss self._batch_size, scale_x_y) File "/home/ubuntu/sagun/PaddleDetection/ppdet/modeling/losses/yolo_loss.py", line 69, in call mask_anchors, self._ignore_thresh) File "/home/ubuntu/sagun/PaddleDetection/ppdet/modeling/anchor_heads/yolo_head.py", line 394, in get_loss self.prefix_name) File "/home/ubuntu/sagun/PaddleDetection/ppdet/modeling/architectures/yolo.py", line 85, in build gt_score, targets) File "/home/ubuntu/sagun/PaddleDetection/ppdet/modeling/architectures/yolo.py", line 159, in train return self.build(feed_vars, mode='train') File "tools/train.py", line 116, in main train_fetches = model.train(feed_vars) File "tools/train.py", line 370, in main()

Error Message Summary:

InvalidArgumentError: The Tensor in the crop Op's Input Variable X(create_parameter_45.w_0) is not initialized. [Hint: Expected t->IsInitialized() == true, but received t->IsInitialized():0 != true:1.] at (/paddle/paddle/fluid/framework/operator.cc:1289) [operator < crop > error]

I tried using coco dataset. But got same error. I am using Cuda 10.2 with cudnn 7.6.5 . I have compiled paddlepaddle properly and and the testarchitectures.py, which show no issues.