// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include "paddle/fluid/framework/new_executor/new_executor_defs.h" #include "paddle/fluid/framework/rw_lock.h" // When in inference scenario, the scopes will not be written by two threads in // a mean time, but a scope may be read by multiple threads concurrently, and // the mutex will cause serious performance issue. // So the mutex is disabled when `ON_INFER`. #ifdef PADDLE_ON_INFERENCE #define SCOPE_VARS_READER_LOCK #define SCOPE_VARS_WRITER_LOCK #else #define SCOPE_VARS_READER_LOCK AutoRDLock auto_lock(&vars_lock_); #define SCOPE_VARS_WRITER_LOCK AutoWRLock auto_lock(&vars_lock_); #endif namespace paddle { namespace framework { InterpretercoreInferShapeContext::InterpretercoreInferShapeContext( const OperatorBase& op, const RuntimeContext& ctx) : op_(op), ctx_(ctx), can_skip_lod_(false) {} bool InterpretercoreInferShapeContext::HasInput(const std::string& name) const { // has only one input const auto& ins = ctx_.inputs; auto it = ins.find(name); if (it == ins.end()) { return false; } const auto& in = it->second; if (in.size() == 0) return false; PADDLE_ENFORCE_EQ( in.size(), 1UL, platform::errors::InvalidArgument( "Input %s should not contain more than one inputs.", name)); return in[0] != nullptr; } bool InterpretercoreInferShapeContext::HasOutput( const std::string& name) const { // has only one output const auto& outs = ctx_.outputs; auto it = outs.find(name); if (it == outs.end()) { return false; } const auto& out = it->second; if (out.size() == 0) { return false; } PADDLE_ENFORCE_EQ( out.size(), 1UL, platform::errors::InvalidArgument( "Output %s should not contain more than one outputs.", name)); return out[0] != nullptr; } bool InterpretercoreInferShapeContext::HasInputs( const std::string& name) const { const auto& ins = ctx_.inputs; auto it = ins.find(name); if (it == ins.end() || it->second.empty()) { return false; } for (auto& input : it->second) { if (input == nullptr) { return false; } } return true; } bool InterpretercoreInferShapeContext::HasOutputs( const std::string& name) const { const auto& outs = ctx_.outputs; auto it = outs.find(name); if (it == outs.end() || it->second.empty()) { return false; } for (auto& output : it->second) { if (output == nullptr) { return false; } } return true; } AttrReader InterpretercoreInferShapeContext::Attrs() const { return AttrReader(op_.Attrs()); } std::vector InterpretercoreInferShapeContext::Inputs( const std::string& name) const { return op_.Inputs(name); } std::vector InterpretercoreInferShapeContext::Outputs( const std::string& name) const { return op_.Outputs(name); } std::string InterpretercoreInferShapeContext::GetInputNameByIdx( size_t idx) const { auto& op_proto = paddle::framework::OpInfoMap::Instance().Get(op_.Type()).proto_; PADDLE_ENFORCE_LT(idx, op_proto->inputs().size(), platform::errors::OutOfRange( "The index should be less than the size of inputs of " "operator %s, but got index is %d and size is %d", op_.Type(), idx, op_proto->inputs().size())); return op_proto->inputs()[idx].name(); } std::string InterpretercoreInferShapeContext::GetOutputNameByIdx( size_t idx) const { auto& op_proto = paddle::framework::OpInfoMap::Instance().Get(op_.Type()).proto_; PADDLE_ENFORCE_LT(idx, op_proto->outputs().size(), platform::errors::OutOfRange( "The index should be less than the size of outputs of " "operator %s, but got index is %d and size is %d", op_.Type(), idx, op_proto->outputs().size())); return op_proto->outputs()[idx].name(); } void InterpretercoreInferShapeContext::ShareDim(const std::string& in, const std::string& out, size_t i, size_t j) { auto in_it = ctx_.inputs.find(in); auto out_it = ctx_.outputs.find(out); PADDLE_ENFORCE_NE(in_it, ctx_.inputs.end(), platform::errors::NotFound("Input %s does not exist.", in)); PADDLE_ENFORCE_NE( out_it, ctx_.outputs.end(), platform::errors::NotFound("Output %s does not exist.", out)); PADDLE_ENFORCE_LT(i, in_it->second.size(), platform::errors::InvalidArgument( "The index of input dimension is out of range, " "excepted index less than %zu, but received %zu.", in_it->second.size(), i)); PADDLE_ENFORCE_LT(j, out_it->second.size(), platform::errors::InvalidArgument( "The index of output dimension is out of range, " "excepted index less than %zu, but received %zu.", out_it->second.size(), j)); Variable* in_var = in_it->second[i]; Variable* out_var = out_it->second[j]; PADDLE_ENFORCE_EQ( in_var->Type(), out_var->Type(), platform::errors::InvalidArgument( "The type of input (%s) and output (%s) are inconsistent.", in, out)); if (in_var->IsType()) { auto& in_sele_rows = in_var->Get(); auto out_sele_rows = out_var->GetMutable(); out_sele_rows->mutable_value()->Resize(in_sele_rows.value().dims()); out_sele_rows->set_rows(in_sele_rows.rows()); out_sele_rows->set_height(in_sele_rows.height()); } else if (in_var->IsType()) { auto& in_lod_tensor = in_var->Get(); auto* out_lod_tensor = out_var->GetMutable(); out_lod_tensor->Resize(in_lod_tensor.dims()); } else { PADDLE_THROW(platform::errors::Unimplemented( "Currently, the input type of ShareDim only can be LoDTensor " "or SelectedRows.")); } } void InterpretercoreInferShapeContext::ShareAllLoD( const std::string& in, const std::string& out) const { auto in_it = ctx_.inputs.find(in); auto out_it = ctx_.outputs.find(out); PADDLE_ENFORCE_NE(in_it, ctx_.inputs.end(), platform::errors::NotFound( "Input [%s] found error in Op [%s]", in, op_.Type())); PADDLE_ENFORCE_NE(out_it, ctx_.outputs.end(), platform::errors::NotFound( "Output [%s] found error in Op [%s]", out, op_.Type())); auto& in_var_list = in_it->second; auto& out_var_list = out_it->second; PADDLE_ENFORCE_EQ( in_var_list.size(), out_var_list.size(), platform::errors::PreconditionNotMet( "Op [%s]: Input var size should be equal with output var size", op_.Type())); auto& out_var_names = op_.Outputs(out); for (size_t i = 0; i < in_var_list.size(); ++i) { if (out_var_names[i] == framework::kEmptyVarName) { continue; } Variable* in_var = in_var_list[i]; if (!in_var->IsType()) return; Variable* out_var = out_var_list[i]; PADDLE_ENFORCE_EQ(out_var->IsType(), true, platform::errors::PreconditionNotMet( "The %d-th output of Output(%s) must be LoDTensor.", i, out_var_names[i])); auto& in_tensor = in_var->Get(); auto* out_tensor = out_var->GetMutable(); out_tensor->set_lod(in_tensor.lod()); #ifdef PADDLE_WITH_MKLDNN if (in_tensor.layout() != DataLayout::kMKLDNN) #endif out_tensor->set_layout(in_tensor.layout()); } } void InterpretercoreInferShapeContext::ShareLoD(const std::string& in, const std::string& out, size_t i, size_t j) const { if (can_skip_lod_) { return; } auto in_it = ctx_.inputs.find(in); auto out_it = ctx_.outputs.find(out); PADDLE_ENFORCE_NE(in_it, ctx_.inputs.end(), platform::errors::NotFound("Input %s does not exist.", in)); PADDLE_ENFORCE_NE( out_it, ctx_.outputs.end(), platform::errors::NotFound("Output %s does not exist.", out)); PADDLE_ENFORCE_LT(i, in_it->second.size(), platform::errors::InvalidArgument( "The index of input dimension is out of range, " "excepted index less than %zu, but received %zu.", in_it->second.size(), i)); PADDLE_ENFORCE_LT(j, out_it->second.size(), platform::errors::InvalidArgument( "The index of output dimension is out of range, " "excepted index less than %zu, but received %zu.", out_it->second.size(), j)); Variable* in_var = in_it->second.at(i); if (!in_var->IsType()) return; Variable* out_var = out_it->second.at(j); PADDLE_ENFORCE_EQ( out_var->IsType(), true, platform::errors::InvalidArgument( "The %zu-th output of Output(%s) must be LoDTensor.", j, out)); auto& in_tensor = in_var->Get(); auto* out_tensor = out_var->GetMutable(); out_tensor->set_lod(in_tensor.lod()); // TODO(dzhwinter) : reuse ShareLoD in most operators. // Need to call ShareLayout explicitly in sequence related ops. // Shall we have a better method to shared info between in/out Tensor? #ifdef PADDLE_WITH_MKLDNN // Fix me: ugly workaround below // Correct solution: // set_layout() should NOT be called here (i.e. ShareLoD). Instead, // layout of output tensor should be set "manually" in Compute() // of each OPKernel. The reason layout should NOT be shared between // input and output "automatically" (now by InferShape()->ShareLoD()) // is that layout transform may occur after InferShape(). // Workaround: // Skip set_layout() when input layout is kMKLDNN // This is to avoid kMKLDNN is populated wrongly into a non-MKLDNN // OPKernel. In all MKLDNN OPkernel, set_layout(kMKLDNN) should be called // in Compute() if (in_tensor.layout() != DataLayout::kMKLDNN) #endif out_tensor->set_layout(in_tensor.layout()); } int32_t InterpretercoreInferShapeContext::GetLoDLevel(const std::string& in, size_t i) const { PADDLE_THROW(platform::errors::PreconditionNotMet( "GetLoDLevel is only used in compile time. The calculation of " "output's actual lod is different among operators so that should be " "set in the runtime kernel.")); } void InterpretercoreInferShapeContext::SetLoDLevel(const std::string& out, int32_t lod_level, size_t j) const { PADDLE_THROW(platform::errors::PreconditionNotMet( "SetLoDLevel is only used in compile time. The calculation of " "output's actual lod is different among operators so that should be " "set in the runtime kernel.")); } bool InterpretercoreInferShapeContext::IsRuntime() const { return true; } // TODO(paddle-dev): Can this be template? std::vector InterpretercoreInferShapeContext::GetInputVarPtrs( const std::string& name) const { const std::vector& vars = InputVars(name); std::vector res; res.reserve(vars.size()); res.insert(res.begin(), vars.begin(), vars.end()); return res; } std::vector InterpretercoreInferShapeContext::GetOutputVarPtrs( const std::string& name) const { const std::vector& vars = OutputVars(name); std::vector res; res.reserve(vars.size()); res.insert(res.begin(), vars.begin(), vars.end()); return res; } DDim InterpretercoreInferShapeContext::GetInputDim( const std::string& name) const { const std::vector& vars = InputVars(name); PADDLE_ENFORCE_EQ( vars.size(), 1UL, platform::errors::InvalidArgument( "Input(%s) should hold one element, but now it holds %zu elements.", name, vars.size())); return this->GetDim(vars[0]); } std::vector InterpretercoreInferShapeContext::GetInputsDim( const std::string& name) const { const std::vector& vars = InputVars(name); return GetDims(vars); } std::vector InterpretercoreInferShapeContext::GetInputsVarType( const std::string& name) const { return GetVarTypes(InputVars(name)); } std::vector InterpretercoreInferShapeContext::GetOutputsVarType( const std::string& name) const { return GetVarTypes(OutputVars(name)); } void InterpretercoreInferShapeContext::SetOutputDim(const std::string& name, const DDim& dim) { auto& vars = OutputVars(name); PADDLE_ENFORCE_EQ(vars.size(), 1UL, platform::errors::InvalidArgument( "Output(%s) should hold one element, " "but now it holds %zu elements.", name, vars.size())); SetDim(vars[0], dim); } void InterpretercoreInferShapeContext::SetOutputsDim( const std::string& name, const std::vector& dims) { auto& vars = OutputVars(name); SetDims(vars, dims); } void InterpretercoreInferShapeContext::SetSkipLoD(bool skip) { can_skip_lod_ = skip; } DDim InterpretercoreInferShapeContext::GetDim(Variable* var) const { PADDLE_ENFORCE_NOT_NULL( var, platform::errors::InvalidArgument("Input variable is nullptr.")); if (var->IsType()) { return var->Get().dims(); } else if (var->IsType()) { return var->Get().GetCompleteDims(); } else { PADDLE_THROW(platform::errors::InvalidArgument( "Only LoDTensor or SelectedRows support 'GetDim', but input " "Variable's type is %s.", ToTypeName(var->Type()))); } } std::vector InterpretercoreInferShapeContext::GetDims( const std::vector& vars) const { std::vector ret; ret.reserve(vars.size()); std::transform(vars.begin(), vars.end(), std::back_inserter(ret), [this](Variable* var) { return this->GetDim(var); }); return ret; } std::vector InterpretercoreInferShapeContext::GetRepeatedDims( const std::string& name) const { PADDLE_THROW(platform::errors::PreconditionNotMet( "GetRepeatedDims method only ban be used in compile time.")); } void InterpretercoreInferShapeContext::SetDim(Variable* var, const DDim& dim) { if (var->IsType()) { var->GetMutable()->Resize(dim); } else if (var->IsType()) { var->GetMutable()->set_height(dim[0]); } else { PADDLE_THROW(platform::errors::Unimplemented( "Variable type error, expect LoDTensor or SelectedRows, but received " "(%s).", ToTypeName(var->Type()))); } } void InterpretercoreInferShapeContext::SetDims( const std::vector& vars, const std::vector& dims) { size_t length = vars.size(); PADDLE_ENFORCE_EQ(length, dims.size(), platform::errors::InvalidArgument( "The number of input variables do not match the " "number of input dimensions, the number of variables " "is %zu, the number of dimensions is %zu.", length, dims.size())); for (size_t i = 0; i < length; ++i) { if (vars[i] == nullptr) { continue; } SetDim(vars[i], dims[i]); } } void InterpretercoreInferShapeContext::SetRepeatedDims( const std::string& name, const std::vector& dims) { PADDLE_THROW(platform::errors::PreconditionNotMet( "SetRepeatedDims method only can be used in compile time.")); } std::vector InterpretercoreInferShapeContext::GetVarTypes( const std::vector& vars) const { std::vector retv; retv.resize(vars.size()); std::transform( vars.begin(), vars.end(), retv.begin(), std::bind(std::mem_fn(&InterpretercoreInferShapeContext::GetVarType), this, std::placeholders::_1)); return retv; } proto::VarType::Type InterpretercoreInferShapeContext::GetVarType( Variable* var) const { return ToVarType(var->Type()); } const std::vector& InterpretercoreInferShapeContext::InputVars( const std::string& name) const { auto it = ctx_.inputs.find(name); PADDLE_ENFORCE_NE( it, ctx_.inputs.end(), platform::errors::NotFound("Operator (%s) does not have the input (%s).", op_.Type(), name)); return it->second; } const std::vector& InterpretercoreInferShapeContext::OutputVars( const std::string& name) const { auto it = ctx_.outputs.find(name); PADDLE_ENFORCE_NE( it, ctx_.outputs.end(), platform::errors::NotFound( "Operator (%s) does not have the outputs (%s).", op_.Type(), name)); return it->second; } VariableScope::VariableScope(Scope* scope) { // for @EMPTY@ variable var_list_.push_back(nullptr); name2id_[kEmptyVarName] = kEmptyVarIndex; vec_meta_info_.emplace_back(0, nullptr); scope_ = scope; PADDLE_ENFORCE_NE( scope, nullptr, platform::errors::PreconditionNotMet( "You have passed a nullptr to construct VariableScope.")); listener_ = std::make_shared(this); scope->AddListener(listener_); } VariableScope::~VariableScope() { if (scope_ && listener_) { scope_->DelListener(listener_); } } Scope* VariableScope::GetMutableScope() const { return scope_; } Scope* VariableScope::GetMutableLocalScope() const { return local_scope_; } void VariableScope::SetLocalScope(Scope* local_scope) { VLOG(4) << "Set local scope: " << local_scope; local_scope_ = local_scope; } Variable* VariableScope::FindVar(const std::string& name) const { auto it = name2id_.find(name); if (it != name2id_.end()) { PADDLE_ENFORCE_LT(it->second, var_list_.size(), platform::errors::NotFound( "The id(%d) of variable(%s) should not be larger " "than the size of variable list(%d).", it->second, name, var_list_.size())); return var_list_[it->second]; } return nullptr; } // Get variable id by name, return -1 if not found int VariableScope::GetIdByName(const std::string& name) const { auto it = name2id_.find(name); if (it != name2id_.end()) { return it->second; } return -1; } // Get variable name by id, return "" if not found std::string VariableScope::GetNameById(int id) const { // NOTE(zhiqiu): do not use vec_meta_info_[id].vardesc_->Name() since // vec_meta_info_[id] may be nullptr, // typically when the target variable is not existed in the original program // desc, but created by interpretercore. // For example, created and used by d2h_copy or h2d_copy operator. auto it = std::find_if(name2id_.begin(), name2id_.end(), [id](const auto& pair) { return pair.second == id; }); if (it != name2id_.end()) { return it->first; } return ""; } bool VariableScope::HasVar(const std::string& name) const { return name2id_.find(name) != name2id_.end(); } int VariableScope::VarId(const std::string& name) const { CheckExist(name); return name2id_.at(name); } Variable* VariableScope::Var(int id) const { return var_list_.at(id); } Variable* VariableScope::Var(const std::string& name) const { return var_list_.at(VarId(name)); } size_t VariableScope::VarSize() const { return var_list_.size(); } void VariableScope::AddVar(const std::string& name, framework::VarDesc* var_desc, bool local_scope) { // NOLINT auto v = local_scope ? local_scope_->Var(name) : scope_->Var(name); if (nullptr == var_desc) { v->GetMutable(); } else { InitializeVariable( v, var_desc ->GetType()); // Scope don't initialize variable recently created } SetVarDesc(name, var_desc); } void VariableScope::AddVar(const std::string& name, const Variable& var) { // NOLINT // Though name existed in outer_scope_, we need // add again to create name2id map. scope_->Var(name); } void VariableScope::SetVarDesc(const std::string& name, framework::VarDesc* var_desc) { CheckExist(name); vec_meta_info_[VarId(name)].var_desc_ = var_desc; } paddle::framework::VarDesc* VariableScope::VarDesc( const std::string& name) const { return VarDesc(VarId(name)); } paddle::framework::VarDesc* VariableScope::VarDesc(int id) const { CheckExist(id); return vec_meta_info_[id].var_desc_; } void VariableScope::SetVarSikpInplace(const std::string& name, bool skip) { CheckExist(name); vec_meta_info_[VarId(name)].sikp_inplace_ = skip; } bool VariableScope::GetVarSikpInplace(int id) const { CheckExist(id); return vec_meta_info_[id].sikp_inplace_; } void VariableScope::CheckExist(int id) const { PADDLE_ENFORCE_LT(id, var_list_.size(), platform::errors::PreconditionNotMet( "Required var_id < %d, but received var_id = %d.", var_list_.size(), id)); } void VariableScope::CheckExist(const std::string& name) const { PADDLE_ENFORCE_EQ(HasVar(name), true, platform::errors::NotFound( "%s not in VariableScope.", name)); } VariableScopeListener::VariableScopeListener(VariableScope* var_scope) { var_scope_ = var_scope; } void VariableScopeListener::onCreateVariable(const std::string& name, Variable* v) { if (!var_scope_->HasVar(name)) { // may exist in variable scope. VLOG(4) << "Calling VariableScope::onCreateVariable with var_name: " << name; var_scope_->name2id_[name] = var_scope_->VarSize(); var_scope_->var_list_.emplace_back(v); var_scope_->vec_meta_info_.emplace_back(0, nullptr); } } void VariableScopeListener::onDeleteVariable(const std::string& name) { if (var_scope_->HasVar(name)) { VLOG(4) << "Calling VariableScope::onDeleteVariable with var_name: " << name; } } void VariableScopeListener::onRenameVariable(const std::string& old_name, const std::string& new_name) {} void VariableScopeListener::onCreateScope(Scope* Scope) {} void VariableScopeListener::onDeleteScope(Scope* Scope) {} void VariableScopeListener::onClear() {} Instruction::Instruction(size_t id, OpFuncNode&& op_func_node, const platform::DeviceContext& dev_ctx) : id_(id), op_func_node_(op_func_node), dev_ctx_(dev_ctx) { PADDLE_ENFORCE_GE(id, 0, platform::errors::PreconditionNotMet( "Required id >= 0, but received id = %d", id)); } size_t Instruction::Id() const { return id_; } const std::map>& Instruction::Inputs() const { return op_func_node_.input_index; } const std::map>& Instruction::Outputs() const { return op_func_node_.output_index; } const std::unordered_set& Instruction::NoDataTransformVars() const { return op_func_node_.no_data_transform_index; } OpKernelComputeFunc Instruction::KernelFunc() const { return op_func_node_.kernel_func_; } OpFuncType Instruction::KernelType() const { return op_func_node_.type_; } OperatorBase* Instruction::OpBase() const { auto op_base = op_func_node_.operator_base_; PADDLE_ENFORCE_NOT_NULL(op_base, platform::errors::PreconditionNotMet( "op_base shall not be nullptr.")); return op_base.get(); } NextInstruction& Instruction::NextInstructions() { return next_instruction_; } const NextInstruction& Instruction::NextInstructions() const { return next_instruction_; } void Instruction::AddGCCheckVar(size_t id) { gc_check_var_list_.push_back(id); } const std::vector& Instruction::GCCheckVars() const { return gc_check_var_list_; } void Instruction::ResetContext(const VariableValueMap& in_vars, const VariableValueMap& out_vars) { runtime_ctx_.reset(new RuntimeContext(in_vars, out_vars)); infershape_ctx_.reset( new InterpretercoreInferShapeContext(*OpBase(), *runtime_ctx_.get())); // NOTE: Because execution_ctx_ is constructed by `scope&`, so we fake an // empty here to avoid illegal local reference. static framework::Scope scope_; execution_ctx_.reset( new ExecutionContext(*OpBase(), scope_, dev_ctx_, *runtime_ctx_.get())); } std::shared_ptr Instruction::InnerRuntimeContext() const { return runtime_ctx_; } std::shared_ptr Instruction::InnerInferShapeContext() const { return infershape_ctx_; } std::shared_ptr Instruction::InnerExecutionContext() const { return execution_ctx_; } const platform::DeviceContext& Instruction::DeviceContext() const { return dev_ctx_; } const std::vector>& Instruction::InplaceInfo() const { return vec_inplace_in_to_out_; } void Instruction::AddInplace(Variable* in, Variable* out) { vec_inplace_in_to_out_.emplace_back(in, out); } const std::vector& Instruction::InputEvents() const { return intput_events_; } const std::vector& Instruction::OutputEvents() const { return output_events_; } void Instruction::AddInputEvent(size_t var_id, std::shared_ptr event, platform::DeviceType waiter_type) { intput_events_.emplace_back(var_id, event, waiter_type); } void Instruction::AddOutputEvent(size_t var_id, std::shared_ptr event, platform::DeviceType waiter_type) { output_events_.emplace_back(var_id, event, waiter_type); } } // namespace framework } // namespace paddle