Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 70ea88bf 编写于 作者: 姜 姜永久 提交者： GitHub
浏览文件
操作

Yj/rm core ops exp (#49490) 

* rm op_function_generator

* rm op_func_generator.h

* rm op_function

* modify cmake

* rm op_function.h

* rm check for op_function_generator.cc

* reset imperative

* rm python part

* fix imperative

* lint

* lint

* modify legacy_c

* review

* modify

* modify legacy

* rm gen op_functions code

* reset framework

* rm core.ops for test

* core.ops->core.eager.ops.legacy

* not raiseError for xpu
上级 192eb4d5
隐藏空白更改
内联 并排
Showing with 11 addition and 1030 deletion
paddle/fluid/eager/auto_code_generator/eager_generator.cc
浏览文件 @ 70ea88bf
...@@ -23,7 +23,7 @@ ...@@ -23,7 +23,7 @@
#include "paddle/fluid/framework/operator.h" #include "paddle/fluid/framework/operator.h"
#include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/framework/program_desc.h"
#include "paddle/fluid/framework/variable.h" #include "paddle/fluid/framework/variable.h"
#include "paddle/fluid/pybind/op_function_generator.h" #include "paddle/fluid/pybind/eager_generator.h"
#include "paddle/fluid/pybind/pybind.h" #include "paddle/fluid/pybind/pybind.h"
#include "paddle/fluid/string/string_helper.h" #include "paddle/fluid/string/string_helper.h"
......
paddle/fluid/pybind/.gitignore
浏览文件 @ 70ea88bf
pybind.h pybind.h
op_function1.cc
op_function2.cc
op_function3.cc
op_function4.cc
op_function5.cc
op_function6.cc
op_function7.cc
op_function8.cc
eager_op_function.cc eager_op_function.cc
eager_legacy_op_function.cc eager_legacy_op_function.cc
paddle/fluid/pybind/CMakeLists.txt
浏览文件 @ 70ea88bf
...@@ -138,15 +138,7 @@ set(PYBIND_SRCS ...@@ -138,15 +138,7 @@ set(PYBIND_SRCS
cuda_streams_py.cc cuda_streams_py.cc
xpu_streams_py.cc xpu_streams_py.cc
jit.cc jit.cc
auto_parallel_py.cc auto_parallel_py.cc)
op_function1.cc
op_function2.cc
op_function3.cc
op_function4.cc
op_function5.cc
op_function6.cc
op_function7.cc
op_function8.cc)
if(WITH_CUSTOM_DEVICE) if(WITH_CUSTOM_DEVICE)
set(PYBIND_DEPS ${PYBIND_DEPS} phi_capi) set(PYBIND_DEPS ${PYBIND_DEPS} phi_capi)
...@@ -295,8 +287,6 @@ if(WITH_PYTHON) ...@@ -295,8 +287,6 @@ if(WITH_PYTHON)
list(APPEND OP_FUNCTION_GENERETOR_DEPS ${PYTHON_LIBRARIES}) list(APPEND OP_FUNCTION_GENERETOR_DEPS ${PYTHON_LIBRARIES})
endif() endif()
add_executable(op_function_generator op_function_generator.cc)
target_link_libraries(op_function_generator ${OP_FUNCTION_GENERETOR_DEPS})
add_executable(eager_legacy_op_function_generator add_executable(eager_legacy_op_function_generator
eager_legacy_op_function_generator.cc) eager_legacy_op_function_generator.cc)
target_link_libraries(eager_legacy_op_function_generator target_link_libraries(eager_legacy_op_function_generator
...@@ -308,32 +298,14 @@ if(WITH_PYTHON) ...@@ -308,32 +298,14 @@ if(WITH_PYTHON)
endif() endif()
get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES) get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
target_link_libraries(op_function_generator ${os_dependency_modules})
target_link_libraries(eager_legacy_op_function_generator target_link_libraries(eager_legacy_op_function_generator
${os_dependency_modules}) ${os_dependency_modules})
if(WITH_ROCM) if(WITH_ROCM)
target_link_libraries(op_function_generator ${ROCM_HIPRTC_LIB})
target_link_libraries(eager_legacy_op_function_generator ${ROCM_HIPRTC_LIB}) target_link_libraries(eager_legacy_op_function_generator ${ROCM_HIPRTC_LIB})
target_link_libraries(kernel_signature_generator ${ROCM_HIPRTC_LIB}) target_link_libraries(kernel_signature_generator ${ROCM_HIPRTC_LIB})
endif() endif()
set(op_function_output_path ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/) set(op_function_output_path ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/)
set(impl_file1 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function1.cc)
set(tmp_impl_file1 ${impl_file1}.tmp)
set(impl_file2 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function2.cc)
set(tmp_impl_file2 ${impl_file2}.tmp)
set(impl_file3 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function3.cc)
set(tmp_impl_file3 ${impl_file3}.tmp)
set(impl_file4 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function4.cc)
set(tmp_impl_file4 ${impl_file4}.tmp)
set(impl_file5 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function5.cc)
set(tmp_impl_file5 ${impl_file5}.tmp)
set(impl_file6 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function6.cc)
set(tmp_impl_file6 ${impl_file6}.tmp)
set(impl_file7 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function7.cc)
set(tmp_impl_file7 ${impl_file7}.tmp)
set(impl_file8 ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function8.cc)
set(tmp_impl_file8 ${impl_file8}.tmp)
set(CODE_GEN_SPLIT_FILE_COUNT "8") set(CODE_GEN_SPLIT_FILE_COUNT "8")
set(eager_impl_file set(eager_impl_file
${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/eager_legacy_op_function.cc) ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/eager_legacy_op_function.cc)
...@@ -346,7 +318,6 @@ if(WITH_PYTHON) ...@@ -346,7 +318,6 @@ if(WITH_PYTHON)
"${PADDLE_SOURCE_DIR}/paddle/fluid/pybind/" "${PADDLE_SOURCE_DIR}/paddle/fluid/pybind/"
"${CODE_GEN_SPLIT_FILE_COUNT}") "${CODE_GEN_SPLIT_FILE_COUNT}")
set(OP_IMPL_DEPS op_function_generator)
set(EAGER_OP_IMPL_DEPS eager_legacy_op_function_generator set(EAGER_OP_IMPL_DEPS eager_legacy_op_function_generator
eager_python_c_codegen) eager_python_c_codegen)
...@@ -357,25 +328,6 @@ if(WITH_PYTHON) ...@@ -357,25 +328,6 @@ if(WITH_PYTHON)
set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}") set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}")
endif() endif()
file(
WRITE
${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat
""
"set build_times=1\n"
":retry\n"
"ECHO op_function_generator run %build_times% time\n"
"taskkill /f /im op_function_generator.exe 2>NUL\n"
"${op_impl_path}/op_function_generator.exe ${op_function_output_path} ${CODE_GEN_SPLIT_FILE_COUNT}\n"
"if %ERRORLEVEL% NEQ 0 (\n"
" set /a build_times=%build_times%+1\n"
" if %build_times% GEQ 10 (\n"
" exit /b 1\n"
" ) else (\n"
" goto :retry\n"
" )\n"
")\n"
"exit /b 0")
file( file(
WRITE WRITE
${CMAKE_BINARY_DIR}/paddle/fluid/pybind/eager_legacy_op_function_generator_retry.bat ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/eager_legacy_op_function_generator_retry.bat
...@@ -439,35 +391,6 @@ if(WITH_PYTHON) ...@@ -439,35 +391,6 @@ if(WITH_PYTHON)
${CMAKE_CURRENT_BINARY_DIR}/onnxruntime.dll) ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime.dll)
endif() endif()
add_custom_command(
OUTPUT op_function
COMMAND
${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file1}
${impl_file1}
COMMENT "copy_if_different ${tmp_impl_file1} to ${impl_file1}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file2}
${impl_file2}
COMMENT "copy_if_different ${tmp_impl_file2} to ${impl_file2}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file3}
${impl_file3}
COMMENT "copy_if_different ${tmp_impl_file3} to ${impl_file3}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file4}
${impl_file4}
COMMENT "copy_if_different ${tmp_impl_file4} to ${impl_file4}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file5}
${impl_file5}
COMMENT "copy_if_different ${tmp_impl_file5} to ${impl_file5}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file6}
${impl_file6}
COMMENT "copy_if_different ${tmp_impl_file6} to ${impl_file6}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file7}
${impl_file7}
COMMENT "copy_if_different ${tmp_impl_file7} to ${impl_file7}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file8}
${impl_file8}
COMMENT "copy_if_different ${tmp_impl_file8} to ${impl_file8}"
DEPENDS ${OP_IMPL_DEPS})
if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
add_custom_command( add_custom_command(
OUTPUT ${eager_impl_file} OUTPUT ${eager_impl_file}
...@@ -524,38 +447,6 @@ if(WITH_PYTHON) ...@@ -524,38 +447,6 @@ if(WITH_PYTHON)
list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0) list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0)
list(APPEND EAGER_OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0) list(APPEND EAGER_OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0)
endif() endif()
add_custom_command(
OUTPUT op_function
COMMAND
${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:."
"${CMAKE_CURRENT_BINARY_DIR}/op_function_generator"
"${op_function_output_path}" "${CODE_GEN_SPLIT_FILE_COUNT}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file1}
${impl_file1}
COMMENT "copy_if_different ${tmp_impl_file1} to ${impl_file1}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file2}
${impl_file2}
COMMENT "copy_if_different ${tmp_impl_file2} to ${impl_file2}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file3}
${impl_file3}
COMMENT "copy_if_different ${tmp_impl_file3} to ${impl_file3}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file4}
${impl_file4}
COMMENT "copy_if_different ${tmp_impl_file4} to ${impl_file4}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file5}
${impl_file5}
COMMENT "copy_if_different ${tmp_impl_file5} to ${impl_file5}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file6}
${impl_file6}
COMMENT "copy_if_different ${tmp_impl_file6} to ${impl_file6}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file7}
${impl_file7}
COMMENT "copy_if_different ${tmp_impl_file7} to ${impl_file7}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file8}
${impl_file8}
COMMENT "copy_if_different ${tmp_impl_file8} to ${impl_file8}"
DEPENDS ${OP_IMPL_DEPS}
VERBATIM)
if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
add_custom_command( add_custom_command(
OUTPUT ${eager_impl_file} OUTPUT ${eager_impl_file}
...@@ -570,7 +461,6 @@ if(WITH_PYTHON) ...@@ -570,7 +461,6 @@ if(WITH_PYTHON)
VERBATIM) VERBATIM)
endif() endif()
endif() endif()
add_custom_target(op_function_generator_cmd ALL DEPENDS op_function)
if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) if(NOT ((NOT WITH_PYTHON) AND ON_INFER))
add_custom_target(eager_legacy_op_function_generator_cmd ALL add_custom_target(eager_legacy_op_function_generator_cmd ALL
DEPENDS ${eager_impl_file}) DEPENDS ${eager_impl_file})
...@@ -647,6 +537,5 @@ if(WITH_PYTHON) ...@@ -647,6 +537,5 @@ if(WITH_PYTHON)
get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES) get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
target_link_libraries(${SHARD_LIB_NAME} ${os_dependency_modules}) target_link_libraries(${SHARD_LIB_NAME} ${os_dependency_modules})
add_dependencies(${SHARD_LIB_NAME} op_function_generator_cmd)
endif() endif()
paddle/fluid/pybind/eager_legacy_op_function_generator.cc
浏览文件 @ 70ea88bf
...@@ -31,7 +31,7 @@ ...@@ -31,7 +31,7 @@
#ifdef PADDLE_WITH_ASCEND_CL #ifdef PADDLE_WITH_ASCEND_CL
#include "paddle/fluid/framework/fleet/ascend_wrapper.h" #include "paddle/fluid/framework/fleet/ascend_wrapper.h"
#endif #endif
#include "paddle/fluid/pybind/op_function_generator.h" #include "paddle/fluid/pybind/eager_generator.h"
// phi // phi
#include "paddle/phi/kernels/declarations.h" #include "paddle/phi/kernels/declarations.h"
......
paddle/fluid/pybind/generate_file_structures.py
浏览文件 @ 70ea88bf
...@@ -23,11 +23,6 @@ if __name__ == "__main__": ...@@ -23,11 +23,6 @@ if __name__ == "__main__":
empty_files = [os.path.join(pybind_dir, "eager_legacy_op_function.cc")] empty_files = [os.path.join(pybind_dir, "eager_legacy_op_function.cc")]
empty_files.append(os.path.join(pybind_dir, "eager_op_function.cc")) empty_files.append(os.path.join(pybind_dir, "eager_op_function.cc"))
for i in range(split_count):
empty_files.append(
os.path.join(pybind_dir, "op_function" + str(i + 1) + ".cc")
)
for path in empty_files: for path in empty_files:
if not os.path.exists(path): if not os.path.exists(path):
open(path, 'a').close() open(path, 'a').close()
paddle/fluid/pybind/imperative.cc
浏览文件 @ 70ea88bf
...@@ -53,7 +53,6 @@ limitations under the License. */ ...@@ -53,7 +53,6 @@ limitations under the License. */
#include "paddle/fluid/operators/utils.h" #include "paddle/fluid/operators/utils.h"
#include "paddle/fluid/pybind/cuda_streams_py.h" #include "paddle/fluid/pybind/cuda_streams_py.h"
#include "paddle/fluid/pybind/eager_utils.h" #include "paddle/fluid/pybind/eager_utils.h"
#include "paddle/fluid/pybind/op_function.h"
#include "paddle/fluid/pybind/pybind_variant_caster.h" #include "paddle/fluid/pybind/pybind_variant_caster.h"
#include "paddle/fluid/pybind/slice_utils.h" #include "paddle/fluid/pybind/slice_utils.h"
#include "paddle/fluid/pybind/tensor_py.h" #include "paddle/fluid/pybind/tensor_py.h"
...@@ -498,15 +497,6 @@ static void VarBaseCopy(std::shared_ptr<imperative::VarBase> &src, // NOLINT ...@@ -498,15 +497,6 @@ static void VarBaseCopy(std::shared_ptr<imperative::VarBase> &src, // NOLINT
void BindImperative(py::module *m_ptr) { void BindImperative(py::module *m_ptr) {
auto &m = *m_ptr; auto &m = *m_ptr;
BindOpFunctions1(&m);
BindOpFunctions2(&m);
BindOpFunctions3(&m);
BindOpFunctions4(&m);
BindOpFunctions5(&m);
BindOpFunctions6(&m);
BindOpFunctions7(&m);
BindOpFunctions8(&m);
#ifndef _WIN32 #ifndef _WIN32
// Dygraph DataLoader signal handler // Dygraph DataLoader signal handler
m.def("_set_process_pids", [](int64_t key, py::object &obj) { m.def("_set_process_pids", [](int64_t key, py::object &obj) {
......
paddle/fluid/pybind/op_function.h 已删除 100644 → 0
浏览文件 @ 192eb4d5
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <pybind11/chrono.h>
#include <pybind11/complex.h>
#include <pybind11/functional.h>
#include <pybind11/stl.h>
#include <memory>
#include <string>
#include <vector>
#include "paddle/fluid/framework/attribute.h"
#include "paddle/fluid/framework/op_info.h"
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/framework/variable.h"
#include "paddle/fluid/imperative/tracer.h"
#include "paddle/fluid/imperative/type_defs.h"
#include "paddle/fluid/pybind/exception.h"
#include "paddle/fluid/pybind/imperative.h"
#include "paddle/fluid/pybind/op_function_common.h"
namespace py = pybind11;
namespace paddle {
namespace pybind {
static inline std::shared_ptr<imperative::VarBase> CastPyHandleToVarBase(
const std::string& op_type,
const std::string& arg_name,
int arg_idx,
const py::handle& handle,
bool dispensable = false) {
PyObject* py_obj = handle.ptr(); // get underlying PyObject
if (!py_obj || py_obj == Py_None) {
if (!dispensable) {
PADDLE_THROW(platform::errors::InvalidArgument(
"%s(): argument '%s' (position %d) must be Tensor, but got "
"%s",
op_type,
arg_name,
arg_idx,
Py_TYPE(py_obj)->tp_name));
}
return nullptr;
}
try {
return py::cast<std::shared_ptr<imperative::VarBase>>(py::handle(py_obj));
} catch (py::cast_error&) {
PADDLE_THROW(platform::errors::InvalidArgument(
"%s(): argument '%s' (position %d) must be Tensor, but got "
"%s",
op_type,
arg_name,
arg_idx,
Py_TYPE(py_obj)->tp_name));
}
}
static inline std::vector<std::shared_ptr<imperative::VarBase>>
CastPyHandleToVarBaseList(const std::string& op_type,
const std::string& arg_name,
int arg_idx,
const py::handle& handle,
bool dispensable = false) {
PyObject* py_obj = handle.ptr(); // get underlying PyObject
if (!py_obj || py_obj == Py_None) {
if (!dispensable) {
PADDLE_THROW(platform::errors::InvalidArgument(
"%s(): argument '%s' (position %d) must be Tensor, but got "
"%s",
op_type,
arg_name,
arg_idx,
Py_TYPE(py_obj)->tp_name));
}
return {};
}
std::vector<std::shared_ptr<imperative::VarBase>> result;
if (PyList_Check(py_obj) || PyTuple_Check(py_obj)) {
auto size = PyTuple_Check(py_obj) ? PyTuple_GET_SIZE(py_obj)
: PyList_GET_SIZE(py_obj);
for (auto i = 0; i < size; ++i) {
PyObject* item = PyTuple_Check(py_obj) ? PyTuple_GET_ITEM(py_obj, i)
: PyList_GET_ITEM(py_obj, i);
if (!item || item == Py_None) {
result.emplace_back(nullptr);
continue;
}
try {
result.emplace_back(
py::cast<std::shared_ptr<imperative::VarBase>>(py::handle(item)));
} catch (py::cast_error&) {
PADDLE_THROW(platform::errors::InvalidArgument(
"%s(): argument '%s' (position %d) must be list of "
"Tensors, but "
"got %s in list (item %d)",
op_type,
arg_name,
arg_idx,
Py_TYPE(item)->tp_name,
i));
}
}
} else {
PADDLE_THROW(platform::errors::InvalidArgument(
"%s(): argument '%s' (position %d) must be list of Tensors, but got "
"%s",
op_type,
arg_name,
arg_idx,
Py_TYPE(py_obj)->tp_name));
}
return result;
} // namespace pybind
static inline std::vector<std::shared_ptr<imperative::VarBase>>
ConstructDuplicableOutput(const size_t num) {
auto tracer = imperative::GetCurrentTracer();
std::vector<std::shared_ptr<imperative::VarBase>> res;
res.reserve(num);
for (size_t i = 0; i < num; i++) {
auto var_base_name = tracer->GenerateUniqueName();
res.emplace_back(new imperative::VarBase(var_base_name));
}
return res;
}
static inline void HandleViewBetweenInputAndOutput(
const std::shared_ptr<imperative::VarBase>& input_var,
const std::shared_ptr<imperative::VarBase>& view_output_var) {
PADDLE_ENFORCE_EQ(
input_var->Var().IsInitialized(),
true,
platform::errors::InvalidArgument("Tensor %s has not been initialized!",
input_var->Name()));
if (input_var->Var().IsType<phi::DenseTensor>()) {
const auto& input_tensor = input_var->Var().Get<phi::DenseTensor>();
PADDLE_ENFORCE_EQ(
input_tensor.IsInitialized(),
true,
platform::errors::InvalidArgument(
"LoDTensor %s has not been initialized!", input_var->Name()));
auto* view_output_tensor =
view_output_var->MutableVar()->GetMutable<phi::DenseTensor>();
view_output_tensor->ShareBufferWith(input_tensor);
view_output_tensor->ShareInplaceVersionCounterWith(input_tensor);
VLOG(3) << "Perform View between Output Var(" << view_output_var->Name()
<< ") and Input Var(" << input_var->Name()
<< "), share allocation and inplace version.";
}
}
static inline PyObject* MakeReturnPyObject(
const std::shared_ptr<paddle::imperative::VarBase>& out) {
return ::pybind11::detail::type_caster_base<imperative::VarBase>::cast_holder(
::pybind11::detail::holder_helper<
std::shared_ptr<imperative::VarBase>>::get(out),
&out)
.ptr();
}
static inline PyObject* MakeReturnPyObject(
const std::vector<std::shared_ptr<imperative::VarBase>>& out) {
PyObject* result = PyList_New((Py_ssize_t)out.size());
for (size_t i = 0; i < out.size(); i++) {
PyList_SET_ITEM(
result,
(Py_ssize_t)i,
::pybind11::detail::type_caster_base<imperative::VarBase>::cast_holder(
::pybind11::detail::holder_helper<
std::shared_ptr<imperative::VarBase>>::get(out[i]),
&out[i])
.ptr()); // NOLINT
}
return result;
}
template <typename Tuple, size_t N>
struct TupleVarBasesResult {
static void Run(const Tuple& out, PyObject* result) {
TupleVarBasesResult<Tuple, N - 1>::Run(out, result);
PyTuple_SET_ITEM(result, N - 1, MakeReturnPyObject(std::get<N - 1>(out)));
}
};
template <typename Tuple>
struct TupleVarBasesResult<Tuple, 1> {
static void Run(const Tuple& out, PyObject* result) {
PyTuple_SET_ITEM(result, 0, MakeReturnPyObject(std::get<0>(out)));
}
};
template <typename... Args>
PyObject* MakeReturnPyObject(const std::tuple<Args...>& out) {
auto len = sizeof...(Args);
PyObject* result = PyTuple_New(len);
TupleVarBasesResult<decltype(out), sizeof...(Args)>::Run(out, result);
return result;
}
void BindOpFunctions1(pybind11::module* module);
void BindOpFunctions2(pybind11::module* module);
void BindOpFunctions3(pybind11::module* module);
void BindOpFunctions4(pybind11::module* module);
void BindOpFunctions5(pybind11::module* module);
void BindOpFunctions6(pybind11::module* module);
void BindOpFunctions7(pybind11::module* module);
void BindOpFunctions8(pybind11::module* module);
} // namespace pybind
} // namespace paddle
paddle/fluid/pybind/op_function_generator.cc 已删除 100644 → 0
浏览文件 @ 192eb4d5
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/pybind/op_function_generator.h"
#include <algorithm>
#include <fstream>
#include <iostream>
#include <string>
#ifndef _WIN32
#include <unistd.h>
#endif
#include "paddle/fluid/framework/op_info.h"
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/framework/operator.h"
#include "paddle/fluid/framework/variable.h"
#include "paddle/fluid/pybind/pybind.h"
#include "paddle/fluid/string/string_helper.h"
#ifdef PADDLE_WITH_ASCEND_CL
#include "paddle/fluid/framework/fleet/ascend_wrapper.h"
#endif
// phi
#include "paddle/phi/kernels/declarations.h"
static std::string LegalizeVarName(const std::string& var_name) {
std::string ret = var_name;
std::replace(ret.begin(), ret.end(), '@', '_'); // replace all '-' to '_'
return ret;
}
// NOTE(pangyoki): Inplace OP with duplicable input.
// The set includes inplace ops that have duplicable input.
// The first Varbase in input needs to be specified for the inplace strategy
// and share Varbase with the output.
std::set<std::string> inplace_op_duplicable_ins_set = {
"sum",
};
// clang-format off
const char* OUT_INITIALIZER_TEMPLATE =
R"({"%s", {std::shared_ptr<imperative::VarBase>(new imperative::VarBase("auto_"+std::to_string(VarBaseUniqueNameID++)+"_"))}})";
const char* OUT_DUPLICABLE_INITIALIZER_TEMPLATE = R"({"%s", ConstructDuplicableOutput(%s)})";
const char* INPUT_INITIALIZER_TEMPLATE = R"({"%s", {%s}})";
const char* INPUT_LIST_INITIALIZER_TEMPLATE = R"({"%s", %s})";
const char* INPUT_INITIALIZER_TEMPLATE_WITH_NULL = R"(
if (%s != nullptr) {
ins["%s"] = {%s};
}
)";
const char* INPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST = R"(
if (%s.size() != 0) {
ins["%s"] = %s;
}
)";
const char* OUTPUT_INITIALIZER_TEMPLATE_WITH_NULL = R"(
outs["%s"] = {%s};
)";
const char* OUTPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST = R"(
outs["%s"] = %s;
)";
// if inputs is list, no need {}
const char* ARG_OUT_NUM = R"(%sNum)";
const char* ARG_OUT_NUM_TYPE = R"(size_t )";
const char* IN_VAR_TYPE = R"(py::handle)";
const char* IN_VAR_LIST_TYPE = R"(py::handle)";
const char* OUT_VAR_TYPE = R"(std::shared_ptr<imperative::VarBase>)";
const char* OUT_VAR_LIST_TYPE = R"(std::vector<std::shared_ptr<imperative::VarBase>>)";
const char* CAST_VAR_TEMPLATE = R"(
auto %s = GetVarBaseFromArgs(op_type, "%s", args, %d, %s);)";
const char* CAST_VAR_LIST_TEMPLATE = R"(
auto %s = GetVarBaseListFromArgs(op_type, "%s", args, %d, %s);)";
const char* CAST_SIZE_T_TEMPLATE = R"(
auto %s = GetUnsignedLongFromArgs(op_type, "%s", args, %d, %s);)";
const char* ARG_TEMPLATE = R"(const %s& %s)";
const char* RETURN_TUPLE_TYPE = R"(std::tuple<%s>)";
const char* RETURN_TUPLE_TEMPLATE = R"(std::make_tuple(%s))";
const char* RETURN_LIST_TEMPLATE = R"(outs["%s"])";
const char* RETURN_TEMPLATE = R"(outs["%s"][0])";
const char* FUNCTION_ARGS = R"(%s, const py::args& args)";
const char* FUNCTION_ARGS_NO_INPUT = R"(const py::args& args)";
const char* HANDLE_VIEW_BETWEEN_INPUT_AND_OUTPUT = R"(
if (ins.count("%s") && outs.count("%s")) {
HandleViewBetweenInputAndOutput(ins["%s"][0], outs["%s"][0]);
})";
const char* INPLACE_DUPLICABLE_INPUT = R"([0])";
const char* INPLACE_LEAF_ERROR_MESSAGE = R"(Leaf Var (%s) that doesn't stop gradient can't use inplace strategy.)";
const char* INPLACE_STRATEGY_TEMPLATE =
R"(
PADDLE_ENFORCE_EQ(
%s->IsLeaf() && !%s->OverridedStopGradient(), false,
platform::errors::InvalidArgument("%s", %s->Name()));
%s->BumpInplaceVersion();
VLOG(3) << "Var(" << %s->Name() << ") uses Inplace Strategy.";
)";
const char* INPLACE_MAPPING_TEMPLATE = R"({"%s", "%s"})";
const char* OP_FUNCTION_TEMPLATE =
R"(
static PyObject * %s(PyObject *self, PyObject *args, PyObject *kwargs)
{
PyThreadState *tstate = nullptr;
try
{
std::string op_type = "%s";
platform::RecordEvent op_type_record_event("%s pybind_imperative_func");
%s
framework::AttributeMap attrs;
ConstructAttrMapFromPyArgs(op_type, args, %d, PyTuple_GET_SIZE(args) , attrs);
tstate = PyEval_SaveThread();
%s
imperative::NameVarBaseMap outs = %s;
imperative::NameVarBaseMap ins = %s;
%s
imperative::GetCurrentTracer()->TraceOp(op_type, ins, outs, attrs, {%s});
PyEval_RestoreThread(tstate);
tstate = nullptr;
%s
}
catch(...) {
if (tstate) {
PyEval_RestoreThread(tstate);
}
ThrowExceptionToPython(std::current_exception());
return nullptr;
}
})";
const char* PYBIND_ITEM_TEMPLATE = R"( {"%s", (PyCFunction)(void(*)(void))%s, METH_VARARGS | METH_KEYWORDS, "C++ interface function for %s in dygraph."},)";
// clang-format on
static inline bool FindInsMap(const std::string& op_type,
const std::string& in_name) {
return op_ins_map[op_type].count(in_name);
}
static inline bool FindOutsMap(const std::string& op_type,
const std::string& out_name) {
return op_outs_map[op_type].count(out_name);
}
static inline bool FindPassingOutsMap(const std::string& op_type,
const std::string& out_name) {
return op_passing_outs_map[op_type].count(out_name);
}
static inline bool FindDuplicableInputInplaceOpSet(const std::string& op_type) {
return inplace_op_duplicable_ins_set.count(op_type);
}
static inline bool FindViewOpMap(const std::string& op_type) {
return view_op_map.count(op_type);
}
static inline std::string TempName(const std::string& name) {
return name + '_';
}
std::string GenerateOpFunctionsBody(
const paddle::framework::proto::OpProto* op_proto,
std::string func_name,
bool use_inplace_strategy = false,
std::map<std::string, std::string> inplace_map = {}) {
auto& op_type = op_proto->type();
std::string input_args = "";
std::string ins_initializer = "{";
std::string ins_initializer_with_null = "";
std::string py_arg = "";
int arg_idx = 0;
int input_args_num = 0;
std::string ins_cast_str = "";
std::string view_strategy_str = "";
std::string inplace_strategy_str = "";
for (auto& input : op_proto->inputs()) {
auto& in_name = input.name();
// skip those dispensable inputs, like ResidualData in conv2d
if (input.dispensable() && !FindInsMap(op_type, in_name)) {
continue;
}
const auto in_type = input.duplicable() ? IN_VAR_LIST_TYPE : IN_VAR_TYPE;
auto input_arg = paddle::string::Sprintf(
ARG_TEMPLATE, in_type, LegalizeVarName(TempName(in_name)));
input_args += input_arg;
input_args += ",";
input_args_num++;
const auto in_cast_type =
input.duplicable() ? CAST_VAR_LIST_TEMPLATE : CAST_VAR_TEMPLATE;
auto dispensable = input.dispensable() ? "true" : "false";
ins_cast_str += paddle::string::Sprintf(in_cast_type,
LegalizeVarName(in_name),
in_name,
arg_idx++,
dispensable);
if (input.dispensable()) {
const auto in_template = input.duplicable()
? INPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST
: INPUT_INITIALIZER_TEMPLATE_WITH_NULL;
ins_initializer_with_null +=
paddle::string::Sprintf(in_template,
LegalizeVarName(in_name),
in_name,
LegalizeVarName(in_name));
} else {
const auto in_template = input.duplicable()
? INPUT_LIST_INITIALIZER_TEMPLATE
: INPUT_INITIALIZER_TEMPLATE;
ins_initializer += paddle::string::Sprintf(
in_template, in_name, LegalizeVarName(in_name));
ins_initializer += ",";
}
}
if (ins_initializer.back() == ',') {
ins_initializer.pop_back();
}
ins_initializer += "}";
if (!input_args.empty() && input_args.back() == ',') {
input_args.pop_back();
}
// Generate outs initializer
std::string outs_initializer = "{";
std::string outs_initializer_with_null = "";
std::string inplace_mapping_str = "";
std::string return_str = "";
int outs_num = 0;
for (auto& output : op_proto->outputs()) {
auto& out_name = output.name();
// skip those dispensable oututs
if (output.dispensable() && !FindOutsMap(op_type, out_name)) {
continue;
}
const auto out_type =
output.duplicable() ? OUT_VAR_LIST_TYPE : OUT_VAR_TYPE;
const auto return_template =
output.duplicable() ? RETURN_LIST_TEMPLATE : RETURN_TEMPLATE;
if (FindPassingOutsMap(op_type, out_name)) {
if (input_args != "") {
input_args += ",";
}
input_args += out_type;
input_args += LegalizeVarName(out_name);
input_args_num++;
if (output.dispensable()) {
const auto out_template =
output.duplicable() ? OUTPUT_INITIALIZER_TEMPLATE_WITH_NULL_LIST
: OUTPUT_INITIALIZER_TEMPLATE_WITH_NULL;
outs_initializer_with_null +=
paddle::string::Sprintf(out_template, out_name, out_name);
} else {
const auto out_template = output.duplicable()
? INPUT_LIST_INITIALIZER_TEMPLATE
: INPUT_INITIALIZER_TEMPLATE;
outs_initializer += paddle::string::Sprintf(
out_template, out_name, LegalizeVarName(out_name));
outs_initializer += ",";
}
const auto in_cast_type =
output.duplicable() ? CAST_VAR_LIST_TEMPLATE : CAST_VAR_TEMPLATE;
auto dispensable = output.dispensable() ? "true" : "false";
ins_cast_str += paddle::string::Sprintf(in_cast_type,
LegalizeVarName(out_name),
out_name,
arg_idx++,
dispensable);
} else if (use_inplace_strategy && inplace_map.count(out_name)) {
PADDLE_ENFORCE_NE(
inplace_map[out_name],
"",
paddle::platform::errors::InvalidArgument(
"Inplace op %s has no input corresponding to output %s.",
op_type,
out_name));
// TODO(pangyoki): Inplace op don't have duplicable output in temporary,
// so don't support duplicable output now.
const auto out_template = INPUT_INITIALIZER_TEMPLATE;
auto inplace_input_name = inplace_map[out_name];
inplace_mapping_str += paddle::string::Sprintf(
INPLACE_MAPPING_TEMPLATE, inplace_input_name, out_name);
inplace_mapping_str += ",";
// If inplace op has duplicable input, the first Varbase in input will
// share Varbase with output.
if (FindDuplicableInputInplaceOpSet(op_type)) {
inplace_input_name += INPLACE_DUPLICABLE_INPUT;
}
// Leaf Var that doesn't stop gradient can't use inplace strategy.
// Increase inplace_version.
inplace_strategy_str +=
paddle::string::Sprintf(INPLACE_STRATEGY_TEMPLATE,
LegalizeVarName(inplace_input_name),
LegalizeVarName(inplace_input_name),
INPLACE_LEAF_ERROR_MESSAGE,
LegalizeVarName(inplace_input_name),
LegalizeVarName(inplace_input_name),
LegalizeVarName(inplace_input_name));
outs_initializer += paddle::string::Sprintf(
out_template, out_name, LegalizeVarName(inplace_input_name));
outs_initializer += ",";
} else {
// There are few Operators that have duplicable output, like `Out` in
// split op. We need to specify the number of variables for the
// duplicable output, as the argument OutNum;
if (output.duplicable()) {
if (input_args != "") {
input_args += ",";
}
auto out_num_str =
paddle::string::Sprintf(ARG_OUT_NUM, LegalizeVarName(out_name));
input_args += ARG_OUT_NUM_TYPE;
input_args += out_num_str;
input_args_num++;
outs_initializer += paddle::string::Sprintf(
OUT_DUPLICABLE_INITIALIZER_TEMPLATE, out_name, out_num_str);
auto dispensable = output.dispensable() ? "true" : "false";
ins_cast_str += paddle::string::Sprintf(CAST_SIZE_T_TEMPLATE,
out_num_str,
out_num_str,
arg_idx++,
dispensable);
} else {
outs_initializer +=
paddle::string::Sprintf(OUT_INITIALIZER_TEMPLATE, out_name);
}
outs_initializer += ",";
}
return_str += paddle::string::Sprintf(return_template, out_name);
return_str += ",";
outs_num += 1;
}
if (outs_initializer.back() == ',') {
outs_initializer.pop_back();
return_str.pop_back();
}
outs_initializer += "}";
if (!inplace_mapping_str.empty() && inplace_mapping_str.back() == ',') {
inplace_mapping_str.pop_back();
}
if (!use_inplace_strategy && FindViewOpMap(op_type)) {
std::string viwe_input_name = view_op_map[op_type].first;
std::string viwe_output_name = view_op_map[op_type].second;
view_strategy_str +=
paddle::string::Sprintf(HANDLE_VIEW_BETWEEN_INPUT_AND_OUTPUT,
viwe_input_name,
viwe_output_name,
viwe_input_name,
viwe_output_name);
}
if (outs_num == 0) {
return_str = "RETURN_PY_NONE";
} else if (outs_num == 1) {
return_str = "return MakeReturnPyObject(" + return_str + ");";
} else {
return_str = "return MakeReturnPyObject(" +
paddle::string::Sprintf(RETURN_TUPLE_TEMPLATE, return_str) +
");";
}
std::string function_args = "";
if (input_args == "") {
function_args = FUNCTION_ARGS_NO_INPUT;
} else {
function_args = paddle::string::Sprintf(FUNCTION_ARGS, input_args);
}
// generate op funtcion body
auto op_function_str = paddle::string::Sprintf(
OP_FUNCTION_TEMPLATE,
func_name,
op_type,
op_type,
ins_cast_str,
input_args_num,
inplace_strategy_str,
outs_initializer,
ins_initializer,
ins_initializer_with_null + outs_initializer_with_null +
view_strategy_str,
inplace_mapping_str,
return_str);
return op_function_str;
}
static std::vector<
std::tuple<std::vector<std::string>, std::vector<std::string>>>
GenerateOpFunctions(int split_count) {
auto& op_info_map = paddle::framework::OpInfoMap::Instance().map();
std::vector<std::tuple<std::vector<std::string>, std::vector<std::string>>>
result;
std::vector<std::string> op_function_list, bind_function_list;
auto& all_kernels = paddle::framework::OperatorWithKernel::AllOpKernels();
paddle::flat_hash_map<std::string, paddle::framework::OpInfo>
op_info_map_need_gen;
for (auto& pair : op_info_map) {
auto& op_info = pair.second;
auto op_proto = op_info.proto_;
if (op_proto == nullptr) {
continue;
}
auto& op_type = op_proto->type();
// Skip operator which is not inherit form OperatorWithKernel, like while,
// since only OperatorWithKernel can run in dygraph mode.
// if the phi lib contains op kernel, we still generate ops method
if (!all_kernels.count(op_type) &&
!phi::KernelFactory::Instance().HasCompatiblePhiKernel(op_type)) {
continue;
}
// Skip the sparse op
if (op_type.compare(0, 7, "sparse_") == 0 && op_type != "sparse_momentum" &&
op_type != "sparse_attention") {
continue;
}
op_info_map_need_gen.emplace(pair);
}
int cc_file_api_size = op_info_map_need_gen.size() / split_count;
if (op_info_map_need_gen.size() % split_count != 0) {
cc_file_api_size++;
}
int api_index = 0;
int file_index = 0;
for (auto& pair : op_info_map_need_gen) {
auto& op_info = pair.second;
auto op_proto = op_info.proto_;
auto& op_type = op_proto->type();
// NOTE(pangyoki): Inplace Strategy.
// In this case, output will reuse input varbase.
// Dygraph mode needs to be aligned with the in-place strategy in static
// mode, and the mapping relationships between output and input that have
// been defined in static graph mode should be used in dygraph mode.
// Find which ops need to use Inplace strategy in static graph mode, and get
// the mapping relationship between Inplace output and input.
auto& infer_inplace =
paddle::framework::OpInfoMap::Instance().Get(op_type).infer_inplace_;
std::map<std::string, std::string> inplace_map;
if (infer_inplace) {
auto in_to_outs = infer_inplace(true);
for (auto& inplace_pair : in_to_outs) {
inplace_map[inplace_pair.second] = inplace_pair.first;
}
}
std::string func_name = "imperative_" + op_type;
std::string op_function_str = GenerateOpFunctionsBody(op_proto, func_name);
// generate pybind item
auto bind_function_str = paddle::string::Sprintf(
PYBIND_ITEM_TEMPLATE, op_type, func_name, op_type);
op_function_list.emplace_back(std::move(op_function_str));
bind_function_list.emplace_back(std::move(bind_function_str));
if (infer_inplace) {
// Reuse Varbase Inplace OP: op_type_.
// The inplace OP needs a new implementation method.
std::string inplace_op_type = op_type + "_";
std::string inplace_func_name = "imperative_" + inplace_op_type;
std::string inplace_op_function_str = GenerateOpFunctionsBody(
op_proto, inplace_func_name, true, inplace_map);
// generate pybind item
auto inplace_bind_function_str =
paddle::string::Sprintf(PYBIND_ITEM_TEMPLATE,
inplace_op_type,
inplace_func_name,
inplace_op_type);
op_function_list.emplace_back(std::move(inplace_op_function_str));
bind_function_list.emplace_back(std::move(inplace_bind_function_str));
}
api_index++;
if (api_index / cc_file_api_size > file_index) {
file_index++;
result.push_back(std::make_tuple(op_function_list, bind_function_list));
op_function_list.clear();
bind_function_list.clear();
}
}
result.push_back(std::make_tuple(op_function_list, bind_function_list));
return result;
}
int main(int argc, char* argv[]) {
if (argc != 3) {
std::cerr << "argc must be 3" << std::endl;
return -1;
}
#ifdef PADDLE_WITH_ASCEND_CL
auto ascend_ptr = paddle::framework::AscendInstance::GetInstance();
ascend_ptr->InitGEForUT();
#endif
std::vector<std::string> headers{"\"paddle/fluid/imperative/tracer.h\"",
"\"paddle/fluid/platform/profiler.h\"",
"\"pybind11/numpy.h\"",
"\"pybind11/pybind11.h\"",
"\"pybind11/detail/common.h\"",
"\"paddle/fluid/pybind/eager_utils.h\"",
"\"paddle/fluid/pybind/op_function.h\"",
"<Python.h>"};
std::string path = argv[1];
int split_count = atoi(argv[2]);
auto op_funcs = GenerateOpFunctions(split_count);
for (size_t i = 0; i < op_funcs.size(); i++) {
std::ofstream out(path + "op_function" + std::to_string(i + 1) + ".cc.tmp",
std::ios::out);
out << "#if defined(_MSC_VER)\n"
<< "#include <BaseTsd.h>\n"
<< "typedef SSIZE_T ssize_t;\n"
<< "#endif\n";
for (auto& header : headers) {
out << "#include " + header + "\n";
}
out << "\n\n";
out << "namespace paddle {\n"
<< "namespace pybind {\n\n";
out << "extern std::atomic<int> VarBaseUniqueNameID;\n";
out << paddle::string::join_strings(std::get<0>(op_funcs[i]), '\n');
out << "\n\n";
out << "static PyMethodDef ExtestMethods[] = {\n"
<< paddle::string::join_strings(std::get<1>(op_funcs[i]), '\n')
<< "\n {nullptr,nullptr,0,nullptr}"
<< "};\n\n";
out << "void BindOpFunctions" << i + 1 << "(pybind11::module *module) {\n"
<< " auto m = module->def_submodule(\"ops\");\n"
<< " if (PyModule_AddFunctions(m.ptr(), ExtestMethods) < 0) {\n"
<< " PADDLE_THROW(platform::errors::Fatal (\"Add functions to "
"core.ops failed!\"));\n"
<< " }\n\n"
<< " InitOpsAttrTypeMap();"
<< "}\n\n"
<< "} // namespace pybind\n"
<< "} // namespace paddle\n";
out.close();
}
#ifdef PADDLE_WITH_ASCEND_CL
ge::GEFinalize();
#endif
return 0;
}
paddle/scripts/paddle_build.bat
浏览文件 @ 70ea88bf
...@@ -41,10 +41,8 @@ taskkill /f /im python.exe /t 2>NUL ...@@ -41,10 +41,8 @@ taskkill /f /im python.exe /t 2>NUL
taskkill /f /im nvcc.exe /t 2>NUL taskkill /f /im nvcc.exe /t 2>NUL
taskkill /f /im cicc.exe /t 2>NUL taskkill /f /im cicc.exe /t 2>NUL
taskkill /f /im ptxas.exe /t 2>NUL taskkill /f /im ptxas.exe /t 2>NUL
taskkill /f /im op_function_generator.exe /t 2>NUL
taskkill /f /im eager_generator.exe /t 2>NUL taskkill /f /im eager_generator.exe /t 2>NUL
taskkill /f /im eager_legacy_op_function_generator.exe /t 2>NUL taskkill /f /im eager_legacy_op_function_generator.exe /t 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="eager_generator.exe" call terminate 2>NUL wmic process where name="eager_generator.exe" call terminate 2>NUL
wmic process where name="eager_legacy_op_function_generator.exe" call terminate 2>NUL wmic process where name="eager_legacy_op_function_generator.exe" call terminate 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL wmic process where name="cvtres.exe" call terminate 2>NUL
...@@ -533,10 +531,8 @@ taskkill /f /im csc.exe /t 2>NUL ...@@ -533,10 +531,8 @@ taskkill /f /im csc.exe /t 2>NUL
taskkill /f /im nvcc.exe /t 2>NUL taskkill /f /im nvcc.exe /t 2>NUL
taskkill /f /im cicc.exe /t 2>NUL taskkill /f /im cicc.exe /t 2>NUL
taskkill /f /im ptxas.exe /t 2>NUL taskkill /f /im ptxas.exe /t 2>NUL
taskkill /f /im op_function_generator.exe /t 2>NUL
taskkill /f /im eager_generator.exe /t 2>NUL taskkill /f /im eager_generator.exe /t 2>NUL
taskkill /f /im eager_legacy_op_function_generator.exe /t 2>NUL taskkill /f /im eager_legacy_op_function_generator.exe /t 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="eager_generator.exe" call terminate 2>NUL wmic process where name="eager_generator.exe" call terminate 2>NUL
wmic process where name="eager_legacy_op_function_generator.exe" call terminate 2>NUL wmic process where name="eager_legacy_op_function_generator.exe" call terminate 2>NUL
wmic process where name="cmake.exe" call terminate 2>NUL wmic process where name="cmake.exe" call terminate 2>NUL
...@@ -936,10 +932,8 @@ taskkill /f /im python.exe /t 2>NUL ...@@ -936,10 +932,8 @@ taskkill /f /im python.exe /t 2>NUL
taskkill /f /im nvcc.exe /t 2>NUL taskkill /f /im nvcc.exe /t 2>NUL
taskkill /f /im cicc.exe /t 2>NUL taskkill /f /im cicc.exe /t 2>NUL
taskkill /f /im ptxas.exe /t 2>NUL taskkill /f /im ptxas.exe /t 2>NUL
taskkill /f /im op_function_generator.exe /t 2>NUL
taskkill /f /im eager_generator.exe /t 2>NUL taskkill /f /im eager_generator.exe /t 2>NUL
taskkill /f /im eager_legacy_op_function_generator.exe /t 2>NUL taskkill /f /im eager_legacy_op_function_generator.exe /t 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="eager_generator.exe" call terminate 2>NUL wmic process where name="eager_generator.exe" call terminate 2>NUL
wmic process where name="eager_legacy_op_function_generator.exe" call terminate 2>NUL wmic process where name="eager_legacy_op_function_generator.exe" call terminate 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL wmic process where name="cvtres.exe" call terminate 2>NUL
......
python/paddle/_legacy_C_ops.py
浏览文件 @ 70ea88bf
...@@ -12,45 +12,11 @@ ...@@ -12,45 +12,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from paddle.fluid import core
from .fluid import framework from paddle.fluid import core
__all__ = [] __all__ = []
_already_switch_to_eager_ = False for name in dir(core.eager.ops.legacy):
globals()[name] = getattr(core.eager.ops.legacy, name)
if not framework._in_eager_mode_: __all__.append(name)
for name in dir(core.ops):
globals()[name] = getattr(core.ops, name)
__all__.append(name)
_already_switch_to_eager_ = False
else:
for name in dir(core.eager.ops.legacy):
globals()[name] = getattr(core.eager.ops.legacy, name)
__all__.append(name)
_already_switch_to_eager_ = True
def switch_to_core_ops():
global _already_switch_to_eager_
if _already_switch_to_eager_:
for name in dir(core.eager.ops.legacy):
del globals()[name]
__all__.remove(name)
for name in dir(core.ops):
globals()[name] = getattr(core.ops, name)
__all__.append(name)
_already_switch_to_eager_ = False
def switch_to_eager_ops():
global _already_switch_to_eager_
if not _already_switch_to_eager_:
for name in dir(core.ops):
del globals()[name]
__all__.remove(name)
for name in dir(core.eager.ops.legacy):
globals()[name] = getattr(core.eager.ops.legacy, name)
__all__.append(name)
_already_switch_to_eager_ = True
python/paddle/distributed/fleet/layers/mpu/mp_layers.py
浏览文件 @ 70ea88bf
...@@ -29,10 +29,7 @@ __all__ = [] ...@@ -29,10 +29,7 @@ __all__ = []
def is_fused_matmul_bias_supported(): def is_fused_matmul_bias_supported():
if paddle.is_compiled_with_cuda() and not paddle.is_compiled_with_rocm(): return hasattr(core.eager.ops.legacy, 'fused_gemm_epilogue')
return hasattr(core.ops, 'fused_gemm_epilogue')
else:
return False
class VocabParallelEmbedding(Layer): class VocabParallelEmbedding(Layer):
......
python/paddle/fluid/framework.py
浏览文件 @ 70ea88bf
...@@ -124,7 +124,6 @@ def _update_monkey_methods(is_eager): ...@@ -124,7 +124,6 @@ def _update_monkey_methods(is_eager):
assert isinstance(is_eager, bool) assert isinstance(is_eager, bool)
# switch into eager mode # switch into eager mode
if is_eager: if is_eager:
_legacy_C_ops.switch_to_eager_ops()
if not _already_patch_eager_tensor: if not _already_patch_eager_tensor:
monkey_patch_varbase() monkey_patch_varbase()
monkey_patch_math_varbase() monkey_patch_math_varbase()
...@@ -132,7 +131,6 @@ def _update_monkey_methods(is_eager): ...@@ -132,7 +131,6 @@ def _update_monkey_methods(is_eager):
_already_patch_eager_tensor = True _already_patch_eager_tensor = True
# switch back into legacy mode # switch back into legacy mode
else: else:
_legacy_C_ops.switch_to_core_ops()
if not _already_patch_varbase: if not _already_patch_varbase:
monkey_patch_varbase() monkey_patch_varbase()
monkey_patch_math_varbase() monkey_patch_math_varbase()
......
python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py
浏览文件 @ 70ea88bf
...@@ -148,7 +148,7 @@ def multiclass_nms( ...@@ -148,7 +148,7 @@ def multiclass_nms(
'normalized', 'normalized',
normalized, normalized,
) )
output, index, nms_rois_num = core.ops.multiclass_nms3( output, index, nms_rois_num = core.eager.ops.legacy.multiclass_nms3(
bboxes, scores, rois_num, *attrs bboxes, scores, rois_num, *attrs
) )
if not return_index: if not return_index:
......
python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py
浏览文件 @ 70ea88bf
...@@ -17,16 +17,13 @@ import unittest ...@@ -17,16 +17,13 @@ import unittest
import numpy as np import numpy as np
import paddle import paddle
import paddle.fluid.core as core from paddle.fluid import core
from paddle.incubate.nn import FusedLinear from paddle.incubate.nn import FusedLinear
from paddle.incubate.nn.functional import fused_linear, fused_matmul_bias from paddle.incubate.nn.functional import fused_linear, fused_matmul_bias
def is_fused_matmul_bias_supported(): def is_fused_matmul_bias_supported():
if paddle.is_compiled_with_cuda() and not paddle.is_compiled_with_rocm(): return hasattr(core.eager.ops.legacy, 'fused_gemm_epilogue')
return hasattr(core.ops, 'fused_gemm_epilogue')
else:
return False
def matmul(x, y, bias, trans_x, trans_y): def matmul(x, y, bias, trans_x, trans_y):
......
tools/check_file_diff_approvals.sh
浏览文件 @ 70ea88bf
...@@ -44,7 +44,6 @@ API_FILES=("CMakeLists.txt" ...@@ -44,7 +44,6 @@ API_FILES=("CMakeLists.txt"
"python/paddle/fluid/backward.py" "python/paddle/fluid/backward.py"
"paddle/fluid/operators/distributed/send_recv.proto.in" "paddle/fluid/operators/distributed/send_recv.proto.in"
"paddle/fluid/framework/unused_var_check.cc" "paddle/fluid/framework/unused_var_check.cc"
"paddle/fluid/pybind/op_function_generator.cc"
"python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py" "python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py"
"python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py" "python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py"
"python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py" "python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py"
...@@ -143,9 +142,6 @@ for API_FILE in ${API_FILES[*]}; do ...@@ -143,9 +142,6 @@ for API_FILE in ${API_FILES[*]}; do
elif [ "${API_FILE}" == "paddle/fluid/framework/unused_var_check.cc" ];then elif [ "${API_FILE}" == "paddle/fluid/framework/unused_var_check.cc" ];then
echo_line="You must have one RD (zhiqiu (Recommend) or chenwhql) approval for the changes of paddle/fluid/framework/unused_var_check.cc, which manages the allow list of operators that have unused input variables. Before change the allow list, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/OP-Should-Not-Have-Unused-Input] and try to refine code first. \n" echo_line="You must have one RD (zhiqiu (Recommend) or chenwhql) approval for the changes of paddle/fluid/framework/unused_var_check.cc, which manages the allow list of operators that have unused input variables. Before change the allow list, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/OP-Should-Not-Have-Unused-Input] and try to refine code first. \n"
check_approval 1 6888866 22561442 check_approval 1 6888866 22561442
elif [ "${API_FILE}" == "paddle/fluid/pybind/op_function_generator.cc" ];then
echo_line="You must have one RD (zhiqiu (Recommend) , phlrain) approval for the changes of paddle/fluid/pybind/op_function_generator.cc, which manages the logic of automatic generating op functions for dygraph. \n"
check_approval 1 6888866 43953930
elif [ "${API_FILE}" == "python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py" ];then elif [ "${API_FILE}" == "python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py" ];then
echo_line="It is an Op accuracy problem, please take care of it. You must have one RD (hong19860320 (Recommend), luotao1, phlrain) approval for the changes of check_shape_white_list.py, which manages the white list of operators with limited input size. Inputs size of all cases in the op test must be greater than or equal to 100. For more information, please refer to: https://github.com/PaddlePaddle/Paddle/wiki/OP-Test-Input-Shape-Requirements. \n" echo_line="It is an Op accuracy problem, please take care of it. You must have one RD (hong19860320 (Recommend), luotao1, phlrain) approval for the changes of check_shape_white_list.py, which manages the white list of operators with limited input size. Inputs size of all cases in the op test must be greater than or equal to 100. For more information, please refer to: https://github.com/PaddlePaddle/Paddle/wiki/OP-Test-Input-Shape-Requirements. \n"
check_approval 1 9973393 6836917 43953930 check_approval 1 9973393 6836917 43953930
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或