未验证 提交 15d93394 编写于 作者: R risemeup1 提交者: GitHub

Upgrade protobuf to 4.21.x (#49168)

* upgrade protobuf to 3.19.0 in cmake

* recover protobuf python version

* fix distribute compile

* fix

* fix framework.data_feed_pb2

* fix macos ifdef

* fix lite

* test

* update protoc from 3.19.0 t0 3.20.0

* test

* debug

* test

* test

* debug

* debug

* debug

* debug

* test

* debug

* update protocol from 3.20.0 to 4.21.12

* modify graph_brpc_client.h

* modify graph_brpc_client.h

* test

* test

* test

* fix third_party cache problem on build ci

* updata proto

* test

* test

* test

* test

* test

* test

* fix coverage failed test

* try to fix test_exe_fleet_model_run

* fix cinn bug

* fix windows compile problem

* fix python/requirements

---------
Co-authored-by: Npangyoki <pangyoki@126.com>
上级 7f7e9320
...@@ -17,8 +17,8 @@ if(NOT WITH_CINN) ...@@ -17,8 +17,8 @@ if(NOT WITH_CINN)
endif() endif()
if(NOT CINN_GIT_TAG) if(NOT CINN_GIT_TAG)
# 2023.01.28 commit # 2023.02.09 commit
set(CINN_GIT_TAG 1449890f7724babf2a343c6f8073bd28a7bbc683) set(CINN_GIT_TAG 63cf4267a0f8756df61b0dca5fb30bf3748ba2f4)
endif() endif()
message(STATUS "CINN version: " ${CINN_GIT_TAG}) message(STATUS "CINN version: " ${CINN_GIT_TAG})
......
...@@ -120,7 +120,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR) ...@@ -120,7 +120,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
${LITE_PREFIX_DIR}/src/extern_lite/cmake/os/armlinux.cmake ${LITE_PREFIX_DIR}/src/extern_lite/cmake/os/armlinux.cmake
UPDATE_COMMAND "" UPDATE_COMMAND ""
BUILD_COMMAND ${LITE_BUILD_COMMAND} BUILD_COMMAND ${LITE_BUILD_COMMAND}
INSTALL_COMMAND "" INSTALL_COMMAND
cp ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.cc
${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/ && cp
${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS} -DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS}
...@@ -169,7 +173,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR) ...@@ -169,7 +173,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
"s?NNadapter_bridges_path = os.path.abspath('..')+\"\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?NNadapter_bridges_path = os.path.abspath(\'..\')+\"\/extern_lite\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?" "s?NNadapter_bridges_path = os.path.abspath('..')+\"\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?NNadapter_bridges_path = os.path.abspath(\'..\')+\"\/extern_lite\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?"
${LITE_PREFIX_DIR}/src/extern_lite//lite/tools/cmake_tools/record_supported_kernel_op.py ${LITE_PREFIX_DIR}/src/extern_lite//lite/tools/cmake_tools/record_supported_kernel_op.py
BUILD_COMMAND ${LITE_BUILD_COMMAND} BUILD_COMMAND ${LITE_BUILD_COMMAND}
INSTALL_COMMAND "" INSTALL_COMMAND
cp ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.cc
${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/ && cp
${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS} -DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS}
......
...@@ -75,8 +75,8 @@ function(protobuf_generate_python SRCS) ...@@ -75,8 +75,8 @@ function(protobuf_generate_python SRCS)
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py") list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py")
add_custom_command( add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py" OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py"
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PADDLE_BINARY_DIR}
${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} -I${PADDLE_SOURCE_DIR} ${ABS_FIL}
DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE}
COMMENT "Running Python protocol buffer compiler on ${FIL}" COMMENT "Running Python protocol buffer compiler on ${FIL}"
VERBATIM) VERBATIM)
...@@ -236,24 +236,24 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST) ...@@ -236,24 +236,24 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11) if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11)
set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git) set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git)
set(PROTOBUF_TAG v3.8.0) set(PROTOBUF_TAG v21.12)
elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11) elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11)
set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git) set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git)
set(PROTOBUF_TAG v3.8.0) set(PROTOBUF_TAG v21.12)
elseif(WITH_IPU) elseif(WITH_IPU)
set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git) set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
set(PROTOBUF_TAG d750fbf648256c7c631f51ffdbf67d7c18b0114e) set(PROTOBUF_TAG v21.12)
elseif(WIN32) elseif(WIN32)
set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git) set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
# Change the tag to support building with vs2019 # Change the tag to support building with vs2019
set(PROTOBUF_TAG 01a05a53f40ca2ac5f0af10c6cc0810bee39b792) set(PROTOBUF_TAG 01a05a53f40ca2ac5f0af10c6cc0810bee39b792)
else() else()
set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git) set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
set(PROTOBUF_TAG 9f75c5aa851cd877fb0d93ccc31b8567a6706546) set(PROTOBUF_TAG v21.12)
if(WITH_GPU) if(WITH_GPU)
if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0
AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0) AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
set(PROTOBUF_TAG 2dc747c574b68a808ea4699d26942c8132fe2b09) set(PROTOBUF_TAG v21.12)
endif() endif()
endif() endif()
endif() endif()
...@@ -321,17 +321,17 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST) ...@@ -321,17 +321,17 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
endfunction() endfunction()
if(WITH_ASCEND OR WITH_ASCEND_CL) if(WITH_ASCEND OR WITH_ASCEND_CL)
set(PROTOBUF_VERSION 3.8.0) set(PROTOBUF_VERSION 21.12)
elseif(WITH_IPU) elseif(WITH_IPU)
set(PROTOBUF_VERSION 3.6.1) set(PROTOBUF_VERSION 21.12)
elseif(WITH_ARM_BRPC) elseif(WITH_ARM_BRPC)
set(PROTOBUF_VERSION 3.7.1-baidu-ee-common) set(PROTOBUF_VERSION 21.12-baidu-ee-common)
else() else()
set(PROTOBUF_VERSION 3.1.0) set(PROTOBUF_VERSION 21.12)
if(WITH_GPU) if(WITH_GPU)
if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0
AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0) AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
set(PROTOBUF_VERSION 3.16.0) set(PROTOBUF_VERSION 21.12)
endif() endif()
endif() endif()
endif() endif()
......
...@@ -1063,8 +1063,8 @@ function(paddle_protobuf_generate_cpp SRCS HDRS) ...@@ -1063,8 +1063,8 @@ function(paddle_protobuf_generate_cpp SRCS HDRS)
add_custom_command( add_custom_command(
OUTPUT "${_protobuf_protoc_src}" "${_protobuf_protoc_hdr}" OUTPUT "${_protobuf_protoc_src}" "${_protobuf_protoc_hdr}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}" COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}"
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} -I${CMAKE_CURRENT_SOURCE_DIR} COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} -I${PADDLE_SOURCE_DIR} --cpp_out
--cpp_out "${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL} "${PADDLE_BINARY_DIR}" ${ABS_FIL}
# Set `EXTERN_PROTOBUF_DEPEND` only if need to compile `protoc.exe`. # Set `EXTERN_PROTOBUF_DEPEND` only if need to compile `protoc.exe`.
DEPENDS ${ABS_FIL} ${EXTERN_PROTOBUF_DEPEND} DEPENDS ${ABS_FIL} ${EXTERN_PROTOBUF_DEPEND}
COMMENT "Running C++ protocol buffer compiler on ${FIL}" COMMENT "Running C++ protocol buffer compiler on ${FIL}"
......
...@@ -433,6 +433,7 @@ bool DistModel::PrepareFleetExe() { ...@@ -433,6 +433,7 @@ bool DistModel::PrepareFleetExe() {
executor_desc_ = FleetExecutorDesc(); executor_desc_ = FleetExecutorDesc();
executor_desc_.set_cur_rank(config_.local_rank); executor_desc_.set_cur_rank(config_.local_rank);
std::unordered_map<int64_t, int64_t> id_to_rank; std::unordered_map<int64_t, int64_t> id_to_rank;
for (int i = 0; i < config_.nranks; ++i) { for (int i = 0; i < config_.nranks; ++i) {
RankInfo *rank_info = executor_desc_.add_cluster_info(); RankInfo *rank_info = executor_desc_.add_cluster_info();
rank_info->set_rank(i); rank_info->set_rank(i);
......
...@@ -49,11 +49,13 @@ class GraphPsService_Stub : public PsService_Stub { ...@@ -49,11 +49,13 @@ class GraphPsService_Stub : public PsService_Stub {
} }
virtual ~GraphPsService_Stub() {} virtual ~GraphPsService_Stub() {}
GraphPsService_Stub(const GraphPsService_Stub&);
GraphPsService_Stub& operator=(const GraphPsService_Stub&);
// implements PsService ------------------------------------------ // implements PsService ------------------------------------------
GraphBrpcService* graph_service; GraphBrpcService* graph_service;
std::shared_ptr<::ThreadPool> task_pool; std::shared_ptr<::ThreadPool> task_pool;
::google::protobuf::RpcChannel* local_channel; ::google::protobuf::RpcChannel* local_channel;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(GraphPsService_Stub);
void service(::google::protobuf::RpcController* controller, void service(::google::protobuf::RpcController* controller,
const ::paddle::distributed::PsRequestMessage* request, const ::paddle::distributed::PsRequestMessage* request,
::paddle::distributed::PsResponseMessage* response, ::paddle::distributed::PsResponseMessage* response,
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "paddle/fluid/framework/ir/is_test_pass.h" #include "paddle/fluid/framework/ir/is_test_pass.h"
#ifdef _WIN32 #if defined _WIN32 || defined __APPLE__
#undef FALSE #undef FALSE
#undef TRUE #undef TRUE
#endif #endif
......
...@@ -14,7 +14,7 @@ limitations under the License. */ ...@@ -14,7 +14,7 @@ limitations under the License. */
syntax = "proto2"; syntax = "proto2";
import "framework.proto"; import "paddle/fluid/framework/framework.proto";
package paddle.framework.proto; package paddle.framework.proto;
message OpDef { message OpDef {
......
...@@ -11,7 +11,7 @@ limitations under the License. */ ...@@ -11,7 +11,7 @@ limitations under the License. */
syntax = "proto2"; syntax = "proto2";
import "framework.proto"; import "paddle/fluid/framework/framework.proto";
package paddle.framework.proto; package paddle.framework.proto;
// Describes one subsitute subgraph. // Describes one subsitute subgraph.
......
...@@ -14,8 +14,8 @@ limitations under the License. */ ...@@ -14,8 +14,8 @@ limitations under the License. */
syntax = "proto2"; syntax = "proto2";
option optimize_for = LITE_RUNTIME; option optimize_for = LITE_RUNTIME;
import "data_feed.proto"; import "paddle/fluid/framework/data_feed.proto";
import "framework.proto"; import "paddle/fluid/framework/framework.proto";
package paddle.framework; package paddle.framework;
message TrainerDesc { message TrainerDesc {
......
...@@ -30,9 +30,8 @@ inline __device__ bool AttemptInsert( ...@@ -30,9 +30,8 @@ inline __device__ bool AttemptInsert(
static_cast<unsigned int>(-1), static_cast<unsigned int>(-1),
static_cast<unsigned int>(id)); static_cast<unsigned int>(id));
if (key == -1 || key == id) { if (key == -1 || key == id) {
atomicMin( atomicMin(reinterpret_cast<unsigned int*>(&key_index[pos]), // NOLINT
reinterpret_cast<unsigned long long int*>(&key_index[pos]), // NOLINT static_cast<unsigned int>(index)); // NOLINT
static_cast<unsigned long long int>(index)); // NOLINT
return true; return true;
} else { } else {
return false; return false;
...@@ -43,9 +42,8 @@ inline __device__ bool AttemptInsert( ...@@ -43,9 +42,8 @@ inline __device__ bool AttemptInsert(
static_cast<unsigned long long int>(-1), // NOLINT static_cast<unsigned long long int>(-1), // NOLINT
static_cast<unsigned long long int>(id)); // NOLINT static_cast<unsigned long long int>(id)); // NOLINT
if (key == -1 || key == id) { if (key == -1 || key == id) {
atomicMin( atomicMin(reinterpret_cast<unsigned int*>(&key_index[pos]), // NOLINT
reinterpret_cast<unsigned long long int*>(&key_index[pos]), // NOLINT static_cast<unsigned int>(index)); // NOLINT
static_cast<unsigned long long int>(index)); // NOLINT
return true; return true;
} else { } else {
return false; return false;
......
...@@ -210,4 +210,4 @@ message ProgramDesc { ...@@ -210,4 +210,4 @@ message ProgramDesc {
repeated BlockDesc blocks = 1; repeated BlockDesc blocks = 1;
optional Version version = 4; optional Version version = 4;
optional OpCompatibleMap op_compatible_map = 3; optional OpCompatibleMap op_compatible_map = 3;
} }
\ No newline at end of file
...@@ -3429,7 +3429,6 @@ function build_pr_and_develop() { ...@@ -3429,7 +3429,6 @@ function build_pr_and_develop() {
mkdir ${PADDLE_ROOT}/build/pr_whl && cp ${PADDLE_ROOT}/build/python/dist/*.whl ${PADDLE_ROOT}/build/pr_whl mkdir ${PADDLE_ROOT}/build/pr_whl && cp ${PADDLE_ROOT}/build/python/dist/*.whl ${PADDLE_ROOT}/build/pr_whl
rm -f ${PADDLE_ROOT}/build/python/dist/*.whl && rm -f ${PADDLE_ROOT}/build/python/build/.timestamp rm -f ${PADDLE_ROOT}/build/python/dist/*.whl && rm -f ${PADDLE_ROOT}/build/python/build/.timestamp
git fetch upstream develop git fetch upstream develop
git checkout develop git checkout develop
dev_commit=`git log -1|head -1|awk '{print $2}'` dev_commit=`git log -1|head -1|awk '{print $2}'`
......
...@@ -28,7 +28,9 @@ import subprocess ...@@ -28,7 +28,9 @@ import subprocess
import multiprocessing import multiprocessing
import sys import sys
import logging import logging
from .proto import framework_pb2
from .proto import framework_pb2, data_feed_pb2
from . import core from . import core
from . import unique_name from . import unique_name
......
requests>=2.20.0 requests>=2.20.0
numpy>=1.13 numpy>=1.13
protobuf>=3.1.0, <=3.20.2 protobuf>=3.1.0 ; platform_system != "Windows"
protobuf>=3.1.0, <=3.20.2 ; platform_system == "Windows"
Pillow Pillow
decorator decorator
astor astor
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册