Upgrade protobuf to 4.21.x (#49168)

* upgrade protobuf to 3.19.0 in cmake

* recover protobuf python version

* fix distribute compile

* fix

* fix framework.data_feed_pb2

* fix macos ifdef

* fix lite

* test

* update protoc from 3.19.0 t0 3.20.0

* test

* debug

* test

* test

* debug

* debug

* debug

* debug

* test

* debug

* update protocol from 3.20.0 to 4.21.12

* modify graph_brpc_client.h

* modify graph_brpc_client.h

* test

* test

* test

* fix third_party cache problem on build ci

* updata proto

* test

* test

* test

* test

* test

* test

* fix coverage failed test

* try to fix test_exe_fleet_model_run

* fix cinn bug

* fix windows compile problem

* fix python/requirements

---------
Co-authored-by: Npangyoki <pangyoki@126.com>

cmake/external/cinn.cmake

@@ -17,8 +17,8 @@ if(NOT WITH_CINN)
 endif()
 
 if(NOT CINN_GIT_TAG)
-  # 2023.01.28 commit
-  set(CINN_GIT_TAG 1449890f7724babf2a343c6f8073bd28a7bbc683)
+  # 2023.02.09 commit
+  set(CINN_GIT_TAG 63cf4267a0f8756df61b0dca5fb30bf3748ba2f4)
 endif()
 
 message(STATUS "CINN version: " ${CINN_GIT_TAG})

cmake/external/lite.cmake

@@ -120,7 +120,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
         ${LITE_PREFIX_DIR}/src/extern_lite/cmake/os/armlinux.cmake
       UPDATE_COMMAND ""
       BUILD_COMMAND ${LITE_BUILD_COMMAND}
-      INSTALL_COMMAND ""
+      INSTALL_COMMAND
+        cp ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.cc
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/ && cp
+        ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/
       CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
                  -DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS}
@@ -169,7 +173,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
         "s?NNadapter_bridges_path = os.path.abspath('..')+\"\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?NNadapter_bridges_path = os.path.abspath(\'..\')+\"\/extern_lite\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?"
         ${LITE_PREFIX_DIR}/src/extern_lite//lite/tools/cmake_tools/record_supported_kernel_op.py
       BUILD_COMMAND ${LITE_BUILD_COMMAND}
-      INSTALL_COMMAND ""
+      INSTALL_COMMAND
+        cp ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.cc
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/ && cp
+        ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/
       CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
                  -DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS}

cmake/external/protobuf.cmake

@@ -75,8 +75,8 @@ function(protobuf_generate_python SRCS)
     list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py")
     add_custom_command(
       OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py"
-      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out
-              ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PADDLE_BINARY_DIR}
+              -I${PADDLE_SOURCE_DIR} ${ABS_FIL}
       DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE}
       COMMENT "Running Python protocol buffer compiler on ${FIL}"
       VERBATIM)
@@ -236,24 +236,24 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
 
   if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11)
     set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git)
-    set(PROTOBUF_TAG v3.8.0)
+    set(PROTOBUF_TAG v21.12)
   elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11)
     set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git)
-    set(PROTOBUF_TAG v3.8.0)
+    set(PROTOBUF_TAG v21.12)
   elseif(WITH_IPU)
     set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
-    set(PROTOBUF_TAG d750fbf648256c7c631f51ffdbf67d7c18b0114e)
+    set(PROTOBUF_TAG v21.12)
   elseif(WIN32)
     set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
     # Change the tag to support building with vs2019
     set(PROTOBUF_TAG 01a05a53f40ca2ac5f0af10c6cc0810bee39b792)
   else()
     set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
-    set(PROTOBUF_TAG 9f75c5aa851cd877fb0d93ccc31b8567a6706546)
+    set(PROTOBUF_TAG v21.12)
     if(WITH_GPU)
       if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0
          AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
-        set(PROTOBUF_TAG 2dc747c574b68a808ea4699d26942c8132fe2b09)
+        set(PROTOBUF_TAG v21.12)
       endif()
     endif()
   endif()
@@ -321,17 +321,17 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
 endfunction()
 
 if(WITH_ASCEND OR WITH_ASCEND_CL)
-  set(PROTOBUF_VERSION 3.8.0)
+  set(PROTOBUF_VERSION 21.12)
 elseif(WITH_IPU)
-  set(PROTOBUF_VERSION 3.6.1)
+  set(PROTOBUF_VERSION 21.12)
 elseif(WITH_ARM_BRPC)
-  set(PROTOBUF_VERSION 3.7.1-baidu-ee-common)
+  set(PROTOBUF_VERSION 21.12-baidu-ee-common)
 else()
-  set(PROTOBUF_VERSION 3.1.0)
+  set(PROTOBUF_VERSION 21.12)
   if(WITH_GPU)
     if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0
        AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
-      set(PROTOBUF_VERSION 3.16.0)
+      set(PROTOBUF_VERSION 21.12)
     endif()
   endif()
 endif()

cmake/generic.cmake

@@ -1063,8 +1063,8 @@ function(paddle_protobuf_generate_cpp SRCS HDRS)
     add_custom_command(
       OUTPUT "${_protobuf_protoc_src}" "${_protobuf_protoc_hdr}"
       COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}"
-      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} -I${CMAKE_CURRENT_SOURCE_DIR}
-              --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL}
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} -I${PADDLE_SOURCE_DIR} --cpp_out
+              "${PADDLE_BINARY_DIR}" ${ABS_FIL}
       # Set `EXTERN_PROTOBUF_DEPEND` only if need to compile `protoc.exe`.
       DEPENDS ${ABS_FIL} ${EXTERN_PROTOBUF_DEPEND}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"

paddle/fluid/distributed/fleet_executor/dist_model.cc

@@ -433,6 +433,7 @@ bool DistModel::PrepareFleetExe() {
   executor_desc_ = FleetExecutorDesc();
   executor_desc_.set_cur_rank(config_.local_rank);
   std::unordered_map<int64_t, int64_t> id_to_rank;
+
   for (int i = 0; i < config_.nranks; ++i) {
     RankInfo *rank_info = executor_desc_.add_cluster_info();
     rank_info->set_rank(i);

paddle/fluid/distributed/ps/service/graph_brpc_client.h

@@ -49,11 +49,13 @@ class GraphPsService_Stub : public PsService_Stub {
   }
   virtual ~GraphPsService_Stub() {}
 
+  GraphPsService_Stub(const GraphPsService_Stub&);
+  GraphPsService_Stub& operator=(const GraphPsService_Stub&);
+
   // implements PsService ------------------------------------------
   GraphBrpcService* graph_service;
   std::shared_ptr<::ThreadPool> task_pool;
   ::google::protobuf::RpcChannel* local_channel;
-  GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(GraphPsService_Stub);
   void service(::google::protobuf::RpcController* controller,
                const ::paddle::distributed::PsRequestMessage* request,
                ::paddle::distributed::PsResponseMessage* response,

paddle/fluid/framework/ir/is_test_pass_tester.cc

@@ -14,7 +14,7 @@
 #include <gtest/gtest.h>
 
 #include "paddle/fluid/framework/ir/is_test_pass.h"
-#ifdef _WIN32
+#if defined _WIN32 || defined __APPLE__
 #undef FALSE
 #undef TRUE
 #endif

paddle/fluid/framework/op_def.proto

@@ -14,7 +14,7 @@ limitations under the License. */
 
 syntax = "proto2";
 
-import "framework.proto";
+import "paddle/fluid/framework/framework.proto";
 package paddle.framework.proto;
 
 message OpDef {

paddle/fluid/framework/pass_desc.proto

@@ -11,7 +11,7 @@ limitations under the License. */
 
 syntax = "proto2";
 
-import "framework.proto";
+import "paddle/fluid/framework/framework.proto";
 package paddle.framework.proto;
 
 // Describes one subsitute subgraph.

paddle/fluid/framework/trainer_desc.proto

@@ -14,8 +14,8 @@ limitations under the License. */
 
 syntax = "proto2";
 option optimize_for = LITE_RUNTIME;
-import "data_feed.proto";
-import "framework.proto";
+import "paddle/fluid/framework/data_feed.proto";
+import "paddle/fluid/framework/framework.proto";
 package paddle.framework;
 
 message TrainerDesc {

paddle/fluid/operators/graph_khop_sampler_imp.h

@@ -30,9 +30,8 @@ inline __device__ bool AttemptInsert(
                                  static_cast<unsigned int>(-1),
                                  static_cast<unsigned int>(id));
     if (key == -1 || key == id) {
-      atomicMin(
-          reinterpret_cast<unsigned long long int*>(&key_index[pos]),  // NOLINT
-          static_cast<unsigned long long int>(index));                 // NOLINT
+      atomicMin(reinterpret_cast<unsigned int*>(&key_index[pos]),  // NOLINT
+                static_cast<unsigned int>(index));                 // NOLINT
       return true;
     } else {
       return false;
@@ -43,9 +42,8 @@ inline __device__ bool AttemptInsert(
         static_cast<unsigned long long int>(-1),                // NOLINT
         static_cast<unsigned long long int>(id));               // NOLINT
     if (key == -1 || key == id) {
-      atomicMin(
-          reinterpret_cast<unsigned long long int*>(&key_index[pos]),  // NOLINT
-          static_cast<unsigned long long int>(index));                 // NOLINT
+      atomicMin(reinterpret_cast<unsigned int*>(&key_index[pos]),  // NOLINT
+                static_cast<unsigned int>(index));                 // NOLINT
       return true;
     } else {
       return false;

paddle/infrt/paddle/framework.proto

@@ -210,4 +210,4 @@ message ProgramDesc {
   repeated BlockDesc blocks = 1;
   optional Version version = 4;
   optional OpCompatibleMap op_compatible_map = 3;
-}
\ No newline at end of file
+}

paddle/scripts/paddle_build.sh

@@ -3429,7 +3429,6 @@ function build_pr_and_develop() {
     mkdir ${PADDLE_ROOT}/build/pr_whl && cp ${PADDLE_ROOT}/build/python/dist/*.whl ${PADDLE_ROOT}/build/pr_whl
     rm -f ${PADDLE_ROOT}/build/python/dist/*.whl && rm -f ${PADDLE_ROOT}/build/python/build/.timestamp
 
-
     git fetch upstream develop
     git checkout develop
     dev_commit=`git log -1|head -1|awk '{print $2}'`

python/paddle/fluid/framework.py

@@ -28,7 +28,9 @@ import subprocess
 import multiprocessing
 import sys
 import logging
-from .proto import framework_pb2
+
+from .proto import framework_pb2, data_feed_pb2
+
 
 from . import core
 from . import unique_name

python/requirements.txt

 requests>=2.20.0
 numpy>=1.13
-protobuf>=3.1.0, <=3.20.2
+protobuf>=3.1.0 ; platform_system != "Windows"
+protobuf>=3.1.0, <=3.20.2 ; platform_system == "Windows"
 Pillow
 decorator
 astor