From a01663ca1fb77e574553136383d66b5f4b49991a Mon Sep 17 00:00:00 2001 From: zhouwei25 <52485244+zhouwei25@users.noreply.github.com> Date: Wed, 25 Dec 2019 10:03:25 +0800 Subject: [PATCH] remove patch command and file of cares to Improved quality of Paddle Repo (#21776) --- CMakeLists.txt | 1 - cmake/external/cares.cmake | 48 ---- cmake/third_party.cmake | 1 - .../fluid/operators/collective/CMakeLists.txt | 2 +- .../operators/distributed/CMakeLists.txt | 2 +- .../operators/distributed_ops/CMakeLists.txt | 2 +- paddle/scripts/paddle_build.sh | 3 +- patches/cares/ares_parse_ptr_reply.c.txt | 221 ------------------ 8 files changed, 5 insertions(+), 275 deletions(-) delete mode 100644 cmake/external/cares.cmake delete mode 100644 patches/cares/ares_parse_ptr_reply.c.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 69fad78a10..f5b9412f9a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,7 +133,6 @@ endif() include(third_party) # download, build, install third_party if(WITH_DISTRIBUTE) - include(external/cares) if(WITH_GRPC) message(STATUS "Use grpc framework.") include(external/grpc) diff --git a/cmake/external/cares.cmake b/cmake/external/cares.cmake deleted file mode 100644 index de199da3e6..0000000000 --- a/cmake/external/cares.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: c-ares is needed when linking with grpc. - -include (ExternalProject) - -SET(CARES_SOURCES_DIR ${THIRD_PARTY_PATH}/cares) -SET(CARES_INSTALL_DIR ${THIRD_PARTY_PATH}/install/cares) -SET(CARES_INCLUDE_DIR "${CARES_INSTALL_DIR}/include/" CACHE PATH "cares include directory." FORCE) - -if(NOT APPLE AND NOT WIN32) - if(${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 8.0) - set(PATCH_COMMAND_CARES cp ${PADDLE_SOURCE_DIR}/patches/cares/ares_parse_ptr_reply.c.txt ${CARES_SOURCES_DIR}/src/extern_cares/ares_parse_ptr_reply.c) - endif() -endif() - -ExternalProject_Add( - extern_cares - ${SHALLOW_CLONE} - GIT_REPOSITORY "https://github.com/c-ares/c-ares.git" - GIT_TAG "cares-1_13_0" - PREFIX ${CARES_SOURCES_DIR} - UPDATE_COMMAND "" - CONFIGURE_COMMAND ./buildconf && ./configure --disable-shared --prefix=${CARES_INSTALL_DIR} - BUILD_IN_SOURCE 1 - PATCH_COMMAND ${PATCH_COMMAND_CARES} - BUILD_COMMAND make -j $(nproc) - INSTALL_COMMAND make install -) - -ADD_LIBRARY(cares STATIC IMPORTED GLOBAL) -SET_PROPERTY(TARGET cares PROPERTY IMPORTED_LOCATION - "${CARES_INSTALL_DIR}/lib/libcares.a") - -include_directories(${CARES_INCLUDE_DIR}) -ADD_DEPENDENCIES(cares extern_cares) diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index 5c27e5f951..43c24ed7e1 100644 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -249,7 +249,6 @@ if(WITH_BOX_PS) endif(WITH_BOX_PS) if(WITH_DISTRIBUTE) - list(APPEND third_party_deps extern_cares) if(WITH_GRPC) list(APPEND third_party_deps extern_grpc) else() diff --git a/paddle/fluid/operators/collective/CMakeLists.txt b/paddle/fluid/operators/collective/CMakeLists.txt index 75185d5365..c3b3f60eaf 100644 --- a/paddle/fluid/operators/collective/CMakeLists.txt +++ b/paddle/fluid/operators/collective/CMakeLists.txt @@ -2,7 +2,7 @@ include(operators) set(COLLECTIVE_DEPS "") if(WITH_GRPC) - set(COLLECTIVE_DEPS sendrecvop_rpc parameter_send parameter_recv communicator async_sparse_param_update_recorder grpc++_unsecure grpc_unsecure gpr cares zlib protobuf node) + set(COLLECTIVE_DEPS sendrecvop_rpc parameter_send parameter_recv communicator async_sparse_param_update_recorder grpc++_unsecure grpc_unsecure gpr zlib protobuf node) else() set(COLLECTIVE_DEPS sendrecvop_rpc parameter_send parameter_recv communicator async_sparse_param_update_recorder brpc leveldb protobuf ssl crypto zlib node) if(WITH_BRPC_RDMA) diff --git a/paddle/fluid/operators/distributed/CMakeLists.txt b/paddle/fluid/operators/distributed/CMakeLists.txt index 197bdb6298..5aa91733fe 100644 --- a/paddle/fluid/operators/distributed/CMakeLists.txt +++ b/paddle/fluid/operators/distributed/CMakeLists.txt @@ -18,7 +18,7 @@ cc_test(heart_beat_monitor_test SRCS heart_beat_monitor_test.cc DEPS heart_beat_ # FIXME(typhoonzero): use add_subdirectory once we clean the dependency of these files set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor") if(WITH_GRPC) - set(GRPC_DEPS grpc++_unsecure grpc_unsecure gpr cares zlib protobuf) + set(GRPC_DEPS grpc++_unsecure grpc_unsecure gpr zlib protobuf) set(GRPC_SRCS grpc/grpc_client.cc grpc/grpc_server.cc grpc/grpc_serde.cc grpc/grpc_bytebuffer_stream.cc grpc/grpc_variable_response.cc) grpc_library(sendrecvop_rpc SRCS sendrecvop_utils.cc request_handler_impl.cc rpc_client.cc rpc_server.cc diff --git a/paddle/fluid/operators/distributed_ops/CMakeLists.txt b/paddle/fluid/operators/distributed_ops/CMakeLists.txt index 43287c7f25..d1a9c65c9c 100644 --- a/paddle/fluid/operators/distributed_ops/CMakeLists.txt +++ b/paddle/fluid/operators/distributed_ops/CMakeLists.txt @@ -2,7 +2,7 @@ include(operators) set(DISTRIBUTE_DEPS "") if(WITH_GRPC) - set(DISTRIBUTE_DEPS sendrecvop_rpc parameter_send parameter_recv communicator async_sparse_param_update_recorder grpc++_unsecure grpc_unsecure gpr cares zlib protobuf node) + set(DISTRIBUTE_DEPS sendrecvop_rpc parameter_send parameter_recv communicator async_sparse_param_update_recorder grpc++_unsecure grpc_unsecure gpr zlib protobuf node) else() set(DISTRIBUTE_DEPS sendrecvop_rpc parameter_send parameter_recv communicator async_sparse_param_update_recorder brpc leveldb protobuf ssl crypto zlib node) if(WITH_BRPC_RDMA) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 9f5ceb1fc1..ccb138ec2d 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -474,7 +474,8 @@ function generate_upstream_develop_api_spec() { generate_api_spec "$1" "DEV" git checkout $cur_branch git branch -D develop_base_pr - ENABLE_MAKE_CLEAN="OFF" + ENABLE_MAKE_CLEAN="ON" + rm -rf ${PADDLE_ROOT}/build/Makefile ${PADDLE_ROOT}/build/CMakeCache.txt } function generate_api_spec() { diff --git a/patches/cares/ares_parse_ptr_reply.c.txt b/patches/cares/ares_parse_ptr_reply.c.txt deleted file mode 100644 index c590add9e7..0000000000 --- a/patches/cares/ares_parse_ptr_reply.c.txt +++ /dev/null @@ -1,221 +0,0 @@ - -/* Copyright 1998 by the Massachusetts Institute of Technology. - * - * Permission to use, copy, modify, and distribute this - * software and its documentation for any purpose and without - * fee is hereby granted, provided that the above copyright - * notice appear in all copies and that both that copyright - * notice and this permission notice appear in supporting - * documentation, and that the name of M.I.T. not be used in - * advertising or publicity pertaining to distribution of the - * software without specific, written prior permission. - * M.I.T. makes no representations about the suitability of - * this software for any purpose. It is provided "as is" - * without express or implied warranty. - */ - -#include "ares_setup.h" - -#ifdef HAVE_NETINET_IN_H -# include -#endif -#ifdef HAVE_NETDB_H -# include -#endif -#ifdef HAVE_ARPA_NAMESER_H -# include -#else -# include "nameser.h" -#endif -#ifdef HAVE_ARPA_NAMESER_COMPAT_H -# include -#endif - -#ifdef HAVE_STRINGS_H -# include -#endif - -#include "ares.h" -#include "ares_dns.h" -#include "ares_nowarn.h" -#include "ares_private.h" - -int ares_parse_ptr_reply(const unsigned char *abuf, int alen, const void *addr, - int addrlen, int family, struct hostent **host) -{ - unsigned int qdcount, ancount; - int status, i, rr_type, rr_class, rr_len; - long len; - const unsigned char *aptr; - char *ptrname, *hostname, *rr_name, *rr_data; - struct hostent *hostent; - int aliascnt = 0; - int alias_alloc = 8; - char ** aliases; - size_t rr_data_len; - - /* Set *host to NULL for all failure cases. */ - *host = NULL; - - /* Give up if abuf doesn't have room for a header. */ - if (alen < HFIXEDSZ) - return ARES_EBADRESP; - - /* Fetch the question and answer count from the header. */ - qdcount = DNS_HEADER_QDCOUNT(abuf); - ancount = DNS_HEADER_ANCOUNT(abuf); - if (qdcount != 1) - return ARES_EBADRESP; - - /* Expand the name from the question, and skip past the question. */ - aptr = abuf + HFIXEDSZ; - status = ares__expand_name_for_response(aptr, abuf, alen, &ptrname, &len); - if (status != ARES_SUCCESS) - return status; - if (aptr + len + QFIXEDSZ > abuf + alen) - { - ares_free(ptrname); - return ARES_EBADRESP; - } - aptr += len + QFIXEDSZ; - - /* Examine each answer resource record (RR) in turn. */ - hostname = NULL; - aliases = ares_malloc(alias_alloc * sizeof(char *)); - if (!aliases) - { - ares_free(ptrname); - return ARES_ENOMEM; - } - for (i = 0; i < (int)ancount; i++) - { - /* Decode the RR up to the data field. */ - status = ares__expand_name_for_response(aptr, abuf, alen, &rr_name, &len); - if (status != ARES_SUCCESS) - break; - aptr += len; - if (aptr + RRFIXEDSZ > abuf + alen) - { - ares_free(rr_name); - status = ARES_EBADRESP; - break; - } - rr_type = DNS_RR_TYPE(aptr); - rr_class = DNS_RR_CLASS(aptr); - rr_len = DNS_RR_LEN(aptr); - aptr += RRFIXEDSZ; - if (aptr + rr_len > abuf + alen) - { - ares_free(rr_name); - status = ARES_EBADRESP; - break; - } - - if (rr_class == C_IN && rr_type == T_PTR - && strcasecmp(rr_name, ptrname) == 0) - { - /* Decode the RR data and set hostname to it. */ - status = ares__expand_name_for_response(aptr, abuf, alen, &rr_data, - &len); - if (status != ARES_SUCCESS) - { - ares_free(rr_name); - break; - } - if (hostname) - ares_free(hostname); - hostname = rr_data; - rr_data_len = strlen(rr_data)+1; - aliases[aliascnt] = ares_malloc(rr_data_len * sizeof(char)); - if (!aliases[aliascnt]) - { - ares_free(rr_name); - status = ARES_ENOMEM; - break; - } - strncpy(aliases[aliascnt], rr_data, rr_data_len); - aliascnt++; - if (aliascnt >= alias_alloc) { - char **ptr; - alias_alloc *= 2; - ptr = ares_realloc(aliases, alias_alloc * sizeof(char *)); - if(!ptr) { - ares_free(rr_name); - status = ARES_ENOMEM; - break; - } - aliases = ptr; - } - } - - if (rr_class == C_IN && rr_type == T_CNAME) - { - /* Decode the RR data and replace ptrname with it. */ - status = ares__expand_name_for_response(aptr, abuf, alen, &rr_data, - &len); - if (status != ARES_SUCCESS) - { - ares_free(rr_name); - break; - } - ares_free(ptrname); - ptrname = rr_data; - } - - ares_free(rr_name); - aptr += rr_len; - if (aptr > abuf + alen) - { /* LCOV_EXCL_START: already checked above */ - status = ARES_EBADRESP; - break; - } /* LCOV_EXCL_STOP */ - } - - if (status == ARES_SUCCESS && !hostname) - status = ARES_ENODATA; - if (status == ARES_SUCCESS) - { - /* We got our answer. Allocate memory to build the host entry. */ - hostent = ares_malloc(sizeof(struct hostent)); - if (hostent) - { - hostent->h_addr_list = ares_malloc(2 * sizeof(char *)); - if (hostent->h_addr_list) - { - hostent->h_addr_list[0] = ares_malloc(addrlen); - if (hostent->h_addr_list[0]) - { - hostent->h_aliases = ares_malloc((aliascnt+1) * sizeof (char *)); - if (hostent->h_aliases) - { - /* Fill in the hostent and return successfully. */ - hostent->h_name = hostname; - for (i=0 ; ih_aliases[i] = aliases[i]; - hostent->h_aliases[aliascnt] = NULL; - hostent->h_addrtype = aresx_sitoss(family); - hostent->h_length = aresx_sitoss(addrlen); - memcpy(hostent->h_addr_list[0], addr, addrlen); - hostent->h_addr_list[1] = NULL; - *host = hostent; - ares_free(aliases); - ares_free(ptrname); - return ARES_SUCCESS; - } - ares_free(hostent->h_addr_list[0]); - } - ares_free(hostent->h_addr_list); - } - ares_free(hostent); - } - status = ARES_ENOMEM; - } - for (i=0 ; i