Skip to content

MegEngine
MegEngine

MegEngine 天元 / MegEngine
1 年多 前同步成功

通知 403
Star 4705
Fork 582
MegEngine
MegEngine
提交 d74a4ee9 编写于 作者: M Megvii Engine Team
浏览文件
操作

fix(mge/build): use consistent genfiles/megbrain_build_config.h 

Modify get_dynamic_info to work with cmake changes

Also rename get_dynamic_info as get_mem_allocation_info

GitOrigin-RevId: 383affe3884fadce913211bd0d80c42fae85e39b
上级 720b6b49
显示空白变更内容
内联 并排
Showing with 234 addition and 135 deletion
CMakeLists.txt
浏览文件 @ d74a4ee9
......@@ -49,6 +49,13 @@ option(MGE_WITH_TRT "Build MegEngine with TensorRT." ON)
option(MGE_USE_SYSTEM_LIB "Build MegEngine with system libraries." OFF)
option(MGB_WITH_FLATBUFFERS "Build MegBrain with FlatBuffers serialization support." ON)
if(NOT MGE_WITH_JIT)
if(MGE_WITH_HALIDE)
message(WARNING "MGE_WITH_HALIDE is set to OFF with MGE_WITH_JIT disabled")
set(MGE_WITH_HALIDE OFF)
endif()
endif()
if(MGE_WITH_CUDA)
include(CheckLanguage)
check_language(CUDA)
......@@ -77,9 +84,6 @@ if(${CMAKE_THREAD_LIBS_INIT} STREQUAL "-pthread" AND MGE_WITH_CUDA)
PROPERTY INTERFACE_COMPILE_OPTIONS "$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-pthread>"
"$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:-pthread>")
endif()
if(CMAKE_THREAD_LIBS_INIT)
add_definitions(-DMGB_HAVE_THREAD=1)
endif()
set(MGE_BLAS MKL CACHE STRING "BLAS implementaion used by MegEngine.")
......@@ -93,12 +97,6 @@ option(MGE_ENABLE_RTTI "Build with RTTI" ON)
option(MGE_ENABLE_LOGGING "Build with logging" ON)
option(MGE_DEBUG_UTIL "Enable debug utility" ON)
if(MGE_DEBUG_UTIL)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMGB_ENABLE_DEBUG_UTIL=1")
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMGB_ENABLE_DEBUG_UTIL=0")
endif()
if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.")
set(CMAKE_BUILD_TYPE RelWithDebInfo)
......@@ -113,33 +111,6 @@ if(NOT MGE_ENABLE_EXCEPTIONS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exception")
endif()
# RTTI
if(MGE_ENABLE_RTTI)
add_definitions(-DMEGDNN_ENABLE_MANGLING=0 -DMEGDNN_ENABLE_RTTI=1)
else()
add_definitions(-DMEGDNN_ENABLE_MANGLING=1 -DMEGDNN_ENABLE_RTTI=0)
endif()
# Logging
if(MGE_ENABLE_LOGGING)
add_definitions(-DMEGDNN_ENABLE_LOGGING=1 -DMGB_ENABLE_LOGGING=1 -DMGB_ENABLE_JSON=1)
else()
add_definitions(-DMEGDNN_ENABLE_LOGGING=0 -DMGB_ENABLE_LOGGING=0 -DMGB_ENABLE_JSON=0)
endif()
# Exception
if(MGE_ENABLE_EXCEPTIONS)
add_definitions(-DMEGDNN_ENABLE_EXCEPTIONS=1)
else()
message(STATUS "Exceptions disabled; MegEngine would kill itself when it is supposed to throw an exception.")
add_definitions(-DMEGDNN_ENABLE_EXCEPTIONS=0)
endif()
if(MGE_WITH_JIT AND MGE_WITH_HALIDE)
set(HALIDE_SHARED_LIBRARY OFF CACHE BOOL "Build as a shared library")
include(cmake/Halide.cmake)
add_definitions(-DMGB_JIT_HALIDE=1)
endif()
option(MGE_WITH_TEST "Enable test for MegEngine." OFF)
if(MGE_WITH_TEST)
......@@ -319,25 +290,78 @@ if(MGE_WITH_MKLDNN AND ${MGE_ARCH} STREQUAL "x86_64")
include(cmake/MKL_DNN.cmake)
endif()
# RTTI
if(MGE_ENABLE_RTTI)
add_definitions(-DMEGDNN_ENABLE_MANGLING=0 -DMEGDNN_ENABLE_RTTI=1)
else()
add_definitions(-DMEGDNN_ENABLE_MANGLING=1 -DMEGDNN_ENABLE_RTTI=0)
endif()
set(MGB_VERBOSE_TYPEINFO_NAME ${MGE_ENABLE_RTTI})
add_subdirectory(dnn)
# Logging
if(MGE_ENABLE_LOGGING)
add_definitions(-DMEGDNN_ENABLE_LOGGING=1)
else()
add_definitions(-DMEGDNN_ENABLE_LOGGING=0)
endif()
set(MGB_ENABLE_LOGGING ${MGE_ENABLE_LOGGING})
set(MGB_ENABLE_JSON ${MGE_ENABLE_LOGGING})
# Exception
if(MGE_ENABLE_EXCEPTIONS)
add_definitions(-DMEGDNN_ENABLE_EXCEPTIONS=1)
else()
message(STATUS "Exceptions disabled; MegEngine would kill itself when it is supposed to throw an exception.")
add_definitions(-DMEGDNN_ENABLE_EXCEPTIONS=0)
endif()
set(MGB_ENABLE_EXCEPTION ${MGE_ENABLE_EXCEPTIONS})
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DMGB_ASSERT_LOC=1")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DMGB_ASSERT_LOC=0")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -DMGB_ASSERT_LOC=1")
set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -DMGB_ASSERT_LOC=0")
# JIT
if(MGE_WITH_JIT AND MGE_WITH_HALIDE)
set(HALIDE_SHARED_LIBRARY OFF CACHE BOOL "Build as a shared library")
include(cmake/Halide.cmake)
endif()
set(MGB_JIT ${MGE_WITH_JIT})
set(MGB_JIT_HALIDE ${MGE_WITH_HALIDE})
if(MGE_ENABLE_RTTI)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMGB_VERBOSE_TYPEINFO_NAME=1")
# Thread
if(CMAKE_THREAD_LIBS_INIT)
set(MGB_HAVE_THREAD 1)
endif()
if(MGE_ENABLE_EXCEPTIONS)
add_definitions(-DMGB_ENABLE_EXCEPTION=1)
# CUDA
set(MGB_CUDA ${MGE_WITH_CUDA})
# Debug info
if(${CMAKE_BUILD_TYPE} STREQUAL "Debug" OR ${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo")
set(MGB_ASSERT_LOC 1)
set(MGB_ENABLE_DEBUG_UTIL 1)
else()
add_definitions(-DMGB_ENABLE_EXCEPTION=0)
set(MGB_ASSERT_LOC 0)
set(MGB_ENABLE_DEBUG_UTIL 0)
endif()
# TensorRT
set(MGB_ENABLE_TENSOR_RT ${MGE_WITH_TRT})
# Inference only
if(MGE_INFERENCE_ONLY)
set(MGB_ENABLE_GRAD 0)
set(MGB_BUILD_SLIM_SERVING 1)
else()
set(MGB_ENABLE_GRAD 1)
set(MGB_BUILD_SLIM_SERVING 0)
endif()
# Distributed communication
set(MGB_ENABLE_OPR_MM ${MGE_WITH_DISTRIBUTED})
# Write out megbrain_build_config.h
configure_file(src/core/include/megbrain_build_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/genfiles/megbrain_build_config.h)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/genfiles/megbrain_build_config.h DESTINATION include)
add_subdirectory(dnn)
list(APPEND MGB_OPR_PARAM_DEFS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/tools/param_defs/mgb_opr_param_defs.py)
set(MGB_OPR_PARAM_DEFS_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/dnn/scripts/gen_param_defs.py)
......@@ -402,24 +426,3 @@ if(TARGET _mgb)
)
endif()
set(MGB_CUDA ${MGE_WITH_CUDA})
if(${CMAKE_BUILD_TYPE} STREQUAL "Debug" OR ${CMAKE_BUILD_TYPE} STREQUAL "RelWithDebInfo")
set(MGB_ASSERT_LOC 1)
else()
set(MGB_ASSERT_LOC 0)
endif()
set(MGB_ENABLE_DEBUG_UTIL ${MGE_DEBUG_UTIL})
set(MGB_ENABLE_LOGGING ${MGE_ENABLE_LOGGING})
set(MGB_VERBOSE_TYPEINFO_NAME ${MGE_ENABLE_RTTI})
set(MGB_ENABLE_EXCEPTION ${MGE_ENABLE_EXCEPTIONS})
set(MGB_JIT ${MGE_WITH_JIT})
set(MGB_JIT_HALIDE ${MGE_WITH_HALIDE})
set(MGB_ENABLE_TENSOR_RT ${MGE_WITH_TRT})
set(MGB_ENABLE_JSON ${MGE_ENABLE_LOGGING})
set(MGB_ENABLE_GRAD NOT ${MGE_INFERENCE_ONLY})
set(MGB_BUILD_SLIM_SERVING ${MGE_INFERENCE_ONLY})
configure_file(src/core/include/megbrain_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/genfiles/megbrain_build_config.h)
file(READ src/core/include/megbrain_build_config.h _CONTENT)
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/genfiles/megbrain_build_config.h ${_CONTENT})
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/genfiles/megbrain_build_config.h DESTINATION include)
python_module/CMakeLists.txt
浏览文件 @ d74a4ee9
......@@ -10,9 +10,6 @@ find_package(Numpy REQUIRED)
find_package(SWIG REQUIRED)
set(SWIG_SRC src/swig/mgb.i)
set(CMAKE_SWIG_FLAGS -Wall -threads -py3 -modern -DSWIGWORDSIZE64)
if(MGB_ENABLE_JSON)
set(CMAKE_SWIG_FLAGS ${CMAKE_SWIG_FLAGS} -DMGB_ENABLE_JSON)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
if(MGE_WITH_DISTRIBUTED)
......@@ -105,5 +102,6 @@ add_custom_command(
COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/src/serialization/include ${CMAKE_CURRENT_BINARY_DIR}/megengine/_internal/include
COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/src/plugin/include ${CMAKE_CURRENT_BINARY_DIR}/megengine/_internal/include
COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/dnn/include ${CMAKE_CURRENT_BINARY_DIR}/megengine/_internal/include
COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/genfiles/megbrain_build_config.h ${CMAKE_CURRENT_BINARY_DIR}/megengine/_internal/include/megbrain_build_config.h
)
python_module/src/swig/comp_graph.i
浏览文件 @ d74a4ee9
......@@ -69,13 +69,8 @@ class CompGraph {
return reinterpret_cast<size_t>(&$self->get());
}
std::string get_dynamic_info() const {
#ifdef MGB_ENABLE_JSON
auto jsonstr = self->get().get_dynamic_info();
return jsonstr->to_string();
#else
return std::string("");
#endif
std::string get_mem_allocation_info() const {
return self->get().get_mem_allocation_info();
}
std::string __repr__() const {
......
python_module/test/unit/core/test_dynamic_profiling.py
浏览文件 @ d74a4ee9
......@@ -14,7 +14,8 @@ import megengine.functional as F
from megengine import graph, tensor
def test_dynmaic_profiling():
def test_dynamic_profiling():
with graph.Graph():
sz = 16
cg = graph.get_default_graph()
......@@ -22,19 +23,21 @@ def test_dynmaic_profiling():
x = tensor(np.arange(0, sz, dtype=np.float32))
y = F.relu(x)
str1 = cg.get_dynamic_info()
str1 = cg.get_mem_allocation_info()
if str1 == "":
return
json_str1 = json.loads(str1)
z = F.add_update(x, y)
json_str2 = json.loads(cg.get_dynamic_info())
json_str2 = json.loads(cg.get_mem_allocation_info())
diff = lambda l1, l2: [x for x in l1 if x not in l2]
jdiff = diff(json_str2, json_str1)
assert len(jdiff) == 1, "add_update operator should produce only one opr internally"
assert (
len(jdiff) == 1
), "add_update operator should produce only one opr internally"
dest_key = list(jdiff[0].keys())[0]
assert (
......
src/CMakeLists.txt
浏览文件 @ d74a4ee9
......@@ -2,19 +2,9 @@
file(GLOB_RECURSE SOURCES core/impl/*.cpp gopt/impl/*.cpp opr/impl/*.cpp plugin/impl/*.cpp serialization/impl/*.cpp core/impl/*.inl gopt/impl/*.inl opr/impl/*.inl plugin/impl/*.inl serialization/impl/*.inl)
set(MGB_DEF)
if(MGE_WITH_JIT)
list(APPEND MGB_DEF -DMGB_JIT=1)
file(GLOB_RECURSE SOURCES_ jit/impl/*.cpp jit/impl/*.inl)
list(APPEND SOURCES ${SOURCES_})
else()
list(APPEND MGB_DEF -DMGB_JIT=0)
endif()
if(MGE_INFERENCE_ONLY)
list(APPEND MGB_DEF -DMGB_ENABLE_GRAD=0 -DMGB_BUILD_SLIM_SERVING=1)
else()
list(APPEND MGB_DEF -DMGB_ENABLE_GRAD=1 -DMGB_BUILD_SLIM_SERVING=0)
endif()
if(MGE_WITH_DISTRIBUTED)
......@@ -22,7 +12,7 @@ if(MGE_WITH_DISTRIBUTED)
list(APPEND SOURCES ${SOURCES_})
endif()
set(MGB_INC core/include gopt/include opr/include plugin/include serialization/include)
set(MGB_INC ${PROJECT_BINARY_DIR}/genfiles core/include gopt/include opr/include plugin/include serialization/include)
if(MGE_WITH_JIT)
list(APPEND MGB_INC jit/include)
......@@ -32,25 +22,13 @@ if(MGE_WITH_JIT)
endif()
if(MGE_WITH_DISTRIBUTED)
list(APPEND MGB_DEF -DMGB_ENABLE_OPR_MM=1)
list(APPEND MGB_INC opr-mm/include)
else()
list(APPEND MGB_DEF -DMGB_ENABLE_OPR_MM=0)
endif()
if(MGE_WITH_CUDA)
list(APPEND MGB_DEF -DMGB_CUDA=1)
else()
list(APPEND MGB_DEF -DMGB_CUDA=0)
endif()
if(MGE_WITH_CUDA AND MGE_WITH_TRT)
list(APPEND MGB_DEF -DMGB_ENABLE_TENSOR_RT=1)
list(APPEND MGB_INC tensorrt/include)
file(GLOB_RECURSE SOURCES_ tensorrt/impl/*.cpp tensorrt/impl/*.inl)
list(APPEND SOURCES ${SOURCES_})
else()
list(APPEND MGB_DEF -DMGB_ENABLE_TENSOR_RT=0)
endif()
set(MGB_DEF ${MGB_DEF} PARENT_SCOPE)
......
src/core/impl/graph/cg_impl.cpp
浏览文件 @ d74a4ee9
......@@ -726,8 +726,8 @@ std::string ComputingGraphImpl::VarReceiverInfo::to_string() const {
allow_empty_value);
}
std::string ComputingGraphImpl::get_mem_allocation_info() const {
#if MGB_ENABLE_JSON
std::shared_ptr<json::Value> ComputingGraphImpl::get_dynamic_info() const {
auto make_var_json = [](VarNode* single_var) {
auto &&cur_mem_plan = single_var->mem_plan();
if (cur_mem_plan.valid())
......@@ -766,8 +766,11 @@ std::shared_ptr<json::Value> ComputingGraphImpl::get_dynamic_info() const {
objlist->add(obj);
}
return objlist;
}
return objlist->to_string();
#endif // MGB_ENABLE_JSON
mgb_log_warn("mgb is not configured with MGB_ENABLE_JSON on,"
"get_mem_allocation_info returns null string");
return std::string();
}
// vim: syntax=cpp.doxygen foldmethod=marker foldmarker=f{{{,f}}}
src/core/impl/graph/cg_impl.h
浏览文件 @ d74a4ee9
......@@ -146,9 +146,7 @@ public:
return m_var_receiver.at(var);
}
#if MGB_ENABLE_JSON
std::shared_ptr<json::Value> get_dynamic_info() const override;
#endif
std::string get_mem_allocation_info() const override;
VarNode* find_var_by_id(size_t id) const override;
......
src/core/include/megbrain/graph/cg.h
浏览文件 @ d74a4ee9
......@@ -175,9 +175,7 @@ class ComputingGraph : public std::enable_shared_from_this<ComputingGraph>,
virtual const VarReceiverInfo& var_receiver_in_current_comp_seq(
const VarNode *var) const = 0;
#if MGB_ENABLE_JSON
virtual std::shared_ptr<json::Value> get_dynamic_info() const = 0;
#endif
virtual std::string get_mem_allocation_info() const = 0;
/*!
* \brief find var node by its ID
......
src/core/include/megbrain_build_config.h.in 0 → 100644
浏览文件 @ d74a4ee9
/**
* \file src/core/include/megbrain_build_config.h
* MegEngine is Licensed under the Apache License, Version 2.0 (the "License")
*
* Copyright (c) 2014-2020 Megvii Inc. All rights reserved.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
#ifndef _HEADER_MGB_BUILD_CONFIG
#define _HEADER_MGB_BUILD_CONFIG
#cmakedefine01 MGB_CUDA
#cmakedefine01 MGB_ASSERT_LOC
#cmakedefine01 MGB_ENABLE_DEBUG_UTIL
#cmakedefine01 MGB_ENABLE_LOGGING
#cmakedefine01 MGB_ENABLE_GRAD
#cmakedefine01 MGB_VERBOSE_TYPEINFO_NAME
#cmakedefine01 MGB_BUILD_SLIM_SERVING
#cmakedefine01 MGB_ENABLE_EXCEPTION
#cmakedefine01 MGB_JIT
#cmakedefine01 MGB_JIT_HALIDE
#cmakedefine01 MGB_ENABLE_TENSOR_RT
#cmakedefine01 MGB_ENABLE_JSON
#cmakedefine01 MGB_HAVE_THREAD
#cmakedefine01 MGB_ENABLE_OPR_MM
// whether cuda is available
#ifndef MGB_CUDA
#define MGB_CUDA 1
#endif
// whether to include file/line location for assert message
#ifndef MGB_ASSERT_LOC
#define MGB_ASSERT_LOC 1
#endif
// whether to enable utils/debug.h and other debug methods
#ifndef MGB_ENABLE_DEBUG_UTIL
#define MGB_ENABLE_DEBUG_UTIL 1
#endif
// whether to enable logging
#ifndef MGB_ENABLE_LOGGING
#define MGB_ENABLE_LOGGING 1
#endif
// whether to enable registering opr grad functions
#ifndef MGB_ENABLE_GRAD
#define MGB_ENABLE_GRAD 1
#endif
// whether to include actual class name in mgb::Typeinfo object; if this is
// disabled, mgb::serialization::OprRegistry::find_opr_by_name would not work.
#ifndef MGB_VERBOSE_TYPEINFO_NAME
#define MGB_VERBOSE_TYPEINFO_NAME 1
#endif
// whether to enbale configuing megbrain internals through env vars
#ifndef MGB_ENABLE_GETENV
#define MGB_ENABLE_GETENV 1
#endif
// whether to remove unnecessary features when used for serving
#ifndef MGB_BUILD_SLIM_SERVING
#define MGB_BUILD_SLIM_SERVING 0
#endif
// whether to enable exception
#ifndef MGB_ENABLE_EXCEPTION
#if __EXCEPTIONS
#define MGB_ENABLE_EXCEPTION 1
#else
#define MGB_ENABLE_EXCEPTION 0
#endif
#endif
// whether <thread> is available and usable
#ifndef MGB_HAVE_THREAD
#define MGB_HAVE_THREAD 1
#endif
// whether to trade thread safety for memory usage
#ifndef MGB_THREAD_SAFE
#define MGB_THREAD_SAFE MGB_HAVE_THREAD
#endif
// whether to enable JIT
#ifndef MGB_JIT
#define MGB_JIT 1
#endif
#ifndef MGB_JIT_HALIDE
#define MGB_JIT_HALIDE 0
#endif
// whether to enable TensorRT support
#ifndef MGB_ENABLE_TENSOR_RT
#define MGB_ENABLE_TENSOR_RT MGB_CUDA
#endif
// whether to enable fastrun profile
#ifndef MGB_ENABLE_FASTRUN
#define MGB_ENABLE_FASTRUN 1
#endif
/* ================= following are more finegrind controls ================= */
// whether to enable json dumper
#ifndef MGB_ENABLE_JSON
#define MGB_ENABLE_JSON !MGB_BUILD_SLIM_SERVING
#endif
// whether to enable distributed communication
#ifndef MGB_ENABLE_OPR_MM
#define MGB_ENABLE_OPR_MM 0
#endif
#endif // _HEADER_MGB_BUILD_CONFIG
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或