提交 ef239f83 编写于 作者: M Megvii Engine Team 提交者: Xinran Xu

feat(windows/python_whl): make windows HAPPY for build megbrain python package

GitOrigin-RevId: 92b2c07bf939efcac6432bf4c93cd5c46910f129
上级 bf6cbc1d
...@@ -439,6 +439,7 @@ endif() ...@@ -439,6 +439,7 @@ endif()
set(MGB_JIT ${MGE_WITH_JIT}) set(MGB_JIT ${MGE_WITH_JIT})
set(MGB_JIT_HALIDE ${MGE_WITH_HALIDE}) set(MGB_JIT_HALIDE ${MGE_WITH_HALIDE})
# Thread
IF(APPLE) IF(APPLE)
set(CMAKE_THREAD_LIBS_INIT "-lpthread") set(CMAKE_THREAD_LIBS_INIT "-lpthread")
set(CMAKE_HAVE_THREADS_LIBRARY 1) set(CMAKE_HAVE_THREADS_LIBRARY 1)
...@@ -447,8 +448,14 @@ IF(APPLE) ...@@ -447,8 +448,14 @@ IF(APPLE)
set(THREADS_PREFER_PTHREAD_FLAG ON) set(THREADS_PREFER_PTHREAD_FLAG ON)
ENDIF() ENDIF()
# Thread if(MSVC OR WIN32)
if(CMAKE_THREAD_LIBS_INIT) set(CMAKE_HAVE_THREADS_LIBRARY 1)
set(CMAKE_USE_WIN32_THREADS_INIT 1)
set(CMAKE_USE_PTHREADS_INIT 1)
set(THREADS_PREFER_PTHREAD_FLAG ON)
endif()
if(CMAKE_THREAD_LIBS_INIT OR CMAKE_USE_WIN32_THREADS_INIT)
set(MGB_HAVE_THREAD 1) set(MGB_HAVE_THREAD 1)
endif() endif()
...@@ -471,12 +478,6 @@ else() ...@@ -471,12 +478,6 @@ else()
set(MGB_ENABLE_DEBUG_UTIL 0) set(MGB_ENABLE_DEBUG_UTIL 0)
endif() endif()
# FIXME: remove this after imp DEBUG UTIL for windows
if(MSVC OR WIN32)
set(MGB_ENABLE_DEBUG_UTIL 0)
message(" -- disable MGB_ENABLE_DEBUG_UTIL in windows build")
endif()
# TensorRT # TensorRT
set(MGB_ENABLE_TENSOR_RT ${MGE_WITH_TRT}) set(MGB_ENABLE_TENSOR_RT ${MGE_WITH_TRT})
......
...@@ -11,7 +11,13 @@ find_package(NumPy REQUIRED) ...@@ -11,7 +11,13 @@ find_package(NumPy REQUIRED)
find_package(SWIG REQUIRED) find_package(SWIG REQUIRED)
set(SWIG_SRC src/swig/mgb.i) set(SWIG_SRC src/swig/mgb.i)
set(CMAKE_SWIG_FLAGS -Wall -threads -py3 -modern -DSWIGWORDSIZE64) if(MSVC OR WIN32)
set(CMAKE_SWIG_FLAGS -Wall -threads -py3 -DSWIGWORDSIZE64)
message("WARN: swig have some define issue at windows(64) env")
message("Please refs scripts/whl/BUILD_PYTHON_WHL_README.md to init windows build env")
else()
set(CMAKE_SWIG_FLAGS -Wall -threads -py3 -modern -DSWIGWORDSIZE64)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
file(GLOB_RECURSE OPR_DECL_SRCS "${PROJECT_SOURCE_DIR}/src/**/*.oprdecl") file(GLOB_RECURSE OPR_DECL_SRCS "${PROJECT_SOURCE_DIR}/src/**/*.oprdecl")
...@@ -69,6 +75,8 @@ set_target_properties(mgb PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BI ...@@ -69,6 +75,8 @@ set_target_properties(mgb PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BI
if (APPLE) if (APPLE)
target_link_libraries(mgb megbrain megdnn) target_link_libraries(mgb megbrain megdnn)
set_target_properties(mgb PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") set_target_properties(mgb PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
elseif (MSVC OR WIN32)
target_link_libraries(mgb megbrain megdnn)
else() else()
target_link_libraries(mgb megbrain megdnn -Wl,--version-script=${VERSION_SCRIPT}) target_link_libraries(mgb megbrain megdnn -Wl,--version-script=${VERSION_SCRIPT})
endif() endif()
......
...@@ -13,6 +13,7 @@ import atexit ...@@ -13,6 +13,7 @@ import atexit
import collections import collections
import json import json
import os import os
import platform
import signal import signal
import struct import struct
...@@ -163,7 +164,11 @@ class GlobalInfkernFinder: ...@@ -163,7 +164,11 @@ class GlobalInfkernFinder:
further investigation further investigation
""" """
_signal = None
if platform.system() != "Windows":
_signal = signal.SIGUSR1 _signal = signal.SIGUSR1
else:
_signal = signal.CTRL_C_EVENT
_registry = [] _registry = []
_shell_maker = None _shell_maker = None
...@@ -197,7 +202,7 @@ class GlobalInfkernFinder: ...@@ -197,7 +202,7 @@ class GlobalInfkernFinder:
from IPython.terminal.embed import InteractiveShellEmbed from IPython.terminal.embed import InteractiveShellEmbed
cls._shell_maker = InteractiveShellEmbed cls._shell_maker = InteractiveShellEmbed
fast_signal_hander(signal.SIGUSR1, cls._on_signal) fast_signal_hander(cls._signal, cls._on_signal)
cls._registry.append(finder) cls._registry.append(finder)
......
...@@ -7,10 +7,13 @@ ...@@ -7,10 +7,13 @@
# software distributed under the License is distributed on an # software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
import platform import platform
import resource
import sys import sys
import threading import threading
# Windows do not imp resource package
if platform.system() != "Windows":
import resource
class AlternativeRecursionLimit: class AlternativeRecursionLimit:
r"""A reentrant context manager for setting global recursion limits. r"""A reentrant context manager for setting global recursion limits.
...@@ -29,6 +32,7 @@ class AlternativeRecursionLimit: ...@@ -29,6 +32,7 @@ class AlternativeRecursionLimit:
with self.lock: with self.lock:
if self.count == 0: if self.count == 0:
self.orig_py_limit = sys.getrecursionlimit() self.orig_py_limit = sys.getrecursionlimit()
if platform.system() != "Windows":
( (
self.orig_rlim_stack_soft, self.orig_rlim_stack_soft,
self.orig_rlim_stack_hard, self.orig_rlim_stack_hard,
...@@ -43,6 +47,7 @@ class AlternativeRecursionLimit: ...@@ -43,6 +47,7 @@ class AlternativeRecursionLimit:
except ValueError as exc: except ValueError as exc:
if platform.system() != "Darwin": if platform.system() != "Darwin":
raise exc raise exc
# increase recursion limit # increase recursion limit
sys.setrecursionlimit(self.new_py_limit) sys.setrecursionlimit(self.new_py_limit)
self.count += 1 self.count += 1
...@@ -52,6 +57,8 @@ class AlternativeRecursionLimit: ...@@ -52,6 +57,8 @@ class AlternativeRecursionLimit:
self.count -= 1 self.count -= 1
if self.count == 0: if self.count == 0:
sys.setrecursionlimit(self.orig_py_limit) sys.setrecursionlimit(self.orig_py_limit)
if platform.system() != "Windows":
try: try:
resource.setrlimit( resource.setrlimit(
resource.RLIMIT_STACK, resource.RLIMIT_STACK,
......
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
import os import os
import re import re
import pathlib import pathlib
import platform
from distutils.file_util import copy_file from distutils.file_util import copy_file
from setuptools import setup, find_packages, Extension from setuptools import setup, find_packages, Extension
from setuptools.command.build_ext import build_ext as _build_ext from setuptools.command.build_ext import build_ext as _build_ext
...@@ -25,6 +26,9 @@ class build_ext(_build_ext): ...@@ -25,6 +26,9 @@ class build_ext(_build_ext):
extdir.parent.mkdir(parents=True, exist_ok=True) extdir.parent.mkdir(parents=True, exist_ok=True)
modpath = self.get_ext_fullname(ext.name).split('.') modpath = self.get_ext_fullname(ext.name).split('.')
if platform.system() == "Windows":
modpath[-1] += '.pyd'
else:
modpath[-1] += '.so' modpath[-1] += '.so'
modpath = str(pathlib.Path(*modpath).resolve()) modpath = str(pathlib.Path(*modpath).resolve())
......
...@@ -21,7 +21,13 @@ ...@@ -21,7 +21,13 @@
#include <cstdarg> #include <cstdarg>
#include <Python.h> #include <Python.h>
#ifdef WIN32
#include <windows.h>
#include <stdio.h>
#else
#include <unistd.h> #include <unistd.h>
#endif
namespace { namespace {
...@@ -50,8 +56,35 @@ class Init { ...@@ -50,8 +56,35 @@ class Init {
}; };
Init Init::inst; Init Init::inst;
int fork_exec_impl(const std::string &arg0, const std::string &arg1, int fork_exec_impl(const std::string& arg0, const std::string& arg1,
const std::string &arg2) { const std::string& arg2) {
#ifdef WIN32
STARTUPINFO si;
PROCESS_INFORMATION pi;
ZeroMemory(&si, sizeof(si));
si.cb = sizeof(si);
ZeroMemory(&pi, sizeof(pi));
auto args_str = " " + arg1 + " " + arg2;
// Start the child process.
if (!CreateProcess(arg0.c_str(), // exe name
const_cast<char*>(args_str.c_str()), // Command line
NULL, // Process handle not inheritable
NULL, // Thread handle not inheritable
FALSE, // Set handle inheritance to FALSE
0, // No creation flags
NULL, // Use parent's environment block
NULL, // Use parent's starting directory
&si, // Pointer to STARTUPINFO structure
&pi) // Pointer to PROCESS_INFORMATION structure
) {
mgb_log_warn("CreateProcess failed (%lu).\n", GetLastError());
fprintf(stderr, "[megbrain] failed to execl %s [%s, %s]\n",
arg0.c_str(), arg1.c_str(), arg2.c_str());
__builtin_trap();
}
return pi.dwProcessId;
#else
auto pid = fork(); auto pid = fork();
if (!pid) { if (!pid) {
execl(arg0.c_str(), arg0.c_str(), arg1.c_str(), arg2.c_str(), nullptr); execl(arg0.c_str(), arg0.c_str(), arg1.c_str(), arg2.c_str(), nullptr);
...@@ -62,6 +95,7 @@ int fork_exec_impl(const std::string &arg0, const std::string &arg1, ...@@ -62,6 +95,7 @@ int fork_exec_impl(const std::string &arg0, const std::string &arg1,
} }
mgb_assert(pid > 0, "failed to fork: %s", std::strerror(errno)); mgb_assert(pid > 0, "failed to fork: %s", std::strerror(errno));
return pid; return pid;
#endif
} }
} // anonymous namespace } // anonymous namespace
......
...@@ -17,7 +17,32 @@ ...@@ -17,7 +17,32 @@
#include <set> #include <set>
#if defined(WIN32)
#include <io.h>
#include <windows.h>
#define F_OK 0
#define RTLD_LAZY 0
#define RTLD_GLOBAL 0
#define RTLD_NOLOAD 0
#define access(a, b) false
static void* dlopen(const char* file, int) {
return static_cast<void*>(LoadLibrary(file));
}
static void* dlerror() {
const char* errmsg = "dlerror not aviable in windows";
return const_cast<char*>(errmsg);
}
static void* dlsym(void* handle, const char* name) {
FARPROC symbol = GetProcAddress((HMODULE)handle, name);
return reinterpret_cast<void*>(symbol);
}
#else
#include <dlfcn.h> #include <dlfcn.h>
#endif
#if MGB_ENABLE_OPR_MM #if MGB_ENABLE_OPR_MM
#include "megbrain/opr/mm_handler.h" #include "megbrain/opr/mm_handler.h"
...@@ -274,8 +299,7 @@ void _config::load_opr_library(const char* self_path, const char* lib_path) { ...@@ -274,8 +299,7 @@ void _config::load_opr_library(const char* self_path, const char* lib_path) {
} }
} }
std::vector<std::pair<unsigned long int, std::string>> std::vector<std::pair<size_t, std::string>> _config::dump_registered_oprs() {
_config::dump_registered_oprs() {
#if MGB_ENABLE_DEBUG_UTIL #if MGB_ENABLE_DEBUG_UTIL
return serialization::OprRegistry::dump_registries(); return serialization::OprRegistry::dump_registries();
#else #else
......
...@@ -62,7 +62,7 @@ class _config { ...@@ -62,7 +62,7 @@ class _config {
static void load_opr_library( static void load_opr_library(
const char* self_path, const char* lib_path); const char* self_path, const char* lib_path);
static std::vector<std::pair<unsigned long int, std::string>> static std::vector<std::pair<size_t, std::string>>
dump_registered_oprs(); dump_registered_oprs();
static int create_mm_server(const std::string& server_addr, int port); static int create_mm_server(const std::string& server_addr, int port);
......
...@@ -404,11 +404,18 @@ void CompGraphCallbackValueProxy::do_copy() { ...@@ -404,11 +404,18 @@ void CompGraphCallbackValueProxy::do_copy() {
m_copy_event->record(); m_copy_event->record();
} }
#if defined(WIN32)
#include <windows.h>
#include <stdio.h>
#undef CONST
#define usleep Sleep
#endif
void CompGraphCallbackValueProxy::sync() const { void CompGraphCallbackValueProxy::sync() const {
mgb_assert(!m_use_raw_hv); mgb_assert(!m_use_raw_hv);
RealTimer t0; RealTimer t0;
double next_warn_time = 2, warn_time_delta = 1; double next_warn_time = 2, warn_time_delta = 1;
while (!m_copy_event->finished()) { while (!m_copy_event->finished()) {
//! sleep 1ms or sleep 1us no difference for performance on win32
usleep(1); usleep(1);
if (t0.get_secs() >= next_warn_time) { if (t0.get_secs() >= next_warn_time) {
mgb_log_warn("wait d2h copy for more than %.3f secs", mgb_log_warn("wait d2h copy for more than %.3f secs",
......
...@@ -16,10 +16,15 @@ ...@@ -16,10 +16,15 @@
#include <thread> #include <thread>
#include <cstring> #include <cstring>
#include <sstream>
#ifdef WIN32
#include <windows.h>
#else
#include <pthread.h> #include <pthread.h>
#include <signal.h>
#include <unistd.h> #include <unistd.h>
#endif
#include <signal.h>
/* ================= _InfkernFinderImpl ================= */ /* ================= _InfkernFinderImpl ================= */
size_t _InfkernFinderImpl::sm_id = 0; size_t _InfkernFinderImpl::sm_id = 0;
...@@ -72,23 +77,28 @@ class _FastSignal::Impl { ...@@ -72,23 +77,28 @@ class _FastSignal::Impl {
bool m_worker_started = false; bool m_worker_started = false;
std::mutex m_mtx; std::mutex m_mtx;
std::thread m_worker_hdl; std::thread m_worker_hdl;
#ifdef WIN32
SECURITY_ATTRIBUTES win_sa = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE};
HANDLE pipe_r, pipe_w;
DWORD bytes_r_w;
#else
int m_pfd[2]; //! pipe fds; write signal handlers, -1 for exit int m_pfd[2]; //! pipe fds; write signal handlers, -1 for exit
#endif
std::unordered_map<int, HandlerCallback> m_handler_callbacks; std::unordered_map<int, HandlerCallback> m_handler_callbacks;
void worker() { void worker() {
#ifdef __APPLE__ std::ostringstream oss;
uint64_t tid; oss << std::this_thread::get_id() << std::endl;
pthread_threadid_np(NULL, &tid); mgb_log("fast signal worker started in thread %s", oss.str().c_str());
mgb_log("fast signal worker started in thread 0x%zx",
static_cast<size_t>(tid));
#else
mgb_log("fast signal worker started in thread 0x%zx",
static_cast<size_t>(pthread_self()));
#endif
mgb::sys::set_thread_name("fastsgl"); mgb::sys::set_thread_name("fastsgl");
int signum; int signum;
for (; ; ) { for (;;) {
#ifdef WIN32
if (ReadFile(pipe_r, &signum, sizeof(int), &bytes_r_w, NULL) ==
NULL) {
#else
if (read(m_pfd[0], &signum, sizeof(int)) != sizeof(int)) { if (read(m_pfd[0], &signum, sizeof(int)) != sizeof(int)) {
#endif
if (errno == EINTR) if (errno == EINTR)
continue; continue;
mgb_log_error("fast signal worker: " mgb_log_error("fast signal worker: "
...@@ -114,10 +124,17 @@ class _FastSignal::Impl { ...@@ -114,10 +124,17 @@ class _FastSignal::Impl {
if (m_worker_started) if (m_worker_started)
return; return;
#ifdef WIN32
if (!CreatePipe(&pipe_r, &pipe_w, &win_sa, 0)) {
throw mgb::MegBrainError(mgb::ssprintf("failed to create pipe: %s",
strerror(errno)));
}
#else
if (pipe(m_pfd)) { if (pipe(m_pfd)) {
throw mgb::MegBrainError(mgb::ssprintf( throw mgb::MegBrainError(mgb::ssprintf("failed to create pipe: %s",
"failed to create pipe: %s", strerror(errno))); strerror(errno)));
} }
#endif
std::thread t(std::bind(&Impl::worker, this)); std::thread t(std::bind(&Impl::worker, this));
m_worker_hdl.swap(t); m_worker_hdl.swap(t);
m_worker_started = true; m_worker_started = true;
...@@ -125,7 +142,11 @@ class _FastSignal::Impl { ...@@ -125,7 +142,11 @@ class _FastSignal::Impl {
void write_pipe(int v) { void write_pipe(int v) {
mgb_assert(m_worker_started); mgb_assert(m_worker_started);
#ifdef WIN32
if (WriteFile(pipe_w, &v, sizeof(int), &bytes_r_w, NULL) == NULL) {
#else
if (write(m_pfd[1], &v, sizeof(int)) != sizeof(int)) { if (write(m_pfd[1], &v, sizeof(int)) != sizeof(int)) {
#endif
mgb_log_error("fast signal: failed to write to self pipe: %s", mgb_log_error("fast signal: failed to write to self pipe: %s",
strerror(errno)); strerror(errno));
} }
...@@ -169,8 +190,13 @@ class _FastSignal::Impl { ...@@ -169,8 +190,13 @@ class _FastSignal::Impl {
return; return;
write_pipe(-1); write_pipe(-1);
m_worker_hdl.join(); m_worker_hdl.join();
#ifdef WIN32
CloseHandle(pipe_r);
CloseHandle(pipe_w);
#else
close(m_pfd[0]); close(m_pfd[0]);
close(m_pfd[1]); close(m_pfd[1]);
#endif
m_handler_callbacks.clear(); m_handler_callbacks.clear();
m_worker_started = false; m_worker_started = false;
} }
...@@ -192,11 +218,19 @@ void _FastSignal::signal_hander(int signum) { ...@@ -192,11 +218,19 @@ void _FastSignal::signal_hander(int signum) {
} }
void _FastSignal::register_handler(int signum, PyObject *func) { void _FastSignal::register_handler(int signum, PyObject *func) {
#ifdef WIN32
//! up to now we can only use CTRL_C_EVENT to unix signal.SIGUSR1/2
//FIXME: how to coherence signal number at python side
// https://docs.microsoft.com/en-gb/cpp/c-runtime-library/reference/signal?view=vs-2017
mgb_assert(signum == CTRL_C_EVENT, "only allow register CTRL_C_EVENT as unix signal.SIGUSR1/2 now");
signal(signum, signal_hander);
#else
struct sigaction action; struct sigaction action;
memset(&action, 0, sizeof(action)); memset(&action, 0, sizeof(action));
action.sa_handler = &signal_hander; action.sa_handler = &signal_hander;
int ret = sigaction(signum, &action, nullptr); int ret = sigaction(signum, &action, nullptr);
mgb_assert(!ret, "sigaction failed: %s", strerror(errno)); mgb_assert(!ret, "sigaction failed: %s", strerror(errno));
#endif
sm_impl.register_handler(signum, func); sm_impl.register_handler(signum, func);
} }
......
...@@ -45,7 +45,7 @@ std::string demangle_typeid(const char* name) { ...@@ -45,7 +45,7 @@ std::string demangle_typeid(const char* name) {
namespace { namespace {
// does nothing if not g++ // does nothing if not g++
std::string mgb::demangle_typeid(const char* name) { std::string demangle_typeid(const char* name) {
return name; return name;
} }
} }
......
...@@ -31,13 +31,7 @@ void _init_bfloat16_types(PyObject *m); // implemented in bfloat16.cpp ...@@ -31,13 +31,7 @@ void _init_bfloat16_types(PyObject *m); // implemented in bfloat16.cpp
%template(_VectorString) std::vector<std::string>; %template(_VectorString) std::vector<std::string>;
%template(_PairStringSizeT) std::pair<std::string, size_t>; %template(_PairStringSizeT) std::pair<std::string, size_t>;
%template(_PairSizeTSizeT) std::pair<size_t, size_t>; %template(_PairSizeTSizeT) std::pair<size_t, size_t>;
/* %template(_VectorPairSizeTString) std::vector<std::pair<size_t, std::string>>;
* swig use uint64_t have compat build issue with
* clang at osx env, so we use unsigned long to
* replace uint64_t,more detail refs stdint.i
*
*/
%template(_VectorPairUint64String) std::vector<std::pair<unsigned long int, std::string>>;
%pythoncode %{ %pythoncode %{
import numpy as np import numpy as np
......
...@@ -89,6 +89,9 @@ def start_workers(worker, world_size, trace=False): ...@@ -89,6 +89,9 @@ def start_workers(worker, world_size, trace=False):
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
def test_distributed(): def test_distributed():
start_workers(worker, 2, trace=True) start_workers(worker, 2, trace=True)
start_workers(worker, 2, trace=False) start_workers(worker, 2, trace=False)
...@@ -10,10 +10,10 @@ ignore_list="--ignore test/unit/module/test_pytorch.py \ ...@@ -10,10 +10,10 @@ ignore_list="--ignore test/unit/module/test_pytorch.py \
test_dirs="megengine test" test_dirs="megengine test"
pushd $(dirname "${BASH_SOURCE[0]}")/.. >/dev/null pushd $(dirname "${BASH_SOURCE[0]}")/.. >/dev/null
pytest -xv -m 'isolated_distributed' \ python3 -m pytest -xv -m 'isolated_distributed' \
--json-report --json-report-file=time_python_test.json \ --json-report --json-report-file=time_python_test.json \
$ignore_list $test_dirs $ignore_list $test_dirs
pytest -xv -m 'not internet and not isolated_distributed' \ python3 -m pytest -xv -m 'not internet and not isolated_distributed' \
--json-report --json-report-file=time_python_test.json \ --json-report --json-report-file=time_python_test.json \
$ignore_list $test_dirs $ignore_list $test_dirs
popd >/dev/null popd >/dev/null
...@@ -29,6 +29,9 @@ def _init_process_group_wrapper(world_size, rank, dev, backend, q): ...@@ -29,6 +29,9 @@ def _init_process_group_wrapper(world_size, rank, dev, backend, q):
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_reduce_sum(): def test_reduce_sum():
world_size = 2 world_size = 2
...@@ -68,6 +71,9 @@ def test_reduce_sum(): ...@@ -68,6 +71,9 @@ def test_reduce_sum():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_gather(): def test_gather():
world_size = 2 world_size = 2
...@@ -107,6 +113,9 @@ def test_gather(): ...@@ -107,6 +113,9 @@ def test_gather():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_broadcast(): def test_broadcast():
world_size = 2 world_size = 2
...@@ -142,6 +151,9 @@ def test_broadcast(): ...@@ -142,6 +151,9 @@ def test_broadcast():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_scatter(): def test_scatter():
world_size = 2 world_size = 2
...@@ -181,6 +193,9 @@ def test_scatter(): ...@@ -181,6 +193,9 @@ def test_scatter():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_all_to_all(): def test_all_to_all():
world_size = 2 world_size = 2
...@@ -218,6 +233,9 @@ def test_all_to_all(): ...@@ -218,6 +233,9 @@ def test_all_to_all():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_all_gather(): def test_all_gather():
world_size = 2 world_size = 2
...@@ -254,6 +272,9 @@ def test_all_gather(): ...@@ -254,6 +272,9 @@ def test_all_gather():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_reduce_scatter_sum(): def test_reduce_scatter_sum():
world_size = 2 world_size = 2
...@@ -294,6 +315,9 @@ def test_reduce_scatter_sum(): ...@@ -294,6 +315,9 @@ def test_reduce_scatter_sum():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_all_reduce_sum(): def test_all_reduce_sum():
world_size = 2 world_size = 2
...@@ -330,6 +354,9 @@ def test_all_reduce_sum(): ...@@ -330,6 +354,9 @@ def test_all_reduce_sum():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_all_reduce_max(): def test_all_reduce_max():
world_size = 2 world_size = 2
...@@ -366,6 +393,9 @@ def test_all_reduce_max(): ...@@ -366,6 +393,9 @@ def test_all_reduce_max():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_all_reduce_min(): def test_all_reduce_min():
world_size = 2 world_size = 2
...@@ -402,6 +432,9 @@ def test_all_reduce_min(): ...@@ -402,6 +432,9 @@ def test_all_reduce_min():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_bcast_param(): def test_bcast_param():
world_size = 2 world_size = 2
......
...@@ -45,6 +45,9 @@ def _init_process_group_wrapper(world_size, rank, dev, backend, q): ...@@ -45,6 +45,9 @@ def _init_process_group_wrapper(world_size, rank, dev, backend, q):
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_create_mm_server(): def test_create_mm_server():
def worker(): def worker():
...@@ -67,6 +70,9 @@ def test_create_mm_server(): ...@@ -67,6 +70,9 @@ def test_create_mm_server():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_init_process_group(): def test_init_process_group():
world_size = 2 world_size = 2
...@@ -102,6 +108,9 @@ def test_init_process_group(): ...@@ -102,6 +108,9 @@ def test_init_process_group():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_group_barrier(): def test_group_barrier():
world_size = 2 world_size = 2
...@@ -137,6 +146,9 @@ def test_group_barrier(): ...@@ -137,6 +146,9 @@ def test_group_barrier():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_synchronized(): def test_synchronized():
world_size = 2 world_size = 2
......
...@@ -22,6 +22,9 @@ from megengine.test import assertTensorClose ...@@ -22,6 +22,9 @@ from megengine.test import assertTensorClose
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
@pytest.mark.isolated_distributed @pytest.mark.isolated_distributed
def test_syncbn(): def test_syncbn():
nr_chan = 8 nr_chan = 8
...@@ -143,6 +146,9 @@ def test_batchnorm(): ...@@ -143,6 +146,9 @@ def test_batchnorm():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
def test_syncbn1d(): def test_syncbn1d():
nr_chan = 8 nr_chan = 8
data_shape = (3, nr_chan, 4) data_shape = (3, nr_chan, 4)
...@@ -241,6 +247,9 @@ def test_batchnorm2d(): ...@@ -241,6 +247,9 @@ def test_batchnorm2d():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
def test_syncbn2d(): def test_syncbn2d():
nr_chan = 8 nr_chan = 8
data_shape = (3, nr_chan, 16, 16) data_shape = (3, nr_chan, 16, 16)
...@@ -315,6 +324,9 @@ def test_batchnorm_no_stats(): ...@@ -315,6 +324,9 @@ def test_batchnorm_no_stats():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
def test_syncbn_no_stats(): def test_syncbn_no_stats():
nr_chan = 8 nr_chan = 8
data_shape = (3, nr_chan, 4) data_shape = (3, nr_chan, 4)
...@@ -367,6 +379,9 @@ def test_batchnorm2d_no_stats(): ...@@ -367,6 +379,9 @@ def test_batchnorm2d_no_stats():
@pytest.mark.skipif( @pytest.mark.skipif(
platform.system() == "Darwin", reason="do not imp GPU mode at macos now" platform.system() == "Darwin", reason="do not imp GPU mode at macos now"
) )
@pytest.mark.skipif(
platform.system() == "Windows", reason="do not imp GPU mode at Windows now"
)
def test_syncbn2d_no_stats(): def test_syncbn2d_no_stats():
nr_chan = 8 nr_chan = 8
data_shape = (3, nr_chan, 16, 16) data_shape = (3, nr_chan, 16, 16)
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
# Unless required by applicable law or agreed to in writing, # Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an # software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
import os
import tempfile import tempfile
from collections import OrderedDict from collections import OrderedDict
from io import BytesIO from io import BytesIO
...@@ -367,8 +368,13 @@ def test_dump_model(): ...@@ -367,8 +368,13 @@ def test_dump_model():
data.set_value(np.random.random(data_shape)) data.set_value(np.random.random(data_shape))
mlp = MLP() mlp = MLP()
pred = mlp(data) pred = mlp(data)
with tempfile.NamedTemporaryFile() as f: f = tempfile.NamedTemporaryFile(delete=False)
mge.dump(pred, f.name) f_name = f.name
try:
mge.dump(pred, f_name)
finally:
f.close()
os.unlink(f_name)
def test_load_quantized(): def test_load_quantized():
......
...@@ -166,6 +166,18 @@ function prepare_env_for_windows_build() { ...@@ -166,6 +166,18 @@ function prepare_env_for_windows_build() {
echo "put vcvarsall.bat path to PATH env.." echo "put vcvarsall.bat path to PATH env.."
} }
WINDOWS_BUILD_TARGET="Ninja all > build.log"
if [[ -z ${MAKE_DEVELOP} ]]
then
MAKE_DEVELOP="false"
fi
function config_windows_build_target() {
if [ ${MAKE_DEVELOP} = "true" ]; then
echo "build all and develop for pytest test"
WINDOWS_BUILD_TARGET="Ninja all > build.log && Ninja develop"
fi
}
function cmake_build_windows() { function cmake_build_windows() {
# windows do not support long path, so we cache the BUILD_DIR ASAP # windows do not support long path, so we cache the BUILD_DIR ASAP
prepare_env_for_windows_build prepare_env_for_windows_build
...@@ -201,11 +213,12 @@ function cmake_build_windows() { ...@@ -201,11 +213,12 @@ function cmake_build_windows() {
${EXTRA_CMAKE_ARGS} \ ${EXTRA_CMAKE_ARGS} \
../../.. && \ ../../.. && \
echo \"start Ninja build log to build.log, may take serval min...\" && \ echo \"start Ninja build log to build.log, may take serval min...\" && \
Ninja load_and_run > build.log" ${WINDOWS_BUILD_TARGET}"
} }
if [[ $OS =~ "NT" ]]; then if [[ $OS =~ "NT" ]]; then
config_windows_build_target
cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
else else
cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
......
# python whl package build support status # python whl package build support status
* windows build (not ok) * windows build (ok,cpu only)
* linux build (ok, cpu or gpu) * linux build (ok, cpu or gpu)
* macos build (ok,cpu only) * macos build (ok,cpu only)
# build env prepare # build env prepare
...@@ -17,6 +17,48 @@ ...@@ -17,6 +17,48 @@
``` ```
./scripts/whl/macos/macos_whl_env_prepare.sh ./scripts/whl/macos/macos_whl_env_prepare.sh
``` ```
## windows
```
1: refs scripts/cmake-build/BUILD_README.md windows section build for base windows build
2: install several python or install u care python version, default install dir: /c/Users/${USER}/mge_whl_python_env
a: mkdir /c/Users/${USER}/mge_whl_python_env
b: download python 64-bit install exe
https://www.python.org/ftp/python/3.5.4/python-3.5.4-amd64.exe
https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe
https://www.python.org/ftp/python/3.7.7/python-3.7.7-amd64.exe
https://www.python.org/ftp/python/3.8.3/python-3.8.3-amd64.exe
c: install python-3.5.4-amd64.exe to /c/Users/${USER}/mge_whl_python_env/3.5.4 from install gui
d: install python-3.6.8-amd64.exe to /c/Users/${USER}/mge_whl_python_env/3.6.8 from install gui
e: install python-3.7.7-amd64.exe to /c/Users/${USER}/mge_whl_python_env/3.7.7 from install gui
f: install python-3.8.3-amd64.exe to /c/Users/${USER}/mge_whl_python_env/3.8.3 from install gui
3: rename python.exe to python3.exe
a: mv /c/Users/${USER}/mge_whl_python_env/3.5.4/python.exe /c/Users/${USER}/mge_whl_python_env/3.5.4/python3.exe
b: mv /c/Users/${USER}/mge_whl_python_env/3.6.8/python.exe /c/Users/${USER}/mge_whl_python_env/3.6.8/python3.exe
c: mv /c/Users/${USER}/mge_whl_python_env/3.7.7/python.exe /c/Users/${USER}/mge_whl_python_env/3.7.7/python3.exe
d: mv /c/Users/${USER}/mge_whl_python_env/3.8.3/python.exe /c/Users/${USER}/mge_whl_python_env/3.8.3/python3.exe
4: install needed package for build python whl package
a0: /c/Users/${USER}/mge_whl_python_env/3.5.4/python3.exe -m pip install --upgrade pip
a1: /c/Users/${USER}/mge_whl_python_env/3.5.4/python3.exe -m pip install -r python_module/requires-test.txt
a2: /c/Users/${USER}/mge_whl_python_env/3.5.4/python3.exe -m pip install numpy wheel requests tqdm tabulate
b0: /c/Users/${USER}/mge_whl_python_env/3.6.8/python3.exe -m pip install --upgrade pip
b1: /c/Users/${USER}/mge_whl_python_env/3.6.8/python3.exe -m pip install -r python_module/requires-test.txt
b2: /c/Users/${USER}/mge_whl_python_env/3.6.8/python3.exe -m pip install numpy wheel requests tqdm tabulate
c0: /c/Users/${USER}/mge_whl_python_env/3.7.7/python3.exe -m pip install --upgrade pip
c1: /c/Users/${USER}/mge_whl_python_env/3.7.7/python3.exe -m pip install -r python_module/requires-test.txt
c2: /c/Users/${USER}/mge_whl_python_env/3.7.7/python3.exe -m pip install numpy wheel requests tqdm tabulate
d0: /c/Users/${USER}/mge_whl_python_env/3.8.3/python3.exe -m pip install --upgrade pip
d1: /c/Users/${USER}/mge_whl_python_env/3.8.3/python3.exe -m pip install -r python_module/requires-test.txt
d2: /c/Users/${USER}/mge_whl_python_env/3.8.3/python3.exe -m pip install numpy wheel requests tqdm tabulate
5: install swig from install gui
a: download swig: https://nchc.dl.sourceforge.net/project/swig/swigwin/swigwin-4.0.2/swigwin-4.0.2.zip
b: install swig to /c/Users/${USER}/swigwin-4.0.2
c: apply scripts/whl/windows/fix-ptr-define-issue.patch to c/Users/${USER}/swigwin-4.0.2
```
# how to build # how to build
## build for linux ## build for linux
MegBrain delivers `wheel` package with `manylinux2010` tag defined in [PEP-571](https://www.python.org/dev/peps/pep-0571/). MegBrain delivers `wheel` package with `manylinux2010` tag defined in [PEP-571](https://www.python.org/dev/peps/pep-0571/).
...@@ -36,9 +78,22 @@ ...@@ -36,9 +78,22 @@
``` ```
ALL_PYTHON=35m ./build_wheel.sh cpu ALL_PYTHON=35m ./build_wheel.sh cpu
``` ```
## build for macos ## build for macos
``` ```
./scripts/whl/macos/macos_build_whl.sh ./scripts/whl/macos/macos_build_whl.sh
``` ```
If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg:
```
ALL_PYTHON=3.5.9 ./scripts/whl/macos/macos_build_whl.sh
```
## build for windows
```
./scripts/whl/windows/windows_build_whl.sh
```
If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg:
```
ALL_PYTHON=3.5.4 ./scripts/whl/windows/windows_build_whl.sh
```
...@@ -149,13 +149,16 @@ function do_build() { ...@@ -149,13 +149,16 @@ function do_build() {
} }
function third_party_prepare() { function third_party_prepare() {
if [[ -z ${ALREADY_INSTALL_THIRD_PARTY} ]]
then
echo "init third_party..." echo "init third_party..."
${SRC_DIR}/third_party/prepare.sh ${SRC_DIR}/third_party/prepare.sh
if [[ -z ${ALREADY_INSTALL_MKL} ]]
then
echo "init third_party..."
${SRC_DIR}/third_party/install-mkl.sh ${SRC_DIR}/third_party/install-mkl.sh
else else
echo "skip init third_party..." echo "skip init mkl internal"
fi fi
} }
......
diff --git a/Lib/stdint.i b/Lib/stdint.i
index 14fe619..45337c0 100644
--- a/Lib/stdint.i
+++ b/Lib/stdint.i
@@ -86,8 +86,8 @@ typedef unsigned long long int uint_fast64_t;
/* Types for `void *' pointers. */
#if defined(SWIGWORDSIZE64)
-typedef long int intptr_t;
-typedef unsigned long int uintptr_t;
+typedef long long intptr_t;
+typedef unsigned long long uintptr_t;
#else
typedef int intptr_t;
typedef unsigned int uintptr_t;
diff --git a/Lib/swigarch.i b/Lib/swigarch.i
index bf4ee8e..bf6b5c3 100644
--- a/Lib/swigarch.i
+++ b/Lib/swigarch.i
@@ -53,7 +53,7 @@
#ifndef LONG_MAX
#include <limits.h>
#endif
-#if (__WORDSIZE == 32) || (LONG_MAX == INT_MAX)
+#if (__WORDSIZE == 32)
# error "SWIG wrapped code invalid in 32 bit architecture, regenerate code using -DSWIGWORDSIZE32"
#endif
%}
#!/bin/bash -e
NT=$(echo `uname` | grep "NT")
echo $NT
if [ -z "$NT" ];then
echo "only run at windows bash env"
echo "pls consider install bash-like tools, eg MSYS or git-cmd, etc"
exit -1
fi
function err_env() {
echo "check_env failed: pls refs ${SRC_DIR}/scripts/whl/BUILD_PYTHON_WHL_README.md to init env"
exit -1
}
function append_path_env_and_check() {
echo "export swig pwd to PATH"
export PATH=/c/Users/${USER}/swigwin-4.0.2::$PATH
echo "export vs2019 install path"
export VS_PATH=/c/Program\ Files\ \(x86\)/Microsoft\ Visual\ Studio/2019/Enterprise
# for llvm-strip
export PATH=$VS_PATH/VC/Tools/Llvm/bin/:$PATH
}
append_path_env_and_check
SRC_DIR=$(READLINK -f "`dirname $0`/../../../")
ALL_PYTHON=${ALL_PYTHON}
FULL_PYTHON_VER="3.5.4 3.6.8 3.7.7 3.8.3"
if [[ -z ${ALL_PYTHON} ]]
then
ALL_PYTHON=${FULL_PYTHON_VER}
fi
PYTHON_DIR=
PYTHON_LIBRARY=
PYTHON_INCLUDE_DIR=
WINDOWS_WHL_HOME=${SRC_DIR}/scripts/whl/windows/windows_whl_home
if [ -e "${WINDOWS_WHL_HOME}" ]; then
echo "remove old windows whl file"
rm -rf ${WINDOWS_WHL_HOME}
fi
mkdir -p ${WINDOWS_WHL_HOME}
function config_python_env() {
PYTHON_DIR=/c/Users/${USER}/mge_whl_python_env/$1
PYTHON_BIN=${PYTHON_DIR}
if [ ! -f "$PYTHON_BIN/python3.exe" ]; then
echo "ERR: can not find $PYTHON_BIN , Invalid python package"
echo "now support list: ${FULL_PYTHON_VER}"
err_env
else
echo "put python3 to env..."
export PATH=${PYTHON_BIN}:$PATH
which python3
fi
echo ${ver}
PYTHON_LIBRARY=${PYTHON_DIR}/libs/python3.lib
PYTHON_INCLUDE_DIR=${PYTHON_DIR}/include
}
function do_build() {
for ver in ${ALL_PYTHON}
do
#config
config_python_env ${ver}
#check env
if [ ! -f "$PYTHON_LIBRARY" ]; then
echo "ERR: can not find $PYTHON_LIBRARY , Invalid python package"
err_env
fi
if [ ! -d "$PYTHON_INCLUDE_DIR" ]; then
echo "ERR: can not find $PYTHON_INCLUDE_DIR , Invalid python package"
err_env
fi
echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
#append cmake args for config python
#FIXME: ninja handle err with cmake 3.17 when assgin PYTHON_LIBRARY
#But after put python3.exe to HEAD of PATH by config_python_env, cmake can also handle the
#right PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, at the same time, clang-cl need swig target
#force LINK a real PYTHON_LIBRARY file, after test we do not find the symbols conflict with python
#export EXTRA_CMAKE_ARGS="-DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
#config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
export EXTRA_CMAKE_ARGS=${EXTRA_CMAKE_ARGS}" -DCMAKE_BUILD_TYPE=RelWithDebInfo "
#call build and install
#FIXME: cmake do not triger update python config, after
#change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
#-r to remove build cache after a new ver build, which
#will be more slow build than without -r
${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
#call setup.py
BUILD_DIR=${SRC_DIR}/build_dir/host/build/
cd ${BUILD_DIR}
if [ -d "staging" ]; then
echo "remove old build cache file"
rm -rf staging
fi
mkdir -p staging
cp -a python_module/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
cd ${BUILD_DIR}/staging/megengine/_internal
llvm-strip -s _mgb.pyd
cd ${BUILD_DIR}/staging
${PYTHON_DIR}/python3 setup.py bdist_wheel
cp ${BUILD_DIR}/staging/dist/Meg*.whl ${WINDOWS_WHL_HOME}/
echo ""
echo "##############################################################################################"
echo "windows whl package location: ${WINDOWS_WHL_HOME}"
ls ${WINDOWS_WHL_HOME}
echo "##############################################################################################"
done
}
function third_party_prepare() {
echo "init third_party..."
${SRC_DIR}/third_party/prepare.sh
if [[ -z ${ALREADY_INSTALL_MKL} ]]
then
echo "init third_party..."
${SRC_DIR}/third_party/install-mkl.sh
else
echo "skip init mkl internal"
fi
}
######################
third_party_prepare
do_build
...@@ -598,11 +598,9 @@ void run_test_st(Args &env) { ...@@ -598,11 +598,9 @@ void run_test_st(Args &env) {
auto format = auto format =
serialization::GraphLoader::identify_graph_dump_format(*inp_file); serialization::GraphLoader::identify_graph_dump_format(*inp_file);
if (!format.valid()) { mgb_assert(format.valid(),
printf("invalid model: unknown model format, please make sure input " "invalid model: unknown model format, please make sure input "
"file is generated by GraphDumper\n"); "file is generated by GraphDumper");
return;
}
auto loader = auto loader =
serialization::GraphLoader::make(std::move(inp_file), format.val()); serialization::GraphLoader::make(std::move(inp_file), format.val());
RealTimer timer; RealTimer timer;
......
...@@ -35,16 +35,21 @@ using namespace debug; ...@@ -35,16 +35,21 @@ using namespace debug;
#include <cuda_runtime.h> #include <cuda_runtime.h>
#endif #endif
#ifndef WIN32
#include <pthread.h> #include <pthread.h>
#include <unistd.h>
#endif
#include <signal.h> #include <signal.h>
#include <sys/types.h> #include <sys/types.h>
#include <unistd.h>
#ifdef __ANDROID__ #ifdef __ANDROID__
#include <unwind.h> #include <unwind.h>
#else #else
#ifndef WIN32
#include <execinfo.h> #include <execinfo.h>
#endif #endif
#endif
#ifdef __ANDROID__ #ifdef __ANDROID__
namespace { namespace {
...@@ -121,6 +126,8 @@ void get_mem_map( ...@@ -121,6 +126,8 @@ void get_mem_map(
fclose(fin); fclose(fin);
} }
#ifndef WIN32
//FIXME: imp SigHandlerInit backtrace for windows
class SigHandlerInit { class SigHandlerInit {
static void death_handler(int signum) { static void death_handler(int signum) {
char msg0[] = char msg0[] =
...@@ -157,6 +164,7 @@ public: ...@@ -157,6 +164,7 @@ public:
std::set_terminate([]() { death_handler(-1); }); std::set_terminate([]() { death_handler(-1); });
} }
}; };
#endif
#if MGB_CUDA #if MGB_CUDA
class CudaCheckOnFork { class CudaCheckOnFork {
...@@ -201,7 +209,9 @@ class InitCaller { ...@@ -201,7 +209,9 @@ class InitCaller {
static InitCaller inst; static InitCaller inst;
InitCaller() { InitCaller() {
#ifndef WIN32
SigHandlerInit::init_for_segv(); SigHandlerInit::init_for_segv();
#endif
#if MGB_CUDA #if MGB_CUDA
CudaCheckOnFork::init(); CudaCheckOnFork::init();
#endif #endif
...@@ -216,6 +226,7 @@ void (*ForkAfterCudaError::throw_)() = throw_fork_cuda_exc; ...@@ -216,6 +226,7 @@ void (*ForkAfterCudaError::throw_)() = throw_fork_cuda_exc;
std::atomic_size_t ScopedForkWarningSupress::sm_depth{0}; std::atomic_size_t ScopedForkWarningSupress::sm_depth{0};
BacktraceResult mgb::debug::backtrace(int nr_exclude) { BacktraceResult mgb::debug::backtrace(int nr_exclude) {
#ifndef WIN32
static bool thread_local recursive_call = false; static bool thread_local recursive_call = false;
if (recursive_call) { if (recursive_call) {
fprintf(stderr, "recursive call to backtrace()!\n"); fprintf(stderr, "recursive call to backtrace()!\n");
...@@ -265,6 +276,11 @@ BacktraceResult mgb::debug::backtrace(int nr_exclude) { ...@@ -265,6 +276,11 @@ BacktraceResult mgb::debug::backtrace(int nr_exclude) {
recursive_call = false; recursive_call = false;
return result; return result;
#else
//FIXME: imp Backtrace for windows
BacktraceResult result;
return result;
#endif
} }
void BacktraceResult::fmt_to_str(std::string& dst) { void BacktraceResult::fmt_to_str(std::string& dst) {
......
...@@ -197,6 +197,11 @@ void __log__(LogLevel level, const char *file, const char *func, int line, ...@@ -197,6 +197,11 @@ void __log__(LogLevel level, const char *file, const char *func, int line,
#define MGB_GETENV(_name) static_cast<char*>(nullptr) #define MGB_GETENV(_name) static_cast<char*>(nullptr)
#endif #endif
#ifdef WIN32
#define unsetenv(_name) _putenv_s(_name, "");
#define setenv(name,value,overwrite) _putenv_s(name,value)
#endif
// use some macro tricks to get lock guard with unique variable name // use some macro tricks to get lock guard with unique variable name
#define MGB_TOKENPASTE(x, y) x ## y #define MGB_TOKENPASTE(x, y) x ## y
#define MGB_TOKENPASTE2(x, y) MGB_TOKENPASTE(x, y) #define MGB_TOKENPASTE2(x, y) MGB_TOKENPASTE(x, y)
......
...@@ -159,10 +159,9 @@ void OprRegistry::add_using_dynamic_loader( ...@@ -159,10 +159,9 @@ void OprRegistry::add_using_dynamic_loader(
} }
#if MGB_ENABLE_DEBUG_UTIL #if MGB_ENABLE_DEBUG_UTIL
std::vector<std::pair<unsigned long int, std::string>> std::vector<std::pair<size_t, std::string>> OprRegistry::dump_registries() {
OprRegistry::dump_registries() {
auto&& id2reg = static_data().id2reg; auto&& id2reg = static_data().id2reg;
std::vector<std::pair<unsigned long int, std::string>> result; std::vector<std::pair<size_t, std::string>> result;
for (auto iter = id2reg.begin(); iter != id2reg.end(); ++iter) { for (auto iter = id2reg.begin(); iter != id2reg.end(); ++iter) {
if (iter->second.name.size() == 0) if (iter->second.name.size() == 0)
result.push_back({iter->first, "<special>"}); result.push_back({iter->first, "<special>"});
......
...@@ -76,8 +76,7 @@ namespace serialization { ...@@ -76,8 +76,7 @@ namespace serialization {
#if MGB_ENABLE_DEBUG_UTIL #if MGB_ENABLE_DEBUG_UTIL
//! dump registered oprs //! dump registered oprs
static std::vector<std::pair<unsigned long int, std::string>> static std::vector<std::pair<size_t, std::string>> dump_registries();
dump_registries();
#endif #endif
}; };
......
...@@ -17,13 +17,6 @@ ...@@ -17,13 +17,6 @@
#include <iostream> #include <iostream>
#if defined(WIN32)
static inline void unsetenv(std::string name) {
name += "=";
_putenv(name.c_str());
}
#define setenv(name,value,overwrite) _putenv_s(name,value)
#endif
#if !MGB_ENABLE_EXCEPTION #if !MGB_ENABLE_EXCEPTION
#pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wunused-variable"
#endif #endif
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册