From c3417fd21759e049967e62ee21082d7ee4fa413f Mon Sep 17 00:00:00 2001 From: ronnywang Date: Mon, 17 Apr 2023 10:50:47 +0800 Subject: [PATCH] [CustomDevice] fix custom cpu unittests failure when multiple python exist in CI (#52923) * [CustomDevice] fix custom cpu unittests failure when multiple python exist in CI * Update test_custom_op_setup.py --- test/custom_runtime/CMakeLists.txt | 4 +++- test/custom_runtime/test_collective_process_group_xccl.py | 8 ++++++-- test/custom_runtime/test_custom_cpu_plugin.py | 7 +++++-- test/custom_runtime/test_custom_cpu_profiler_plugin.py | 7 +++++-- test/custom_runtime/test_custom_cpu_to_static.py | 7 +++++-- test/custom_runtime/test_custom_op_setup.py | 3 ++- test/custom_runtime/test_fleet_launch_custom_device.sh | 4 ++-- 7 files changed, 28 insertions(+), 12 deletions(-) diff --git a/test/custom_runtime/CMakeLists.txt b/test/custom_runtime/CMakeLists.txt index d1885f9f9df..768dfb8fa48 100644 --- a/test/custom_runtime/CMakeLists.txt +++ b/test/custom_runtime/CMakeLists.txt @@ -19,10 +19,12 @@ if(WITH_CUSTOM_DEVICE AND NOT WITH_GPU) START_BASH test_fleet_launch_custom_device.sh ENVS + PYTHONPATH="" FLAGS_allocator_strategy=naive_best_fit PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR} PLUGIN_URL=${PLUGIN_URL} - PLUGIN_TAG=${PLUGIN_TAG}) + PLUGIN_TAG=${PLUGIN_TAG} + PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}) set_tests_properties(test_custom_cpu_plugin PROPERTIES TIMEOUT 120) set_tests_properties(test_custom_cpu_profiler_plugin PROPERTIES TIMEOUT 120) diff --git a/test/custom_runtime/test_collective_process_group_xccl.py b/test/custom_runtime/test_collective_process_group_xccl.py index d96072cbc65..a0489028987 100644 --- a/test/custom_runtime/test_collective_process_group_xccl.py +++ b/test/custom_runtime/test_collective_process_group_xccl.py @@ -15,6 +15,7 @@ import copy import os import subprocess +import sys import tempfile import time import unittest @@ -154,8 +155,11 @@ class TestProcessGroup(TestMultipleCustomCPU): && git fetch origin \ && git checkout {} -b dev \ && cd backends/custom_cpu \ - && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + && mkdir build && cd build && cmake .. -DPython_EXECUTABLE={} && make -j8'.format( + self.temp_dir.name, + os.getenv('PLUGIN_URL'), + os.getenv('PLUGIN_TAG'), + sys.executable, ) os.system(cmd) diff --git a/test/custom_runtime/test_custom_cpu_plugin.py b/test/custom_runtime/test_custom_cpu_plugin.py index f9606fb240d..c92a26b3dc8 100755 --- a/test/custom_runtime/test_custom_cpu_plugin.py +++ b/test/custom_runtime/test_custom_cpu_plugin.py @@ -31,8 +31,11 @@ class TestCustomCPUPlugin(unittest.TestCase): && git fetch origin \ && git checkout {} -b dev \ && cd backends/custom_cpu \ - && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + && mkdir build && cd build && cmake .. -DPython_EXECUTABLE={} && make -j8'.format( + self.temp_dir.name, + os.getenv('PLUGIN_URL'), + os.getenv('PLUGIN_TAG'), + sys.executable, ) os.system(cmd) diff --git a/test/custom_runtime/test_custom_cpu_profiler_plugin.py b/test/custom_runtime/test_custom_cpu_profiler_plugin.py index 79630f4f505..ea6f4331bd9 100644 --- a/test/custom_runtime/test_custom_cpu_profiler_plugin.py +++ b/test/custom_runtime/test_custom_cpu_profiler_plugin.py @@ -29,8 +29,11 @@ class TestCustomCPUProfilerPlugin(unittest.TestCase): && git fetch origin \ && git checkout {} -b dev \ && cd backends/custom_cpu \ - && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + && mkdir build && cd build && cmake .. -DPython_EXECUTABLE={} && make -j8'.format( + self.temp_dir.name, + os.getenv('PLUGIN_URL'), + os.getenv('PLUGIN_TAG'), + sys.executable, ) os.system(cmd) diff --git a/test/custom_runtime/test_custom_cpu_to_static.py b/test/custom_runtime/test_custom_cpu_to_static.py index 8cf475fc0ff..d395c8c622d 100644 --- a/test/custom_runtime/test_custom_cpu_to_static.py +++ b/test/custom_runtime/test_custom_cpu_to_static.py @@ -112,8 +112,11 @@ class TestCustomCPUPlugin(unittest.TestCase): && git fetch origin \ && git checkout {} -b dev \ && cd backends/custom_cpu \ - && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + && mkdir build && cd build && cmake .. -DPython_EXECUTABLE={} && make -j8'.format( + self.temp_dir.name, + os.getenv('PLUGIN_URL'), + os.getenv('PLUGIN_TAG'), + sys.executable, ) os.system(cmd) diff --git a/test/custom_runtime/test_custom_op_setup.py b/test/custom_runtime/test_custom_op_setup.py index 5f0e3d3ef1a..cb920654646 100644 --- a/test/custom_runtime/test_custom_op_setup.py +++ b/test/custom_runtime/test_custom_op_setup.py @@ -109,11 +109,12 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): && git fetch origin \ && git checkout {} -b dev \ && cd backends/custom_cpu \ - && mkdir build && cd build && cmake .. && make -j8 \ + && mkdir build && cd build && cmake .. -DPython_EXECUTABLE={} && make -j8 \ && cd {}'.format( self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG'), + sys.executable, self.cur_dir, ) os.system(cmd) diff --git a/test/custom_runtime/test_fleet_launch_custom_device.sh b/test/custom_runtime/test_fleet_launch_custom_device.sh index 5269cd32120..370e763e270 100644 --- a/test/custom_runtime/test_fleet_launch_custom_device.sh +++ b/test/custom_runtime/test_fleet_launch_custom_device.sh @@ -21,9 +21,9 @@ pushd ${temp_dir} \ && git clone ${PLUGIN_URL} \ && pushd PaddleCustomDevice/ \ && git fetch origin \ -&& git checkout ${PLUGIN_TAG} -b dev \ +&& git checkout ${PLUGIN_TAG} -b dev \ && pushd backends/custom_cpu \ -&& mkdir build && pushd build && cmake .. && make -j8 && popd && popd && popd && popd +&& mkdir build && pushd build && cmake .. -DPython_EXECUTABLE=${PYTHON_EXECUTABLE} && make -j8 && popd && popd && popd && popd echo "begin test use custom_cpu" -- GitLab