move test_*tokenizer to /test/tokenizer (#52658)

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -552,8 +552,6 @@ if((WITH_GPU) AND (CUDA_VERSION GREATER_EQUAL 11.6))
 endif()
 
 set_tests_properties(test_conv2d_op PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
-set_tests_properties(test_faster_tokenizer_op PROPERTIES LABELS
-                                                         "RUN_TYPE=EXCLUSIVE")
 set_tests_properties(test_conv2d_op_depthwise_conv
                      PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
 set_tests_properties(test_conv2d_api PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")

test/CMakeLists.txt

@@ -126,7 +126,7 @@ if(WITH_TESTING)
   add_subdirectory(rpc)
   # add_subdirectory(sequence)
   add_subdirectory(standalone_executor)
-  # add_subdirectory(tokenizer)
+  add_subdirectory(tokenizer)
   # add_subdirectory(white_list)
   add_subdirectory(xpu)
 endif()

test/tokenizer/CMakeLists.txt

+file(
+  GLOB TEST_OPS
+  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+  "test_*.py")
+string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
+
+foreach(src ${TEST_OPS})
+  py_test(${src} SRCS ${src}.py)
+endforeach()
+
+set_tests_properties(test_faster_tokenizer_op PROPERTIES LABELS
+                                                         "RUN_TYPE=EXCLUSIVE")

python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py → test/tokenizer/test_faster_tokenizer_op.py

@@ -13,20 +13,17 @@
 # limitations under the License.
 
 import os
-import sys
 import tempfile
 import unittest
 
 import numpy as np
+from bert_tokenizer import BertTokenizer
 
 import paddle
 from paddle import _legacy_C_ops, nn
 from paddle.fluid.framework import _non_static_mode, core
 from paddle.fluid.layer_helper import LayerHelper
 
-sys.path.append("./tokenizer")
-from tokenizer.bert_tokenizer import BertTokenizer
-
 
 def to_string_tensor(string_values, name):
     """