# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import queue import sys import threading import time taskQueue = queue.Queue() def worker(fun): while True: temp = taskQueue.get() fun(temp) taskQueue.task_done() def threadPool(threadPoolNum): threadPool = [] for i in range(threadPoolNum): thread = threading.Thread( target=worker, args={ doFun, }, ) thread.daemon = True threadPool.append(thread) return threadPool def get_h_file_md5(rootPath): h_cu_files = '%s/tools/h_cu_files.log' % rootPath f = open(h_cu_files) lines = f.readlines() for line in lines: line = line.strip() os.system('md5sum %s >> %s/tools/h_cu_md5.log' % (line, rootPath)) def insert_pile_to_h_file(rootPath): h_cu_files = '%s/tools/h_cu_files.log' % rootPath f = open(h_cu_files) lines = f.readlines() for line in lines: line = line.strip() func = line.replace('/', '_').replace('.', '_') os.system('echo "\n#ifndef _PRECISE%s_\n" >> %s' % (func.upper(), line)) os.system('echo "#define _PRECISE%s_" >> %s' % (func.upper(), line)) os.system('echo "\n#include \n" >> %s' % line) os.system( 'echo "__attribute__((constructor)) static void calledFirst%s()\n{" >> %s' % (func, line) ) os.system( 'echo \' fprintf(stderr,"precise test map fileeee: %%s\\\\n", __FILE__);\n}\' >> %s' % line ) os.system('echo "\n#endif" >> %s' % line) def add_simple_cxx_test(rootPath): variant_test_path = '%s/paddle/utils/variant_test.cc' % rootPath variant_test_cmakeflie_path = '%s/paddle/utils/CMakeLists.txt' % rootPath if os.path.exists(variant_test_path) and os.path.exists( variant_test_cmakeflie_path ): simple_test_path = '%s/paddle/utils/simple_precision_test.cc' % rootPath os.system('touch %s' % simple_test_path) os.system( "echo '#include \"gtest/gtest.h\"\n' >> %s" % simple_test_path ) os.system( 'echo "TEST(interface_test, type) { }\n" >> %s' % simple_test_path ) os.system('echo "cc_test(" >> %s' % variant_test_cmakeflie_path) os.system( 'echo " simple_precision_test" >> %s' % variant_test_cmakeflie_path ) os.system( 'echo " SRCS simple_precision_test.cc" >> %s' % variant_test_cmakeflie_path ) os.system('echo " DEPS gtest)\n" >> %s' % variant_test_cmakeflie_path) def remove_pile_from_h_file(rootPath): h_cu_files = '%s/tools/h_cu_files.log' % rootPath f = open(h_cu_files) lines = f.readlines() count = 12 for line in lines: line = line.strip() while count > 0: os.system("sed -i '$d' %s" % line) count = count - 1 count = 12 def get_h_cu_file(file_path): rootPath = file_path[0] dir_path = file_path[1] filename = file_path[2] ut = filename.replace('^', '').replace('$', '').replace('.log', '') ut_path = "%s/build/ut_map/%s" % (rootPath, ut) if os.path.exists(ut_path): os.system( "cat %s/%s | grep 'precise test map fileeee:'| uniq >> %s/build/ut_map/%s/related_%s.txt" % (dir_path, filename, rootPath, ut, ut) ) else: print("%s has failed,no has direcotory" % ut) def doFun(file_path): get_h_cu_file(file_path) def main(rootPath, dir_path): """ get useful message """ startTime = int(time.time()) test_h_cu_dict = {} pool = threadPool(23) for i in range(pool.__len__()): pool[i].start() files = os.listdir(dir_path) for filename in files: file_path = [rootPath, dir_path, filename] taskQueue.put(file_path) taskQueue.join() endTime = int(time.time()) print('analy h/cu file cost Time: %s' % (endTime - startTime)) if __name__ == "__main__": func = sys.argv[1] if func == 'get_h_file_md5': rootPath = sys.argv[2] get_h_file_md5(rootPath) elif func == 'insert_pile_to_h_file': rootPath = sys.argv[2] insert_pile_to_h_file(rootPath) add_simple_cxx_test(rootPath) elif func == 'analy_h_cu_file': dir_path = sys.argv[2] rootPath = sys.argv[3] main(rootPath, dir_path) elif func == 'remove_pile_from_h_file': rootPath = sys.argv[2] remove_pile_from_h_file(rootPath)