diff --git a/CV/find_face.py b/CV/find_face.py new file mode 100644 index 0000000000000000000000000000000000000000..b480267b6e6df08465cbe1d85f82ec8bc525958e --- /dev/null +++ b/CV/find_face.py @@ -0,0 +1,175 @@ +import face_recognition +import os +import shutil +import datetime +import concurrent.futures +import cv2 +import numpy as np +import random +import string + +all_img_cnt = 0 + +def random_str(length=12): + """ + Generate secret key from alpha and digit. + :param length: length of secret key. + :return: [length] long secret key. + """ + key = '' + while length: + key += random.choice(string.ascii_letters + string.digits) + length -= 1 + return key + +def get_bar(percent,num = 25): + # graphs = ' ▏▎▍▋▊▉' + percent = round(percent) + bar = '[' + for i in range(num): + if i < round(percent/int((100/num))): + bar += '#' + else: + bar += '-' + bar += ']' + return bar + +def lapulase(resImg): + img2gray = cv2.cvtColor(resImg, cv2.COLOR_BGR2GRAY) # 将图片压缩为单通道的灰度图 + res = cv2.Laplacian(img2gray, cv2.CV_64F) + score = res.var() + return score + +def resize(img,size): + h, w = img.shape[:2] + if min(h, w) ==size: + return img + if w >= h: + res = cv2.resize(img,(int(size*w/h), size),interpolation=cv2.INTER_LANCZOS4) + else: + res = cv2.resize(img,(size, int(size*h/w)),interpolation=cv2.INTER_LANCZOS4) + return res + +def Traversal(filedir): + file_list=[] + for root,dirs,files in os.walk(filedir): + for file in files: + file_list.append(os.path.join(root,file)) + for dir in dirs: + Traversal(dir) + return file_list + +def is_img(ext): + ext = ext.lower() + if ext in ['.jpg','.png','.jpeg','.bmp']: + return True + else: + return False + +def picture_select(file_list): + imgpath_list=[] + for pic in file_list: + if is_img(os.path.splitext(pic)[1]): + imgpath_list.append(pic) + return imgpath_list + +def find_save_resize_face(input_path): + try: + + filename,extension = os.path.splitext(os.path.split(input_path)[1]) + # image = face_recognition.load_image_file(input_path) + image = cv2.imread(input_path) + h,w = image.shape[:2] + #print(image.dtype) + # Find all the faces in the image and print + face_locations = face_recognition.face_locations(image) + # print("found {} face(s) in this photograph.".format(len(face_locations))) + + origin_image = cv2.imread(input_path) + mask = np.zeros(origin_image.shape[:2],dtype = "uint8") + + count=0 + for face_location in face_locations: + # Print the location of each face in this image + top, right, bottom, left = face_location + #print("A face is located at pixel location Top: {}, Left: {}, Bottom: {}, Right: {}".format(top, left, bottom, right)) + # You can access the actual face itself like this: + if IS_random_EXTEND: + ex=int(((EXTEND-1+0.2*random.random())*(bottom-top))/2) + else: + ex=int(((EXTEND-1)*(bottom-top))/2) + # ex_face=int((1*(bottom-top))/2) + if ((bottom-top)>MINSIZE) and 0.950 and (bottom+ex)0 and (right+ex)Del_Blur_Score: + #print(os.path.join(outdir_face,random_str()+'.jpg')) + cv2.imwrite(os.path.join(outdir_face,random_str()+'.jpg'),face) + count = count+1 + #mask = cv2.rectangle(mask,(left-ex,top-ex),(right+ex,bottom+ex),255,-1) + #mask + # if count > 0: + # mask = resize(mask,256) + # origin_image = resize(origin_image,1024) + # cv2.imwrite(os.path.join(outdir_ori,outname+filename+'.jpg'),origin_image) + # cv2.imwrite(os.path.join(outdir_mask,outname+filename+'.png'),mask) + # print(output_path) + return count + + except Exception as e: + #print(input_path,e) + return 0 + + +filedir = (input("filedir:").strip()).replace("'","") +outname = (input("outname:").strip()).replace("'","") #outname = 'star' +MINSIZE = int((input("min_face_size(defult=256):").strip()).replace("'","")) +WORKERS = int((input("cpu_workers(defult=4):").strip()).replace("'","")) + +# EXTEND=1.4 +EXTEND = 1.6 +Del_Blur_Score = 50 +IS_random_EXTEND = False +# WORKERS = 4 +# MINSIZE = 256 + +outdir='./output/'+outname + +outdir_ori = os.path.join(outdir,'origin_image') +outdir_face = os.path.join(outdir,'face') +outdir_mask = os.path.join(outdir,'mask') +if not os.path.isdir(outdir): + os.makedirs(outdir) + os.makedirs(outdir_ori) + os.makedirs(outdir_face) + os.makedirs(outdir_mask) + +file_list = Traversal(filedir) +imgpath_list = picture_select(file_list) +random.shuffle(imgpath_list) +all_length = len(imgpath_list) + +print("Find picture:"+" "+str(all_length)) +print('Begining......') + + +starttime = datetime.datetime.now() +face_cnt=0 +with concurrent.futures.ProcessPoolExecutor(max_workers=WORKERS) as executor: + for i,imgpath,count in zip(range(1,len(imgpath_list)+1), + imgpath_list, + executor.map(find_save_resize_face,imgpath_list)): + # print(imgpath) + face_cnt+=count + if i%100==0: + endtime = datetime.datetime.now() + used_time = (endtime-starttime).seconds + percent = round(100*i/all_length,2) + + print('\r','','Ok:'+str(i),'Face:'+str(face_cnt)+' ', + str(percent)+'%'+get_bar(percent,30), + ' Used/All:'+str(int(used_time))+'s/'+str(int(used_time/i*all_length))+'s',end= " ") + #starttime_show = datetime.datetime.now() + + print('\nFinished!','Finall find face:',face_cnt,' Cost time:',(datetime.datetime.now()-starttime).seconds,'s') \ No newline at end of file diff --git a/README.md b/README.md index 705743f65ecc6401790a687f83bf8a8bd7aa21b0..df5c2bd05804fade045e5638d5a331e157761963 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,45 @@ -# Haystack +## Haystack Gadgets for learning and research +### System +* monitor_linux.py +A simple Deep Learning Server performance monitor. Written by python without any dependencies. +```bash +(base) hypo@lthpc:~$ python monitor_linux.py +Cpu Temp: 48.0C | Freq: 2499.9MHz +Util: 85.6% [###########################################-------] + +Memory Mem: 34354MB/64301MB | Swap: 0MB/62498MB +Mem: 53.0% [#############------------] Swap: 0.0% [-------------------------] + +Gpu0: TITAN V Temp: 45.0C | Power: 11w/250w | Mem: 0MB/12066MB | Fan: 31% +Util:0.0% [-------------------------] Mem:0.0% [-------------------------] + +Gpu1: TITAN V Temp: 45.0C | Power: 12w/250w | Mem: 0MB/12066MB | Fan: 31% +Util:0.0% [-------------------------] Mem:0.0% [-------------------------] + +Network ↑ all:10.2GB ↓ all:173.8GB ↑ :147.6Kb/s ↓ :6000.5Kb/s + +Filesystem Mounted on Used/Total Used% +udev /dev 0/32G 0% +/dev/sda7 / 9.9G/143G 8% +tmpfs /dev/shm 0/32G 0% +tmpfs /sys/fs/cgroup 0/32G 0% +/dev/sdb1 /home 103G/2.7T 4% +/dev/sdc1 /media/usb 3.6T/3.7T 99% +``` + +### CV +* find_face.py +FInd face in images, then save them. +Depend on face_recognition, opencv-python. +```bash +(pytorch) hypo@hypo-7820:/media/hypo/OS/MyProject/Haystack/CV$ python find_face.py +filedir:'/media/hypo/OS/MyProject/BaiduImageSpider/download/Angelababy' +outname:star +min_face_size(defult=256):256 +cpu_workers(defult=4):4 +Find picture: 504 +Begining...... + Ok:500 Face:50 99.21%[##############################] Used/All:143s/144s +Finished! Finall find face: 50 Cost time: 145 s +``` \ No newline at end of file diff --git a/System/monitor_linux.py b/System/monitor_linux.py new file mode 100644 index 0000000000000000000000000000000000000000..f028641b5dd079aff8901343a69f379d85dad134 --- /dev/null +++ b/System/monitor_linux.py @@ -0,0 +1,244 @@ +import os +import time +# -*- coding:utf-8 -*- + +#Cpu +############################################################### +# Cpu use +last_worktime=0 +last_idletime=0 +def get_cpu_use(): + global last_worktime, last_idletime + f=open("/proc/stat","r") + line="" + while not "cpu " in line: + line=f.readline() + f.close() + spl=line.split(" ") + worktime=int(spl[2])+int(spl[3])+int(spl[4]) + idletime=int(spl[5]) + dworktime=(worktime-last_worktime) + didletime=(idletime-last_idletime) + rate=float(dworktime)/(didletime+dworktime) + cpu_t = rate*100 + last_worktime=worktime + last_idletime=idletime + if(last_worktime==0): + return 0 + return round(cpu_t,2) + +# Cpu freq +def get_cpu_freq(): + freq_str = os.popen('cat /proc/cpuinfo | grep MHz | uniq').read() + freqs =[] + while freq_str.find('\n') != -1: + freqs.append(float(freq_str[freq_str.find(':')+2:freq_str.find('\n')])) + freq_str=freq_str[freq_str.find('\n')+1:] + return max(freqs) + +# Cpu freq +def get_cpu_temp(): + temp_str = os.popen('cat /sys/class/hwmon/hwmon0/device/hwmon/hwmon0/temp1_input').read() + return (float(temp_str)/1000) + +# Mem +############################################################### +def get_mem_use(): + mem_str = os.popen('free').read() + if '内存'in mem_str : + mem_str = mem_str[mem_str.find('内存：')+3:mem_str.find('交换')] + else: + mem_str = mem_str[mem_str.find('Mem:')+4:mem_str.find('Swap')] + mem_str = mem_str.split() + total = int(float(mem_str[0])/1024) + used = int(float(mem_str[1])/1024) + percent = int(used/total*100) + return total,used,percent + +def get_swap_use(): + mem_str = os.popen('free').read() + if '内存'in mem_str : + mem_str = mem_str[mem_str.find('交换：')+3:] + else: + mem_str = mem_str[mem_str.find('Swap:')+5:] + mem_str = mem_str.split() + total = int(float(mem_str[0])/1024) + used = int(float(mem_str[1])/1024) + percent = int(used/total*100) + return total,used,percent + +#gpu +############################################################### +gpus_str = os.popen('nvidia-smi -L').read() +gpus =[] +while gpus_str.find('\n') != -1: + gpus.append(gpus_str[gpus_str.find(':')+2:gpus_str.find('(')-1]) + gpus_str=gpus_str[gpus_str.find('\n')+1:] + +def get_gpu_use(): + + gpu_infos = [] + for gpu in gpus: + infos_str = os.popen('nvidia-smi').read() + infos_str = infos_str[infos_str.find(gpu):] + infos_str = infos_str[infos_str.find('\n')+1:] + infos_str = infos_str[:infos_str.find('\n')+1] + infos_str = infos_str.split() + #['|', '50%', '42C', 'P0', '19W', '/', '75W', '|', '929MiB', '/', '5050MiB', '|', '14%', 'Default', '|'] + fan = int(infos_str[1].replace('%','')) # % + temp = int(infos_str[2].replace('C','')) # C + if infos_str[4] == 'N/A': + power_used = -1 + power_max = -1 + else: + power_used = int(infos_str[4].replace('W','')) # W + power_max = int(infos_str[6].replace('W','')) # W + mem_used = int(infos_str[8].replace('MiB','')) # MB + mem_max = int(infos_str[10].replace('MiB','')) # MB + util_used = int(infos_str[12].replace('%','')) # % + gpu_infos.append([fan,temp,power_used,power_max,mem_used,mem_max,util_used]) + return gpu_infos + + +#Net +############################################################### +net_infos_history = [0,0] +def get_net_use(network_adapter = 'all'): + net_str = os.popen('cat /proc/net/dev').read() + infos = [] + net_infos = [0,0,0,0] + while net_str.find('\n') != -1: + if net_str[:50].find(':') != -1: + infos.append((net_str[net_str.find(':')+2:net_str.find('\n')-1]).split()) + net_str=net_str[net_str.find('\n')+1:] + net_rxs=[];net_txs=[] + for info in infos: + net_rxs.append(int(info[0])) + net_txs.append(int(info[8])) + if network_adapter == 'all': + net_infos[0] = (sum(net_rxs)/1024) + net_infos[1] = (sum(net_txs)/1024) + elif network_adapter == 'auto': + net_infos[0] = (max(net_rxs)/1024) + net_infos[1] = (max(net_txs)/1024) + if net_infos_history[0] == 0: + net_infos[2] = 0 + net_infos[3] = 0 + else: + net_infos[2] = net_infos[0]-net_infos_history[0] + net_infos[3] = net_infos[1]-net_infos_history[1] + net_infos_history[0] = net_infos[0] + net_infos_history[1] = net_infos[1] + return net_infos + +#Disk +############################################################### +def get_disk_use(): + disk_str = os.popen('df -h').read() + disk_str = disk_str.split('\n') + disk_infos = [] + for line in disk_str: + if 'G' in line[:24]: + line = line.split() + if float(line[1].replace('G',''))>20: + disk_infos.append(line) + elif 'T' in line: + line = line.split() + disk_infos.append(line) + return disk_infos + +def get_bar(percent,num = 25): + # graphs = ' ▏▎▍▋▊▉' + percent = round(percent) + bar = '[' + for i in range(num): + if i < round(percent/int((100/num))): + bar += '#' + else: + bar += '-' + bar += ']' + return bar + +def fill_str(string,num): + if len(string)>num: + return string[:num-3]+'...' + for i in range(num-len(string)): + string +=' ' + return string + +def main(): + while(1): + #cpu + cpu_used = get_cpu_use() + cpu_freq = get_cpu_freq() + cpu_temp = get_cpu_temp() + cpu_used_bar = get_bar(cpu_used,num=50) + + #memory + mem_total,mem_used,mem_percent = get_mem_use() + mem_used_bar = get_bar(mem_percent) + swap_total,swap_used,swap_percent = get_swap_use() + swap_used_bar = get_bar(swap_percent) + + #gpu + gpu_infoss = [];util_used_bars=[];gpu_mem_bars=[] + for i in range(len(gpus)): + gpu_infos = get_gpu_use() + gpu_infoss.append(gpu_infos) + util_used_bars.append(get_bar(gpu_infos[0][6])) + gpu_mem_bars.append(get_bar(100*gpu_infos[0][4]/gpu_infos[0][5])) + + #net + net_infos = get_net_use() + + #disk + disk_infos = get_disk_use() + + ####################################print################################# + os.system('clear') + #cpu + print(('Cpu Temp: {0:.1f}C | Freq: {1:.1f}MHz').format(cpu_temp,cpu_freq)) + print(('Util: {0:.1f}% '+cpu_used_bar).format(cpu_used)) + + #memory + print(('\nMemory Mem: {0:d}MB/{1:d}MB | Swap: {2:d}MB/{3:d}MB').format( + mem_used,mem_total,swap_used,swap_total)) + print(('Mem: {0:.1f}% '+mem_used_bar+' Swap: {1:.1f}% '+swap_used_bar).format(mem_percent,swap_percent)) + + #gpu + for i in range(len(gpus)): + print(('\n'+'Gpu'+'{0:d}'+': '+gpus[i]+' Temp: {1:.1f}C | Power: {2:d}w/{3:d}w | Mem: {4:d}MB/{5:d}MB | Fan: {6:d}%').format( + i,gpu_infoss[i][0][1],gpu_infoss[i][0][2],gpu_infoss[i][0][3], + gpu_infoss[i][0][4],gpu_infoss[i][0][5],gpu_infoss[i][0][0])) + print(('Util:{0:.1f}% '+util_used_bars[i]+' Mem:{1:.1f}% '+gpu_mem_bars[i]).format( + gpu_infoss[i][0][6],100*gpu_infoss[i][0][4]/gpu_infoss[i][0][5])) + + #net + print(('\nNetwork ↑ all:{0:.1f}GB ↓ all:{1:.1f}GB ↑ :{2:.1f}Kb/s ↓ :{3:.1f}Kb/s').format( + net_infos[1]/1024/1024,net_infos[0]/1024/1024,net_infos[3],net_infos[2])) + + #disk + print('\nFilesystem Mounted on Used/Total Used%') + for disk_info in disk_infos: + print(fill_str(disk_info[0], 20),fill_str(disk_info[5], 20), + fill_str(disk_info[2]+'/'+disk_info[1], 20),fill_str(disk_info[4], 15)) + + time.sleep(1) + +if __name__ == '__main__': + main() +# print(disk_infos) +# disk_str = tmp +# tmp = [] +# for line in disk_str: +# info = +# tmp.append(line) +# print(disk_str) + + +# total = + +# cpu_used = top_str[top_str.find('%Cpu(s):'):top_str.find('%Cpu(s):')+5] +# print(cpu_used) +# out_string = os.popen('nvidia-smi').read() +# print(out_string) \ No newline at end of file