add find_face.py,monitor_linux.py

CV/find_face.py

+import face_recognition
+import os
+import shutil
+import datetime
+import concurrent.futures
+import cv2
+import numpy as np
+import random
+import string
+
+all_img_cnt = 0
+
+def random_str(length=12):
+    """
+    Generate secret key from alpha and digit.
+    :param length: length of secret key.
+    :return: [length] long secret key.
+    """
+    key = ''
+    while length:
+        key += random.choice(string.ascii_letters + string.digits)
+        length -= 1
+    return key
+
+def get_bar(percent,num = 25):
+    # graphs = ' ▏▎▍▋▊▉'
+    percent = round(percent)
+    bar = '['
+    for i in range(num):
+        if i < round(percent/int((100/num))):
+            bar += '#'
+        else:
+            bar += '-'
+    bar += ']'
+    return bar
+
+def lapulase(resImg):
+    img2gray = cv2.cvtColor(resImg, cv2.COLOR_BGR2GRAY)  # 将图片压缩为单通道的灰度图
+    res = cv2.Laplacian(img2gray, cv2.CV_64F)
+    score = res.var()
+    return score
+
+def resize(img,size):
+    h, w = img.shape[:2]
+    if min(h, w) ==size:
+        return img
+    if w >= h:
+        res = cv2.resize(img,(int(size*w/h), size),interpolation=cv2.INTER_LANCZOS4)
+    else:
+        res = cv2.resize(img,(size, int(size*h/w)),interpolation=cv2.INTER_LANCZOS4)
+    return res
+
+def Traversal(filedir):
+    file_list=[]
+    for root,dirs,files in os.walk(filedir): 
+        for file in files:
+            file_list.append(os.path.join(root,file)) 
+        for dir in dirs:
+            Traversal(dir)
+    return file_list
+
+def is_img(ext):
+    ext = ext.lower()
+    if ext in ['.jpg','.png','.jpeg','.bmp']:
+        return True
+    else:
+        return False
+
+def picture_select(file_list):
+    imgpath_list=[]
+    for pic in file_list:
+        if is_img(os.path.splitext(pic)[1]):
+            imgpath_list.append(pic)
+    return imgpath_list
+
+def find_save_resize_face(input_path):
+    try:
+
+        filename,extension = os.path.splitext(os.path.split(input_path)[1])
+        # image = face_recognition.load_image_file(input_path)
+        image = cv2.imread(input_path)
+        h,w = image.shape[:2]
+        #print(image.dtype)
+        # Find all the faces in the image and print
+        face_locations = face_recognition.face_locations(image)
+        # print("found {} face(s) in this photograph.".format(len(face_locations)))
+        
+        origin_image = cv2.imread(input_path)
+        mask = np.zeros(origin_image.shape[:2],dtype = "uint8")
+
+        count=0
+        for face_location in face_locations:
+            # Print the location of each face in this image
+            top, right, bottom, left = face_location
+            #print("A face is located at pixel location Top: {}, Left: {}, Bottom: {}, Right: {}".format(top, left, bottom, right))
+            # You can access the actual face itself like this:
+            if IS_random_EXTEND:
+                ex=int(((EXTEND-1+0.2*random.random())*(bottom-top))/2)
+            else:
+                ex=int(((EXTEND-1)*(bottom-top))/2)
+            # ex_face=int((1*(bottom-top))/2)
+            if ((bottom-top)>MINSIZE) and 0.95<abs((bottom-top)/(left-right))<1.05 and (top-ex)>0 and (bottom+ex)<h and (left-ex)>0 and (right+ex)<w:
+                face = origin_image[top-ex:bottom+ex, left-ex:right+ex]
+                face = cv2.resize(face, (512,512),interpolation=cv2.INTER_LANCZOS4)
+                # cv2.imwrite(os.path.join(outdir_face,outname+filename+'_'+str(count)+'.jpg'),face)
+                if lapulase(face)>Del_Blur_Score:
+                    #print(os.path.join(outdir_face,random_str()+'.jpg'))
+                    cv2.imwrite(os.path.join(outdir_face,random_str()+'.jpg'),face)
+                    count = count+1
+            #mask = cv2.rectangle(mask,(left-ex,top-ex),(right+ex,bottom+ex),255,-1)  
+        #mask         
+        # if count > 0:
+        #     mask = resize(mask,256)
+        #     origin_image = resize(origin_image,1024)
+        #     cv2.imwrite(os.path.join(outdir_ori,outname+filename+'.jpg'),origin_image)
+        #     cv2.imwrite(os.path.join(outdir_mask,outname+filename+'.png'),mask)
+        # print(output_path)
+        return count
+
+    except Exception as e:
+        #print(input_path,e)
+        return 0
+
+
+filedir = (input("filedir:").strip()).replace("'","")
+outname = (input("outname:").strip()).replace("'","") #outname = 'star'
+MINSIZE = int((input("min_face_size(defult=256):").strip()).replace("'",""))
+WORKERS = int((input("cpu_workers(defult=4):").strip()).replace("'",""))
+
+# EXTEND=1.4
+EXTEND = 1.6
+Del_Blur_Score = 50
+IS_random_EXTEND = False
+# WORKERS = 4
+# MINSIZE = 256
+
+outdir='./output/'+outname
+
+outdir_ori = os.path.join(outdir,'origin_image')
+outdir_face = os.path.join(outdir,'face')
+outdir_mask = os.path.join(outdir,'mask')
+if not os.path.isdir(outdir):
+    os.makedirs(outdir)
+    os.makedirs(outdir_ori)
+    os.makedirs(outdir_face)
+    os.makedirs(outdir_mask)
+
+file_list = Traversal(filedir)
+imgpath_list = picture_select(file_list)
+random.shuffle(imgpath_list)
+all_length = len(imgpath_list)
+
+print("Find picture:"+" "+str(all_length))
+print('Begining......')
+
+
+starttime = datetime.datetime.now()
+face_cnt=0
+with concurrent.futures.ProcessPoolExecutor(max_workers=WORKERS) as executor:
+    for i,imgpath,count in zip(range(1,len(imgpath_list)+1),
+                               imgpath_list,
+                               executor.map(find_save_resize_face,imgpath_list)):
+        # print(imgpath)
+        face_cnt+=count
+        if i%100==0:
+            endtime = datetime.datetime.now()
+            used_time = (endtime-starttime).seconds
+            percent = round(100*i/all_length,2)
+
+            print('\r','','Ok:'+str(i),'Face:'+str(face_cnt)+' ',
+                str(percent)+'%'+get_bar(percent,30),
+                ' Used/All:'+str(int(used_time))+'s/'+str(int(used_time/i*all_length))+'s',end= " ")
+            #starttime_show = datetime.datetime.now()
+
+    print('\nFinished!','Finall find face:',face_cnt,' Cost time:',(datetime.datetime.now()-starttime).seconds,'s')
\ No newline at end of file

README.md

-# Haystack
+## Haystack
 Gadgets for learning and research
+### System
+* monitor_linux.py
+A simple Deep Learning Server performance monitor. Written by python without any dependencies.
+```bash
+(base) hypo@lthpc:~$ python monitor_linux.py
+Cpu   Temp: 48.0C | Freq: 2499.9MHz
+Util: 85.6%  [###########################################-------]
+
+Memory   Mem: 34354MB/64301MB | Swap: 0MB/62498MB
+Mem: 53.0% [#############------------] Swap: 0.0% [-------------------------]
+
+Gpu0: TITAN V   Temp: 45.0C | Power: 11w/250w | Mem: 0MB/12066MB | Fan: 31%
+Util:0.0% [-------------------------]   Mem:0.0% [-------------------------]
+
+Gpu1: TITAN V   Temp: 45.0C | Power: 12w/250w | Mem: 0MB/12066MB | Fan: 31%
+Util:0.0% [-------------------------]   Mem:0.0% [-------------------------]
+
+Network    ↑ all:10.2GB ↓ all:173.8GB     ↑ :147.6Kb/s ↓ :6000.5Kb/s
+
+Filesystem           Mounted on           Used/Total           Used%
+udev                 /dev                 0/32G                0%             
+/dev/sda7            /                    9.9G/143G            8%             
+tmpfs                /dev/shm             0/32G                0%             
+tmpfs                /sys/fs/cgroup       0/32G                0%             
+/dev/sdb1            /home                103G/2.7T            4%             
+/dev/sdc1            /media/usb           3.6T/3.7T            99%    
+```
+
+### CV
+* find_face.py
+FInd face in images, then save them.
+Depend on face_recognition, opencv-python.
+```bash
+(pytorch) hypo@hypo-7820:/media/hypo/OS/MyProject/Haystack/CV$ python find_face.py
+filedir:'/media/hypo/OS/MyProject/BaiduImageSpider/download/Angelababy' 
+outname:star
+min_face_size(defult=256):256
+cpu_workers(defult=4):4
+Find picture: 504
+Begining......
+  Ok:500 Face:50  99.21%[##############################]  Used/All:143s/144s 
+Finished! Finall find face: 50  Cost time: 145 s
+```
\ No newline at end of file

System/monitor_linux.py

+import os
+import time
+# -*- coding:utf-8 -*- 
+
+#Cpu
+###############################################################
+# Cpu use
+last_worktime=0
+last_idletime=0
+def get_cpu_use():
+    global last_worktime, last_idletime
+    f=open("/proc/stat","r")
+    line=""
+    while not "cpu " in line:
+        line=f.readline()
+    f.close()
+    spl=line.split(" ")
+    worktime=int(spl[2])+int(spl[3])+int(spl[4])
+    idletime=int(spl[5])
+    dworktime=(worktime-last_worktime)
+    didletime=(idletime-last_idletime)
+    rate=float(dworktime)/(didletime+dworktime)
+    cpu_t = rate*100
+    last_worktime=worktime
+    last_idletime=idletime
+    if(last_worktime==0): 
+        return 0
+    return round(cpu_t,2)
+
+# Cpu freq
+def get_cpu_freq():
+    freq_str = os.popen('cat /proc/cpuinfo | grep MHz | uniq').read()
+    freqs =[]
+    while freq_str.find('\n') != -1:
+        freqs.append(float(freq_str[freq_str.find(':')+2:freq_str.find('\n')]))
+        freq_str=freq_str[freq_str.find('\n')+1:]
+    return max(freqs)
+
+# Cpu freq
+def get_cpu_temp():
+    temp_str = os.popen('cat /sys/class/hwmon/hwmon0/device/hwmon/hwmon0/temp1_input').read()
+    return (float(temp_str)/1000)
+
+# Mem
+###############################################################
+def get_mem_use():
+    mem_str = os.popen('free').read()
+    if '内存'in mem_str :
+        mem_str = mem_str[mem_str.find('内存：')+3:mem_str.find('交换')]
+    else:
+        mem_str = mem_str[mem_str.find('Mem:')+4:mem_str.find('Swap')]
+    mem_str = mem_str.split()
+    total = int(float(mem_str[0])/1024)
+    used = int(float(mem_str[1])/1024)
+    percent = int(used/total*100)
+    return total,used,percent
+
+def get_swap_use():
+    mem_str = os.popen('free').read()
+    if '内存'in mem_str :
+        mem_str = mem_str[mem_str.find('交换：')+3:]
+    else:
+        mem_str = mem_str[mem_str.find('Swap:')+5:]
+    mem_str = mem_str.split()
+    total = int(float(mem_str[0])/1024)
+    used = int(float(mem_str[1])/1024)
+    percent = int(used/total*100)
+    return total,used,percent
+
+#gpu 
+###############################################################
+gpus_str = os.popen('nvidia-smi -L').read()
+gpus =[]
+while gpus_str.find('\n') != -1:
+    gpus.append(gpus_str[gpus_str.find(':')+2:gpus_str.find('(')-1])
+    gpus_str=gpus_str[gpus_str.find('\n')+1:]
+
+def get_gpu_use():
+    
+    gpu_infos = []
+    for gpu in gpus:
+        infos_str = os.popen('nvidia-smi').read()
+        infos_str = infos_str[infos_str.find(gpu):]
+        infos_str = infos_str[infos_str.find('\n')+1:]
+        infos_str = infos_str[:infos_str.find('\n')+1]
+        infos_str = infos_str.split()
+        #['|', '50%', '42C', 'P0', '19W', '/', '75W', '|', '929MiB', '/', '5050MiB', '|', '14%', 'Default', '|']
+        fan = int(infos_str[1].replace('%',''))        # %
+        temp = int(infos_str[2].replace('C',''))       # C
+        if infos_str[4] == 'N/A':
+            power_used = -1
+            power_max = -1
+        else:
+            power_used = int(infos_str[4].replace('W','')) # W
+            power_max = int(infos_str[6].replace('W',''))  # W
+        mem_used = int(infos_str[8].replace('MiB','')) # MB
+        mem_max = int(infos_str[10].replace('MiB',''))  # MB
+        util_used = int(infos_str[12].replace('%',''))  # %
+        gpu_infos.append([fan,temp,power_used,power_max,mem_used,mem_max,util_used])
+    return gpu_infos
+
+
+#Net
+###############################################################
+net_infos_history = [0,0]
+def get_net_use(network_adapter = 'all'):
+    net_str = os.popen('cat /proc/net/dev').read()
+    infos = []
+    net_infos = [0,0,0,0]
+    while net_str.find('\n') != -1:
+        if net_str[:50].find(':') != -1:
+            infos.append((net_str[net_str.find(':')+2:net_str.find('\n')-1]).split())
+        net_str=net_str[net_str.find('\n')+1:]
+    net_rxs=[];net_txs=[]
+    for info in infos:
+        net_rxs.append(int(info[0]))
+        net_txs.append(int(info[8]))
+    if network_adapter == 'all':
+        net_infos[0] = (sum(net_rxs)/1024)
+        net_infos[1] = (sum(net_txs)/1024)
+    elif network_adapter == 'auto':
+        net_infos[0] = (max(net_rxs)/1024)
+        net_infos[1] = (max(net_txs)/1024)
+    if net_infos_history[0] == 0:
+        net_infos[2] = 0
+        net_infos[3] = 0
+    else:
+        net_infos[2] = net_infos[0]-net_infos_history[0]
+        net_infos[3] = net_infos[1]-net_infos_history[1]
+    net_infos_history[0] = net_infos[0]
+    net_infos_history[1] = net_infos[1]
+    return net_infos
+
+#Disk
+###############################################################
+def get_disk_use():
+    disk_str = os.popen('df -h').read()
+    disk_str = disk_str.split('\n')
+    disk_infos = []
+    for line in disk_str:
+        if 'G' in line[:24]:
+            line = line.split()
+            if float(line[1].replace('G',''))>20:
+                disk_infos.append(line)
+        elif 'T' in line:
+            line = line.split()
+            disk_infos.append(line)
+    return disk_infos
+
+def get_bar(percent,num = 25):
+    # graphs = ' ▏▎▍▋▊▉'
+    percent = round(percent)
+    bar = '['
+    for i in range(num):
+        if i < round(percent/int((100/num))):
+            bar += '#'
+        else:
+            bar += '-'
+    bar += ']'
+    return bar
+
+def fill_str(string,num):
+    if len(string)>num:
+        return string[:num-3]+'...'
+    for i in range(num-len(string)):
+        string +=' '
+    return string
+
+def main():
+    while(1):
+        #cpu
+        cpu_used = get_cpu_use()
+        cpu_freq = get_cpu_freq()
+        cpu_temp = get_cpu_temp()
+        cpu_used_bar = get_bar(cpu_used,num=50)
+
+        #memory
+        mem_total,mem_used,mem_percent = get_mem_use()
+        mem_used_bar = get_bar(mem_percent)
+        swap_total,swap_used,swap_percent = get_swap_use()
+        swap_used_bar = get_bar(swap_percent)
+
+        #gpu
+        gpu_infoss = [];util_used_bars=[];gpu_mem_bars=[]
+        for i in range(len(gpus)):
+            gpu_infos = get_gpu_use()
+            gpu_infoss.append(gpu_infos)
+            util_used_bars.append(get_bar(gpu_infos[0][6]))
+            gpu_mem_bars.append(get_bar(100*gpu_infos[0][4]/gpu_infos[0][5]))
+
+        #net
+        net_infos = get_net_use()
+
+        #disk
+        disk_infos = get_disk_use()
+
+        ####################################print#################################
+        os.system('clear')
+        #cpu
+        print(('Cpu   Temp: {0:.1f}C | Freq: {1:.1f}MHz').format(cpu_temp,cpu_freq))
+        print(('Util: {0:.1f}%  '+cpu_used_bar).format(cpu_used))
+
+        #memory
+        print(('\nMemory   Mem: {0:d}MB/{1:d}MB | Swap: {2:d}MB/{3:d}MB').format(
+            mem_used,mem_total,swap_used,swap_total))
+        print(('Mem: {0:.1f}% '+mem_used_bar+' Swap: {1:.1f}% '+swap_used_bar).format(mem_percent,swap_percent))
+
+        #gpu
+        for i in range(len(gpus)):
+            print(('\n'+'Gpu'+'{0:d}'+': '+gpus[i]+'   Temp: {1:.1f}C | Power: {2:d}w/{3:d}w | Mem: {4:d}MB/{5:d}MB | Fan: {6:d}%').format(
+                i,gpu_infoss[i][0][1],gpu_infoss[i][0][2],gpu_infoss[i][0][3],
+                gpu_infoss[i][0][4],gpu_infoss[i][0][5],gpu_infoss[i][0][0]))
+            print(('Util:{0:.1f}% '+util_used_bars[i]+'   Mem:{1:.1f}% '+gpu_mem_bars[i]).format(
+                gpu_infoss[i][0][6],100*gpu_infoss[i][0][4]/gpu_infoss[i][0][5]))
+
+        #net
+        print(('\nNetwork    ↑ all:{0:.1f}GB ↓ all:{1:.1f}GB     ↑ :{2:.1f}Kb/s ↓ :{3:.1f}Kb/s').format(
+            net_infos[1]/1024/1024,net_infos[0]/1024/1024,net_infos[3],net_infos[2]))
+        
+        #disk
+        print('\nFilesystem           Mounted on           Used/Total           Used%')
+        for disk_info in disk_infos:
+            print(fill_str(disk_info[0], 20),fill_str(disk_info[5], 20),
+                fill_str(disk_info[2]+'/'+disk_info[1], 20),fill_str(disk_info[4], 15))
+
+        time.sleep(1)
+
+if __name__ == '__main__':
+    main()
+# print(disk_infos)
+# disk_str = tmp
+# tmp = []
+# for line in disk_str:
+#     info = 
+#         tmp.append(line)
+# print(disk_str)
+
+
+# total = 
+
+# cpu_used = top_str[top_str.find('%Cpu(s):'):top_str.find('%Cpu(s):')+5]
+# print(cpu_used)
+# out_string = os.popen('nvidia-smi').read()
+# print(out_string)
\ No newline at end of file