Add PlayMusic

.gitignore

@@ -129,6 +129,7 @@ dmypy.json
 .pyre/
 
 # myrules
+tmp/
 /DualVectorFoil
 /GuiChu
 /SkyScreen

PlayMusic/README.md

+# 根据简谱自动生成音乐
+## Example :TwinkleTwinkleLittleStar.txt
+```python
+D   　　　 　#调号
+2/4　　　　#节拍
+108　　　　#播放速度
+0.8　　　　#每个音符的相对持续时间
+1,1　             #　do ,１拍
+1,1　             #　do ,１拍
+5,1　             #　so,１拍
+5,1　             #　so,１拍
+6,1　             #　la,１拍
+6,1　             #　la,１拍
+5,1　             #　so,１拍
+1,0.5　         #　do ,0.5拍
+-,1　             #　-,１拍
+1+　             #　升8度do,１拍
+1--　             #　降两个8度do,１拍
+```
\ No newline at end of file

PlayMusic/TwinkleTwinkleLittleStar.txt

+D
+2/4
+108
+0.8
+1,1
+1,1
+5,1
+5,1
+6,1
+6,1
+5,1
+-,1
+4,1
+4,1
+3,1
+3,0.5,3,0.5
+2,1
+2,1
+1,1
+-,1
+5,1
+5,1
+4,1
+4,1
+3,1
+3,1
+2,1
+-,1
+5,1
+5,1
+4,1
+4,1
+3,1
+3,1
+2,1
+-,1
+1,1
+1,1
+5,1
+5,1
+6,1
+6,1
+5,1
+-,1
+4,1
+4,1
+3,1
+3,0.5
+3,0.5
+2,1
+2,1
+1,1
+-,1
\ No newline at end of file

PlayMusic/notation.py

+import os
+import sys
+import random
+import time
+import numpy as np
+import matplotlib.pylab as plt
+
+sys.path.append("..")
+from Util import util,ffmpeg,dsp,sound
+from Util import array_operation as arrop
+
+
+"""音调对应的频率
+* C大调中就是1=C中C唱做do,而在1=D中D要唱作do
+* C D E F G A B -> do re mi fa so la xi
+* 暂时不支持C# D# F# A#
+"""
+NOTES = ['C','D','E','F','G','A','B']
+base_pitch = [16.352,18.354,20.602,21.827,24.500,27.500,30.868]
+#Frequency in hertz (semitones above or below middle C)
+OPS = np.zeros(7*10) 
+for i in range(10):
+    for j in range(7):
+        OPS[i*7+j] = base_pitch[j]*(2**i)
+
+def getfreq(note,PN):
+    if note[0] == '-':
+        return 0
+    index = 4*7-1 + NOTES.index(PN) + int(note[0]) #1 = C or D.....
+    if '+' in note:
+        index += (len(note)-1)+7
+    elif '-' in note:
+        index -= (len(note)-1)-7
+    freq = OPS[index]
+    return freq
+
+def wave(f, fs, time, mode='sin'):
+    length = int(fs*time)
+    signal = dsp.wave(f, fs, time, mode)
+    weight = np.zeros(length)
+    weight[0:length//10] = np.hanning(length//10*2)[0:length//10]
+    weight[length//10:] = np.hanning(int(length*0.9*2))[int(length*0.9):]
+    return signal*weight
+
+
+"""拍子强度
+strengths = {'2/4':['++',''],
+             '3/4':['++','',''],
+             '4/4':['++','','+',''],
+             '3/8':['++','',''],
+             '6/8':['++','','','+','','']}
+"""
+
+def getstrength(i,BN,x):
+    if int(BN[0]) == 2:
+        if i%2 == 0:
+            x = x*1.25
+    elif int(BN[0]) == 3:
+        if i%3 == 0:
+            x = x*1.25
+    elif int(BN[0]) == 4:
+        if i%4 == 0:
+            x = x*1.25
+        elif i%4 == 2:
+            x = x*1.125
+    return x
+
+def readscore(path):
+    notations = {}
+    notations['data'] =[]
+    for i,line in enumerate(open(path),0):
+        line = line.strip('\n')
+        if i==0:
+            notations['PN'] = line
+        elif i == 1:
+            notations['BN'] = line
+        elif i == 2:
+            notations['BPM'] = float(line)
+        elif i == 3:
+            notations['PNLT'] = float(line)
+        else:
+            notations['data'].append(line.split(','))
+    return notations
+
+
+def notations2music(notations, mode = 'sin', isplot = False):
+    BPM = notations['BPM']
+    BN = notations['BN']
+    PNLT = notations['PNLT']
+    interval = 60.0/BPM 
+    fs = 44100
+    time = 0
+
+    music = np.zeros(int(fs*(len(notations['data'])+1)*interval))
+
+    for i in range(len(notations['data'])):
+        for j in range(len(notations['data'][i])//2):
+            freq = getfreq(notations['data'][i][j*2],notations['PN'])
+            if freq != 0:
+                music[int(time*fs):int(time*fs)+int(PNLT*fs)] += getstrength(i,BN,wave(freq, fs, PNLT, mode = mode))
+            
+            if isplot:
+                plot_data = music[int(time*fs):int(time*fs)+int(PNLT*fs)]
+                plt.clf()
+                plt.subplot(221)
+                plt.plot(np.linspace(0, len(plot_data)/fs,len(plot_data)),plot_data)
+                plt.title('Current audio waveform')
+                plt.xlabel('Time')
+                plt.ylim((-1.5,1.5))
+
+                plt.subplot(222)
+                _plot_data = plot_data[int(len(plot_data)*0.2):int(len(plot_data)*0.25)]
+                plt.plot(np.linspace(0, len(_plot_data)/fs,len(_plot_data)),_plot_data)
+                plt.title('Partial audio waveform')
+                plt.xlabel('Time')
+                plt.ylim((-1.5,1.5))
+
+                plt.subplot(223)
+                f,k = dsp.showfreq(plot_data, 44100, 2000)
+                plt.plot(f,k)
+                plt.title('FFT')
+                plt.xlabel('Hz')
+                plt.ylim((-1000,10000))
+                plt.pause(interval-0.125)
+
+            time += float(notations['data'][i][j*2+1])*interval
+
+    return (arrop.sigmoid(music)-0.5)*65536
+
+notations = readscore('./TwinkleTwinkleLittleStar.txt')
+print(notations)
+# sin triangle square
+music = notations2music(notations,mode='sin',isplot=False)
+import threading
+t=threading.Thread(target=sound.playtest,args=(music,))
+t.start()
+music = notations2music(notations,mode='sin',isplot=True)

README.md

@@ -11,7 +11,10 @@ B站入口项目仓库分流页
 [[项目地址](https://github.com/HypoX64/ShellPlayer)] [[效果演示:蔡徐坤]](https://www.bilibili.com/video/BV1Ag4y1z7BB) [[效果演示:大威天龙]](https://www.bilibili.com/video/BV1dT4y1u76w)
 
 ## GuiChu —— 鬼畜视频生成器
-[[猫和老鼠X余生一个浪(大威天龙)]](https://www.bilibili.com/video/BV1c54y1Q7RG)
+[[效果演示:猫和老鼠X余生一个浪(大威天龙)]](https://www.bilibili.com/video/BV1c54y1Q7RG)
+
+## 根据简谱自动生成音乐
+[[项目地址](./PlayMusic/README.md)]
 
 ## syseye —— 轻量级深度学习服务器性能监视器
 [[项目地址](https://github.com/HypoX64/syseye)]
\ No newline at end of file

Util/array_operation.py

+import numpy as np
+
+def interp(y,length):
+    xp = np.linspace(0, len(y)-1,num = len(y))
+    fp = y
+    x = np.linspace(0, len(y)-1,num = length)
+    return np.interp(x, xp, fp)
+
+def sigmoid(x):
+    return 1.0 / (1.0 + np.exp(-x));
+
+def pad(data,padding):
+    pad_data = np.zeros(padding, dtype=data.dtype)
+    data = np.append(data, pad_data)
+    return data
+
+def normliaze(data, mode = 'norm', sigma = 0, dtype=np.float32, truncated = 2):
+    '''
+    mode: norm | std | maxmin | 5_95
+    dtype : np.float64,np.float16...
+    '''
+    data = data.astype(dtype)
+    data_calculate = data.copy()
+    if mode == 'norm':
+        result = (data-np.mean(data_calculate))/sigma
+    elif mode == 'std':
+        mu = np.mean(data_calculate)
+        sigma = np.std(data_calculate)
+        result = (data - mu) / sigma
+    elif mode == 'maxmin':
+        result = (data-np.mean(data_calculate))/(max(np.max(data_calculate),np.abs(np.min(data_calculate))))
+    elif mode == '5_95':
+        data_sort = np.sort(data_calculate,axis=None)
+        th5 = data_sort[int(0.05*len(data_sort))]
+        th95 = data_sort[int(0.95*len(data_sort))]
+        baseline = (th5+th95)/2
+        sigma = (th95-th5)/2
+        if sigma == 0:
+            sigma = 1e-06
+        result = (data-baseline)/sigma
+
+    if truncated > 1:
+        result = np.clip(result, (-truncated), (truncated))
+
+    return result.astype(dtype)
+
+def diff1d(indata,stride=1,padding=1,bias=False):
+
+    pad = np.zeros(padding)
+    indata = np.append(indata, pad)
+    if bias:
+        if np.min(indata)<0:
+            indata = indata - np.min(indata)
+
+    outdata = np.zeros(int(len(indata)/stride)-1)
+    for i in range(int(len(indata)/stride)-1):
+        outdata[i]=indata[i*stride+stride]-indata[i*stride]
+    return outdata
+
+
+def findpeak(indata, ismax=False, interval=10, threshold=0.1, reverse=False):
+    '''
+    return:indexs
+    '''
+    indexs = []
+    if reverse:
+        indata = -1*indata
+        indexs = [0,len(indata)-1]
+    else:
+        indata = np.clip(indata, np.max(indata)*threshold, np.max(indata))
+    diff = diff1d(indata)
+    if ismax:
+        return np.array([np.argmax(indata)])
+
+    rise = True
+    if diff[0] <=0:
+        rise = False
+    for i in range(len(diff)):
+        if rise==True and diff[i]<=0:
+            index = i
+            ok_flag = True
+            for x in range(interval):
+                if indata[np.clip(index-x,0,len(indata)-1)]>indata[index] or indata[np.clip(index+x,0,len(indata)-1)]>indata[index]:
+                    ok_flag = False
+            if ok_flag:
+                indexs.append(index)
+
+        if diff[i] <=0:
+            rise = False
+        else:
+            rise = True
+
+    return np.sort(np.array(indexs))
+
+def crop(data,indexs,size):
+    if len(data) < size:
+        data = pad(data, size-len(data))
+
+    crop_result = []
+    for index in indexs:
+        if index-int(size/2)<0:
+            crop_result.append(data[0:size])
+        elif index+int(size/2)<0:
+            crop_result.append(data[len(data)-size:len(data)])
+        else:
+            crop_result.append(data[index-int(size/2):index+int(size/2)])
+    return np.array(crop_result)
+
+def cropbyindex(data,indexs,reverse_indexs):
+    crop_result = []
+    crop_index = []
+    for index in indexs:
+        for i in range(len(reverse_indexs)-1):
+            if reverse_indexs[i] < index:
+                if reverse_indexs[i+1] > index:
+                    crop_result.append(data[reverse_indexs[i]:reverse_indexs[i+1]])
+                    crop_index.append([reverse_indexs[i],reverse_indexs[i+1]])
+    return np.array(crop_result),np.array(crop_index)
+
+def get_crossing(line1,line2):
+    cross_pos = []
+    dif = line1-line2
+    flag = 1
+    if dif[0]<0:
+        dif = -dif        
+    for i in range(int(len(dif))):
+        if flag == 1:
+            if dif[i] <= 0:
+                cross_pos.append(i)
+                flag = 0
+        else:
+            if dif[i] >= 0:
+                cross_pos.append(i)
+                flag = 1
+    return cross_pos
+
+def get_y(indexs,fun):
+    y = []
+    for index in indexs:
+        y.append(fun[index])
+    return np.array(y)
+
+def fillnone(arr_in,flag,num = 7):
+    arr = arr_in.copy()
+    index = np.linspace(0,len(arr)-1,len(arr),dtype='int')
+    cnt = 0
+    for i in range(2,len(arr)-2):
+        if arr[i] != flag:
+            arr[i] = arr[i]
+            if cnt != 0:
+                if cnt <= num*2:
+                    arr[i-cnt:round(i-cnt/2)] = arr[i-cnt-1-2]
+                    arr[round(i-cnt/2):i] = arr[i+2]
+                    index[i-cnt:round(i-cnt/2)] = i-cnt-1-2
+                    index[round(i-cnt/2):i] = i+2
+                else:
+                    arr[i-cnt:i-cnt+num] = arr[i-cnt-1-2]
+                    arr[i-num:i] = arr[i+2] 
+                    index[i-cnt:i-cnt+num] = i-cnt-1-2
+                    index[i-num:i] = i+2
+                cnt = 0
+        else:
+            cnt += 1
+    return arr,index
+
+
+def closest(a,array):
+    '''
+    return:index
+    '''
+
+    corrcoefs = []
+    for i in range(len(array)):
+        #corrcoefs.append(np.linalg.norm(a-array[i], ord=1))
+        corrcoefs.append(np.corrcoef(a,array[i])[0][1])
+    return corrcoefs.index(np.max(corrcoefs))
+
+def match(src,dst):
+    '''
+    return:dstindexs
+    '''
+    indexs = []
+    for i in range(len(src)):
+        indexs.append(closest(src[i], dst))
+    return np.array(indexs)
+
+
+
+def main():
+    a = [0,2,4,6,8,10]
+    print(interp(a, 6))
+if __name__ == '__main__':
+    main()
\ No newline at end of file

Util/dsp.py

+import scipy
+import scipy.signal
+import scipy.fftpack
+import numpy as np
+from .array_operation import *
+import matplotlib.pylab as plt
+
+def sin(f,fs,time,theta=0):
+    x = np.linspace(0, 2*np.pi*f*time, int(fs*time))
+    return np.sin(x+theta)
+
+def wave(f,fs,time,mode='sin'):
+    f,fs,time = float(f),float(fs),float(time)
+    if mode == 'sin':
+        return sin(f,fs,time,theta=0)
+    elif mode == 'square':
+        half_T_num = int(time*f)*2 + 1
+        half_T_point = int(fs/f/2)
+        x = np.zeros(int(fs*time)+2*half_T_point)
+        for i in range(half_T_num):
+            if i%2 == 0:
+                x[i*half_T_point:(i+1)*half_T_point] = -1
+            else:
+                x[i*half_T_point:(i+1)*half_T_point] = 1
+        return x[:int(fs*time)]
+    elif mode == 'triangle':
+        half_T_num = int(time*f)*2 + 1
+        half_T_point = int(fs/f/2)
+        up = np.linspace(-1, 1, half_T_point)
+        down = np.linspace(1, -1, half_T_point)
+        x = np.zeros(int(fs*time)+2*half_T_point)
+        for i in range(half_T_num):
+            if i%2 == 0:
+                x[i*half_T_point:(i+1)*half_T_point] = up.copy()
+            else:
+                x[i*half_T_point:(i+1)*half_T_point] = down.copy()
+        return x[:int(fs*time)]
+
+def downsample(signal,fs1=0,fs2=0,alpha=0,mod = 'just_down'):
+    if alpha ==0:
+        alpha = int(fs1/fs2)
+    if mod == 'just_down':
+        return signal[::alpha]
+    elif mod == 'avg':
+        result = np.zeros(int(len(signal)/alpha))
+        for i in range(int(len(signal)/alpha)):
+            result[i] = np.mean(signal[i*alpha:(i+1)*alpha])
+        return result       
+
+def medfilt(signal,x):
+    return scipy.signal.medfilt(signal,x)
+
+def cleanoffset(signal):
+    return signal - np.mean(signal)
+
+def BPF_FIR(signal,fs,fc1,fc2,numtaps=101):
+    b=scipy.signal.firwin(numtaps, [fc1, fc2], pass_zero=False,fs=fs)
+    result = scipy.signal.lfilter(b, 1, signal)
+    return result
+
+def fft_filter(signal,fs,fc=[],type = 'bandpass'):
+    '''
+    signal: Signal
+    fs: Sampling frequency
+    fc: [fc1,fc2...] Cut-off frequency 
+    type: bandpass | bandstop
+    '''
+    k = []
+    N=len(signal)#get N
+
+    for i in range(len(fc)):
+        k.append(int(fc[i]*N/fs))
+
+    #FFT
+    signal_fft=scipy.fftpack.fft(signal)
+    #Frequency truncation
+
+    if type == 'bandpass':
+        a = np.zeros(N)
+        for i in range(int(len(fc)/2)):
+            a[k[2*i]:k[2*i+1]] = 1
+            a[N-k[2*i+1]:N-k[2*i]] = 1
+    elif type == 'bandstop':
+        a = np.ones(N)
+        for i in range(int(len(fc)/2)):
+            a[k[2*i]:k[2*i+1]] = 0
+            a[N-k[2*i+1]:N-k[2*i]] = 0
+    signal_fft = a*signal_fft
+    signal_ifft=scipy.fftpack.ifft(signal_fft)
+    result = signal_ifft.real
+    return result
+
+def basefreq(signal,fs,fc=0):
+    if fc==0:
+        kc = int(len(signal)/2)
+    else:   
+        kc = int(len(signal)/fs*fc)
+    length = len(signal)
+    signal_fft = np.abs(scipy.fftpack.fft(signal))[:kc]
+    _sum = np.sum(signal_fft)/2
+    tmp_sum = 0
+    for i in range(kc):
+        tmp_sum += signal_fft[i]
+        if tmp_sum > _sum:
+            return i/(length/fs)
+
+def showfreq(signal,fs,fc=0):
+    if fc==0:
+        kc = int(len(signal)/2)
+    else:   
+        kc = int(len(signal)/fs*fc)
+    signal_fft = np.abs(scipy.fftpack.fft(signal))
+    f = np.linspace(0,fs/2,num=int(len(signal_fft)/2))
+    return f[:kc],signal_fft[0:int(len(signal_fft)/2)][:kc]
+
+def rms(signal):
+    signal = signal.astype('float64')
+    return np.mean((signal*signal))**0.5
+
+def energy(signal,kernel_size,stride,padding = 0):
+    _signal = np.zeros(len(signal)+padding)
+    _signal[0:len(signal)] = signal
+    signal = _signal
+    out_len = int((len(signal)+1-kernel_size)/stride)
+    energy = np.zeros(out_len)
+    for i in range(out_len):
+        energy[i] = rms(signal[i*stride:i*stride+kernel_size]) 
+    return energy
+
+def envelope_demodulation(signal,kernel_size,alpha = 0.9,mod='max'):
+    out_len = int(len(signal)/kernel_size)
+    envelope = np.zeros(out_len)
+    for i in range(out_len):
+        # envelope[i] = np.max(signal[i*kernel_size:(i+1)*kernel_size])
+        envelope[i] = np.sort(signal[i*kernel_size:(i+1)*kernel_size])[int(alpha*kernel_size)]
+    return envelope
+
+def main():
+    print(downsample(piano,alpha=9))
+if __name__ == '__main__':
+    main()
+

Util/ffmpeg.py

+import os,json
+import subprocess
+
+# ffmpeg 3.4.6
+
+def args2cmd(args):
+    cmd = ''
+    for arg in args:
+        cmd += (arg+' ')
+    return cmd
+
+def run(args,mode = 0):
+
+    if mode == 0:
+        cmd = args2cmd(args)
+        os.system(cmd)
+
+    elif mode == 1:
+        '''
+        out_string = os.popen(cmd_str).read()
+        For chinese path in Windows
+        https://blog.csdn.net/weixin_43903378/article/details/91979025
+        '''
+        cmd = args2cmd(args)
+        stream = os.popen(cmd)._stream
+        sout = stream.buffer.read().decode(encoding='utf-8')
+        return sout
+
+    elif mode == 2:
+        p = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout = p.stdout.readlines()
+        return sout
+
+
+def video2image(videopath, imagepath, fps=0, start_time=0, last_time=0):
+    args = ['ffmpeg', '-i', '"'+videopath+'"']
+    if last_time!=0:
+        args += ['-ss', start_time]
+        args += ['-t', last_time]
+    if fps != 0:
+        args += ['-r', fps]
+    args += ['-f', 'image2','-q:v','-0',imagepath]
+    run(args)
+        
+def video2voice(videopath,voicepath,samplingrate=0):
+    args = ['ffmpeg', '-i', '"'+videopath+'"']
+    if samplingrate != 0:
+        args += ['-ar', str(samplingrate)]
+    args += [voicepath]
+    run(args)
+
+def image2video(fps,imagepath,voicepath,videopath):
+    os.system('ffmpeg -y -r '+str(fps)+' -i '+imagepath+' -vcodec libx264 '+'./tmp/video_tmp.mp4')
+    os.system('ffmpeg -i ./tmp/video_tmp.mp4 -i "'+voicepath+'" -vcodec copy -acodec copy '+videopath)
+
+def get_video_infos(videopath):
+    args =  ['ffprobe -v quiet -print_format json -show_format -show_streams', '-i', '"'+videopath+'"']
+
+    out_string = run(args,mode=1)
+
+    infos = json.loads(out_string)
+    try:
+        fps = eval(infos['streams'][0]['avg_frame_rate'])
+        endtime = float(infos['format']['duration'])
+        width = int(infos['streams'][0]['width'])
+        height = int(infos['streams'][0]['height'])
+    except Exception as e:
+        fps = eval(infos['streams'][1]['r_frame_rate'])
+        endtime = float(infos['format']['duration'])
+        width = int(infos['streams'][1]['width'])
+        height = int(infos['streams'][1]['height'])
+
+    return fps,endtime,height,width
+
+def cut_video(in_path,start_time,last_time,out_path,vcodec='h265'):
+    if vcodec == 'copy':
+        os.system('ffmpeg -ss '+start_time+' -t '+last_time+' -i "'+in_path+'" -vcodec copy -acodec copy '+out_path)
+    elif vcodec == 'h264':    
+        os.system('ffmpeg -ss '+start_time+' -t '+last_time+' -i "'+in_path+'" -vcodec libx264 -b 12M '+out_path)
+    elif vcodec == 'h265':
+        os.system('ffmpeg -ss '+start_time+' -t '+last_time+' -i "'+in_path+'" -vcodec libx265 -b 12M '+out_path)
+
+def continuous_screenshot(videopath,savedir,fps):
+    '''
+    videopath: input video path
+    savedir:   images will save here
+    fps:       save how many images per second
+    '''
+    videoname = os.path.splitext(os.path.basename(videopath))[0]
+    os.system('ffmpeg -i "'+videopath+'" -vf fps='+str(fps)+' -q:v -0 '+savedir+'/'+videoname+'_%06d.jpg')

Util/image_processing.py

+import cv2
+import numpy as np
+import random
+
+import platform
+system_type = 'Linux'
+if 'Windows' in platform.platform():
+    system_type = 'Windows'
+
+def imread(file_path,mod = 'normal'):
+    '''
+    mod = 'normal' | 'gray' | 'all'
+    '''
+    if system_type == 'Linux':
+        if mod == 'normal':
+            img = cv2.imread(file_path)
+        elif mod == 'gray':
+            img = cv2.imread(file_path,0)
+        elif mod == 'all':
+            img = cv2.imread(file_path,-1)
+    
+    #For chinese path, use cv2.imdecode in windows.
+    #It will loss EXIF, I can't fix it
+    else: 
+        if mod == 'gray':
+            img = cv2.imdecode(np.fromfile(file_path,dtype=np.uint8),0)
+        else:
+            img = cv2.imdecode(np.fromfile(file_path,dtype=np.uint8),-1)
+
+    return img
+
+def imwrite(file_path,img):
+    '''
+    in other to save chinese path images in windows,
+    this fun just for save final output images
+    '''
+    if system_type == 'Linux':
+        cv2.imwrite(file_path, img)
+    else:
+        cv2.imencode('.jpg', img)[1].tofile(file_path)
+
+def resize(img,size,interpolation=cv2.INTER_LINEAR):
+    h, w = img.shape[:2]
+    if np.min((w,h)) ==size:
+        return img
+    if w >= h:
+        res = cv2.resize(img,(int(size*w/h), size),interpolation=interpolation)
+    else:
+        res = cv2.resize(img,(size, int(size*h/w)),interpolation=interpolation)
+    return res
+
+def resize_like(img,img_like):
+    h, w = img_like.shape[:2]
+    img = cv2.resize(img, (w,h))
+    return img
+
+def ch_one2three(img):
+    #zeros = np.zeros(img.shape[:2], dtype = "uint8")
+    # ret,thresh = cv2.threshold(img,127,255,cv2.THRESH_BINARY)
+    res = cv2.merge([img, img, img])
+    return res
+
+def color_adjust(img,alpha=1,beta=0,b=0,g=0,r=0,ran = False):
+    '''
+    g(x) = (1+α)g(x)+255*β, 
+    g(x) = g(x[:+b*255,:+g*255,:+r*255])
+    
+    Args:
+        img   : input image
+        alpha : contrast
+        beta  : brightness
+        b     : blue hue
+        g     : green hue
+        r     : red hue
+        ran   : if True, randomly generated color correction parameters
+    Retuens:
+        img   : output image
+    '''
+    img = img.astype('float')
+    if ran:
+        alpha = random.uniform(-0.2,0.2)
+        beta  = random.uniform(-0.2,0.2)
+        b     = random.uniform(-0.1,0.1)
+        g     = random.uniform(-0.1,0.1)
+        r     = random.uniform(-0.1,0.1)
+    img = (1+alpha)*img+255.0*beta
+    bgr = [b*255.0,g*255.0,r*255.0]
+    for i in range(3): img[:,:,i]=img[:,:,i]+bgr[i]
+    
+    return (np.clip(img,0,255)).astype('uint8')
+
+def makedataset(target_image,orgin_image):
+    target_image = resize(target_image,256)
+    orgin_image = resize(orgin_image,256)
+    img = np.zeros((256,512,3), dtype = "uint8")
+    w = orgin_image.shape[1]
+    img[0:256,0:256] = target_image[0:256,int(w/2-256/2):int(w/2+256/2)]
+    img[0:256,256:512] = orgin_image[0:256,int(w/2-256/2):int(w/2+256/2)]
+    return img
+
+def image2folat(img,ch):
+    size=img.shape[0]
+    if ch == 1:
+        img = (img[:,:,0].reshape(1,size,size)/255.0).astype(np.float32)
+    else:
+        img = (img.transpose((2, 0, 1))/255.0).astype(np.float32)
+    return img
+
+def spiltimage(img,size = 128):
+    h, w = img.shape[:2]
+    # size = min(h,w)
+    if w >= h:
+        img1 = img[:,0:size]
+        img2 = img[:,w-size:w]
+    else:
+        img1 = img[0:size,:]
+        img2 = img[h-size:h,:]
+
+    return img1,img2
+
+def mergeimage(img1,img2,orgin_image,size = 128):
+    h, w = orgin_image.shape[:2]
+    new_img1 = np.zeros((h,w), dtype = "uint8")
+    new_img2 = np.zeros((h,w), dtype = "uint8")
+
+    # size = min(h,w)
+    if w >= h:
+        new_img1[:,0:size]=img1
+        new_img2[:,w-size:w]=img2
+    else:
+        new_img1[0:size,:]=img1
+        new_img2[h-size:h,:]=img2
+    result_img = cv2.add(new_img1,new_img2)
+    return result_img
+
+def find_best_ROI(mask):
+    contours,hierarchy=cv2.findContours(mask, cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
+    if len(contours)>0:
+        areas = []
+        for contour in contours:
+            areas.append(cv2.contourArea(contour))
+        index = areas.index(max(areas))
+        mask = np.zeros_like(mask)
+        mask = cv2.fillPoly(mask,[contours[index]],(255))
+    return mask
+
+def boundingSquare(mask,Ex_mul):
+    # thresh = mask_threshold(mask,10,threshold)
+    area = mask_area(mask)
+    if area == 0 :
+        return 0,0,0,0
+
+    x,y,w,h = cv2.boundingRect(mask)
+    
+    center = np.array([int(x+w/2),int(y+h/2)])
+    size = max(w,h)
+    point0=np.array([x,y])
+    point1=np.array([x+size,y+size])
+
+    h, w = mask.shape[:2]
+    if size*Ex_mul > min(h, w):
+        size = min(h, w)
+        halfsize = int(min(h, w)/2)
+    else:
+        size = Ex_mul*size
+        halfsize = int(size/2)
+        size = halfsize*2
+    point0 = center - halfsize
+    point1 = center + halfsize
+    if point0[0]<0:
+        point0[0]=0
+        point1[0]=size
+    if point0[1]<0:
+        point0[1]=0
+        point1[1]=size
+    if point1[0]>w:
+        point1[0]=w
+        point0[0]=w-size
+    if point1[1]>h:
+        point1[1]=h
+        point0[1]=h-size
+    center = ((point0+point1)/2).astype('int')
+    return center[0],center[1],halfsize,area
+
+def mask_threshold(mask,blur,threshold):
+    mask = cv2.threshold(mask,threshold,255,cv2.THRESH_BINARY)[1]
+    mask = cv2.blur(mask, (blur, blur))
+    mask = cv2.threshold(mask,threshold/5,255,cv2.THRESH_BINARY)[1]
+    return mask
+
+def mask_area(mask):
+    mask = cv2.threshold(mask,127,255,0)[1]
+    # contours= cv2.findContours(mask,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)[1] #for opencv 3.4
+    contours= cv2.findContours(mask,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)[0]#updata to opencv 4.0
+    try:
+        area = cv2.contourArea(contours[0])
+    except:
+        area = 0
+    return area
+
+
+def replace_mosaic(img_origin,img_fake,x,y,size,no_father):
+    img_fake = resize(img_fake,size*2)
+    if no_father:
+        img_origin[y-size:y+size,x-size:x+size]=img_fake
+        img_result = img_origin
+    else:
+        #color correction
+        RGB_origin = img_origin[y-size:y+size,x-size:x+size].mean(0).mean(0)
+        RGB_fake = img_fake.mean(0).mean(0)
+        for i in range(3):img_fake[:,:,i] = np.clip(img_fake[:,:,i]+RGB_origin[i]-RGB_fake[i],0,255)      
+        #eclosion
+        eclosion_num = int(size/5)
+        entad = int(eclosion_num/2+2)
+        mask = np.zeros(img_origin.shape, dtype='uint8')
+        mask = cv2.rectangle(mask,(x-size+entad,y-size+entad),(x+size-entad,y+size-entad),(255,255,255),-1)
+        mask = (cv2.blur(mask, (eclosion_num, eclosion_num)))
+        mask = mask/255.0
+
+        img_tmp = np.zeros(img_origin.shape)
+        img_tmp[y-size:y+size,x-size:x+size]=img_fake
+        img_result = img_origin.copy()
+        img_result = (img_origin*(1-mask)+img_tmp*mask).astype('uint8')
+    return img_result
\ No newline at end of file

Util/sound.py

+import time
+import numpy as np
+import scipy.fftpack
+from .array_operation import *
+from .dsp import *
+from .util import *
+import librosa
+from scipy.io import wavfile
+import os
+import matplotlib.pylab as plt
+
+piano = np.array([0,27.5,29.1,30.9,32.7,34.6,36.7,38.9,41.2,
+                43.7,46.2,49.0,51.9,55.0,58.3,61.7,65.4,69.3,
+                73.4,77.8,82.4,87.3,92.5,98.0,103.8,110.0,116.5,
+                123.5,130.8,138.6,146.8,155.6,164.8,174.6,185.0,
+                196.0,207.7,220.0,233.1,246.9,261.6,277.2,293.7,
+                311.1,329.6,349.2,370.0,392.0,415.3,440.0,466.2,
+                493.9,523.3,554.4,587.3,622.3,659.3,698.5,740.0,
+                784.0,830.6,880.0,932.3,987.8,1047,1109,1175,1245,
+                1319,1397,1480,1568,1661,1760,1865,1976,2093,2217,
+                2349,2489,2637,2794,2960,3136,3322,3520,3729,3951,4186,4400])
+
+piano_10 = np.array([0,73.4,207.7,349.2,587.3,987.8,1245,1661,2093,2794,4400])
+
+#------------------------------IO------------------------------
+def numpy2voice(npdata):
+    voice = np.zeros((len(npdata),2))
+    voice[:,0] = npdata
+    voice[:,1] = npdata
+    return voice
+
+def load(path,ch = 0):
+    freq,audio = wavfile.read(path)
+    if ch == 0:
+        audio = audio[:,0]
+    elif ch == 1:
+        audio = audio[:,1]
+    return freq,audio.astype(np.float64)
+
+def write(npdata,path='./tmp/test_output.wav',freq = 44100):
+    voice = numpy2voice(npdata)
+    wavfile.write(path, freq, voice.astype(np.int16))
+
+def play(path):
+    os.system("paplay "+path)
+
+def playtest(npdata,freq = 44100):
+    time.sleep(0.5)
+    makedirs('./tmp/')
+    voice = numpy2voice(npdata)
+    wavfile.write('./tmp/test_output.wav', freq, voice.astype(np.int16))
+    play('./tmp/test_output.wav')
+
+
+#------------------------------DSP------------------------------
+
+def filter(audio,fc=[],fs=44100,win_length = 4096):
+    for i in range(len(audio)//win_length):
+        audio[i*win_length:(i+1)*win_length] = fft_filter(audio[i*win_length:(i+1)*win_length], fs=fs, fc=fc)
+    
+    return audio
+
+# def freq_correct(src,dst,fs=44100,alpha = 0.05,fc=3000):
+#     src_freq = basefreq(src, 44100,3000)
+#     dst_freq = basefreq(dst, 44100,3000)
+#     offset = int((src_freq-dst_freq)/(src_freq*0.05))
+#     out = librosa.effects.pitch_shift(dst.astype(np.float64), 44100, n_steps=offset)
+#     #print('freqloss:',round((basefreq(out, 44100,3000)-basefreq(src, 44100,3000))/basefreq(src, 44100,3000),3))
+#     return out.astype(np.int16)
+
+def freq_correct(src,dst,fs=44100,fc=3000,mode = 'normal',win_length = 1024, alpha = 1):
+    
+    out = np.zeros_like(src)
+    try:
+        if mode == 'normal':
+            src_oct = librosa.hz_to_octs(basefreq(src, fs, fc))
+            dst_oct = librosa.hz_to_octs(basefreq(dst, fs, fc))
+            offset = (dst_oct-src_oct)*12*alpha
+            out = librosa.effects.pitch_shift(src, 44100, n_steps=offset)
+        elif mode == 'track':
+            length = min([len(src),len(dst)])
+            for i in range(length//win_length):
+                src_oct = librosa.hz_to_octs(basefreq(src[i*win_length:(i+1)*win_length], fs, fc))
+                dst_oct = librosa.hz_to_octs(basefreq(dst[i*win_length:(i+1)*win_length], fs, fc))
+                
+                offset = (dst_oct-src_oct)*12*alpha
+                out[i*win_length:(i+1)*win_length] = librosa.effects.pitch_shift(src[i*win_length:(i+1)*win_length], 44100, n_steps=offset)
+        return out
+    except Exception as e:
+        return src
+
+    #print('freqloss:',round((basefreq(out, 44100,3000)-basefreq(src, 44100,3000))/basefreq(src, 44100,3000),3))
+    
+
+def energy_correct(src,dst,mode = 'normal',win_length = 512,alpha=1):
+    """
+    mode: normal | track
+    """
+    out = np.zeros_like(src)
+    if mode == 'normal':
+        src_rms = rms(src)
+        dst_rms = rms(dst)
+        out = src*(dst_rms/src_rms)*alpha
+    elif mode == 'track':
+        length = min([len(src),len(dst)])
+        tracks = []
+        for i in range(length//win_length):
+            src_rms = np.clip(rms(src[i*win_length:(i+1)*win_length]),1e-6,np.inf)
+            dst_rms = rms(dst[i*win_length:(i+1)*win_length])
+            tracks.append((dst_rms/src_rms)*alpha)
+        tracks = np.clip(np.array(tracks),0.1,10)
+        tracks = interp(tracks, length)
+        out = src*tracks
+
+    return np.clip(out,-32760,32760)
+
+def time_correct(src,dst,_min=0.25):
+    src_time = len(src)
+    dst_time = len(dst)
+    rate = np.clip(src_time/dst_time,_min,100)
+    out = librosa.effects.time_stretch(src,rate)
+    return out
+
+def freqfeatures(signal,fs):
+    signal = normliaze(signal,mode = '5_95',truncated=100)
+    signal_fft = np.abs(scipy.fftpack.fft(signal))
+    length = len(signal)
+    features = []
+    for i in range(len(piano_10)-1):
+        k1 = int(length/fs*piano_10[i])
+        k2 = int(length/fs*piano_10[i+1])
+        features.append(np.mean(signal_fft[k1:k2]))
+    return np.array(features)
+
+
+
+def main():
+    xp = [1, 2, 3]
+    fp = [3, 2, 0]
+    x = [0, 1, 1.5, 2.72, 3.14]
+    print(np.interp(x, xp, fp))
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

Util/util.py

+import os
+import shutil
+def Traversal(filedir):
+    file_list=[]
+    for root,dirs,files in os.walk(filedir): 
+        for file in files:
+            file_list.append(os.path.join(root,file)) 
+        for dir in dirs:
+            Traversal(dir)
+    return file_list
+
+def is_img(path):
+    ext = os.path.splitext(path)[1]
+    ext = ext.lower()
+    if ext in ['.jpg','.png','.jpeg','.bmp']:
+        return True
+    else:
+        return False
+
+def is_video(path):
+    ext = os.path.splitext(path)[1]
+    ext = ext.lower()
+    if ext in ['.mp4','.flv','.avi','.mov','.mkv','.wmv','.rmvb','.mts']:
+        return True
+    else:
+        return False
+
+def is_imgs(paths):
+    tmp = []
+    for path in paths:
+        if is_img(path):
+            tmp.append(path)
+    return tmp
+
+def is_videos(paths):
+    tmp = []
+    for path in paths:
+        if is_video(path):
+            tmp.append(path)
+    return tmp  
+
+def  writelog(path,log):
+    f = open(path,'a+')
+    f.write(log+'\n')
+    f.close()
+
+
+def makedirs(path):
+    if os.path.isdir(path):
+        print(path,'existed')
+    else:
+        os.makedirs(path)
+        print('makedir:',path)
+
+def clean_tempfiles(tmp_init=True):
+    if os.path.isdir('./tmp'):   
+        shutil.rmtree('./tmp')
+    if tmp_init:
+        os.makedirs('./tmp')
+        os.makedirs('./tmp/video_voice')
+        os.makedirs('./tmp/music/')
+        os.makedirs('./tmp/video_imgs')
+        os.makedirs('./tmp/output_imgs')
+
+def file_init(opt):
+    if not os.path.isdir(opt.result_dir):
+        os.makedirs(opt.result_dir)
+        print('makedir:',opt.result_dir)
+    clean_tempfiles()
+
+def get_bar(percent,num = 25):
+    bar = '['
+    for i in range(num):
+        if i < round(percent/(100/num)):
+            bar += '#'
+        else:
+            bar += '-'
+    bar += ']'
+    return bar+' '+str(round(percent,2))+'%'
+
+def copyfile(src,dst):
+    try:
+        shutil.copyfile(src, dst)
+    except Exception as e:
+        print(e)
+
+def second2stamp(s):
+    floats = s
+    h = int(s/3600)
+    s = int(s%3600)
+    m = int(s/60)
+    s = int(s%60)
+    floats = floats - int(floats) + s
+
+    return "%02d:%02d:%.3f" % (h, m, floats)