diff --git a/flink-addons/flink-streaming/src/test/resources/Performance/PerformanceTracker.py b/flink-addons/flink-streaming/src/test/resources/Performance/PerformanceTracker.py index ad15e1b3504cf39e96d2f622ac1f99d010865371..423daa83e8c72ff35471035f268f66d54e5fd09e 100755 --- a/flink-addons/flink-streaming/src/test/resources/Performance/PerformanceTracker.py +++ b/flink-addons/flink-streaming/src/test/resources/Performance/PerformanceTracker.py @@ -10,112 +10,107 @@ import pandas as pd import os import operator - linestyles = ['_', '-', '--', ':'] markers=['D','s', '|', '', 'x', '_', '^', ' ', 'd', 'h', '+', '*', ',', 'o', '.', '1', 'p', 'H', 'v', '>']; colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k'] def readFiles(csv_dir): - dataframes=[] - - + counters=[] + for fname in os.listdir(csv_dir): if '.csv' in fname: - dataframes.append((fname.rstrip('.csv'),int(fname.rstrip('.csv').split('-')[-1])-1,pd.read_csv(os.path.join(csv_dir,fname),index_col='Time'))) - return dataframes + counters.append((fname.rstrip('.csv'),int(fname.rstrip('.csv').split('-')[-1])-1,pd.read_csv(os.path.join(csv_dir,fname),index_col='Time'))) + return counters -def plotCounter(csv_dir,name='', smooth=5): - dataframes= readFiles(csv_dir) - +def plotCounter(csv_dir, sname='', smooth=5,savePath=''): + counters= readFiles(csv_dir) + addSpeed(counters) - for dataframe in dataframes: - df=dataframe[2] - speed=[0] - values=list(df.ix[:,0]) - for i in range(1,len(values)): - speed.append(float(values[i]-values[i-1])/float(df.index[i]-df.index[i-1]+0.01)) - df['speed']=speed - plt.figure(figsize=(12, 8), dpi=80) - plt.title('Counter') + - if name=='': - for dataframe in dataframes: - - m=markers[dataframe[1]%len(markers)] - - dataframe[2].ix[:,0].plot(marker=m,markevery=10,markersize=10) - plt.legend([x[0] for x in dataframes]) - + selectedCounters=[] + for (name, number, df) in counters: + if sname in name: + selectedCounters.append((name, number, df)) + if sname=='': + sname='counters' + save=savePath!='' + + plotDfs(selectedCounters,smooth,save,savePath+'/'+sname) + +def plotDfs(counters,smooth,save,saveFile): plt.figure(figsize=(12, 8), dpi=80) - plt.title('dC/dT') - - for dataframe in dataframes: - - m=markers[dataframe[1]%len(markers)] + plt.title('Counter') + for (name, number, df) in counters: - pd.rolling_mean(dataframe[2].speed,smooth).plot(marker=m,markevery=10,markersize=10) - plt.legend([x[0] for x in dataframes]) - else: - df2=[] - for dataframe in dataframes: - if name in dataframe[0]: - df2.append(dataframe) - - - for dataframe in df2: - - m=markers[dataframe[1]%len(markers)] + m=markers[number%len(markers)] - dataframe[2].ix[:,0].plot(marker=m,markevery=10,markersize=10) - plt.legend([x[0] for x in df2]) - + df.ix[:,0].plot(marker=m,markevery=10,markersize=10) + plt.legend([x[0] for x in counters]) + if save: + plt.savefig(saveFile+'C.png') + + plt.figure(figsize=(12, 8), dpi=80) plt.title('dC/dT') + - for dataframe in df2: + for (name, number, df) in counters: - m=markers[dataframe[1]%len(markers)] + m=markers[number%len(markers)] - pd.rolling_mean(dataframe[2].speed,smooth).plot(marker=m,markevery=10,markersize=10) - plt.legend([x[0] for x in df2]) - -def plotThroughput(csv_dir,taskname, smooth=5): - dataframes= readFiles(csv_dir) - - for dataframe in dataframes: - df=dataframe[2] - speed=[0] - values=list(df.ix[:,0]) - for i in range(1,len(values)): - speed.append(float(values[i]-values[i-1])/float(df.index[i]-df.index[i-1]+0.01)) - df['speed']=speed - - selected={} - - for df in dataframes: - if taskname in df[0]: - if df[1] in selected: - selected[df[1]].append(df[2]) - else: - selected[df[1]]=[df[2]] - plt.figure() - plt.title(taskname) - for i in selected: - selected[i]=reduce(operator.add,selected[i]) - m=markers[i%len(markers)] - selected[i].ix[:,0].plot(marker=m,markevery=10,markersize=10) - - - plt.legend(selected.keys()) - - plt.figure() - plt.title(taskname+" - dC/dT") - for i in selected: - m=markers[i%len(markers)] - pd.rolling_mean(selected[i].speed,smooth).plot(marker=m,markevery=10,markersize=10) + pd.rolling_mean(df.speed,smooth).plot(marker=m,markevery=10,markersize=10) + plt.legend([x[0] for x in counters]) + if save: + plt.savefig(saveFile+'D.png') + +def addSpeed(counters): + for (tname, number, df) in counters: + speed=[0] + values=list(df.ix[:,0]) + for i in range(1,len(values)): + speed.append(float(values[i]-values[i-1])/float(df.index[i]-df.index[i-1]+0.01)) + df['speed']=speed + return counters - plt.legend(selected.keys()) +def plotThroughput(csv_dir,tasknames, smooth=5,savePath=''): + if type(tasknames)!=list: + tasknames=[tasknames] + for taskname in tasknames: + counters= readFiles(csv_dir) + addSpeed(counters) + selected={} + + for (tname, number, df) in counters: + if taskname in tname: + if number in selected: + selected[number].append(df) + else: + selected[number]=[df] + plt.figure() + plt.title(taskname) + for i in selected: + if len(selected[i])>1: + selected[i]=reduce(operator.add,selected[i]) + else: + selected[i]=selected[i][0] + m=markers[i%len(markers)] + selected[i].ix[:,0].plot(marker=m,markevery=10,markersize=10) + + + plt.legend(selected.keys()) + if savePath !='': + plt.savefig(savePath+'/'+taskname+'C.png') + plt.figure() + plt.title(taskname+" - dC/dT") + for i in selected: + m=markers[i%len(markers)] + pd.rolling_mean(selected[i].speed,smooth).plot(marker=m,markevery=10,markersize=10) + + plt.legend(selected.keys()) + if savePath !='': + plt.savefig(savePath+'/'+taskname+'D.png') def plotTimer(csv_dir,smooth=5,std=50): dataframes= readFiles(csv_dir)