提交 09fca5a6 编写于 作者: Z zhxfl

revise by review

上级 52cda823
"""This model read the sample from disk.
use multiprocessing to reading samples
push samples from one block to multiprocessing queue
Todos:
1. multiprocess read block from disk
"""
import random
import Queue
import numpy as np
import struct
import data_utils.trans_mean_variance_norm as trans_mean_variance_norm
import data_utils.trans_add_delta as trans_add_delta
class OneBlock(object):
""" struct for one block :
contain label, label desc, feature, feature_desc
Attributes:
label(str) : label path of one block
label_desc(str) : label description path of one block
feature(str) : feature path of on block
feature_desc(str) : feature description path of on block
"""
def __init__(self):
"""the constructor."""
self.label = "label"
self.label_desc = "label_desc"
self.feature = "feature"
self.feature_desc = "feature_desc"
class DataRead(object):
"""
Attributes:
_lblock(obj:`OneBlock`) : the list of OneBlock
_ndrop_sentence_len(int): dropout the sentence which's frame_num large than _ndrop_sentence_len
_que_sample(obj:`Queue`): sample buffer
_nframe_dim(int): the batch sample frame_dim(todo remove)
_nstart_block_idx(int): the start block id
_nload_block_num(int): the block num
"""
def __init__(self, sfeature_lst, slabel_lst, ndrop_sentence_len=512):
"""
Args:
sfeature_lst(str):feature lst path
slabel_lst(str):label lst path
Returns:
None
"""
self._lblock = []
self._ndrop_sentence_len = ndrop_sentence_len
self._que_sample = Queue.Queue()
self._nframe_dim = 120 * 11
self._nstart_block_idx = 0
self._nload_block_num = 1
self._ndrop_frame_len = 256
self._load_list(sfeature_lst, slabel_lst)
def _load_list(self, sfeature_lst, slabel_lst):
""" load list and shuffle
Args:
sfeature_lst(str):feature lst path
slabel_lst(str):label lst path
Returns:
None
"""
lfeature = open(sfeature_lst).readlines()
llabel = open(slabel_lst).readlines()
assert len(llabel) == len(lfeature)
for i in range(0, len(lfeature), 2):
one_block = OneBlock()
one_block.label = llabel[i]
one_block.label_desc = llabel[i + 1]
one_block.feature = lfeature[i]
one_block.feature_desc = lfeature[i + 1]
self._lblock.append(one_block)
random.shuffle(self._lblock)
def _load_one_block(self, lsample, id):
"""read one block by id and push load sample in list lsample
Args:
lsample(list): return sample list
id(int): block id
Returns:
None
"""
if id >= len(self._lblock):
return
slabel_path = self._lblock[id].label.strip()
slabel_desc_path = self._lblock[id].label_desc.strip()
sfeature_path = self._lblock[id].feature.strip()
sfeature_desc_path = self._lblock[id].feature_desc.strip()
llabel_line = open(slabel_desc_path).readlines()
lfeature_line = open(sfeature_desc_path).readlines()
file_lable_bin = open(slabel_path, "r")
file_feature_bin = open(sfeature_path, "r")
sample_num = int(llabel_line[0].split()[1])
assert sample_num == int(lfeature_line[0].split()[1])
llabel_line = llabel_line[1:]
lfeature_line = lfeature_line[1:]
for i in range(sample_num):
# read label
llabel_split = llabel_line[i].split()
nlabel_start = int(llabel_split[2])
nlabel_size = int(llabel_split[3])
nlabel_frame_num = int(llabel_split[4])
file_lable_bin.seek(nlabel_start, 0)
label_bytes = file_lable_bin.read(nlabel_size)
assert nlabel_frame_num * 4 == len(label_bytes)
label_array = struct.unpack('I' * nlabel_frame_num, label_bytes)
label_data = np.array(label_array, dtype="int64")
label_data = label_data.reshape((nlabel_frame_num, 1))
# read feature
lfeature_split = lfeature_line[i].split()
nfeature_start = int(lfeature_split[2])
nfeature_size = int(lfeature_split[3])
nfeature_frame_num = int(lfeature_split[4])
nfeature_frame_dim = int(lfeature_split[5])
file_feature_bin.seek(nfeature_start, 0)
feature_bytes = file_feature_bin.read(nfeature_size)
assert nfeature_frame_num * nfeature_frame_dim * 4 == len(
feature_bytes)
feature_array = struct.unpack('f' * nfeature_frame_num *
nfeature_frame_dim, feature_bytes)
feature_data = np.array(feature_array, dtype="float32")
feature_data = feature_data.reshape(
(nfeature_frame_num, nfeature_frame_dim))
#drop long sentence
if self._ndrop_frame_len < feature_data.shape[0]:
continue
lsample.append((feature_data, label_data))
def get_one_batch(self, nbatch_size):
"""construct one batch(feature, label), batch size is nbatch_size
Args:
nbatch_size(int): batch size
Returns:
None
"""
if self._que_sample.empty():
lsample = self._load_block(
range(self._nstart_block_idx, self._nstart_block_idx +
self._nload_block_num, 1))
self._move_sample(lsample)
self._nstart_block_idx += self._nload_block_num
if self._que_sample.empty():
self._nstart_block_idx = 0
return None
#cal all frame num
ncur_len = 0
lod = [0]
samples = []
bat_feature = np.zeros((nbatch_size, self._nframe_dim))
for i in range(nbatch_size):
# empty clear zero
if self._que_sample.empty():
self._nstart_block_idx = 0
# copy
else:
(one_feature, one_label) = self._que_sample.get()
samples.append((one_feature, one_label))
ncur_len += one_feature.shape[0]
lod.append(ncur_len)
bat_feature = np.zeros((ncur_len, self._nframe_dim), dtype="float32")
bat_label = np.zeros((ncur_len, 1), dtype="int64")
ncur_len = 0
for sample in samples:
one_feature = sample[0]
one_label = sample[1]
nframe_num = one_feature.shape[0]
nstart = ncur_len
nend = ncur_len + nframe_num
bat_feature[nstart:nend, :] = one_feature
bat_label[nstart:nend, :] = one_label
ncur_len += nframe_num
return (bat_feature, bat_label, lod)
def set_trans(self, ltrans):
""" set transform list
Args:
ltrans(list): data tranform list
Returns:
None
"""
self._ltrans = ltrans
def _load_block(self, lblock_id):
"""read blocks
"""
lsample = []
for id in lblock_id:
self._load_one_block(lsample, id)
# transform sample
for (nidx, sample) in enumerate(lsample):
for trans in self._ltrans:
sample = trans.perform_trans(sample)
#print nidx
lsample[nidx] = sample
return lsample
def load_block(self, lblock_id):
"""read blocks
Args:
lblock_id(list):the block list id
Returns:
None
"""
lsample = []
for id in lblock_id:
self._load_one_block(lsample, id)
# transform sample
for (nidx, sample) in enumerate(lsample):
for trans in self._ltrans:
sample = trans.perform_trans(sample)
#print nidx
lsample[nidx] = sample
return lsample
def _move_sample(self, lsample):
"""move sample to queue
Args:
lsample(list): one block of samples read from disk
Returns:
None
"""
# random
random.shuffle(lsample)
for sample in lsample:
self._que_sample.put(sample)
#by zhxfl 2018.01.24
""" @package docstring
load speech data from disk
"""
import random
import Queue
import numpy
import struct
import data_utils.trans_mean_variance_norm as trans_mean_variance_norm
import data_utils.trans_add_delta as trans_add_delta
g_lblock = []
g_que_sample = Queue.Queue()
g_nframe_dim = 120 * 11
g_nstart_block_idx = 0
g_nload_block_num = 1
g_ndrop_frame_len = 256
class OneBlock(object):
""" Documentation for a class.
struct for one block :
contain label, label desc, feature, feature_desc
"""
def __init__(self):
"""The constructor."""
self.label = ""
self.label_desc = ""
self.feature = ""
self.feature_desc = ""
def set_trans(ltrans):
global g_ltrans
g_ltrans = ltrans
def load_list(sfeature_lst, slabel_lst):
""" load list """
global g_lblock
lFeature = open(sfeature_lst).readlines()
lLabel = open(slabel_lst).readlines()
assert len(lLabel) == len(lFeature)
for i in range(0, len(lFeature), 2):
one_block = OneBlock()
one_block.label = lLabel[i]
one_block.label_desc = lLabel[i + 1]
one_block.feature = lFeature[i]
one_block.feature_desc = lFeature[i + 1]
g_lblock.append(one_block)
random.shuffle(g_lblock)
def load_one_block(lsample, id):
"""read one block"""
global g_lblock
if id >= len(g_lblock):
return
slabel_path = g_lblock[id].label.replace("\n", "")
slabel_desc_path = g_lblock[id].label_desc.replace("\n", "")
sfeature_path = g_lblock[id].feature.replace("\n", "")
sfeature_desc_path = g_lblock[id].feature_desc.replace("\n", "")
llabel_line = open(slabel_desc_path).readlines()
lfeature_line = open(sfeature_desc_path).readlines()
file_lable_bin = open(slabel_path, "r")
file_feature_bin = open(sfeature_path, "r")
sample_num = int(llabel_line[0].split()[1])
assert sample_num == int(lfeature_line[0].split()[1])
llabel_line = llabel_line[1:]
lfeature_line = lfeature_line[1:]
for i in range(sample_num):
# read label
llabel_split = llabel_line[i].split()
nlabel_start = int(llabel_split[2])
nlabel_size = int(llabel_split[3])
nlabel_frame_num = int(llabel_split[4])
file_lable_bin.seek(nlabel_start, 0)
label_bytes = file_lable_bin.read(nlabel_size)
assert nlabel_frame_num * 4 == len(label_bytes)
label_array = struct.unpack('I' * nlabel_frame_num, label_bytes)
label_data = numpy.array(label_array, dtype=int)
label_data = label_data.reshape((nlabel_frame_num, 1))
# read feature
lfeature_split = lfeature_line[i].split()
nfeature_start = int(lfeature_split[2])
nfeature_size = int(lfeature_split[3])
nfeature_frame_num = int(lfeature_split[4])
nfeature_frame_dim = int(lfeature_split[5])
file_feature_bin.seek(nfeature_start, 0)
feature_bytes = file_feature_bin.read(nfeature_size)
assert nfeature_frame_num * nfeature_frame_dim * 4 == len(feature_bytes)
feature_array = struct.unpack('f' * nfeature_frame_num *
nfeature_frame_dim, feature_bytes)
feature_data = numpy.array(feature_array, dtype=float)
feature_data = feature_data.reshape(
(nfeature_frame_num, nfeature_frame_dim))
global g_ndrop_frame_len
#drop long sentence
if g_ndrop_frame_len < feature_data.shape[0]:
continue
lsample.append((feature_data, label_data))
def load_block(lblock_id):
"""
read blocks
"""
global g_ltrans
lsample = []
for id in lblock_id:
load_one_block(lsample, id)
# transform sample
for (nidx, sample) in enumerate(lsample):
for trans in g_ltrans:
sample = trans.perform_trans(sample)
print nidx
lsample[nidx] = sample
return lsample
def move_sample(lsample):
"""
move sample to queue
"""
# random
random.shuffle(lsample)
global g_que_sample
for sample in lsample:
g_que_sample.put(sample)
def get_one_batch(nbatch_size):
"""
construct one batch
"""
global g_que_sample
global g_nstart_block_idx
global g_nframe_dim
global g_nload_block_num
if g_que_sample.empty():
lsample = load_block(
range(g_nstart_block_idx, g_nstart_block_idx + g_nload_block_num,
1))
move_sample(lsample)
g_nstart_block_idx += g_nload_block_num
if g_que_sample.empty():
g_nstart_block_idx = 0
return None
#cal all frame num
ncur_len = 0
lod = [0]
samples = []
bat_feature = numpy.zeros((nbatch_size, g_nframe_dim))
for i in range(nbatch_size):
# empty clear zero
if g_que_sample.empty():
g_nstart_block_idx = 0
# copy
else:
(one_feature, one_label) = g_que_sample.get()
samples.append((one_feature, one_label))
ncur_len += one_feature.shape[0]
lod.append(ncur_len)
bat_feature = numpy.zeros((ncur_len, g_nframe_dim), dtype="float32")
bat_label = numpy.zeros((ncur_len, 1), dtype="int64")
ncur_len = 0
for sample in samples:
one_feature = sample[0]
one_label = sample[1]
nframe_num = one_feature.shape[0]
nstart = ncur_len
nend = ncur_len + nframe_num
bat_feature[nstart:nend, :] = one_feature
bat_label[nstart:nend, :] = one_label
ncur_len += nframe_num
return (bat_feature, bat_label, lod)
#by zhxfl 2018.01.29 import numpy as np
import numpy
import math import math
import copy import copy
...@@ -7,6 +6,10 @@ import copy ...@@ -7,6 +6,10 @@ import copy
class TransAddDelta(object): class TransAddDelta(object):
""" add delta of feature data """ add delta of feature data
trans feature for shape(a, b) to shape(a, b * 3) trans feature for shape(a, b) to shape(a, b * 3)
Attributes:
_norder(int):
_window(int):
""" """
def __init__(self, norder=2, nwindow=2): def __init__(self, norder=2, nwindow=2):
...@@ -21,26 +24,30 @@ class TransAddDelta(object): ...@@ -21,26 +24,30 @@ class TransAddDelta(object):
def perform_trans(self, sample): def perform_trans(self, sample):
""" add delta for feature """ add delta for feature
trans feature shape from (a,b) to (a, b * 3) trans feature shape from (a,b) to (a, b * 3)
Args:
sample(object,tuple): contain feature numpy and label numpy
Returns:
(feature, label)
""" """
(feature, label) = sample (feature, label) = sample
frame_dim = feature.shape[1] frame_dim = feature.shape[1]
d_frame_dim = frame_dim * 3 d_frame_dim = frame_dim * 3
head_filled = 5 head_filled = 5
tail_filled = 5 tail_filled = 5
mat = numpy.zeros( mat = np.zeros(
(feature.shape[0] + head_filled + tail_filled, d_frame_dim), (feature.shape[0] + head_filled + tail_filled, d_frame_dim),
dtype="float32") dtype="float32")
#copy first frame #copy first frame
for i in xrange(head_filled): for i in xrange(head_filled):
numpy.copyto(mat[i, 0:frame_dim], feature[0, :]) np.copyto(mat[i, 0:frame_dim], feature[0, :])
numpy.copyto( np.copyto(mat[head_filled:head_filled + feature.shape[0], 0:frame_dim],
mat[head_filled:head_filled + feature.shape[0], 0:frame_dim], feature[:, :])
feature[:, :])
# copy last frame # copy last frame
for i in xrange(head_filled + feature.shape[0], mat.shape[0], 1): for i in xrange(head_filled + feature.shape[0], mat.shape[0], 1):
numpy.copyto(mat[i, 0:frame_dim], feature[feature.shape[0] - 1, :]) np.copyto(mat[i, 0:frame_dim], feature[feature.shape[0] - 1, :])
nframe = feature.shape[0] nframe = feature.shape[0]
start = head_filled start = head_filled
...@@ -65,6 +72,8 @@ class TransAddDelta(object): ...@@ -65,6 +72,8 @@ class TransAddDelta(object):
size: frame dimentional size: frame dimentional
n: frame num n: frame num
step: 3 * (frame num) step: 3 * (frame num)
Returns:
None
""" """
sigma_t2 = 0.0 sigma_t2 = 0.0
delta_window = self._nwindow delta_window = self._nwindow
......
#by zhxfl 2018.01.29 import numpy as np
import numpy
import math import math
class TransMeanVarianceNorm(object): class TransMeanVarianceNorm(object):
""" normalization of mean variance for feature data """ normalization of mean variance for feature data
Attributes:
_mean(numpy.array): the feature mean vector
_var(numpy.array): the feature variance
""" """
def __init__(self, snorm_path): def __init__(self, snorm_path):
...@@ -17,12 +19,14 @@ class TransMeanVarianceNorm(object): ...@@ -17,12 +19,14 @@ class TransMeanVarianceNorm(object):
self._load_norm(snorm_path) self._load_norm(snorm_path)
def _load_norm(self, snorm_path): def _load_norm(self, snorm_path):
""" load global mean var file """ load mean var file
Args:
snorm_path(str):the file path
""" """
lLines = open(snorm_path).readlines() lLines = open(snorm_path).readlines()
nLen = len(lLines) nLen = len(lLines)
self._mean = numpy.zeros((nLen), dtype="float32") self._mean = np.zeros((nLen), dtype="float32")
self._var = numpy.zeros((nLen), dtype="float32") self._var = np.zeros((nLen), dtype="float32")
self._nLen = nLen self._nLen = nLen
for nidx, l in enumerate(lLines): for nidx, l in enumerate(lLines):
s = l.split() s = l.split()
...@@ -34,11 +38,18 @@ class TransMeanVarianceNorm(object): ...@@ -34,11 +38,18 @@ class TransMeanVarianceNorm(object):
def get_mean_var(self): def get_mean_var(self):
""" get mean and var """ get mean and var
Args:
Returns:
(mean, var)
""" """
return (self._mean, self._var) return (self._mean, self._var)
def perform_trans(self, sample): def perform_trans(self, sample):
""" feature = (feature - mean) * var """ feature = (feature - mean) * var
Args:
sample(object):input sample, contain feature numpy and label numpy
Returns:
(feature, label)
""" """
(feature, label) = sample (feature, label) = sample
shape = feature.shape shape = feature.shape
......
#by zhxfl
import numpy
import math
class TransSplit(object):
""" expand feature data from shape (frame_num, frame_dim)
to shape (frame_num, frame_dim * 11)
"""
def __init__(self, nleft_context=5, nright_context=5):
self._nleft_context = nleft_context
self._nright_context = nright_context
#by zhxfl 2018.01.31 import numpy as np
import numpy
import math import math
class TransSplice(object): class TransSplice(object):
""" expand feature data from shape (frame_num, frame_dim) """ copy feature context to construct new feature
expand feature data from shape (frame_num, frame_dim)
to shape (frame_num, frame_dim * 11) to shape (frame_num, frame_dim * 11)
Attributes:
_nleft_context(int): copy left context number
_nright_context(int): copy right context number
""" """
def __init__(self, nleft_context=5, nright_context=5): def __init__(self, nleft_context=5, nright_context=5):
""" init construction """ init construction
Args:
nleft_context(int):
nright_context(int):
""" """
self._nleft_context = nleft_context self._nleft_context = nleft_context
self._nright_context = nright_context self._nright_context = nright_context
def perform_trans(self, sample): def perform_trans(self, sample):
""" splice """ copy feature context
Args:
sample(object): input sample(feature, label)
Return:
(feature, label)
""" """
(feature, label) = sample (feature, label) = sample
nframe_num = feature.shape[0] nframe_num = feature.shape[0]
nframe_dim = feature.shape[1] nframe_dim = feature.shape[1]
nnew_frame_dim = nframe_dim * ( nnew_frame_dim = nframe_dim * (
self._nleft_context + self._nright_context + 1) self._nleft_context + self._nright_context + 1)
mat = numpy.zeros( mat = np.zeros(
(nframe_num + self._nleft_context + self._nright_context, (nframe_num + self._nleft_context + self._nright_context,
nframe_dim), nframe_dim),
dtype="float32") dtype="float32")
ret = numpy.zeros((nframe_num, nnew_frame_dim), dtype="float32") ret = np.zeros((nframe_num, nnew_frame_dim), dtype="float32")
#copy left #copy left
for i in xrange(self._nleft_context): for i in xrange(self._nleft_context):
...@@ -44,7 +54,7 @@ class TransSplice(object): ...@@ -44,7 +54,7 @@ class TransSplice(object):
mat = mat.reshape(mat.shape[0] * mat.shape[1]) mat = mat.reshape(mat.shape[0] * mat.shape[1])
ret = ret.reshape(ret.shape[0] * ret.shape[1]) ret = ret.reshape(ret.shape[0] * ret.shape[1])
for i in xrange(nframe_num): for i in xrange(nframe_num):
numpy.copyto(ret[i * nnew_frame_dim:(i + 1) * nnew_frame_dim], np.copyto(ret[i * nnew_frame_dim:(i + 1) * nnew_frame_dim],
mat[i * nframe_dim:i * nframe_dim + nnew_frame_dim]) mat[i * nframe_dim:i * nframe_dim + nnew_frame_dim])
ret = ret.reshape((nframe_num, nnew_frame_dim)) ret = ret.reshape((nframe_num, nnew_frame_dim))
return (ret, label) return (ret, label)
#by zhxfl 2018.01.31
def to_lodtensor(data, place): def to_lodtensor(data, place):
"""convert tensor to lodtensor """convert tensor to lodtensor
""" """
......
...@@ -9,10 +9,10 @@ import time ...@@ -9,10 +9,10 @@ import time
import paddle.v2 as paddle import paddle.v2 as paddle
import paddle.v2.fluid as fluid import paddle.v2.fluid as fluid
import paddle.v2.fluid.profiler as profiler import paddle.v2.fluid.profiler as profiler
import data_utils.load_data as load_data
import data_utils.trans_mean_variance_norm as trans_mean_variance_norm import data_utils.trans_mean_variance_norm as trans_mean_variance_norm
import data_utils.trans_add_delta as trans_add_delta import data_utils.trans_add_delta as trans_add_delta
import data_utils.trans_splice as trans_splice import data_utils.trans_splice as trans_splice
import data_utils.data_read as reader
def parse_args(): def parse_args():
...@@ -61,6 +61,9 @@ def parse_args(): ...@@ -61,6 +61,9 @@ def parse_args():
'--use_nvprof', '--use_nvprof',
action='store_true', action='store_true',
help='If set, use nvprof for CUDA.') help='If set, use nvprof for CUDA.')
parser.add_argument('--mean_var', type=str, help='mean var path')
parser.add_argument('--feature_lst', type=str, help='mean var path')
parser.add_argument('--label_lst', type=str, help='mean var path')
args = parser.parse_args() args = parser.parse_args()
return args return args
...@@ -159,12 +162,12 @@ def train(args): ...@@ -159,12 +162,12 @@ def train(args):
ltrans = [ ltrans = [
trans_add_delta.TransAddDelta(2, 2), trans_add_delta.TransAddDelta(2, 2),
trans_mean_variance_norm.TransMeanVarianceNorm( trans_mean_variance_norm.TransMeanVarianceNorm(args.mean_var),
"data/global_mean_var_search26kHr"), trans_splice.TransSplice() trans_splice.TransSplice()
] ]
load_data.set_trans(ltrans)
load_data.load_list("/home/disk2/mini_speech_fbank_40/data/feature.lst", data_reader = reader.DataRead(args.feature_lst, args.label_lst)
"/home/disk2/mini_speech_fbank_40/data/label.lst") data_reader.set_trans(ltrans)
res_feature = fluid.LoDTensor() res_feature = fluid.LoDTensor()
res_label = fluid.LoDTensor() res_label = fluid.LoDTensor()
...@@ -175,7 +178,7 @@ def train(args): ...@@ -175,7 +178,7 @@ def train(args):
batch_id = 0 batch_id = 0
while True: while True:
# load_data # load_data
one_batch = load_data.get_one_batch(args.batch_size) one_batch = data_reader.get_one_batch(args.batch_size)
if one_batch == None: if one_batch == None:
break break
(bat_feature, bat_label, lod) = one_batch (bat_feature, bat_label, lod) = one_batch
......
#by zhxfl 2018.01.31 #by zhxfl 2018.01.31
import sys import sys
import unittest import unittest
import numpy import numpy as np
sys.path.append("../") sys.path.append("../")
import data_utils.trans_mean_variance_norm as trans_mean_variance_norm import data_utils.trans_mean_variance_norm as trans_mean_variance_norm
import data_utils.trans_add_delta as trans_add_delta import data_utils.trans_add_delta as trans_add_delta
...@@ -13,7 +13,7 @@ class TestTransMeanVarianceNorm(unittest.TestCase): ...@@ -13,7 +13,7 @@ class TestTransMeanVarianceNorm(unittest.TestCase):
""" """
def test(self): def test(self):
feature = numpy.zeros((2, 120), dtype="float32") feature = np.zeros((2, 120), dtype="float32")
feature.fill(1) feature.fill(1)
trans = trans_mean_variance_norm.TransMeanVarianceNorm( trans = trans_mean_variance_norm.TransMeanVarianceNorm(
"../data/global_mean_var_search26kHr") "../data/global_mean_var_search26kHr")
...@@ -21,7 +21,7 @@ class TestTransMeanVarianceNorm(unittest.TestCase): ...@@ -21,7 +21,7 @@ class TestTransMeanVarianceNorm(unittest.TestCase):
(mean, var) = trans.get_mean_var() (mean, var) = trans.get_mean_var()
feature_flat1 = feature1.flatten() feature_flat1 = feature1.flatten()
feature_flat = feature.flatten() feature_flat = feature.flatten()
one = numpy.ones((1), dtype="float32") one = np.ones((1), dtype="float32")
for idx, val in enumerate(feature_flat1): for idx, val in enumerate(feature_flat1):
cur_idx = idx % 120 cur_idx = idx % 120
self.assertAlmostEqual(val, (one[0] - mean[cur_idx]) * var[cur_idx]) self.assertAlmostEqual(val, (one[0] - mean[cur_idx]) * var[cur_idx])
...@@ -34,7 +34,7 @@ class TestTransAddDelta(unittest.TestCase): ...@@ -34,7 +34,7 @@ class TestTransAddDelta(unittest.TestCase):
def test_regress(self): def test_regress(self):
"""test regress """test regress
""" """
feature = numpy.zeros((14, 120), dtype="float32") feature = np.zeros((14, 120), dtype="float32")
feature[0:5, 0:40].fill(1) feature[0:5, 0:40].fill(1)
feature[0 + 5, 0:40].fill(1) feature[0 + 5, 0:40].fill(1)
feature[1 + 5, 0:40].fill(2) feature[1 + 5, 0:40].fill(2)
...@@ -59,7 +59,7 @@ class TestTransAddDelta(unittest.TestCase): ...@@ -59,7 +59,7 @@ class TestTransAddDelta(unittest.TestCase):
def test_perform(self): def test_perform(self):
"""test perform """test perform
""" """
feature = numpy.zeros((4, 40), dtype="float32") feature = np.zeros((4, 40), dtype="float32")
feature[0, 0:40].fill(1) feature[0, 0:40].fill(1)
feature[1, 0:40].fill(2) feature[1, 0:40].fill(2)
feature[2, 0:40].fill(3) feature[2, 0:40].fill(3)
...@@ -83,7 +83,7 @@ class TestTransSplict(unittest.TestCase): ...@@ -83,7 +83,7 @@ class TestTransSplict(unittest.TestCase):
""" """
def test_perfrom(self): def test_perfrom(self):
feature = numpy.zeros((8, 10), dtype="float32") feature = np.zeros((8, 10), dtype="float32")
for i in xrange(feature.shape[0]): for i in xrange(feature.shape[0]):
feature[i, :].fill(i) feature[i, :].fill(i)
...@@ -104,8 +104,6 @@ class TestTransSplict(unittest.TestCase): ...@@ -104,8 +104,6 @@ class TestTransSplict(unittest.TestCase):
if cur_val < 7: if cur_val < 7:
cur_val += 1.0 cur_val += 1.0
for k in xrange(10): for k in xrange(10):
print i, j, k
print feature[i].reshape(11, 10)
self.assertAlmostEqual(feature[i][j * 10 + k], cur_val) self.assertAlmostEqual(feature[i][j * 10 + k], cur_val)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册