Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Pinoxchio
apollo
提交
5dc83983
A
apollo
项目概览
Pinoxchio
/
apollo
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
A
apollo
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
5dc83983
编写于
2月 07, 2018
作者:
K
kechxu
提交者:
Kecheng Xu
2月 07, 2018
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Prediction: add downsample and merge h5
上级
8ffeff56
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
117 addition
and
3 deletion
+117
-3
modules/tools/prediction/mlp_train/merge_h5.py
modules/tools/prediction/mlp_train/merge_h5.py
+79
-0
modules/tools/prediction/mlp_train/mlp_train.py
modules/tools/prediction/mlp_train/mlp_train.py
+38
-3
未找到文件。
modules/tools/prediction/mlp_train/merge_h5.py
0 → 100644
浏览文件 @
5dc83983
#!/usr/bin/env python
###############################################################################
# Copyright 2018 The Apollo Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
import
os
import
glob
import
argparse
import
datetime
import
numpy
as
np
import
h5py
def
load_hdf5
(
filename
):
"""
load training samples from *.hdf5 file
"""
if
not
(
os
.
path
.
exists
(
filename
)):
print
"file:"
,
filename
,
"does not exist"
os
.
_exit
(
1
)
if
os
.
path
.
splitext
(
filename
)[
1
]
!=
'.h5'
:
print
"file:"
,
filename
,
"is not an hdf5 file"
os
.
_exit
(
1
)
h5_file
=
h5py
.
File
(
filename
,
'r'
)
values
=
h5_file
.
values
()[
0
]
print
"load data size:"
,
values
.
shape
[
0
]
return
values
if
__name__
==
'__main__'
:
parser
=
argparse
.
ArgumentParser
(
description
=
'generate training samples
\
from a specified directory'
)
parser
.
add_argument
(
'directory'
,
type
=
str
,
help
=
'directory contains feature files in .h5'
)
args
=
parser
.
parse_args
()
path
=
args
.
directory
print
"load h5 from directory:"
,
format
(
path
)
if
os
.
path
.
isdir
(
path
):
features
=
None
labels
=
None
h5_files
=
glob
.
glob
(
path
+
'/*.h5'
)
print
"Length of files:"
,
len
(
h5_files
)
for
i
,
h5_file
in
enumerate
(
h5_files
):
print
"Process File"
,
i
,
":"
,
h5_file
feature
=
load_hdf5
(
h5_file
)
if
np
.
any
(
np
.
isinf
(
feature
)):
print
"inf data found"
features
=
np
.
concatenate
((
features
,
feature
),
axis
=
0
)
if
features
is
not
None
\
else
feature
else
:
print
"Fail to find"
,
path
os
.
_exit
(
-
1
)
date
=
datetime
.
datetime
.
now
().
strftime
(
'%Y-%m-%d'
)
sample_dir
=
path
+
'/mlp_merge'
if
not
os
.
path
.
exists
(
sample_dir
):
os
.
makedirs
(
sample_dir
)
sample_file
=
sample_dir
+
'/mlp_'
+
date
+
'.h5'
print
"Save samples file to:"
,
sample_file
h5_file
=
h5py
.
File
(
sample_file
,
'w'
)
h5_file
.
create_dataset
(
'data'
,
data
=
features
)
h5_file
.
close
()
modules/tools/prediction/mlp_train/mlp_train.py
浏览文件 @
5dc83983
...
@@ -81,6 +81,36 @@ def load_data(filename):
...
@@ -81,6 +81,36 @@ def load_data(filename):
return
samples
[
'data'
]
return
samples
[
'data'
]
def
down_sample
(
data
):
cutin_false_drate
=
0.9
go_false_drate
=
0.9
go_true_drate
=
0.7
cutin_true_drate
=
0.0
label
=
data
[:,
-
1
]
size
=
np
.
shape
(
label
)[
0
]
cutin_false_index
=
(
label
==
-
1
)
go_false_index
=
(
label
==
0
)
go_true_index
=
(
label
==
1
)
cutin_true_index
=
(
label
==
2
)
rand
=
np
.
random
.
random
((
size
))
cutin_false_select
=
np
.
logical_and
(
cutin_false_index
,
rand
>
cutin_false_drate
)
cutin_true_select
=
np
.
logical_and
(
cutin_true_index
,
rand
>
cutin_true_drate
)
go_false_select
=
np
.
logical_and
(
go_false_index
,
rand
>
go_false_drate
)
go_true_select
=
np
.
logical_and
(
go_true_index
,
rand
>
go_true_drate
)
all_select
=
np
.
logical_or
(
cutin_false_select
,
cutin_true_select
)
all_select
=
np
.
logical_or
(
all_select
,
go_false_select
)
all_select
=
np
.
logical_or
(
all_select
,
go_true_select
)
data_downsampled
=
data
[
all_select
,
:]
return
data_downsampled
def
get_param_norm
(
feature
):
def
get_param_norm
(
feature
):
"""
"""
Normalize the samples and save normalized parameters
Normalize the samples and save normalized parameters
...
@@ -105,12 +135,12 @@ def setup_model():
...
@@ -105,12 +135,12 @@ def setup_model():
model
.
add
(
Dense
(
dim_hidden_2
,
model
.
add
(
Dense
(
dim_hidden_2
,
init
=
'he_normal'
,
init
=
'he_normal'
,
activation
=
'relu'
,
activation
=
'relu'
,
W_regularizer
=
l2
(
0.0
05
)))
W_regularizer
=
l2
(
0.0
1
)))
model
.
add
(
Dense
(
dim_output
,
model
.
add
(
Dense
(
dim_output
,
init
=
'he_normal'
,
init
=
'he_normal'
,
activation
=
'sigmoid'
,
activation
=
'sigmoid'
,
W_regularizer
=
l2
(
0.0
0
1
)))
W_regularizer
=
l2
(
0.01
)))
model
.
compile
(
loss
=
'binary_crossentropy'
,
model
.
compile
(
loss
=
'binary_crossentropy'
,
optimizer
=
'rmsprop'
,
optimizer
=
'rmsprop'
,
...
@@ -228,11 +258,14 @@ if __name__ == "__main__":
...
@@ -228,11 +258,14 @@ if __name__ == "__main__":
file
=
args
.
filename
file
=
args
.
filename
data
=
load_data
(
file
)
data
=
load_data
(
file
)
data
=
down_sample
(
data
)
print
"Data load success."
print
"Data load success."
print
"data size ="
,
data
.
shape
print
"data size ="
,
data
.
shape
train_data
,
test_data
=
train_test_split
(
data
,
train_data_rate
)
train_data
,
test_data
=
train_test_split
(
data
,
train_data_rate
)
print
"training size ="
,
train_data
.
shape
X_train
=
train_data
[:,
0
:
dim_input
]
X_train
=
train_data
[:,
0
:
dim_input
]
Y_train
=
train_data
[:,
-
1
]
Y_train
=
train_data
[:,
-
1
]
Y_trainc
=
Y_train
>
0.1
Y_trainc
=
Y_train
>
0.1
...
@@ -245,11 +278,13 @@ if __name__ == "__main__":
...
@@ -245,11 +278,13 @@ if __name__ == "__main__":
X_train
=
(
X_train
-
param_norm
[
0
])
/
param_norm
[
1
]
X_train
=
(
X_train
-
param_norm
[
0
])
/
param_norm
[
1
]
X_test
=
(
X_test
-
param_norm
[
0
])
/
param_norm
[
1
]
model
=
setup_model
()
model
=
setup_model
()
model
.
fit
(
X_train
,
Y_trainc
,
model
.
fit
(
X_train
,
Y_trainc
,
shuffle
=
True
,
shuffle
=
True
,
nb_epoch
=
3
0
,
nb_epoch
=
2
0
,
batch_size
=
32
)
batch_size
=
32
)
print
"Model trained success."
print
"Model trained success."
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录