Skip to content

P
PaddleHub

PaddlePaddle / PaddleHub
大约 1 年 前同步成功

通知 282
Star 12117
Fork 2091
P
PaddleHub
提交 87c03753 编写于 作者: Z Zeyu Chen
浏览文件
操作

remove tests

上级 c4641533
隐藏空白更改
内联 并排
Showing with 1 addition and 926 deletion
.travis.yml
浏览文件 @ 87c03753
......@@ -9,7 +9,6 @@ os:
- linux
script:
- /bin/bash ./scripts/test_all_case.sh ./tests/tclist_all
- /bin/bash ./scripts/check_code_style.sh
notifications:
......
demo/senta/senta_demo.py
浏览文件 @ 87c03753
......@@ -4,7 +4,7 @@ import paddlehub as hub
if __name__ == "__main__":
# Load Senta-BiLSTM module
senta = hub.Module(name="senta")
senta = hub.Module(name="senta_bilstm")
test_text = ["这家餐厅很好吃", "这部电影真的很差劲"]
......
tests/tclist_all 已删除 100644 → 0
浏览文件 @ c4641533
#test_downloader
#test_export_n_load_module
#test_module
#test_train_w2v
#test_pyobj_serialize
#test_signature
#test_param_serialize
tests/test_downloader.py 已删除 100644 → 0
浏览文件 @ c4641533
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddlehub as hub
class TestDownloader(unittest.TestCase):
def test_download(self):
link = "https://paddlehub.cdn.bcebos.com/word2vec/word2vec_test_module.tar.gz"
module_path = hub.download_and_uncompress(link)
if __name__ == "__main__":
unittest.main()
tests/test_export_n_load_module.py 已删除 100644 → 0
浏览文件 @ c4641533
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import division
from __future__ import print_function
import numpy as np
import paddle.fluid as fluid
import paddle
import paddlehub as hub
import unittest
import os
from collections import defaultdict
EMBED_SIZE = 16
HIDDEN_SIZE = 256
N = 5
BATCH_SIZE = 64
PASS_NUM = 1000
word_dict = paddle.dataset.imikolov.build_dict()
dict_size = len(word_dict)
data = paddle.dataset.imikolov.train(word_dict, N)
_MOCK_DATA = [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]
def mock_data():
for d in _MOCK_DATA:
yield d
batch_reader = paddle.batch(mock_data, BATCH_SIZE)
#batch_reader = paddle.batch(data, BATCH_SIZE)
batch_size = 0
for d in batch_reader():
batch_size += 1
def word2vec(words, is_sparse, trainable=True):
emb_param_attr = fluid.ParamAttr(name="embedding", trainable=trainable)
embed_first = fluid.layers.embedding(
input=words[0],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr=emb_param_attr)
embed_second = fluid.layers.embedding(
input=words[1],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr=emb_param_attr)
embed_third = fluid.layers.embedding(
input=words[2],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr=emb_param_attr)
embed_fourth = fluid.layers.embedding(
input=words[3],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr=emb_param_attr)
concat_emb = fluid.layers.concat(
input=[embed_first, embed_second, embed_third, embed_fourth], axis=1)
hidden1 = fluid.layers.fc(input=concat_emb, size=HIDDEN_SIZE, act='sigmoid')
pred_prob = fluid.layers.fc(input=hidden1, size=dict_size, act='softmax')
# declare later than predict word
next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64')
cost = fluid.layers.cross_entropy(input=pred_prob, label=next_word)
avg_cost = fluid.layers.mean(cost)
return pred_prob, avg_cost
def get_dictionary(word_dict):
dictionary = defaultdict(int)
w_id = 0
for w in word_dict:
if isinstance(w, bytes):
w = w.decode("ascii")
dictionary[w] = w_id
w_id += 1
return dictionary
def test_create_w2v_module(use_gpu=False):
place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace()
first_word = fluid.layers.data(name='firstw', shape=[1], dtype='int64')
second_word = fluid.layers.data(name='secondw', shape=[1], dtype='int64')
third_word = fluid.layers.data(name='thirdw', shape=[1], dtype='int64')
forth_word = fluid.layers.data(name='fourthw', shape=[1], dtype='int64')
next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64')
word_list = [first_word, second_word, third_word, forth_word, next_word]
pred_prob, avg_cost = word2vec(word_list, is_sparse=True)
main_program = fluid.default_main_program()
startup_program = fluid.default_startup_program()
sgd_optimizer = fluid.optimizer.SGDOptimizer(learning_rate=1e-2)
sgd_optimizer.minimize(avg_cost)
exe = fluid.Executor(place)
exe.run(startup_program) # initialization
step = 0
for epoch in range(0, PASS_NUM):
for mini_batch in batch_reader():
feed_var_list = [
main_program.global_block().var("firstw"),
main_program.global_block().var("secondw"),
main_program.global_block().var("thirdw"),
main_program.global_block().var("fourthw"),
main_program.global_block().var("nextw")
]
feeder = fluid.DataFeeder(feed_list=feed_var_list, place=place)
cost = exe.run(
main_program,
feed=feeder.feed(mini_batch),
fetch_list=[avg_cost])
step += 1
if step % 100 == 0:
print("Epoch={} Step={} Cost={}".format(epoch, step, cost[0]))
saved_module_dir = "./tmp/word2vec_test_module"
# save inference model including feed and fetch variable info
dictionary = get_dictionary(word_dict)
module_inputs = [
main_program.global_block().var("firstw"),
main_program.global_block().var("secondw"),
main_program.global_block().var("thirdw"),
main_program.global_block().var("fourthw"),
]
signature = hub.create_signature(
"default",
inputs=module_inputs,
outputs=[pred_prob],
feed_names=["firstw", "secondw", "thirdw", "fourthw"],
fetch_names=["pred_prob"])
hub.create_module(
sign_arr=signature, module_dir=saved_module_dir, word_dict=dictionary)
def test_load_w2v_module(use_gpu=False):
saved_module_dir = "./tmp/word2vec_test_module"
w2v_module = hub.Module(module_dir=saved_module_dir)
feed_dict, fetch_dict, program = w2v_module(
sign_name="default", trainable=False)
with fluid.program_guard(main_program=program):
pred_prob = fetch_dict["pred_prob"]
pred_word = fluid.layers.argmax(x=pred_prob, axis=1)
# set place, executor, datafeeder
place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
feed_vars = [
feed_dict["firstw"], feed_dict["secondw"], feed_dict["thirdw"],
feed_dict["fourthw"]
]
feeder = fluid.DataFeeder(place=place, feed_list=feed_vars)
word_ids = [[1, 2, 3, 4]]
result = exe.run(
fluid.default_main_program(),
feed=feeder.feed(word_ids),
fetch_list=[pred_word],
return_numpy=True)
print(result)
if __name__ == "__main__":
use_gpu = False
print("test create word2vec module")
test_create_w2v_module(use_gpu)
print("test load word2vec module")
test_load_w2v_module(use_gpu=False)
tests/test_module.py 已删除 100644 → 0
浏览文件 @ c4641533
# coding=utf-8
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddlehub as hub
import paddle.fluid as fluid
class TestModule(unittest.TestCase):
#TODO(ZeyuChen): add setup for test envrinoment prepration
def test_word2vec_module_usage(self):
url = "https://paddlehub.cdn.bcebos.com/word2vec/word2vec_test_module.tar.gz"
w2v_module = hub.Module(module_url=url)
feed_dict, fetch_dict, program = w2v_module(
sign_name="default", trainable=False)
with fluid.program_guard(main_program=program):
pred_prob = fetch_dict["pred_prob"]
pred_word = fluid.layers.argmax(x=pred_prob, axis=1)
# set place, executor, datafeeder
place = fluid.CPUPlace()
exe = fluid.Executor(place)
feed_vars = [
feed_dict["firstw"], feed_dict["secondw"], feed_dict["thirdw"],
feed_dict["fourthw"]
]
feeder = fluid.DataFeeder(place=place, feed_list=feed_vars)
word_ids = [[1, 2, 3, 4]]
result = exe.run(
fluid.default_main_program(),
feed=feeder.feed(word_ids),
fetch_list=[pred_word],
return_numpy=True)
self.assertEqual(result[0], 5)
def test_senta_module_usage(self):
pass
# m = Module(module_dir="./models/bow_net")
# inputs = [["外人", "爸妈", "翻车"], ["金钱", "电量"]]
# tensor = m._preprocess_input(inputs)
# print(tensor)
# result = m({"words": tensor})
# print(result)
if __name__ == "__main__":
unittest.main()
tests/test_param_serialize.py 已删除 100644 → 0
浏览文件 @ c4641533
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddlehub as hub
import paddle.fluid as fluid
from paddlehub.paddle_helper import from_param_to_flexible_data, from_flexible_data_to_param
from paddlehub import module_desc_pb2
from paddlehub.logger import logger
class TestParamAttrSerializeAndDeSerialize(unittest.TestCase):
def test_convert_l1_regularizer(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(
name="fc_w",
regularizer=fluid.regularizer.L1Decay(
regularization_coeff=1)))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.regularizer.__class__ == param_dict[
'regularizer'].__class__, "regularzier type convert error!"
assert fc_w.regularizer._regularization_coeff == param_dict[
'regularizer']._regularization_coeff, "regularzier value convert error!"
def test_convert_l2_regularizer(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(
name="fc_w",
regularizer=fluid.regularizer.L2Decay(
regularization_coeff=1.5)))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.regularizer.__class__ == param_dict[
'regularizer'].__class__, "regularzier type convert error!"
assert fc_w.regularizer._regularization_coeff == param_dict[
'regularizer']._regularization_coeff, "regularzier value convert error!"
def test_convert_error_clip_by_value(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(
name="fc_w",
gradient_clip=fluid.clip.ErrorClipByValue(max=1)))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.gradient_clip_attr.__class__ == param_dict[
'gradient_clip_attr'].__class__, "clip type convert error!"
assert fc_w.gradient_clip_attr.max == param_dict[
'gradient_clip_attr'].max, "clip value convert error!"
assert fc_w.gradient_clip_attr.min == param_dict[
'gradient_clip_attr'].min, "clip value convert error!"
def test_convert_gradient_clip_by_value(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(
name="fc_w",
gradient_clip=fluid.clip.GradientClipByValue(max=1)))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.gradient_clip_attr.__class__ == param_dict[
'gradient_clip_attr'].__class__, "clip type convert error!"
assert fc_w.gradient_clip_attr.max == param_dict[
'gradient_clip_attr'].max, "clip value convert error!"
assert fc_w.gradient_clip_attr.min == param_dict[
'gradient_clip_attr'].min, "clip value convert error!"
def test_convert_gradient_clip_by_normal(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(
name="fc_w",
gradient_clip=fluid.clip.GradientClipByNorm(clip_norm=1)))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.gradient_clip_attr.__class__ == param_dict[
'gradient_clip_attr'].__class__, "clip type convert error!"
assert fc_w.gradient_clip_attr.clip_norm == param_dict[
'gradient_clip_attr'].clip_norm, "clip value convert error!"
def test_convert_gradient_clip_by_global_normal(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(
name="fc_w",
gradient_clip=fluid.clip.GradientClipByGlobalNorm(
clip_norm=1)))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.gradient_clip_attr.__class__ == param_dict[
'gradient_clip_attr'].__class__, "clip type convert error!"
assert fc_w.gradient_clip_attr.clip_norm == param_dict[
'gradient_clip_attr'].clip_norm, "clip value convert error!"
assert fc_w.gradient_clip_attr.group_name == param_dict[
'gradient_clip_attr'].group_name, "clip value convert error!"
def test_convert_trainable(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(name="fc_w", trainable=False))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.trainable.__class__ == param_dict[
'trainable'].__class__, "trainable type convert error!"
assert fc_w.trainable == param_dict[
'trainable'], "trainable value convert error!"
def test_convert_do_model_average(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(name="fc_w", do_model_average=True))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.do_model_average.__class__ == param_dict[
'do_model_average'].__class__, "do_model_average type convert error!"
assert fc_w.do_model_average == param_dict[
'do_model_average'], "do_model_average value convert error!"
def test_convert_optimize_attr(self):
program = fluid.Program()
with fluid.program_guard(program):
input = fluid.layers.data(name="test", shape=[1], dtype="float32")
fluid.layers.fc(
input=input,
size=10,
param_attr=fluid.ParamAttr(name="fc_w", learning_rate=5))
fc_w = [
param for param in
fluid.default_main_program().global_block().iter_parameters()
][0]
flexible_data = module_desc_pb2.FlexibleData()
from_param_to_flexible_data(fc_w, flexible_data)
param_dict = from_flexible_data_to_param(flexible_data)
assert fc_w.optimize_attr.__class__ == param_dict[
'optimize_attr'].__class__, "optimize_attr type convert error!"
assert fc_w.optimize_attr == param_dict[
'optimize_attr'], "optimize_attr value convert error!"
if __name__ == "__main__":
unittest.main()
tests/test_pyobj_serialize.py 已删除 100644 → 0
浏览文件 @ c4641533
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import math
import unittest
import paddlehub as hub
import paddle.fluid as fluid
from paddlehub.utils import from_pyobj_to_flexible_data, from_flexible_data_to_pyobj, get_pykey
from paddlehub import module_desc_pb2
from paddlehub.logger import logger
def _compare_float(a, b):
error_value = 1.0e-9
logger.debug("a is %f and b is %f" % (a, b))
# if python version < 3.5
if sys.version_info < (3, 5):
return abs(a - b) < error_value
else:
return math.isclose(a, b)
def _check_none(pyobj, flexible_data):
assert flexible_data.type == module_desc_pb2.NONE, "type conversion error"
def _check_int(pyobj, flexible_data):
assert flexible_data.type == module_desc_pb2.INT, "type conversion error"
assert flexible_data.i == pyobj, "value convesion error"
def _check_float(pyobj, flexible_data):
assert flexible_data.type == module_desc_pb2.FLOAT, "type conversion error"
assert _compare_float(flexible_data.f, pyobj), "value convesion error"
def _check_str(pyobj, flexible_data):
assert flexible_data.type == module_desc_pb2.STRING, "type conversion error"
assert flexible_data.s == pyobj, "value convesion error"
def _check_bool(pyobj, flexible_data):
assert flexible_data.type == module_desc_pb2.BOOLEAN, "type conversion error"
assert flexible_data.b == pyobj, "value convesion error"
class TestPyobj2FlexibleData(unittest.TestCase):
def test_int_2_flexible_data(self):
input = None
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_none(input, flexible_data)
def test_int_2_flexible_data(self):
input = 1
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_int(input, flexible_data)
def test_float_2_flexible_data(self):
input = 2.012
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_float(input, flexible_data)
def test_string_2_flexible_data(self):
input = "123"
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_str(input, flexible_data)
def test_bool_2_flexible_data(self):
input = False
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_bool(input, flexible_data)
def test_list_2_flexible_data(self):
input = [1, 2, 3]
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.LIST, "type conversion error"
assert len(
flexible_data.list.data) == len(input), "value convesion error"
for index in range(len(input)):
_check_int(input[index], flexible_data.list.data[str(index)])
def test_tuple_2_flexible_data(self):
input = (1, 2, 3)
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.LIST, "type conversion error"
assert len(
flexible_data.list.data) == len(input), "value convesion error"
for index in range(len(input)):
_check_int(input[index], flexible_data.list.data[str(index)])
def test_set_2_flexible_data(self):
input = {1, 2, 3}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.SET, "type conversion error"
assert len(
flexible_data.set.data) == len(input), "value convesion error"
for index in range(len(input)):
assert flexible_data.set.data[str(
index)].i in input, "value convesion error"
def test_dict_2_flexible_data(self):
input = {1: 1, 2: 2, 3: 3}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.MAP, "type conversion error"
assert len(
flexible_data.map.data) == len(input), "value convesion error"
for key, value in flexible_data.map.data.items():
realkey = get_pykey(key, flexible_data.map.keyType[key])
assert realkey in input, "key convesion error"
_check_int(input[realkey], flexible_data.map.data[key])
def test_obj_2_flexible_data(self):
class TestObj:
def __init__(self):
self.a = 1
self.b = 2.0
self.c = "str"
self.d = {'a': 123}
input = TestObj()
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.OBJECT, "type conversion error"
assert len(flexible_data.object.data) == len(
input.__dict__), "value convesion error"
_check_int(input.a, flexible_data.object.data['a'])
_check_float(input.b, flexible_data.object.data['b'])
_check_str(input.c, flexible_data.object.data['c'])
_check_int(input.d['a'], flexible_data.object.data['d'].map.data['a'])
class TestFlexibleData2Pyobj(unittest.TestCase):
def test_flexible_data_2_int(self):
pass
class TestSerializeAndDeSerialize(unittest.TestCase):
def test_convert_none(self):
input = None
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "none convesion error"
def test_convert_int(self):
input = 1
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "int convesion error"
def test_convert_float(self):
input = 2.012
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert _compare_float(input, output), "float convesion error"
def test_convert_str(self):
input = "123"
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "str convesion error"
def test_convert_bool(self):
input = False
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "bool convesion error"
def test_convert_list(self):
input = [1, 2, 3]
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "list convesion error"
def test_convert_tuple(self):
input = (1, 2, 3)
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert list(input) == output, "tuple convesion error"
def test_convert_set(self):
input = {1, 2, 3}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "set convesion error"
def test_convert_dict(self):
input = {1: 1, 2: 2, 3: 3}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "dict convesion error"
def test_convert_compound_object(self):
input = {
False: 1,
'2': 3,
4.0: [5, 6.0, ['7', {
8: 9
}]],
'set': {10},
'dict': {
11: 12
}
}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "dict convesion error"
if __name__ == "__main__":
unittest.main()
tests/test_signature.py 已删除 100644 → 0
浏览文件 @ c4641533
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddlehub as hub
import paddle.fluid as fluid
from paddlehub import create_signature
class TestSignature(unittest.TestCase):
def test_check_signature_info(self):
program = fluid.Program()
with fluid.program_guard(program):
var_1 = fluid.layers.data(name="var_1", dtype="int64", shape=[1])
var_2 = fluid.layers.data(
name="var_2", dtype="float32", shape=[3, 100, 100])
name = "test"
inputs = [var_1]
outputs = [var_2]
feed_names = ["label"]
fetch_names = ["img"]
sign = create_signature(
name=name,
inputs=inputs,
outputs=outputs,
feed_names=feed_names,
fetch_names=fetch_names)
assert sign.get_name() == name, "sign name error"
assert sign.get_inputs() == inputs, "sign inputs error"
assert sign.get_outputs() == outputs, "sign outputs error"
assert sign.get_feed_names() == feed_names, "sign feed_names error"
assert sign.get_fetch_names(
) == fetch_names, "sign fetch_names error"
if __name__ == "__main__":
unittest.main()
tests/test_train_w2v.py 已删除 100644 → 0
浏览文件 @ c4641533
# coding=utf-8
from __future__ import print_function
from __future__ import division
from __future__ import print_function
import paddle
import paddle.fluid as fluid
import paddlehub as hub
import unittest
import os
EMBED_SIZE = 64
HIDDEN_SIZE = 256
N = 5
BATCH_SIZE = 1
PASS_NUM = 100
word_dict = paddle.dataset.imikolov.build_dict()
dict_size = len(word_dict)
_MOCK_DATA = [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]
def mock_data():
for d in _MOCK_DATA:
yield d
batch_reader = paddle.batch(mock_data, BATCH_SIZE)
def word2vec(words, is_sparse):
embed_first = fluid.layers.embedding(
input=words[0],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr='embedding')
embed_second = fluid.layers.embedding(
input=words[1],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr='embedding')
embed_third = fluid.layers.embedding(
input=words[2],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr='embedding')
embed_fourth = fluid.layers.embedding(
input=words[3],
size=[dict_size, EMBED_SIZE],
dtype='float32',
is_sparse=is_sparse,
param_attr='embedding')
concat_emb = fluid.layers.concat(
input=[embed_first, embed_second, embed_third, embed_fourth], axis=1)
hidden1 = fluid.layers.fc(input=concat_emb, size=HIDDEN_SIZE, act='sigmoid')
predict_word = fluid.layers.fc(input=hidden1, size=dict_size, act='softmax')
# declare later than predict word
next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64')
cost = fluid.layers.cross_entropy(input=predict_word, label=next_word)
avg_cost = fluid.layers.mean(cost)
return avg_cost
def train():
place = fluid.CPUPlace()
first_word = fluid.layers.data(name='firstw', shape=[1], dtype='int64')
second_word = fluid.layers.data(name='secondw', shape=[1], dtype='int64')
third_word = fluid.layers.data(name='thirdw', shape=[1], dtype='int64')
forth_word = fluid.layers.data(name='fourthw', shape=[1], dtype='int64')
next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64')
word_list = [first_word, second_word, third_word, forth_word, next_word]
avg_cost = word2vec(word_list, is_sparse=True)
main_program = fluid.default_main_program()
startup_program = fluid.default_startup_program()
sgd_optimizer = fluid.optimizer.SGDOptimizer(learning_rate=1e-3)
sgd_optimizer.minimize(avg_cost)
exe = fluid.Executor(place)
exe.run(startup_program) # initialization
for epoch in range(0, PASS_NUM):
for mini_batch in batch_reader():
feed_var_list = [
main_program.global_block().var("firstw"),
main_program.global_block().var("secondw"),
main_program.global_block().var("thirdw"),
main_program.global_block().var("fourthw"),
main_program.global_block().var("nextw")
]
feeder = fluid.DataFeeder(feed_list=feed_var_list, place=place)
cost = exe.run(
main_program,
feed=feeder.feed(mini_batch),
fetch_list=[avg_cost])
print("Epoch {} Cost = {}".format(epoch, cost[0]))
model_dir = "./tmp/w2v_model"
var_list_to_saved = [main_program.global_block().var("embedding")]
print("saving model to %s" % model_dir)
fluid.io.save_vars(
executor=exe, dirname="./w2v_model/", vars=var_list_to_saved)
if __name__ == "__main__":
train()
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或