diff --git a/paddle_hub/module.py b/paddle_hub/module.py index cc4bbab47c09f1ebc4e1da49b72d915f8e57b688..66828e5790d15dccf0d22fac3507cccd236ada4c 100755 --- a/paddle_hub/module.py +++ b/paddle_hub/module.py @@ -79,10 +79,6 @@ class Module(object): self.config = ModuleConfig(self.module_dir) self.config.load() - # load assets - # self.dict = defaultdict(int) - # self.dict.setdefault(0) - # self._load_assets(module_dir) def _construct_feed_dict(self, inputs): """ Construct feed dict according to user's inputs and module config. @@ -244,10 +240,6 @@ class ModuleConfig(object): """ for w in word_dict: self.dict[w] = word_dict[w] - # mkdir(self.module_dir) - # with open(os.path.join(self.module_dir, DICT_NAME), "w") as fo: - # for w in word_dict: - # self.dict[w] = word_dict[w] def register_feed_signature(self, feed_desc, sign_name="default"): """ Register feed signature to the Module diff --git a/tests/test_export_n_load_module.py b/tests/test_export_n_load_module.py index eefc44b6f8832f80e81f5613b51a992bb5c21790..cfe5107d920b2395ec8121fe15e52351cd86beb5 100755 --- a/tests/test_export_n_load_module.py +++ b/tests/test_export_n_load_module.py @@ -53,12 +53,13 @@ print("imikolov simple dataset batch_size =", batch_size) def module_fn(trainable=False): + # Define module function for saving module # create word input words = fluid.layers.data( name="words", shape=[1], lod_level=1, dtype="int64") # create embedding - emb_name = "embedding" + emb_name = "w2v_emb" emb_param_attr = fluid.ParamAttr(name=emb_name, trainable=trainable) word_emb = fluid.layers.embedding( input=words, @@ -133,11 +134,6 @@ def train(use_cuda=False): exe = fluid.Executor(place) exe.run(startup_program) # initialization - # fluid.io.load_vars( - # executor=exe, - # dirname="./w2v_model", - # vars=[main_program.global_block().var("embedding")]) - # 也可使用predicate方式搜索变量 step = 0 for epoch in range(0, PASS_NUM): for mini_batch in batch_reader(): @@ -159,18 +155,7 @@ def train(use_cuda=False): if step % 100 == 0: print("Epoch={} Step={} Cost={}".format(epoch, step, cost[0])) - model_dir = "./tmp/w2v_model" - # save part of model - var_list_to_saved = [main_program.global_block().var("embedding")] - print("saving model to %s" % model_dir) - fluid.io.save_vars( - executor=exe, dirname=model_dir + "_save_vars", vars=var_list_to_saved) - - # save the whole model - fluid.io.save_persistables( - executor=exe, dirname=model_dir + "_save_persistables") - - saved_model_dir = "./tmp/word2vec_inference_model" + saved_model_dir = "./tmp/word2vec_test_model" # save inference model including feed and fetch variable info fluid.io.save_inference_model( dirname=saved_model_dir, @@ -187,7 +172,7 @@ def train(use_cuda=False): w_id += 1 # save word dict to assets folder - config = hub.ModuleConfig(model_dir) + config = hub.ModuleConfig(saved_model_dir) config.save_dict(word_dict=dictionary) config.dump() @@ -198,28 +183,27 @@ def test_save_module(use_cuda=False): exe = fluid.Executor(place) main_program = fluid.Program() startup_program = fluid.Program() - exe = fluid.Executor(place) with fluid.program_guard(main_program, startup_program): words, word_emb = module_fn() exe.run(startup_program) # load inference embedding parameters - saved_model_dir = "./tmp/word2vec_inference_model" + saved_model_dir = "./tmp/word2vec_test_model" fluid.io.load_inference_model(executor=exe, dirname=saved_model_dir) - feed_var_list = [main_program.global_block().var("words")] - feeder = fluid.DataFeeder(feed_list=feed_var_list, place=place) - results = exe.run( - main_program, - feed=feeder.feed([[[1, 2, 3, 4, 5]]]), - fetch_list=[word_emb], - return_numpy=False) + # feed_var_list = [main_program.global_block().var("words")] + # feeder = fluid.DataFeeder(feed_list=feed_var_list, place=place) + # results = exe.run( + # main_program, + # feed=feeder.feed([[[1, 2, 3, 4, 5]]]), + # fetch_list=[word_emb], + # return_numpy=False) - np_result = np.array(results[0]) - print(np_result) + # np_result = np.array(results[0]) + # print(np_result) # save module_dir - saved_module_dir = "./tmp/word2vec_inference_module" + saved_module_dir = "./tmp/word2vec_test_module" fluid.io.save_inference_model( dirname=saved_module_dir, feeded_var_names=["words"], @@ -233,40 +217,31 @@ def test_save_module(use_cuda=False): w = w.decode("ascii") dictionary[w] = w_id w_id += 1 - # save word dict to assets folder + + input_desc = {"words": words.name} + output_desc = {"emb": word_emb.name} config = hub.ModuleConfig(saved_module_dir) + config.register_feed_signature(input_desc, sign_name="default") + config.register_fetch_signature(output_desc, sign_name="default") config.save_dict(word_dict=dictionary) - config.dump() def test_load_module(use_cuda=False): - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(fluid.CPUPlace()) - saved_module_dir = "./tmp/word2vec_inference_module" - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model( - saved_module_dir, executor=exe) - - # Sequence input in Paddle must be LOD Tensor, so we need to convert them inside Module - word_ids = [[1, 2, 3, 4, 5]] - lod = [[5]] - word_ids_lod_tensor = fluid.create_lod_tensor(word_ids, lod, place) - - results = exe.run( - inference_program, - feed={feed_target_names[0]: word_ids_lod_tensor}, - fetch_list=fetch_targets, - return_numpy=False) + saved_module_dir = "./tmp/word2vec_test_module" + w2v_module = hub.Module(module_dir=saved_module_dir) - print(feed_target_names) - print(fetch_targets) - np_result = np.array(results[0]) - print(np_result) + word_ids = [[1, 2, 3, 4, 5]] # test sequence + word_ids_lod_tensor = w2v_module._preprocess_input(word_ids) + result = w2v_module({"words": word_ids_lod_tensor}) + print(result) if __name__ == "__main__": use_cuda = True + print("train...") train(use_cuda) + print("save module...") test_save_module() + print("load module...") test_load_module() diff --git a/tests/test_module.py b/tests/test_module.py index 1754a1798ec4704fbb417900aaaf90a6aa8d8a8f..6bb3788e074b970088a1fdd137d8f3d9df5c7d52 100755 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -1,3 +1,4 @@ +# coding=utf-8 # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"