From 89d558cc5166e7413e1736cf84d8865b5dc7af1c Mon Sep 17 00:00:00 2001 From: Nicky Date: Thu, 14 Jun 2018 16:32:58 -0700 Subject: [PATCH] Add inference example instead of random data and update per team's comments --- 06.understand_sentiment/README.md | 37 +++++++-- 06.understand_sentiment/index.html | 37 +++++++-- 06.understand_sentiment/train_conv.py | 46 +++++++---- 06.understand_sentiment/train_dyn_rnn.py | 82 +++++++++++-------- 06.understand_sentiment/train_stacked_lstm.py | 82 +++++++++++-------- 5 files changed, 178 insertions(+), 106 deletions(-) diff --git a/06.understand_sentiment/README.md b/06.understand_sentiment/README.md index 6d3a665..f427f13 100644 --- a/06.understand_sentiment/README.md +++ b/06.understand_sentiment/README.md @@ -183,7 +183,7 @@ The above stacked bidirectional LSTM network extracts high-level features and ma To reiterate, we can either invoke `convolution_net` or `stacked_lstm_net`. In below steps, we will go with `convolution_net`. -Next we define a `inference_program` that simply uses `convolution_net` to predict output with the input from `fluid.layer.data`. +Next we define an `inference_program` that simply uses `convolution_net` to predict output with the input from `fluid.layer.data`. ```python def inference_program(word_dict): @@ -200,6 +200,7 @@ Also define `optimizer_func` to specify the optimizer. In the context of supervised learning, labels of the training set are defined in `paddle.layer.data` too. During training, cross-entropy is used as loss function in `paddle.layer.classification_cost` and as the output of the network; During testing, the outputs are the probabilities calculated in the classifier. +First result that returns from the list must be cost. ```python def train_program(word_dict): @@ -309,18 +310,38 @@ inferencer = fluid.Inferencer( place=place) ``` +### Create Lod Tensor with test data + +To do inference, we pick 3 potential reviews out of our mind as testing data. Feel free to modify any of them. +We map each word in the reviews to id from `word_dict`, replaced by 'unknown' if the word is not in `word_dict`. +Then we create lod data with the id list and use `create_lod_tensor` to create lod tensor. + +```python +reviews_str = [ + 'read the book forget the movie', 'this is a great movie', 'this is very bad' +] +reviews = [c.split() for c in reviews_str] + +UNK = word_dict[''] +lod = [] +for c in reviews: + lod.append([word_dict.get(words, UNK) for words in c]) + +base_shape = [[len(c) for c in lod]] + +tensor_words = fluid.create_lod_tensor(lod, base_shape, place) +``` + ### Infer -Now we can infer with inputs that we provide in `feed_order` during training. +Now we can infer and predict probability of positive or negative from each review above. ```python -lod = [[3, 4, 2]] -base_shape = [1] -# The range of random integers is [low, high] -tensor_words = fluid.create_random_int_lodtensor( - lod, base_shape, place, low=0, high=len(word_dict) - 1) results = inferencer.infer({'words': tensor_words}) -print("infer results: ", results) + +for i, r in enumerate(results[0]): + print("Predict probability of ", r[0], " to be positive and ", r[1], " to be negative for review \'", reviews_str[i], "\'") + ``` diff --git a/06.understand_sentiment/index.html b/06.understand_sentiment/index.html index b9f1982..54f0a5b 100644 --- a/06.understand_sentiment/index.html +++ b/06.understand_sentiment/index.html @@ -225,7 +225,7 @@ The above stacked bidirectional LSTM network extracts high-level features and ma To reiterate, we can either invoke `convolution_net` or `stacked_lstm_net`. In below steps, we will go with `convolution_net`. -Next we define a `inference_program` that simply uses `convolution_net` to predict output with the input from `fluid.layer.data`. +Next we define an `inference_program` that simply uses `convolution_net` to predict output with the input from `fluid.layer.data`. ```python def inference_program(word_dict): @@ -242,6 +242,7 @@ Also define `optimizer_func` to specify the optimizer. In the context of supervised learning, labels of the training set are defined in `paddle.layer.data` too. During training, cross-entropy is used as loss function in `paddle.layer.classification_cost` and as the output of the network; During testing, the outputs are the probabilities calculated in the classifier. +First result that returns from the list must be cost. ```python def train_program(word_dict): @@ -351,18 +352,38 @@ inferencer = fluid.Inferencer( place=place) ``` +### Create Lod Tensor with test data + +To do inference, we pick 3 potential reviews out of our mind as testing data. Feel free to modify any of them. +We map each word in the reviews to id from `word_dict`, replaced by 'unknown' if the word is not in `word_dict`. +Then we create lod data with the id list and use `create_lod_tensor` to create lod tensor. + +```python +reviews_str = [ + 'read the book forget the movie', 'this is a great movie', 'this is very bad' +] +reviews = [c.split() for c in reviews_str] + +UNK = word_dict[''] +lod = [] +for c in reviews: + lod.append([word_dict.get(words, UNK) for words in c]) + +base_shape = [[len(c) for c in lod]] + +tensor_words = fluid.create_lod_tensor(lod, base_shape, place) +``` + ### Infer -Now we can infer with inputs that we provide in `feed_order` during training. +Now we can infer and predict probability of positive or negative from each review above. ```python -lod = [[3, 4, 2]] -base_shape = [1] -# The range of random integers is [low, high] -tensor_words = fluid.create_random_int_lodtensor( - lod, base_shape, place, low=0, high=len(word_dict) - 1) results = inferencer.infer({'words': tensor_words}) -print("infer results: ", results) + +for i, r in enumerate(results[0]): + print("Predict probability of ", r[0], " to be positive and ", r[1], " to be negative for review \'", reviews_str[i], "\'") + ``` diff --git a/06.understand_sentiment/train_conv.py b/06.understand_sentiment/train_conv.py index 860a801..61fe18e 100644 --- a/06.understand_sentiment/train_conv.py +++ b/06.understand_sentiment/train_conv.py @@ -69,14 +69,11 @@ def optimizer_func(): def train(use_cuda, train_program, params_dirname): - import time - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() print("Loading IMDB word dict....") word_dict = paddle.dataset.imdb.word_dict() print("Reading training data....") - train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.imdb.train(word_dict), buf_size=25000), @@ -95,18 +92,18 @@ def train(use_cuda, train_program, params_dirname): def event_handler(event): if isinstance(event, fluid.EndStepEvent): - avg_cost, acc = trainer.test( - reader=test_reader, feed_order=feed_order) + if event.step % 10 == 0: + avg_cost, acc = trainer.test( + reader=test_reader, feed_order=feed_order) - print('Step {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( - event.step, avg_cost, acc)) + print('Step {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( + event.step, avg_cost, acc)) - print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) + print("Step {0}, Epoch {1} Metrics {2}".format( + event.step, event.epoch, map(np.array, event.metrics))) - if event.step == 10: # Adjust this number for accuracy - trainer.save_params(params_dirname) - trainer.stop() + elif isinstance(event, fluid.EndEpochEvent): + trainer.save_params(params_dirname) trainer.train( num_epochs=1, @@ -134,13 +131,26 @@ def infer(use_cuda, inference_program, params_dirname=None): # element (word). Hence the LoDTensor will hold data for three sentences of # length 3, 4 and 2, respectively. # Note that lod info should be a list of lists. - lod = [[3, 4, 2]] - base_shape = [1] - # The range of random integers is [low, high] - tensor_words = fluid.create_random_int_lodtensor( - lod, base_shape, place, low=0, high=len(word_dict) - 1) + + reviews_str = [ + 'read the book forget the movie', 'this is a great movie', + 'this is very bad' + ] + reviews = [c.split() for c in reviews_str] + + UNK = word_dict[''] + lod = [] + for c in reviews: + lod.append([word_dict.get(words, UNK) for words in c]) + + base_shape = [[len(c) for c in lod]] + + tensor_words = fluid.create_lod_tensor(lod, base_shape, place) results = inferencer.infer({'words': tensor_words}) - print("infer results: ", results) + + for i, r in enumerate(results[0]): + print("Predict probability of ", r[0], " to be positive and ", r[1], + " to be negative for review \'", reviews_str[i], "\'") def main(use_cuda): diff --git a/06.understand_sentiment/train_dyn_rnn.py b/06.understand_sentiment/train_dyn_rnn.py index f548a27..ef3be26 100644 --- a/06.understand_sentiment/train_dyn_rnn.py +++ b/06.understand_sentiment/train_dyn_rnn.py @@ -87,49 +87,46 @@ def optimizer_func(): def train(use_cuda, train_program, params_dirname): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - + print("Loading IMDB word dict....") word_dict = paddle.dataset.imdb.word_dict() + + print("Reading training data....") + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=25000), + batch_size=BATCH_SIZE) + + print("Reading testing data....") + test_reader = paddle.batch( + paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) + trainer = fluid.Trainer( train_func=partial(train_program, word_dict), place=place, optimizer_func=optimizer_func) + feed_order = ['words', 'label'] + def event_handler(event): - if isinstance(event, fluid.EndEpochEvent): - test_reader = paddle.batch( - paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) - avg_cost, acc = trainer.test( - reader=test_reader, feed_order=['words', 'label']) - - print("avg_cost: %s" % avg_cost) - print("acc : %s" % acc) - - if acc > 0.2: # Smaller value to increase CI speed - trainer.save_params(params_dirname) - trainer.stop() - - else: - print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( - event.epoch + 1, avg_cost, acc)) - if math.isnan(avg_cost): - sys.exit("got NaN loss, training failed.") - elif isinstance(event, fluid.EndStepEvent): - print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) - if event.step == 1: # Run 2 iterations to speed CI - trainer.save_params(params_dirname) - trainer.stop() + if isinstance(event, fluid.EndStepEvent): + if event.step % 10 == 0: + avg_cost, acc = trainer.test( + reader=test_reader, feed_order=feed_order) - train_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.imdb.train(word_dict), buf_size=25000), - batch_size=BATCH_SIZE) + print('Step {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( + event.step, avg_cost, acc)) + + print("Step {0}, Epoch {1} Metrics {2}".format( + event.step, event.epoch, map(np.array, event.metrics))) + + elif isinstance(event, fluid.EndEpochEvent): + trainer.save_params(params_dirname) trainer.train( num_epochs=1, event_handler=event_handler, reader=train_reader, - feed_order=['words', 'label']) + feed_order=feed_order) def infer(use_cuda, inference_program, params_dirname=None): @@ -151,13 +148,26 @@ def infer(use_cuda, inference_program, params_dirname=None): # element (word). Hence the LoDTensor will hold data for three sentences of # length 3, 4 and 2, respectively. # Note that lod info should be a list of lists. - lod = [[3, 4, 2]] - base_shape = [1] - # The range of random integers is [low, high] - tensor_words = fluid.create_random_int_lodtensor( - lod, base_shape, place, low=0, high=len(word_dict) - 1) + + reviews_str = [ + 'read the book forget the movie', 'this is a great movie', + 'this is very bad' + ] + reviews = [c.split() for c in reviews_str] + + UNK = word_dict[''] + lod = [] + for c in reviews: + lod.append([word_dict.get(words, UNK) for words in c]) + + base_shape = [[len(c) for c in lod]] + + tensor_words = fluid.create_lod_tensor(lod, base_shape, place) results = inferencer.infer({'words': tensor_words}) - print("infer results: ", results) + + for i, r in enumerate(results[0]): + print("Predict probability of ", r[0], " to be positive and ", r[1], + " to be negative for review \'", reviews_str[i], "\'") def main(use_cuda): diff --git a/06.understand_sentiment/train_stacked_lstm.py b/06.understand_sentiment/train_stacked_lstm.py index 104215d..351994f 100644 --- a/06.understand_sentiment/train_stacked_lstm.py +++ b/06.understand_sentiment/train_stacked_lstm.py @@ -78,49 +78,46 @@ def optimizer_func(): def train(use_cuda, train_program, params_dirname): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - + print("Loading IMDB word dict....") word_dict = paddle.dataset.imdb.word_dict() + + print("Reading training data....") + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=25000), + batch_size=BATCH_SIZE) + + print("Reading testing data....") + test_reader = paddle.batch( + paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) + trainer = fluid.Trainer( train_func=partial(train_program, word_dict), place=place, optimizer_func=optimizer_func) + feed_order = ['words', 'label'] + def event_handler(event): - if isinstance(event, fluid.EndEpochEvent): - test_reader = paddle.batch( - paddle.dataset.imdb.test(word_dict), batch_size=BATCH_SIZE) - avg_cost, acc = trainer.test( - reader=test_reader, feed_order=['words', 'label']) - - print("avg_cost: %s" % avg_cost) - print("acc : %s" % acc) - - if acc > 0.2: # Smaller value to increase CI speed - trainer.save_params(params_dirname) - trainer.stop() - - else: - print('BatchID {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( - event.epoch + 1, avg_cost, acc)) - if math.isnan(avg_cost): - sys.exit("got NaN loss, training failed.") - elif isinstance(event, fluid.EndStepEvent): - print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) - if event.step == 1: # Run 2 iterations to speed CI - trainer.save_params(params_dirname) - trainer.stop() + if isinstance(event, fluid.EndStepEvent): + if event.step % 10 == 0: + avg_cost, acc = trainer.test( + reader=test_reader, feed_order=feed_order) - train_reader = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.imdb.train(word_dict), buf_size=25000), - batch_size=BATCH_SIZE) + print('Step {0}, Test Loss {1:0.2}, Acc {2:0.2}'.format( + event.step, avg_cost, acc)) + + print("Step {0}, Epoch {1} Metrics {2}".format( + event.step, event.epoch, map(np.array, event.metrics))) + + elif isinstance(event, fluid.EndEpochEvent): + trainer.save_params(params_dirname) trainer.train( num_epochs=1, event_handler=event_handler, reader=train_reader, - feed_order=['words', 'label']) + feed_order=feed_order) def infer(use_cuda, inference_program, params_dirname=None): @@ -142,13 +139,26 @@ def infer(use_cuda, inference_program, params_dirname=None): # element (word). Hence the LoDTensor will hold data for three sentences of # length 3, 4 and 2, respectively. # Note that lod info should be a list of lists. - lod = [[3, 4, 2]] - base_shape = [1] - # The range of random integers is [low, high] - tensor_words = fluid.create_random_int_lodtensor( - lod, base_shape, place, low=0, high=len(word_dict) - 1) + + reviews_str = [ + 'read the book forget the movie', 'this is a great movie', + 'this is very bad' + ] + reviews = [c.split() for c in reviews_str] + + UNK = word_dict[''] + lod = [] + for c in reviews: + lod.append([word_dict.get(words, UNK) for words in c]) + + base_shape = [[len(c) for c in lod]] + + tensor_words = fluid.create_lod_tensor(lod, base_shape, place) results = inferencer.infer({'words': tensor_words}) - print("infer results: ", results) + + for i, r in enumerate(results[0]): + print("Predict probability of ", r[0], " to be positive and ", r[1], + " to be negative for review \'", reviews_str[i], "\'") def main(use_cuda): -- GitLab