From 686b74651d6c123d70dd07157af7ab905d5a4796 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Mon, 20 Feb 2017 16:04:10 +0800 Subject: [PATCH] filter the illegal sentences. --- demo/sentiment/dataprovider.py | 2 ++ demo/sentiment/predict.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/demo/sentiment/dataprovider.py b/demo/sentiment/dataprovider.py index 00f72ceca..4b7f5d0e5 100755 --- a/demo/sentiment/dataprovider.py +++ b/demo/sentiment/dataprovider.py @@ -32,4 +32,6 @@ def process(settings, file_name): word_slot = [ settings.word_dict[w] for w in words if w in settings.word_dict ] + if not word_slot: + continue yield word_slot, label diff --git a/demo/sentiment/predict.py b/demo/sentiment/predict.py index 8ec490f64..64c78e0d6 100755 --- a/demo/sentiment/predict.py +++ b/demo/sentiment/predict.py @@ -138,7 +138,11 @@ def main(): batch = [] for line in sys.stdin: - batch.append([predict.get_index(line)]) + words = predict.get_index(line) + if words: + batch.append([words]) + else: + print('All the words in [%s] are not in the dictionary.' % line) if len(batch) == batch_size: predict.batch_predict(batch) batch = [] -- GitLab