提交 b3f0f3d2 编写于 作者: Q qingqing01 提交者: GitHub

Merge pull request #766 from qingqing01/sentiment

Support predicting the samples from sys.stdin
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import os, sys
import numpy as np
from optparse import OptionParser
from py_paddle import swig_paddle, DataProviderConverter
......@@ -66,35 +66,27 @@ class SentimentPrediction():
for v in open(label_file, 'r'):
self.label[int(v.split('\t')[1])] = v.split('\t')[0]
def get_data(self, data_file):
def get_index(self, data):
"""
Get input data of paddle format.
transform word into integer index according to the dictionary.
"""
with open(data_file, 'r') as fdata:
for line in fdata:
words = line.strip().split()
words = data.strip().split()
word_slot = [
self.word_dict[w] for w in words if w in self.word_dict
]
if not word_slot:
print "all words are not in dictionary: %s", line
continue
yield [word_slot]
return word_slot
def predict(self, data_file):
"""
data_file: file name of input data.
"""
input = self.converter(self.get_data(data_file))
def batch_predict(self, data_batch):
input = self.converter(data_batch)
output = self.network.forwardTest(input)
prob = output[0]["value"]
lab = np.argsort(-prob)
labs = np.argsort(-prob)
for idx, lab in enumerate(labs):
if self.label is None:
print("%s: predicting label is %d" % (data_file, lab[0][0]))
print("predicting label is %d" % (lab[0]))
else:
print("%s: predicting label is %s" %
(data_file, self.label[lab[0][0]]))
print("predicting label is %s" %
(self.label[lab[0]]))
def option_parser():
usage = "python predict.py -n config -w model_dir -d dictionary -i input_file "
......@@ -119,11 +111,13 @@ def option_parser():
default=None,
help="dictionary file")
parser.add_option(
"-i",
"--data",
"-c",
"--batch_size",
type="int",
action="store",
dest="data",
help="data file to predict")
dest="batch_size",
default=1,
help="the batch size for prediction")
parser.add_option(
"-w",
"--model",
......@@ -137,14 +131,21 @@ def option_parser():
def main():
options, args = option_parser()
train_conf = options.train_conf
data = options.data
batch_size = options.batch_size
dict_file = options.dict_file
model_path = options.model_path
label = options.label
swig_paddle.initPaddle("--use_gpu=0")
predict = SentimentPrediction(train_conf, dict_file, model_path, label)
predict.predict(data)
batch = []
for line in sys.stdin:
batch.append([predict.get_index(line)])
if len(batch) == batch_size:
predict.batch_predict(batch)
batch=[]
if len(batch) > 0:
predict.batch_predict(batch)
if __name__ == '__main__':
main()
......@@ -19,9 +19,9 @@ set -e
model=model_output/pass-00002/
config=trainer_config.py
label=data/pre-imdb/labels.list
python predict.py \
-n $config\
-w $model \
-b $label \
-d ./data/pre-imdb/dict.txt \
-i ./data/aclImdb/test/pos/10007_10.txt
cat ./data/aclImdb/test/pos/10007_10.txt | python predict.py \
--tconf=$config\
--model=$model \
--label=$label \
--dict=./data/pre-imdb/dict.txt \
--batch_size=1
......@@ -293,20 +293,21 @@ predict.sh:
model=model_output/pass-00002/
config=trainer_config.py
label=data/pre-imdb/labels.list
python predict.py \
-n $config\
-w $model \
-b $label \
-d data/pre-imdb/dict.txt \
-i data/aclImdb/test/pos/10007_10.txt
```
* `predict.py`: predicting interface.
* -n $config : set network configure.
* -w $model: set model path.
* -b $label: set dictionary about corresponding relation between integer label and string label.
* -d data/pre-imdb/dict.txt: set dictionary.
* -i data/aclImdb/test/pos/10014_7.txt: set one example file to predict.
cat ./data/aclImdb/test/pos/10007_10.txt | python predict.py \
--tconf=$config\
--model=$model \
--label=$label \
--dict=./data/pre-imdb/dict.txt \
--batch_size=1
```
* `cat ./data/aclImdb/test/pos/10007_10.txt` : the input sample.
* `predict.py` : predicting interface.
* `--tconf=$config` : set network configure.
* ` --model=$model` : set model path.
* `--label=$label` : set dictionary about corresponding relation between integer label and string label.
* `--dict=data/pre-imdb/dict.txt` : set dictionary.
* `--batch_size=1` : set batch size.
Note you should make sure the default model path `model_output/pass-00002`
exists or change the model path.
......
......@@ -291,20 +291,21 @@ predict.sh:
model=model_output/pass-00002/
config=trainer_config.py
label=data/pre-imdb/labels.list
python predict.py \
-n $config\
-w $model \
-b $label \
-d data/pre-imdb/dict.txt \
-i data/aclImdb/test/pos/10007_10.txt
```
* `predict.py`: 预测接口脚本。
* -n $config : 设置网络配置。
* -w $model: 设置模型路径。
* -b $label: 设置标签类别字典,这个字典是整数标签和字符串标签的一个对应。
* -d data/pre-imdb/dict.txt: 设置字典文件。
* -i data/aclImdb/test/pos/10014_7.txt: 设置一个要预测的示例文件。
cat ./data/aclImdb/test/pos/10007_10.txt | python predict.py \
--tconf=$config\
--model=$model \
--label=$label \
--dict=./data/pre-imdb/dict.txt \
--batch_size=1
```
* `cat ./data/aclImdb/test/pos/10007_10.txt` : 输入预测样本。
* `predict.py` : 预测接口脚本。
* `--tconf=$config` : 设置网络配置。
* `--model=$model` : 设置模型路径。
* `--label=$label` : 设置标签类别字典,这个字典是整数标签和字符串标签的一个对应。
* `--dict=data/pre-imdb/dict.txt` : 设置字典文件。
* `--batch_size=1` : 设置batch size。
注意应该确保默认模型路径`model_output / pass-00002`存在或更改为其它模型路径。
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册