predict.py

# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import numpy as np
from optparse import OptionParser
from py_paddle import swig_paddle, DataProviderConverter
from paddle.trainer.PyDataProvider2 import integer_value_sequence
from paddle.trainer.config_parser import parse_config
"""
Usage: run following command to show help message.
  python predict.py -h
"""
UNK_IDX = 0


class Prediction():
    def __init__(self, train_conf, dict_file, model_dir, label_file, predicate_dict_file):
        """
        train_conf: trainer configure.
        dict_file: word dictionary file name.
        model_dir: directory of model.
        """

        self.dict = {}
        self.labels = {}
        self.predicate_dict={}
        self.labels_reverse = {}
        self.load_dict_label(dict_file, label_file, predicate_dict_file)

        len_dict = len(self.dict)
        len_label = len(self.labels)
        len_pred = len(self.predicate_dict)

        conf = parse_config(
            train_conf,
            'dict_len=' + str(len_dict) + 
            ',label_len=' + str(len_label) +
            ',pred_len=' + str(len_pred) +
            ',is_predict=True')
        self.network = swig_paddle.GradientMachine.createFromConfigProto(
            conf.model_config)
        self.network.loadParameters(model_dir)

        slots = [
            integer_value_sequence(len_dict),
            integer_value_sequence(len_pred),
            integer_value_sequence(len_dict),
            integer_value_sequence(len_dict),
            integer_value_sequence(len_dict),
            integer_value_sequence(len_dict),
            integer_value_sequence(len_dict), 
            integer_value_sequence(2)
            ]
        self.converter = DataProviderConverter(slots)

    def load_dict_label(self, dict_file, label_file, predicate_dict_file):
        """
        Load dictionary from self.dict_file.
        """
        for line_count, line in enumerate(open(dict_file, 'r')):
            self.dict[line.strip()] = line_count

        for line_count, line in enumerate(open(label_file, 'r')):
            self.labels[line.strip()] = line_count
            self.labels_reverse[line_count] = line.strip()

        for line_count, line in enumerate(open(predicate_dict_file, 'r')):
            self.predicate_dict[line.strip()] = line_count
    def get_data(self, data_file):
        """
        Get input data of paddle format.
        """
        with open(data_file, 'r') as fdata:
            for line in fdata:
                sentence, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, label = line.strip(
                ).split('\t')
                words = sentence.split()
                sen_len = len(words)
                 
                word_slot = [self.dict.get(w, UNK_IDX) for w in words]
                predicate_slot = [self.predicate_dict.get(predicate, UNK_IDX)] * sen_len
                ctx_n2_slot = [self.dict.get(ctx_n2, UNK_IDX)] * sen_len
                ctx_n1_slot = [self.dict.get(ctx_n1, UNK_IDX)] * sen_len
                ctx_0_slot = [self.dict.get(ctx_0, UNK_IDX)] * sen_len
                ctx_p1_slot = [self.dict.get(ctx_p1, UNK_IDX)] * sen_len
                ctx_p2_slot = [self.dict.get(ctx_p2, UNK_IDX)] * sen_len

                marks = mark.split()
                mark_slot = [int(w) for w in marks]
                
                yield word_slot, predicate_slot, ctx_n2_slot, ctx_n1_slot, \
                      ctx_0_slot, ctx_p1_slot, ctx_p2_slot,  mark_slot

    def predict(self, data_file, output_file):
        """
        data_file: file name of input data.
        """
        input = self.converter(self.get_data(data_file))
        output = self.network.forwardTest(input)
        lab = output[0]["id"].tolist()

        with open(data_file, 'r') as fin, open(output_file, 'w') as fout:
            index = 0
            for line in fin:
                sen = line.split('\t')[0]
                len_sen = len(sen.split())
                line_labels = lab[index:index + len_sen]
                index += len_sen
                fout.write(sen + '\t' + ' '.join([self.labels_reverse[
                    i] for i in line_labels]) + '\n')


def option_parser():
    usage = ("python predict.py -c config -w model_dir " 
             "-d word dictionary -l label_file -i input_file  -p pred_dict_file")
    parser = OptionParser(usage="usage: %s [options]" % usage)
    parser.add_option(
        "-c",
        "--tconf",
        action="store",
        dest="train_conf",
        help="network config")
    parser.add_option(
        "-d",
        "--dict",
        action="store",
        dest="dict_file",
        help="dictionary file")
    parser.add_option(
        "-l",
        "--label",
        action="store",
        dest="label_file",
        default=None,
        help="label file")
    parser.add_option(
        "-p",
        "--predict_dict_file",
        action="store",
        dest="predict_dict_file",
        default=None,
        help="predict_dict_file")
    parser.add_option(
        "-i",
        "--data",
        action="store",
        dest="data_file",
        help="data file to predict")
    parser.add_option(
        "-w",
        "--model",
        action="store",
        dest="model_path",
        default=None,
        help="model path")

    parser.add_option(
        "-o",
        "--output_file",
        action="store",
        dest="output_file",
        default=None,
        help="output file")
    return parser.parse_args()


def main():
    options, args = option_parser()
    train_conf = options.train_conf
    data_file = options.data_file
    dict_file = options.dict_file
    model_path = options.model_path
    label_file = options.label_file
    predict_dict_file = options.predict_dict_file
    output_file = options.output_file

    swig_paddle.initPaddle("--use_gpu=0")
    predict = Prediction(train_conf, dict_file, model_path, label_file, predict_dict_file)
    predict.predict(data_file,output_file)


if __name__ == '__main__':
    main()