w2v_evaluate_reader.py 2.9 KB
Newer Older
T
tangwei 已提交
1
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
M
add w2v  
malin10 已提交
2 3 4 5 6 7 8 9 10 11 12 13
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
T
tangwei 已提交
14

M
add w2v  
malin10 已提交
15
import io
T
tangwei 已提交
16

M
add w2v  
malin10 已提交
17
import six
T
tangwei 已提交
18

19 20
from paddlerec.core.reader import Reader
from paddlerec.core.utils import envs
M
add w2v  
malin10 已提交
21 22 23 24


class EvaluateReader(Reader):
    def init(self):
T
for mat  
tangwei 已提交
25
        dict_path = envs.get_global_env("word_id_dict_path", None, "evaluate.reader")
M
add w2v  
malin10 已提交
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
        self.word_to_id = dict()
        self.id_to_word = dict()
        with io.open(dict_path, 'r', encoding='utf-8') as f:
            for line in f:
                self.word_to_id[line.split(' ')[0]] = int(line.split(' ')[1])
                self.id_to_word[int(line.split(' ')[1])] = line.split(' ')[0]
        self.dict_size = len(self.word_to_id)

    def native_to_unicode(self, s):
        if self._is_unicode(s):
            return s
        try:
            return self._to_unicode(s)
        except UnicodeDecodeError:
            res = self._to_unicode(s, ignore_errors=True)
            return res

    def _is_unicode(self, s):
        if six.PY2:
            if isinstance(s, unicode):
                return True
        else:
            if isinstance(s, str):
                return True
        return False
T
for mat  
tangwei 已提交
51

M
add w2v  
malin10 已提交
52 53 54 55 56
    def _to_unicode(self, s, ignore_errors=False):
        if self._is_unicode(s):
            return s
        error_mode = "ignore" if ignore_errors else "strict"
        return s.decode("utf-8", errors=error_mode)
T
for mat  
tangwei 已提交
57

M
add w2v  
malin10 已提交
58 59
    def strip_lines(self, line, vocab):
        return self._replace_oov(vocab, self.native_to_unicode(line))
T
for mat  
tangwei 已提交
60

M
add w2v  
malin10 已提交
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
    def _replace_oov(self, original_vocab, line):
        """Replace out-of-vocab words with "<UNK>".
      This maintains compatibility with published results.
      Args:
        original_vocab: a set of strings (The standard vocabulary for the dataset)
        line: a unicode string - a space-delimited sequence of words.
      Returns:
        a unicode string - a space-delimited sequence of words.
      """
        return u" ".join([
            word if word in original_vocab else u"<UNK>" for word in line.split()
        ])

    def generate_sample(self, line):
        def reader():
            features = self.strip_lines(line.lower(), self.word_to_id)
            features = features.split()
T
for mat  
tangwei 已提交
78 79 80
            yield [('analogy_a', [self.word_to_id[features[0]]]), ('analogy_b', [self.word_to_id[features[1]]]),
                   ('analogy_c', [self.word_to_id[features[2]]]), ('analogy_d', [self.word_to_id[features[3]]])]

M
add w2v  
malin10 已提交
81
        return reader