提交 ba1c9785 编写于 作者: D dangqingqing

close file and compare the result with the old way.

上级 0dd53294
...@@ -12,7 +12,8 @@ ...@@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import paddle.v2.dataset.common #import paddle.v2.dataset.common
import common
import tarfile import tarfile
import gzip import gzip
import itertools import itertools
...@@ -49,11 +50,9 @@ def load_dict(filename): ...@@ -49,11 +50,9 @@ def load_dict(filename):
def corpus_reader(data_path, words_name, props_name): def corpus_reader(data_path, words_name, props_name):
""" """
Read one corpus by corpus name. It returns an iterator. Each element of Read one corpus. It returns an iterator. Each element of
this iterator is a tuple including sentence and labels. The sentence is this iterator is a tuple including sentence and labels. The sentence is
consist of a list of word IDs. The labels include a list of label IDs. consist of a list of word IDs. The labels include a list of label IDs.
:param name: corpus name.
:type name: basestring
:return: a iterator of data. :return: a iterator of data.
:rtype: iterator :rtype: iterator
""" """
...@@ -104,7 +103,8 @@ def corpus_reader(data_path, words_name, props_name): ...@@ -104,7 +103,8 @@ def corpus_reader(data_path, words_name, props_name):
lbl_seq.append('B-' + cur_tag) lbl_seq.append('B-' + cur_tag)
is_in_bracket = True is_in_bracket = True
else: else:
print 'error:', l raise RuntimeError('Unexpected label: %s' %
l)
yield sentences, verb_list[i], lbl_seq yield sentences, verb_list[i], lbl_seq
...@@ -115,6 +115,10 @@ def corpus_reader(data_path, words_name, props_name): ...@@ -115,6 +115,10 @@ def corpus_reader(data_path, words_name, props_name):
sentences.append(word) sentences.append(word)
one_seg.append(label) one_seg.append(label)
pf.close()
wf.close()
tf.close()
return reader return reader
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册