提交 b60e6fd5 编写于 作者: W wangxiao1021

fix bugs

上级 b14bb89c
...@@ -293,10 +293,9 @@ class Reader(object): ...@@ -293,10 +293,9 @@ class Reader(object):
if to_append: if to_append:
batch_records.append(record) batch_records.append(record)
else: else:
ds = ['s'] * 7 batch_pad_records = self._pad_batch_records(batch_records)
for piece in palm.distribute.yield_pieces(\ ds = ['s'] * len(batch_pad_records)
self._pad_batch_records(batch_records), for piece in palm.distribute.yield_pieces(batch_pad_records, ds, batch_size):
ds, batch_size):
yield piece yield piece
batch_records, max_len = [record], len(record.token_ids) batch_records, max_len = [record], len(record.token_ids)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册