未验证 提交 13fd3dca 编写于 作者: S Shaden Smith 提交者: GitHub

Removes DeepSpeedDataSource (#100)

* Removes DeepSpeedDataSource

* dropping unused imports
Co-authored-by: NJeff Rasley <jerasley@microsoft.com>
上级 6d602065
......@@ -3,26 +3,8 @@ Copyright 2019 The Microsoft DeepSpeed Team
'''
import torch
import logging
from torch.utils.data import DataLoader, RandomSampler
from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm
class DeepSpeedDataSource(object):
def __init__(self, filenames):
all_lines = []
for filename in filenames:
logging.info("Start reading file %s" % filename)
with open(filename, "r") as f:
for i, line in enumerate(tqdm(f)):
all_lines.append(line.strip())
self.all_lines = all_lines
self.len = len(self.all_lines)
def __len__(self):
return self.len
class DeepSpeedDataLoader(object):
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册