未验证 提交 dd4eb0cd 编写于 作者: 走神的阿圆's avatar 走神的阿圆 提交者: GitHub

add hdfs --model (#727)

上级 e9613d93
......@@ -99,6 +99,15 @@ class LocalFileSystem(object):
def join(path, *paths):
return os.path.join(path, *paths)
def isfile(self, filename):
return os.path.isfile(filename)
def read_file(self, filename, binary_mode=True):
mode = "rb" if binary_mode else "r"
with open(filename, mode) as reader:
data = reader.read()
return data
def read(self, filename, binary_mode=False, size=None, continue_from=None):
mode = "rb" if binary_mode else "r"
encoding = None if binary_mode else "utf-8"
......@@ -142,6 +151,14 @@ class HDFileSystem(object):
else:
return True
def isfile(self, filename):
return exists(filename)
def read_file(self, filename, binary_mode=True):
with self.cli.read(hdfs_path=filename[7:]) as reader:
data = reader.read()
return data
def makedirs(self, path):
self.cli.makedirs(hdfs_path=path[7:])
......@@ -205,6 +222,13 @@ class BosFileSystem(object):
credentials=BceCredentials(access_key_id, secret_access_key),
endpoint=bos_host)
def isfile(self, filename):
return exists(filename)
def read_file(self, filename, binary=True):
print('BosFileSystem not support --model yet.')
return
@staticmethod
def _get_object_info(path):
path = path[6:]
......@@ -392,6 +416,9 @@ class BFile(object):
def __iter__(self):
return self
def isfile(self, filename):
return self.fs.isfile(filename)
def _read_buffer_to_offset(self, new_buff_offset):
"""Read buffer from index self.buffer_offset to index new_buff_offset.
......@@ -407,6 +434,9 @@ class BFile(object):
self.buff_offset += read_size
return self.buff[old_buff_offset:old_buff_offset + read_size]
def read_file(self, filename, binnary=True):
return self.fs.read_file(filename, binnary)
def read(self, n=None):
"""Read `n` or all contents of self.buff or file.
......
......@@ -70,10 +70,12 @@ class LogReader(object):
@model.setter
def model(self, model_path):
if not os.path.isfile(model_path):
self._model = model_path
with bfile.BFile(model_path, 'rb') as bfp:
if not bfp.isfile(model_path):
print("Model path %s should be file path, please check this path." % model_path)
else:
if os.path.exists(model_path):
if bfile.exists(model_path):
self._model = model_path
else:
print("Model path %s is invalid, please check this path." % model_path)
......
......@@ -18,6 +18,7 @@ import sys
import time
import numpy as np
from visualdl.server.log import logger
from visualdl.io import bfile
from visualdl.utils.string_util import encode_tag, decode_tag
......@@ -192,8 +193,8 @@ def get_histogram(log_reader, run, tag):
def get_graph(log_reader):
result = b""
if log_reader.model:
with open(log_reader.model, "rb") as fp:
result = fp.read()
with bfile.BFile(log_reader.model, 'rb') as bfp:
result = bfp.read_file(log_reader.model)
return result
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册