From 5ff9b7b302d8173e73453596c298677c76762a24 Mon Sep 17 00:00:00 2001 From: ShenYuhan Date: Wed, 26 Aug 2020 10:49:17 +0800 Subject: [PATCH] add `file_name` in LogWriter() for rewriting. (#764) * add `file_name` in LogWriter(0 for rewriting. * fix reviews --- README.md | 2 ++ visualdl/io/bfile.py | 22 +++++++++++++--------- visualdl/writer/record_writer.py | 25 ++++++++++++++++++++++--- visualdl/writer/writer.py | 9 ++++++++- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8dd6c0e5..2450dfba 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ class LogWriter(logdir=None, filename_suffix='', write_to_disk=True, display_name='', + file_name='', **kwargs) ``` @@ -110,6 +111,7 @@ class LogWriter(logdir=None, | filename_suffix | string | 为默认的日志文件名添加后缀 | | write_to_disk | boolean | 是否写入到磁盘 | | display_name | string | 在面板中替换实际显示的`logdir`，当日志所在路径过长或想隐藏日志所在路径时可指定此参数 | +| file_name | string | 指定写入的日志文件名，如果指定的文件名已经存在，则将日志续写在此文件中，文件名必须包括`vdlrecords` | #### 示例 diff --git a/visualdl/io/bfile.py b/visualdl/io/bfile.py index b1328452..baeb8f1b 100644 --- a/visualdl/io/bfile.py +++ b/visualdl/io/bfile.py @@ -130,7 +130,8 @@ class LocalFileSystem(object): self._write(filename, file_content, "ab" if binary_mode else "a") def write(self, filename, file_content, binary_mode=False): - self._write(filename, file_content, "wb" if binary_mode else "w") + self._write(filename, file_content, "ab" if binary_mode else "a") + # self._write(filename, file_content, "wb" if binary_mode else "w") def walk(self, dir): if 'posix' == os.name: @@ -187,7 +188,8 @@ class HDFileSystem(object): self.cli.write(hdfs_path=filename[7:], data=file_content, append=True) def write(self, filename, file_content, binary_mode=False): - self.cli.write(hdfs_path=filename[7:], data=file_content) + self.cli.write(hdfs_path=filename[7:], data=file_content, append=True) + # self.cli.write(hdfs_path=filename[7:], data=file_content) def walk(self, dir): walks = self.cli.walk(hdfs_path=dir[7:]) @@ -326,13 +328,15 @@ class BosFileSystem(object): self._start_append_time = time.time() def write(self, filename, file_content, binary_mode=False): - bucket_name, object_key = BosFileSystem._get_object_info(filename) - - self.bos_client.append_object(bucket_name=bucket_name, - key=object_key, - data=file_content, - content_md5=content_md5(file_content), - content_length=len(file_content)) + self.append(filename, file_content, binary_mode=False) + + # bucket_name, object_key = BosFileSystem._get_object_info(filename) + # + # self.bos_client.append_object(bucket_name=bucket_name, + # key=object_key, + # data=file_content, + # content_md5=content_md5(file_content), + # content_length=len(file_content)) def walk(self, dir): class WalkGenerator(): diff --git a/visualdl/writer/record_writer.py b/visualdl/writer/record_writer.py index 0a3a9b30..e80e0ff9 100644 --- a/visualdl/writer/record_writer.py +++ b/visualdl/writer/record_writer.py @@ -74,12 +74,28 @@ class RecordFileWriter(object): file. """ def __init__(self, logdir, max_queue_size=10, flush_secs=120, - filename_suffix=''): + filename_suffix='', filename=''): self._logdir = logdir if not bfile.exists(logdir): bfile.makedirs(logdir) - self._file_name = bfile.join(logdir, "vdlrecords.%010d.log%s" % ( - time.time(), filename_suffix)) + + if filename: + if 'vdlrecords' in filename: + self._file_name = bfile.join(logdir, filename) + if bfile.exists(self._file_name): + print( + '`{}` is exists, VisualDL will add logs to it.'.format( + self._file_name)) + else: + fn = "vdlrecords.%010d.log%s" % (time.time(), filename_suffix) + self._file_name = bfile.join(logdir, fn) + print( + 'Since the log filename should contain `vdlrecords`, the filename is invalid and `{}` will replace `{}`'.format( + fn, filename)) + else: + self._file_name = bfile.join(logdir, "vdlrecords.%010d.log%s" % ( + time.time(), filename_suffix)) + self._general_file_writer = bfile.BFile(self._file_name, "wb") self._async_writer = _AsyncWriter(RecordWriter( self._general_file_writer), max_queue_size, flush_secs) @@ -91,6 +107,9 @@ class RecordFileWriter(object): def get_logdir(self): return self._logdir + def get_filename(self): + return self._file_name + def add_record(self, record): if not isinstance(record, record_pb2.Record): raise TypeError("Expected an record_pb2.Record proto, " diff --git a/visualdl/writer/writer.py b/visualdl/writer/writer.py index 351d4866..fed40d59 100644 --- a/visualdl/writer/writer.py +++ b/visualdl/writer/writer.py @@ -67,6 +67,7 @@ class LogWriter(object): filename_suffix='', write_to_disk=True, display_name='', + file_name='', **kwargs): """Create a instance of class `LogWriter` and create a vdl log file with given args. @@ -94,6 +95,7 @@ class LogWriter(object): self._filename_suffix = filename_suffix self._write_to_disk = write_to_disk self.kwargs = kwargs + self._file_name = file_name self._file_writer = None self._all_writers = {} @@ -116,10 +118,15 @@ class LogWriter(object): logdir=self._logdir, max_queue_size=self._max_queue, flush_secs=self._flush_secs, - filename_suffix=self._filename_suffix) + filename_suffix=self._filename_suffix, + filename=self._file_name) self._all_writers.update({self._logdir: self._file_writer}) return self._file_writer + @property + def file_name(self): + return self._file_writer.get_filename() + def add_meta(self, tag='meta_data_tag', display_name='', step=0, walltime=None): """Add a meta to vdl record file. -- GitLab