From 660e9ea6dad4f34b331f4d944db8e8b6487da65f Mon Sep 17 00:00:00 2001 From: chenxuyi Date: Sun, 29 Sep 2019 11:05:20 +0800 Subject: [PATCH] + propeller --- propeller/README.en.md | 101 + propeller/README.md | 109 + propeller/__init__.py | 46 + propeller/data/__init__.py | 13 + propeller/data/functional.py | 422 + propeller/paddle/__init__.py | 24 + propeller/paddle/collection.py | 56 + propeller/paddle/data/__init__.py | 19 + propeller/paddle/data/example.proto | 29 + propeller/paddle/data/example_pb2.py | 149 + propeller/paddle/data/feature.proto | 46 + propeller/paddle/data/feature_column.py | 397 + propeller/paddle/data/feature_pb2.py | 549 + propeller/paddle/data/functional.py | 67 + propeller/paddle/service/__init__.py | 13 + propeller/paddle/service/server.py | 119 + propeller/paddle/service/utils.py | 66 + propeller/paddle/summary.py | 36 + propeller/paddle/train/__init__.py | 32 + propeller/paddle/train/distribution.py | 140 + propeller/paddle/train/exporter.py | 104 + propeller/paddle/train/hooks.py | 295 + propeller/paddle/train/metrics.py | 554 + propeller/paddle/train/monitored_executor.py | 355 + propeller/paddle/train/trainer.py | 510 + propeller/service/__init__.py | 13 + propeller/service/client.py | 104 + propeller/service/interface.proto | 46 + propeller/service/interface_pb2.py | 241 + propeller/service/server.py | 170 + propeller/service/utils.py | 113 + propeller/tools/__init__.py | 13 + propeller/tools/ckpt_inspector.py | 123 + propeller/tools/start_server.py | 39 + propeller/train/__init__.py | 13 + propeller/train/model.py | 82 + propeller/types.py | 103 + propeller/util.py | 119 + propeller_xnli_demo.ipynb | 10036 +++++++++++++++++ 39 files changed, 15466 insertions(+) create mode 100644 propeller/README.en.md create mode 100644 propeller/README.md create mode 100644 propeller/__init__.py create mode 100644 propeller/data/__init__.py create mode 100644 propeller/data/functional.py create mode 100644 propeller/paddle/__init__.py create mode 100644 propeller/paddle/collection.py create mode 100644 propeller/paddle/data/__init__.py create mode 100644 propeller/paddle/data/example.proto create mode 100644 propeller/paddle/data/example_pb2.py create mode 100644 propeller/paddle/data/feature.proto create mode 100644 propeller/paddle/data/feature_column.py create mode 100644 propeller/paddle/data/feature_pb2.py create mode 100644 propeller/paddle/data/functional.py create mode 100644 propeller/paddle/service/__init__.py create mode 100644 propeller/paddle/service/server.py create mode 100644 propeller/paddle/service/utils.py create mode 100644 propeller/paddle/summary.py create mode 100644 propeller/paddle/train/__init__.py create mode 100644 propeller/paddle/train/distribution.py create mode 100644 propeller/paddle/train/exporter.py create mode 100644 propeller/paddle/train/hooks.py create mode 100644 propeller/paddle/train/metrics.py create mode 100644 propeller/paddle/train/monitored_executor.py create mode 100644 propeller/paddle/train/trainer.py create mode 100644 propeller/service/__init__.py create mode 100644 propeller/service/client.py create mode 100644 propeller/service/interface.proto create mode 100644 propeller/service/interface_pb2.py create mode 100644 propeller/service/server.py create mode 100644 propeller/service/utils.py create mode 100644 propeller/tools/__init__.py create mode 100644 propeller/tools/ckpt_inspector.py create mode 100644 propeller/tools/start_server.py create mode 100644 propeller/train/__init__.py create mode 100644 propeller/train/model.py create mode 100644 propeller/types.py create mode 100644 propeller/util.py create mode 100644 propeller_xnli_demo.ipynb diff --git a/propeller/README.en.md b/propeller/README.en.md new file mode 100644 index 0000000..8ced3b6 --- /dev/null +++ b/propeller/README.en.md @@ -0,0 +1,101 @@ +[简体中文](./README.md)|English +# Introducing Propeller +This doc introduct Propeller, a high level paddle API for general ML, Propeller encapsulate the following actions:： +- training +- evaluation +- prediction +- export serving + +Propeller provide the following benefits: + +- You can run Propeller-based models on a local host or on a distributed multi-server environment without changing your model. Furthermore, you can run Propeller-based models on CPUs, GPUs without recoding your model. +- Propeller simplify sharing implementations between model developers. +- Propeller do many things for you (logging, hot-start...) +- Propeller buids Program and PyReader or you. +- Propeller provide a safe distributed training loop that controls how and when to: + - build the Program + - initialize variables + - create checkpoint files and recover from failures + - save visualizable results + +## install + +```script +cd propeller && pip install . +``` + +## Getting Started +```python + + #Define model + class BowModel(propeller.Model): + def __init__(self, config, mode): + self.embedding = Embedding(config['emb_size'], config['vocab_size']) + self.fc1 = FC(config['hidden_size']) + self.fc2 = FC(config['hidden_size']) + + def forward(self, features): + q, t = features + q_emb = softsign(self.embedding(q)) + t_emb = softsign(self.embedding(t)) + q_emb = self.fc1(q_emb) + t_emb = self.fc2(t_emn) + prediction = dot(q_emb, emb) + return prediction + + def loss(self, predictions, label): + return sigmoid_cross_entropy_with_logits(predictions, label) + + def backward(self, loss): + opt = AdamOptimizer(1.e-3) + opt.mimize(loss) + + def metrics(self, predictions, label): + auc = atarshi.metrics.Auc(predictions, label) + return {'auc': auc} + + # hyper param comes from files/command line prompt/env vir + run_config = propeller.parse_runconfig(args) + hparams = propeller.parse_hparam(args) + + # Define data + # `FeatureColumns` helps you to organize training/evluation files. + feature_column = propeller.data.FeatureColumns(columns=[ + propeller.data.TextColumn('query', vocab='./vocab'), + propeller.data.TextColumn('title', vocab='./vocab'), + propeller.data.LabelColumn('label'), + ]) + train_ds = feature_column.build_dataset(data_dir='./data', shuffle=True, repeat=True) + eval_ds = feature_column.build_dataset(data_dir='./data', shuffle=False, repeat=False) + + # Start training! + propeller.train_and_eval(BowModel, hparams, run_config, train_ds, eval_ds) +``` +More detail see example/toy/ + +## Main Feature +1. train_and_eval + + according to user-specified `propeller.Model`class，initialize training model in the following 2 modes: 1. TRAIN mode 2. EVAL mode and + perform train_and_eval + +2. FeatureColumns + + `FeatureColumns`is used to ogranize train data. With custmizable `Column` property, it can adaps to many ML tasks（NLP/CV...). + `FeatureColumns` also do the preprocessing for you (tokenization, vocab lookup, serialization, batcing etc.) + + +3. Dataset + + `FeatureColumns` generats `Dataset`，or you can call `propeller.Dataset.from_generator_func` to build your own `Dataset`. + +4. Summary + To trace tensor histogram in training, simply： + ```python + propeller.summary.histogram('loss', tensor) + ``` + + +## Contributing + +1. This project is in alpha stage, any contribution is welcomed. Fill free to create a PR. diff --git a/propeller/README.md b/propeller/README.md new file mode 100644 index 0000000..8e1bb38 --- /dev/null +++ b/propeller/README.md @@ -0,0 +1,109 @@ +简体中文|[English](./README.en.md) +# Introducing paddle-propeller +本文档介绍propeller，一种可极大地简化机器学习编程的高阶 Paddle API。propeller 会封装下列操作： +- 训练 +- 评估 +- 预测 +- 导出以供使用（上线） + +Propeller 具有下列优势： + +- 您可以在本地主机上或分布式多服务器环境中运行基于 Propeller 的模型，而无需更改模型。此外，您可以在 CPU、GPU上运行基于 Propeller 的模型，而无需重新编码模型。 +- Propeller 简化了在模型开发者之间共享实现的过程。 +- 只需关注模型实现以及数据输入，而无需关注其他辅助代码（保存、热启动、打log等） +- Propeller 会为您构建Program以及PyReader。 +- Propeller 提供安全的分布式训练循环，可以控制如何以及何时： + - 构建Program + - 初始化变量 + - 处理异常 + - 创建检查点文件并从故障中恢复 + - 保存可视化的摘要结果 + +## install|安装 + +cd propeller && pip install . + +## Getting Started|快速开始 +```python + + #定义训练模型 + class BowModel(propeller.Model): + def __init__(self, config, mode): + self.embedding = Embedding(config['emb_size'], config['vocab_size']) + self.fc1 = FC(config['hidden_size']) + self.fc2 = FC(config['hidden_size'] + + def forward(self, features): + q, t = features + q_emb = softsign(self.embedding(q)) + t_emb = softsign(self.embedding(t)) + q_emb = self.fc1(q_emb) + t_emb = self.fc2(t_emn) + prediction = dot(q_emb, emb) + return prediction + + def loss(self, predictions, label): + return sigmoid_cross_entropy_with_logits(predictions, label) + + def backward(self, loss): + opt = AdamOptimizer(1.e-3) + opt.mimize(loss) + + def metrics(self, predictions, label): + auc = atarshi.metrics.Auc(predictions, label) + return {'auc': auc} + + # 超参可以来自于文件/ 环境变量/ 命令行 + run_config = propeller.parse_runconfig(args) + hparams = propeller.parse_hparam(args) + + # 定义数据： + # `FeatureColumns` 用于管理训练、预测文件. 会自动进行二进制化. + feature_column = propeller.data.FeatureColumns(columns=[ + propeller.data.TextColumn('query', vocab='./vocab'), + propeller.data.TextColumn('title', vocab='./vocab'), + propeller.data.LabelColumn('label'), + ]) + train_ds = feature_column.build_dataset(data_dir='./data', shuffle=True, repeat=True) + eval_ds = feature_column.build_dataset(data_dir='./data', shuffle=False, repeat=False) + + # 开始训练！ + propeller.train_and_eval(BowModel, hparams, run_config, train_ds, eval_ds) +``` +先洗详细请见example/toy/ + +## 主要构件 +1. train_and_eval + + 会根据用户提供的`propeller.Model`类，实例化两种模式下的训练模型： 1. TRAIN模式 2. EVAL模式。 + 然后开始训练，同时执行评估（Evaluation） + +2. FeatureColumns + + 用`FeatureColumns`来管理训练数据. 根据自定义`Column`来适配多种ML任务（NLP/CV...). + `FeatureColumns`会自动对提供的训练数据进行批量预处理(tokenization, 查词表, etc.)并二进制化，并且生成训练用的dataset + +3. Dataset + + `FeatureColumns`生成`Dataset`，或者您可以调用`propeller.Dataset.from_generator_func`来构造自己的`Dataset`，配合shuffle/ interleave/ padded_batch/ repeat 等方法满足定制化需求. + +4. Summary + 对训练过程中的某些参数进行log追踪，只需要： + ```python + propeller.summary.histogram('loss', tensor) + + ``` + + +## Contributing|贡献 + +1. 本项目处于初期阶段，欢迎贡献！ +2. functional programing is welcomed + + +## TODO + +1. dataset output_types/ output_shapes 自动推断 +2. 自动超参数搜索 +3. propeller server +4. ... diff --git a/propeller/__init__.py b/propeller/__init__.py new file mode 100644 index 0000000..6f598b6 --- /dev/null +++ b/propeller/__init__.py @@ -0,0 +1,46 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import sys +import logging +import six +from time import time + +__version__ = '0.1' + +log = logging.getLogger(__name__) +stream_hdl = logging.StreamHandler(stream=sys.stderr) +formatter = logging.Formatter( + fmt='[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s' +) + +try: + from colorlog import ColoredFormatter + fancy_formatter = ColoredFormatter( + fmt='%(log_color)s[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s' + ) + stream_hdl.setFormatter(fancy_formatter) +except ImportError: + stream_hdl.setFormatter(formatter) + +log.setLevel(logging.INFO) +log.addHandler(stream_hdl) +log.propagate = False + +from propeller.types import * +from propeller.util import ArgumentParser, parse_hparam, parse_runconfig, parse_file diff --git a/propeller/data/__init__.py b/propeller/data/__init__.py new file mode 100644 index 0000000..d0c32e2 --- /dev/null +++ b/propeller/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/propeller/data/functional.py b/propeller/data/functional.py new file mode 100644 index 0000000..3b21c1e --- /dev/null +++ b/propeller/data/functional.py @@ -0,0 +1,422 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys +import logging +import os +import itertools +import random +import inspect +import multiprocessing +from contextlib import contextmanager +import gzip +import struct +import functools + +import six +from six.moves import zip, map, filter +import numpy as np + +from propeller.util import map_structure + +log = logging.getLogger(__name__) + +__all__ = ['Dataset'] + + +@contextmanager +def open_file(filename, format=None): + if format is None: + fd = open(filename, 'rb') + elif format == 'GZIP': + fd = gzip.open(filename, 'rb') + else: + raise ValueError('unkwon file format %s' % format) + yield fd + fd.close() + + +def open_record(filename): + def gen(): + with open_file(filename, format='GZIP') as f: + while True: + data = f.read(struct.calcsize('i')) + if not len(data): + raise StopIteration + l, = struct.unpack('i', data) + data = f.read(l) + yield data + + return gen + + +def shuffle_func(dataset, buffer_size): + def gen(): + buf = [] + iterable = dataset() + try: + while len(buf) < buffer_size: + buf.append(next(iterable)) + while 1: + i = random.randint(0, buffer_size - 1) + n = next(iterable) + yield buf[i] + buf[i] = n + except StopIteration: + if len(buf): + random.shuffle(buf) + for i in buf: + yield i + + return gen + + +def interleave_func(iterable, map_fn, cycle_length, block_length): + def gen(): + ls = itertools.tee(iterable(), cycle_length) + buf = [] + for i, j in enumerate(ls): + j = itertools.islice(j, i, None, cycle_length) + j = map(map_fn, j) + j = (jjj for jj in j for jjj in jj) #flatten + buf.append(j) + + for tup in six.moves.zip_longest(*buf): + for ii in (i for i in tup if i is not None): + yield ii + + return gen + + +def repeat_func(dataset, n): + def gen(): + iterable = dataset() + if n >= 0: + ret = itertools.chain(*itertools.tee(iterable, n)) + else: + ret = itertools.cycle(iterable) + + for i in ret: + yield i + + return gen + + +def filter_func(dataset, fn): + def gen(): + for i in dataset(): + if isinstance(i, tuple) or isinstance(i, list): + if fn(*i) is True: + yield i + else: + if fn(i) is True: + yield i + + return gen + + +def map_func(dataset, fn): + def gen(): + for i in dataset(): + if isinstance(i, tuple) or isinstance(i, list): + yield fn(*i) + else: + yield fn(i) + + return gen + + +def shard_func(dataset, num_shards, index): + def gen(): + iterable = dataset() + ret = itertools.islice(iterable, index, None, num_shards) + for i in ret: + yield i + + return gen + + +def take_func(dataset, count): + def gen(): + iterable = dataset() + ret = itertools.islice(iterable, count) + for i in ret: + yield i + + return gen + + +def buffered_func(dataset, size): + """ + Creates a buffered data reader. + + The buffered data reader will read and save data entries into a + buffer. Reading from the buffered data reader will proceed as long + as the buffer is not empty. + + :param reader: the data reader to read from. + :type reader: callable + :param size: max buffer size. + :type size: int + + :returns: the buffered data reader. + """ + + class EndSignal(): + pass + + end = EndSignal() + + def read_worker(r, q): + for d in r: + q.put(d) + q.put(end) + + def data_reader(): + r = dataset() + q = multiprocessing.Queue(maxsize=size) + t = multiprocessing.Process( + target=read_worker, args=( + r, + q, )) + t.daemon = True + t.start() + e = q.get() + while e != end: + yield e + e = q.get() + + return data_reader + + +def padded_batch_func(dataset, batch_size, pad_value=0, max_seqlen=None): + if not isinstance(batch_size, int): + raise ValueError('unknown batch_size: %s' % repr(batch_size)) + + def gen(): + iterable = dataset() + pad_value_t = pad_value + while True: + buf = list(itertools.islice(iterable, batch_size)) + if not len(buf): + raise StopIteration + buf = list(zip(*buf)) # transpose + if type(pad_value_t) not in [list, tuple]: + pad_value_t = [pad_value_t] * len(buf) + padded = [] + assert len(buf) == len( + pad_value_t), 'pad_value [%d] != element size[%d]' % ( + len(pad_value_t), len(buf)) + for e, pv in zip(buf, pad_value_t): + elem = e[0] + if (not np.isscalar(elem)) and elem.shape != (): + max_len = max(map(len, + e)) if max_seqlen is None else max_seqlen + e = map(lambda i: np.pad(i, [0, max_len - len(i)], 'constant', constant_values=pv) if max_len >= len(i) else i[: max_len], e) + padded.append(np.stack(list(e))) + yield padded + + return gen + + +class Dataset(object): + @classmethod + def from_generator_func(cls, gen, data_shapes=None, data_types=None): + if not inspect.isgeneratorfunction(gen): + raise ValueError('expect generator function, got %s' % repr(gen)) + + def wrapper(): #compat to py3.7 + try: + for item in gen(): + yield item + except RuntimeError as e: + if str(e) != 'generator raised StopIteration': + raise e + + ret = cls() + ret.generator = wrapper + ret.data_shapes = data_shapes + ret.data_types = data_types + return ret + + @classmethod + def from_file(cls, filename, format=None): + if os.path.getsize(filename) == 0: + raise RuntimeError('%s is empty' % filename) + + def gen(): + with open_file(filename, format) as f: + for line in f: + yield line + + ret = cls() + ret.generator = gen + ret.data_shapes = [] + ret.data_types = str + return ret + + @classmethod + def from_record_file(cls, filename): + if os.path.getsize(filename) == 0: + raise RuntimeError('%s is empty' % filename) + gen = open_record(filename) + ret = cls() + ret.generator = gen + ret.data_shapes = [] + ret.data_types = str + return ret + + @classmethod + def from_list(cls, ls): + if not isinstance(ls, list): + raise ValueError('expect list, got %s' % repr(ls)) + + def gen(): + for i in ls: + yield i + + ret = cls() + ret.generator = gen + ret.data_shapes = [] + ret.data_types = str + return ret + + def __init__(self): + self.name = None + self._data_shapes = None + self._data_types = None + self.generator = None + self.pyreader = None + + def __repr__(self): + return 'Dataset: name: %s, data_shapes %s, data_types %s' % ( + self.name, self._data_shapes, self._data_types) + + def __eq__(self, other): + return self.name == other.name and \ + self._data_shapes == other._data_shapes and \ + self._data_types == other._data_types + + def __iter__(self): + return self.generator() + + #def __call__(self): + # return self.generator() + + def _infer_shapes_and_types(self): + if self.generator is not None and self.name is not None: + log.info('Try to infer data shapes & types from generator') + first_value = next(self.generator()) + shapes, types = [], [] + for v in first_value: + if not isinstance(v, np.ndarray): + raise ValueError( + 'dataset generator should use numpy elements, got %s' % + first_value) + shapes.append(v.shape) + types.append(v.dtype.name) + self._data_shapes = shapes + self._data_types = types + log.info('Dataset `%s` has data_shapes: %s data_types: %s' % + (self.name, repr(shapes), repr(types))) + else: + raise ValueError( + 'Try to infer data shapes or types from incomplete Dataset') + + @property + def data_shapes(self): + if self._data_shapes is None: + self._infer_shapes_and_types() + return self._data_shapes + else: + return self._data_shapes + + @data_shapes.setter + def data_shapes(self, val): + self._data_shapes = val + + @property + def data_types(self): + if self._data_types is None: + self._infer_shapes_and_types() + return self._data_types + else: + return self._data_types + + @data_types.setter + def data_types(self, val): + self._data_types = val + + def apply(self, transform_func): + #input_shapes = transform_func.input_shapes + #input_types = transform_func.input_types + #data_shapes = transform_func.data_shapes + #data_types = transform_func.data_types + #assert input_shapes == self._data_shapes + #assert input_types = self._data_types + ret_gen = transform_func(self.generator) + ret = type(self).from_generator_func(ret_gen) + if self.name is not None: + ret.name = self.name + #ret.data_shapes = data_shapes + #ret.data_types = data_types + return ret + + def shuffle(self, buffer_size): + func = functools.partial(shuffle_func, buffer_size=buffer_size) + return self.apply(func) + + def repeat(self, n=-1): + func = functools.partial(repeat_func, n=n) + return self.apply(func) + + def map(self, fn): + func = functools.partial(map_func, fn=fn) + return self.apply(func) + + def filter(self, fn): + func = functools.partial(filter_func, fn=fn) + return self.apply(func) + + def shard(self, num_shards, index): + func = functools.partial( + shard_func, num_shards=num_shards, index=index) + return self.apply(func) + + def interleave(self, map_fn, cycle_length, block_length): + func = functools.partial( + interleave_func, + map_fn=map_fn, + cycle_length=cycle_length, + block_length=block_length) + return self.apply(func) + + def padded_batch(self, batch_size, pad_value=0, max_seqlen=None): + func = functools.partial( + padded_batch_func, + batch_size=batch_size, + pad_value=pad_value, + max_seqlen=max_seqlen) + return self.apply(func) + + def take(self, count=1): + func = functools.partial(take_func, count=count) + return self.apply(func) + + def buffered(self, size=10): + func = functools.partial(buffered_func, size=size) + return self.apply(func) diff --git a/propeller/paddle/__init__.py b/propeller/paddle/__init__.py new file mode 100644 index 0000000..c7a5463 --- /dev/null +++ b/propeller/paddle/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import six +from propeller.types import * +from propeller.util import ArgumentParser, parse_hparam, parse_runconfig, parse_file + +from propeller.paddle import data +from propeller.paddle import train +from propeller.paddle.train import * diff --git a/propeller/paddle/collection.py b/propeller/paddle/collection.py new file mode 100644 index 0000000..c365b09 --- /dev/null +++ b/propeller/paddle/collection.py @@ -0,0 +1,56 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys + +_global_collection = None + + +class Key(object): + """predefine collection keys""" + SUMMARY_SCALAR = 1 + SUMMARY_HISTOGRAM = 2 + SKIP_OPTIMIZE = 3 + + +class Collections(object): + """global collections to record everything""" + + def __init__(self): + self.col = {} + + def __enter__(self): + global _global_collection + _global_collection = self + return self + + def __exit__(self, err_type, err_value, trace): + global _global_collection + _global_collection = None + + def add(self, key, val): + self.col.setdefault(key, []).append(val) + + def get(self, key): + return self.col.get(key, None) + + +def default_collection(): + global _global_collection + if _global_collection is None: + _global_collection = Collections() + return _global_collection diff --git a/propeller/paddle/data/__init__.py b/propeller/paddle/data/__init__.py new file mode 100644 index 0000000..64bd993 --- /dev/null +++ b/propeller/paddle/data/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +from propeller.paddle.data.functional import * +from propeller.paddle.data.feature_column import * diff --git a/propeller/paddle/data/example.proto b/propeller/paddle/data/example.proto new file mode 100644 index 0000000..ba6da96 --- /dev/null +++ b/propeller/paddle/data/example.proto @@ -0,0 +1,29 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Protocol messages for describing input data Examples for machine learning +// model training or inference. +syntax = "proto3"; + +import "propeller/paddle/data/feature.proto"; +package propeller; + +message Example { + Features features = 1; +}; + +message SequenceExample { + Features context = 1; + FeatureLists feature_lists = 2; +}; diff --git a/propeller/paddle/data/example_pb2.py b/propeller/paddle/data/example_pb2.py new file mode 100644 index 0000000..7ea71c7 --- /dev/null +++ b/propeller/paddle/data/example_pb2.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: propeller/paddle/data/example.proto + +import sys +_b = sys.version_info[0] < 3 and (lambda x: x) or ( + lambda x: x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + +from propeller.paddle.data import feature_pb2 as propeller_dot_paddle_dot_data_dot_feature__pb2 + +DESCRIPTOR = _descriptor.FileDescriptor( + name='propeller/paddle/data/example.proto', + package='propeller', + syntax='proto3', + serialized_options=None, + serialized_pb=_b( + '\n#propeller/paddle/data/example.proto\x12\tpropeller\x1a#propeller/paddle/data/feature.proto\"0\n\x07\x45xample\x12%\n\x08\x66\x65\x61tures\x18\x01 \x01(\x0b\x32\x13.propeller.Features\"g\n\x0fSequenceExample\x12$\n\x07\x63ontext\x18\x01 \x01(\x0b\x32\x13.propeller.Features\x12.\n\rfeature_lists\x18\x02 \x01(\x0b\x32\x17.propeller.FeatureListsb\x06proto3' + ), + dependencies=[ + propeller_dot_paddle_dot_data_dot_feature__pb2.DESCRIPTOR, + ]) + +_EXAMPLE = _descriptor.Descriptor( + name='Example', + full_name='propeller.Example', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='features', + full_name='propeller.Example.features', + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=87, + serialized_end=135, ) + +_SEQUENCEEXAMPLE = _descriptor.Descriptor( + name='SequenceExample', + full_name='propeller.SequenceExample', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='context', + full_name='propeller.SequenceExample.context', + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='feature_lists', + full_name='propeller.SequenceExample.feature_lists', + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=137, + serialized_end=240, ) + +_EXAMPLE.fields_by_name[ + 'features'].message_type = propeller_dot_paddle_dot_data_dot_feature__pb2._FEATURES +_SEQUENCEEXAMPLE.fields_by_name[ + 'context'].message_type = propeller_dot_paddle_dot_data_dot_feature__pb2._FEATURES +_SEQUENCEEXAMPLE.fields_by_name[ + 'feature_lists'].message_type = propeller_dot_paddle_dot_data_dot_feature__pb2._FEATURELISTS +DESCRIPTOR.message_types_by_name['Example'] = _EXAMPLE +DESCRIPTOR.message_types_by_name['SequenceExample'] = _SEQUENCEEXAMPLE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Example = _reflection.GeneratedProtocolMessageType( + 'Example', + (_message.Message, ), + dict( + DESCRIPTOR=_EXAMPLE, + __module__='propeller.paddle.data.example_pb2' + # @@protoc_insertion_point(class_scope:propeller.Example) + )) +_sym_db.RegisterMessage(Example) + +SequenceExample = _reflection.GeneratedProtocolMessageType( + 'SequenceExample', + (_message.Message, ), + dict( + DESCRIPTOR=_SEQUENCEEXAMPLE, + __module__='propeller.paddle.data.example_pb2' + # @@protoc_insertion_point(class_scope:propeller.SequenceExample) + )) +_sym_db.RegisterMessage(SequenceExample) + +# @@protoc_insertion_point(module_scope) diff --git a/propeller/paddle/data/feature.proto b/propeller/paddle/data/feature.proto new file mode 100644 index 0000000..564b66b --- /dev/null +++ b/propeller/paddle/data/feature.proto @@ -0,0 +1,46 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package propeller; + +message BytesList { + repeated bytes value = 1; +} +message FloatList { + repeated float value = 1 [packed = true]; +} +message Int64List { + repeated int64 value = 1 [packed = true]; +} + +message Feature { + oneof kind { + BytesList bytes_list = 1; + FloatList float_list = 2; + Int64List int64_list = 3; + } +}; + +message Features { + map feature = 1; +}; + +message FeatureList { + repeated Feature feature = 1; +}; + +message FeatureLists { + map feature_list = 1; +}; diff --git a/propeller/paddle/data/feature_column.py b/propeller/paddle/data/feature_column.py new file mode 100644 index 0000000..5091df1 --- /dev/null +++ b/propeller/paddle/data/feature_column.py @@ -0,0 +1,397 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import sys +import struct +from six.moves import zip, map +import itertools +import gzip +from functools import partial +import multiprocessing +import six +import logging + +import numpy as np +from glob import glob +from propeller.paddle.train import distribution + +from propeller.data.functional import interleave_func +from propeller.paddle.data.functional import Dataset +from propeller.paddle.data import example_pb2, feature_pb2 + +log = logging.getLogger(__name__) + +__all__ = [ + 'FeatureColumns', 'TextColumn', 'TextIDColumn', 'LabelColumn', + 'basic_tokenizer', 'Column' +] + + +def basic_tokenizer(sen): + seg = sen.split(b' ') + seg = filter(lambda i: i != b' ', seg) + return seg + + +class Column(): + def __init__(self, name): + pass + + def raw_to_proto(self, raw): + return feature_pb2.Feature() + + @property + def output_shapes(self): + pass + + @property + def output_types(self): + pass + + def proto_to_instance(self, proto): + raise NotImplementedError() + + def raw_to_instance(self, raw): + raise NotImplementedError() + + +class LabelColumn(Column): + def __init__(self, name, vocab_dict=None, vocab_file=None): + self.name = name + self.vocab = None + if vocab_file: + self.vocab = { + j.strip(): i + for i, j in enumerate(open(vocab_file, 'rb').readlines()) + } + if vocab_dict: + self.vocab = vocab_dict + + @property + def output_shapes(self): + return [1] + + @property + def output_types(self): + return 'int64' + + def raw_to_proto(self, raw): + if self.vocab is None: + ids = [int(raw)] + else: + ids = [self.vocab[raw]] + fe = feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=ids)) + return fe + + def proto_to_instance(self, feature): + ret = np.array(feature.int64_list.value[0], dtype=np.int64) + return ret + + def raw_to_instance(self, raw): + if self.vocab is None: + ids = int(raw) + else: + ids = self.vocab[raw] + return ids + + +class TextColumn(Column): + def __init__(self, + name, + unk_id, + vocab_file=None, + vocab_dict=None, + tokenizer=basic_tokenizer): + self.name = name + self.tokenizer = tokenizer + self.unk_id = unk_id + if not (vocab_file or vocab_dict): + raise ValueError('at least specify vocab_file or vocab_dict') + if vocab_file: + self.vocab = { + j.strip(): i + for i, j in enumerate(open(vocab_file, 'rb').readlines()) + } + if vocab_dict: + self.vocab = vocab_dict + + @property + def output_shapes(self): + return [-1] + + @property + def output_types(self): + return 'int64' + + def raw_to_proto(self, raw): + ids = [self.vocab.get(s, self.unk_id) for s in self.tokenizer(raw)] + fe = feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=ids)) + return fe + + def proto_to_instance(self, feature): + ret = np.array(feature.int64_list.value, dtype=np.int64) + return ret + + def raw_to_instance(self, raw): + ids = [self.vocab.get(s, self.unk_id) for s in self.tokenizer(raw)] + return np.array(ids, dtype=np.int64) + + +class TextIDColumn(Column): + def __init__(self, name): + self.name = name + + @property + def output_shapes(self): + return [-1] + + @property + def output_types(self): + return 'int64' + + def raw_to_proto(self, raw): + ids = [int(s) for s in raw.split(b' ')] + fe = feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=ids)) + return fe + + def proto_to_instance(self, feature): + ret = np.array(feature.int64_list.value, dtype=np.int64) + return ret + + def raw_to_instance(self, raw): + ret = np.array([int(i) for i in raw.split(b' ')], dtype=np.int64) + return ret + + +class FeatureColumns(object): + def __init__(self, columns, pad_id=0): + self._columns = columns + + def raw_files(self, raw_dir): + return [os.path.join(raw_dir, p) for p in os.listdir(raw_dir)] + + def gz_files(self, gz_dir): + return None if gz_dir is None else [ + os.path.join(gz_dir, p) for p in os.listdir(gz_dir) + ] + + def _make_gz_dataset(self, raw_dir, gz_dir): + assert raw_dir or gz_dir, 'data_dir not specified when using gz mode' + if raw_dir is not None: + assert os.path.exists(raw_dir), 'raw_dir not exists: %s' % raw_dir + raw_file = os.listdir(raw_dir) + if gz_dir is None: + gz_dir = '%s_gz' % raw_dir.rstrip('/') + + if not os.path.exists(gz_dir): + os.mkdir(gz_dir) + + if raw_dir is not None: + if len(raw_file) != 0: + log.debug('try making gz') + pool = multiprocessing.Pool() + args = [(os.path.join(raw_dir, f), os.path.join(gz_dir, f), + self._columns, b'\t') for f in raw_file] + pool.map(_make_gz, args) + pool.terminate() + else: + assert len( + os.listdir(gz_dir) + ) != 0, 'cant find gz file or raw-txt file at [%s] and [%s]' % ( + raw_dir, gz_dir) + return gz_dir + + def _read_gz_dataset(self, + gz_files, + shuffle=False, + repeat=True, + shard=False, + **kwargs): + if len(gz_files) == 0: + raise ValueError('reading gz from empty file list: %s' % gz_files) + log.info('reading gz from %s' % '\n'.join(gz_files)) + dataset = Dataset.from_list(gz_files) + if repeat: + dataset = dataset.repeat() + + if shard and distribution.status.mode == distribution.DistributionMode.NCCL: + log.info('Apply dataset sharding in distribution env') + train_ds = train_ds.shard(distribution.status.num_replica, + distribution.status.replica_id) + + if shuffle: + dataset = dataset.shuffle(buffer_size=len(gz_files)) + fn = partial( + interleave_func, + map_fn=lambda filename: Dataset.from_record_file(filename), + cycle_length=len(gz_files), + block_length=1) + dataset = dataset.apply(fn) + if shuffle: + dataset = dataset.shuffle(buffer_size=1000) + + def _parse_gz(record_str): # function that takes python_str as input + ex = example_pb2.Example() + ex.ParseFromString(record_str) + ret = [] + fea_dict = ex.features.feature + for c in self._columns: + ins = c.proto_to_instance(fea_dict[c.name]) + ret.append(ins) + return ret + + dataset = dataset.map(_parse_gz) + return dataset + + def _read_txt_dataset(self, + data_files, + shuffle=False, + repeat=True, + **kwargs): + log.info('reading raw files from %s' % '\n'.join(data_files)) + dataset = Dataset.from_list(data_files) + if repeat: + dataset = dataset.repeat() + if shuffle: + dataset = dataset.shuffle(buffer_size=len(data_files)) + + fn = partial( + interleave_func, + map_fn=lambda filename: Dataset.from_file(filename), + cycle_length=len(data_files), + block_length=1) + dataset = dataset.apply(fn) + if shuffle: + dataset = dataset.shuffle(buffer_size=1000) + + def _parse_txt_file( + record_str): # function that takes python_str as input + features = record_str.strip(b'\n').split(b'\t') + ret = [ + column.raw_to_instance(feature) + for feature, column in zip(features, self._columns) + ] + return ret + + dataset = dataset.map(_parse_txt_file) + return dataset + + def _read_stdin_dataset(self, encoding='utf8', shuffle=False, **kwargs): + log.info('reading raw files stdin') + + def gen(): + if six.PY3: + source = sys.stdin.buffer + else: + source = sys.stdin + while True: + line = source.readline() + if len(line) == 0: + break + yield line, + + dataset = Dataset.from_generator_func(gen) + if shuffle: + dataset = dataset.shuffle(buffer_size=1000) + + def _parse_stdin(record_str): + '''function that takes python_str as input''' + features = record_str.strip(b'\n').split(b'\t') + ret = [ + column.raw_to_instance(feature) + for feature, column in zip(features, self._columns) + ] + return ret + + dataset = dataset.map(_parse_stdin) + return dataset + + def _prepare_dataset(self, + dataset, + map_func_before_batch=None, + map_func_after_batch=None, + shuffle_buffer_size=None, + batch_size=1, + pad_id=0, + prefetch=None, + **kwargs): + + if map_func_before_batch is not None: + dataset = dataset.map(map_func_before_batch) + if batch_size: + dataset = dataset.padded_batch(batch_size, pad_id) + if map_func_after_batch is not None: + dataset = dataset.map(map_func_after_batch) + return dataset + + def build_dataset(self, + name, + use_gz=True, + data_dir=None, + gz_dir=None, + data_file=None, + **kwargs): + if use_gz: + gz_dir = self._make_gz_dataset(data_dir, gz_dir) + gz_files = self.gz_files(gz_dir) + ds = self._read_gz_dataset(gz_files, **kwargs) + else: + if data_dir is not None: + data_files = self.raw_files(data_dir) + elif data_file is not None: + data_files = [data_file] + else: + raise ValueError('data_dir or data_files not specified') + ds = self._read_txt_dataset(data_files, **kwargs) + ds.name = name + return ds + + def build_dataset_from_stdin(self, name, **kwargs): + ds = self._read_stdin_dataset(**kwargs) + ds.name = name + return ds + + +def _make_gz(args): + try: + from_file, to_file, columns, sep = args + if os.path.exists(to_file): + return + with open(from_file, 'rb') as fin, gzip.open(to_file, 'wb') as fout: + log.debug('making gz %s => %s' % (from_file, to_file)) + for i, line in enumerate(fin): + line = line.strip(b'\n').split(sep) + #if i % 10000 == 0: + # log.debug('making gz %s => %s [%d]' % (from_file, to_file, i)) + if len(line) != len(columns): + log.error('columns not match at %s, got %d, expect %d' % + (from_file, len(line), len(columns))) + continue + features = {} + for l, c in zip(line, columns): + features[c.name] = c.raw_to_proto(l) + example = example_pb2.Example(features=feature_pb2.Features( + feature=features)) + serialized = example.SerializeToString() + l = len(serialized) + data = struct.pack('i%ds' % l, l, serialized) + fout.write(data) + log.debug('done making gz %s => %s' % (from_file, to_file)) + except Exception as e: + log.exception(e) + raise e diff --git a/propeller/paddle/data/feature_pb2.py b/propeller/paddle/data/feature_pb2.py new file mode 100644 index 0000000..63ba7c2 --- /dev/null +++ b/propeller/paddle/data/feature_pb2.py @@ -0,0 +1,549 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: propeller/paddle/data/feature.proto + +import sys +_b = sys.version_info[0] < 3 and (lambda x: x) or ( + lambda x: x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + +DESCRIPTOR = _descriptor.FileDescriptor( + name='propeller/paddle/data/feature.proto', + package='propeller', + syntax='proto3', + serialized_options=None, + serialized_pb=_b( + '\n#propeller/paddle/data/feature.proto\x12\tpropeller\"\x1a\n\tBytesList\x12\r\n\x05value\x18\x01 \x03(\x0c\"\x1e\n\tFloatList\x12\x11\n\x05value\x18\x01 \x03(\x02\x42\x02\x10\x01\"\x1e\n\tInt64List\x12\x11\n\x05value\x18\x01 \x03(\x03\x42\x02\x10\x01\"\x95\x01\n\x07\x46\x65\x61ture\x12*\n\nbytes_list\x18\x01 \x01(\x0b\x32\x14.propeller.BytesListH\x00\x12*\n\nfloat_list\x18\x02 \x01(\x0b\x32\x14.propeller.FloatListH\x00\x12*\n\nint64_list\x18\x03 \x01(\x0b\x32\x14.propeller.Int64ListH\x00\x42\x06\n\x04kind\"\x81\x01\n\x08\x46\x65\x61tures\x12\x31\n\x07\x66\x65\x61ture\x18\x01 \x03(\x0b\x32 .propeller.Features.FeatureEntry\x1a\x42\n\x0c\x46\x65\x61tureEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.propeller.Feature:\x02\x38\x01\"2\n\x0b\x46\x65\x61tureList\x12#\n\x07\x66\x65\x61ture\x18\x01 \x03(\x0b\x32\x12.propeller.Feature\"\x9a\x01\n\x0c\x46\x65\x61tureLists\x12>\n\x0c\x66\x65\x61ture_list\x18\x01 \x03(\x0b\x32(.propeller.FeatureLists.FeatureListEntry\x1aJ\n\x10\x46\x65\x61tureListEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.propeller.FeatureList:\x02\x38\x01\x62\x06proto3' + )) + +_BYTESLIST = _descriptor.Descriptor( + name='BytesList', + full_name='propeller.BytesList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', + full_name='propeller.BytesList.value', + index=0, + number=1, + type=12, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=50, + serialized_end=76, ) + +_FLOATLIST = _descriptor.Descriptor( + name='FloatList', + full_name='propeller.FloatList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', + full_name='propeller.FloatList.value', + index=0, + number=1, + type=2, + cpp_type=6, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b('\020\001'), + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=78, + serialized_end=108, ) + +_INT64LIST = _descriptor.Descriptor( + name='Int64List', + full_name='propeller.Int64List', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', + full_name='propeller.Int64List.value', + index=0, + number=1, + type=3, + cpp_type=2, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b('\020\001'), + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=110, + serialized_end=140, ) + +_FEATURE = _descriptor.Descriptor( + name='Feature', + full_name='propeller.Feature', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bytes_list', + full_name='propeller.Feature.bytes_list', + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_list', + full_name='propeller.Feature.float_list', + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int64_list', + full_name='propeller.Feature.int64_list', + index=2, + number=3, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='kind', + full_name='propeller.Feature.kind', + index=0, + containing_type=None, + fields=[]), + ], + serialized_start=143, + serialized_end=292, ) + +_FEATURES_FEATUREENTRY = _descriptor.Descriptor( + name='FeatureEntry', + full_name='propeller.Features.FeatureEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', + full_name='propeller.Features.FeatureEntry.key', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', + full_name='propeller.Features.FeatureEntry.value', + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=_b('8\001'), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=358, + serialized_end=424, ) + +_FEATURES = _descriptor.Descriptor( + name='Features', + full_name='propeller.Features', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='feature', + full_name='propeller.Features.feature', + index=0, + number=1, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[_FEATURES_FEATUREENTRY, ], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=295, + serialized_end=424, ) + +_FEATURELIST = _descriptor.Descriptor( + name='FeatureList', + full_name='propeller.FeatureList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='feature', + full_name='propeller.FeatureList.feature', + index=0, + number=1, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=426, + serialized_end=476, ) + +_FEATURELISTS_FEATURELISTENTRY = _descriptor.Descriptor( + name='FeatureListEntry', + full_name='propeller.FeatureLists.FeatureListEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', + full_name='propeller.FeatureLists.FeatureListEntry.key', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', + full_name='propeller.FeatureLists.FeatureListEntry.value', + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=_b('8\001'), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=559, + serialized_end=633, ) + +_FEATURELISTS = _descriptor.Descriptor( + name='FeatureLists', + full_name='propeller.FeatureLists', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='feature_list', + full_name='propeller.FeatureLists.feature_list', + index=0, + number=1, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[_FEATURELISTS_FEATURELISTENTRY, ], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=479, + serialized_end=633, ) + +_FEATURE.fields_by_name['bytes_list'].message_type = _BYTESLIST +_FEATURE.fields_by_name['float_list'].message_type = _FLOATLIST +_FEATURE.fields_by_name['int64_list'].message_type = _INT64LIST +_FEATURE.oneofs_by_name['kind'].fields.append(_FEATURE.fields_by_name[ + 'bytes_list']) +_FEATURE.fields_by_name[ + 'bytes_list'].containing_oneof = _FEATURE.oneofs_by_name['kind'] +_FEATURE.oneofs_by_name['kind'].fields.append(_FEATURE.fields_by_name[ + 'float_list']) +_FEATURE.fields_by_name[ + 'float_list'].containing_oneof = _FEATURE.oneofs_by_name['kind'] +_FEATURE.oneofs_by_name['kind'].fields.append(_FEATURE.fields_by_name[ + 'int64_list']) +_FEATURE.fields_by_name[ + 'int64_list'].containing_oneof = _FEATURE.oneofs_by_name['kind'] +_FEATURES_FEATUREENTRY.fields_by_name['value'].message_type = _FEATURE +_FEATURES_FEATUREENTRY.containing_type = _FEATURES +_FEATURES.fields_by_name['feature'].message_type = _FEATURES_FEATUREENTRY +_FEATURELIST.fields_by_name['feature'].message_type = _FEATURE +_FEATURELISTS_FEATURELISTENTRY.fields_by_name[ + 'value'].message_type = _FEATURELIST +_FEATURELISTS_FEATURELISTENTRY.containing_type = _FEATURELISTS +_FEATURELISTS.fields_by_name[ + 'feature_list'].message_type = _FEATURELISTS_FEATURELISTENTRY +DESCRIPTOR.message_types_by_name['BytesList'] = _BYTESLIST +DESCRIPTOR.message_types_by_name['FloatList'] = _FLOATLIST +DESCRIPTOR.message_types_by_name['Int64List'] = _INT64LIST +DESCRIPTOR.message_types_by_name['Feature'] = _FEATURE +DESCRIPTOR.message_types_by_name['Features'] = _FEATURES +DESCRIPTOR.message_types_by_name['FeatureList'] = _FEATURELIST +DESCRIPTOR.message_types_by_name['FeatureLists'] = _FEATURELISTS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +BytesList = _reflection.GeneratedProtocolMessageType( + 'BytesList', + (_message.Message, ), + dict( + DESCRIPTOR=_BYTESLIST, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.BytesList) + )) +_sym_db.RegisterMessage(BytesList) + +FloatList = _reflection.GeneratedProtocolMessageType( + 'FloatList', + (_message.Message, ), + dict( + DESCRIPTOR=_FLOATLIST, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.FloatList) + )) +_sym_db.RegisterMessage(FloatList) + +Int64List = _reflection.GeneratedProtocolMessageType( + 'Int64List', + (_message.Message, ), + dict( + DESCRIPTOR=_INT64LIST, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.Int64List) + )) +_sym_db.RegisterMessage(Int64List) + +Feature = _reflection.GeneratedProtocolMessageType( + 'Feature', + (_message.Message, ), + dict( + DESCRIPTOR=_FEATURE, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.Feature) + )) +_sym_db.RegisterMessage(Feature) + +Features = _reflection.GeneratedProtocolMessageType( + 'Features', + (_message.Message, ), + dict( + FeatureEntry=_reflection.GeneratedProtocolMessageType( + 'FeatureEntry', + (_message.Message, ), + dict( + DESCRIPTOR=_FEATURES_FEATUREENTRY, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.Features.FeatureEntry) + )), + DESCRIPTOR=_FEATURES, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.Features) + )) +_sym_db.RegisterMessage(Features) +_sym_db.RegisterMessage(Features.FeatureEntry) + +FeatureList = _reflection.GeneratedProtocolMessageType( + 'FeatureList', + (_message.Message, ), + dict( + DESCRIPTOR=_FEATURELIST, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.FeatureList) + )) +_sym_db.RegisterMessage(FeatureList) + +FeatureLists = _reflection.GeneratedProtocolMessageType( + 'FeatureLists', + (_message.Message, ), + dict( + FeatureListEntry=_reflection.GeneratedProtocolMessageType( + 'FeatureListEntry', + (_message.Message, ), + dict( + DESCRIPTOR=_FEATURELISTS_FEATURELISTENTRY, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.FeatureLists.FeatureListEntry) + )), + DESCRIPTOR=_FEATURELISTS, + __module__='propeller.paddle.data.feature_pb2' + # @@protoc_insertion_point(class_scope:propeller.FeatureLists) + )) +_sym_db.RegisterMessage(FeatureLists) +_sym_db.RegisterMessage(FeatureLists.FeatureListEntry) + +_FLOATLIST.fields_by_name['value']._options = None +_INT64LIST.fields_by_name['value']._options = None +_FEATURES_FEATUREENTRY._options = None +_FEATURELISTS_FEATURELISTENTRY._options = None +# @@protoc_insertion_point(module_scope) diff --git a/propeller/paddle/data/functional.py b/propeller/paddle/data/functional.py new file mode 100644 index 0000000..1f6c989 --- /dev/null +++ b/propeller/paddle/data/functional.py @@ -0,0 +1,67 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import numpy as np +import logging + +import paddle.fluid as F +import paddle.fluid.layers as L + +from propeller.data.functional import Dataset as DatasetBase + +log = logging.getLogger(__name__) + + +class Dataset(DatasetBase): + def placeholders(self): + if self.name is None: + raise ValueError('can not get feature from unnamed Dataset') + + ret = [] + for i, (shape, + types) in enumerate(zip(self.data_shapes, self.data_types)): + ret.append( + L.data( + '%s_placeholder_%d' % (self.name, i), + shape=shape, + append_batch_size=False, + dtype=types)) + return ret + + def features(self): + '''start point of net building. call this in a program scope''' + if self.name is None: + raise ValueError('can not get feature from unnamed Dataset') + + if len(self.data_shapes) != len(self.data_types): + raise ValueError( + 'Dataset shapes and types not match: shape:%s types%s' % + (repr(self._data_shapes), repr(self._data_types))) + return self.placeholders() + + def start(self, places=F.cuda_places()): + #assert self.pyreader is not None, 'use Dataset.features to build net first, then start dataset' + def gen(): + try: + for idx, i in enumerate(self.generator()): + yield i + except Exception as e: + log.exception(e) + raise e + + r = F.io.PyReader( + feed_list=self.placeholders(), capacity=50, iterable=True) + r.decorate_batch_generator(gen, places=places) + return r() diff --git a/propeller/paddle/service/__init__.py b/propeller/paddle/service/__init__.py new file mode 100644 index 0000000..d0c32e2 --- /dev/null +++ b/propeller/paddle/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/propeller/paddle/service/server.py b/propeller/paddle/service/server.py new file mode 100644 index 0000000..3433ee2 --- /dev/null +++ b/propeller/paddle/service/server.py @@ -0,0 +1,119 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import logging +import six +import asyncio +import threading + +import grpc +from propeller.service import interface_pb2 +from propeller.service import interface_pb2_grpc +import propeller.paddle.service.utils as serv_utils + +from concurrent.futures import ThreadPoolExecutor + +import paddle.fluid as F + +from time import sleep, time + +log = logging.getLogger(__name__) + + +def profile(msg): + def decfn(fn): + def retfn(*args, **kwargs): + start = time() + ret = fn(*args, **kwargs) + end = time() + log.debug('%s timecost: %.5f' % (msg, end - start)) + return ret + + return retfn + + return decfn + + +def serve(model_dir, host, num_concurrent=None): + if six.PY2: + raise RuntimeError('propeller service work in python3 only') + num_worker = len(F.cuda_places( + )) if num_concurrent is None else num_concurrent + pool = ThreadPoolExecutor(num_worker) + + class Predictor(object): + def __init__(self, did): + log.debug('create predictor on card %d' % did) + config = F.core.AnalysisConfig(model_dir) + config.enable_use_gpu(5000, did) + self._predictor = F.core.create_paddle_predictor(config) + + @profile('paddle') + def __call__(self, args): + for i, a in enumerate(args): + a.name = 'placeholder_%d' % i + res = self._predictor.run(args) + return res + + predictor_context = {} + + class InferenceService(interface_pb2_grpc.InferenceServicer): + @profile('service') + def Infer(self, request, context): + try: + slots = request.slots + current_thread = threading.current_thread() + log.debug('%d slots received dispatch to thread %s' % + (len(slots), current_thread)) + if current_thread not in predictor_context: + did = list(pool._threads).index(current_thread) + log.debug('spawning worker thread %d' % did) + predictor = Predictor(did) + predictor_context[current_thread] = predictor + else: + predictor = predictor_context[current_thread] + slots = [serv_utils.slot_to_paddlearray(s) for s in slots] + ret = predictor(slots) + response = [serv_utils.paddlearray_to_slot(r) for r in ret] + except Exception as e: + log.exception(e) + raise e + return interface_pb2.Slots(slots=response) + + server = grpc.server(pool) + interface_pb2_grpc.add_InferenceServicer_to_server(InferenceService(), + server) + server.add_insecure_port(host) + server.start() + log.info('server started on %s...' % host) + try: + while True: + sleep(100000) + except KeyboardInterrupt as e: + pass + log.info('server stoped...') + + +if __name__ == '__main__': + from propeller import log + log.setLevel(logging.DEBUG) + serve( + '/home/work/chenxuyi/playground/grpc_play/ernie2.0/', + '10.255.138.19:8334', + num_concurrent=3) diff --git a/propeller/paddle/service/utils.py b/propeller/paddle/service/utils.py new file mode 100644 index 0000000..e295b7b --- /dev/null +++ b/propeller/paddle/service/utils.py @@ -0,0 +1,66 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import struct + +from propeller.service import interface_pb2 +from propeller.service import interface_pb2_grpc + +import paddle.fluid.core as core + + +def slot_to_paddlearray(slot): + if slot.type == interface_pb2.Slot.FP32: + type_str = 'f' + dtype = core.PaddleDType.FLOAT32 + elif slot.type == interface_pb2.Slot.INT32: + type_str = 'i' + dtype = core.PaddleDType.INT32 + elif slot.type == interface_pb2.Slot.INT64: + type_str = 'q' + dtype = core.PaddleDType.INT64 + else: + raise RuntimeError('know type %s' % slot.type) + ret = core.PaddleTensor() + ret.shape = slot.dims + ret.dtype = dtype + num = len(slot.data) // struct.calcsize(type_str) + arr = struct.unpack('%d%s' % (num, type_str), slot.data) + ret.data = core.PaddleBuf(arr) + return ret + + +def paddlearray_to_slot(arr): + if arr.dtype == core.PaddleDType.FLOAT32: + dtype = interface_pb2.Slot.FP32 + type_str = 'f' + arr_data = arr.data.float_data() + elif arr.dtype == core.PaddleDType.INT32: + dtype = interface_pb2.Slot.INT32 + type_str = 'i' + arr_data = arr.data.int32_data() + elif arr.dtype == core.PaddleDType.INT64: + dtype = interface_pb2.Slot.INT64 + type_str = 'q' + arr_data = arr.data.int64_data() + else: + raise RuntimeError('know type %s' % arr.dtype) + data = struct.pack('%d%s' % (len(arr_data), type_str), *arr_data) + pb = interface_pb2.Slot(type=dtype, dims=list(arr.shape), data=data) + return pb diff --git a/propeller/paddle/summary.py b/propeller/paddle/summary.py new file mode 100644 index 0000000..cd19dae --- /dev/null +++ b/propeller/paddle/summary.py @@ -0,0 +1,36 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys + +import paddle.fluid as F +from propeller.paddle.collection import default_collection, Key + + +def scalar(name, tensor): + if not isinstance(tensor, F.framework.Variable): + raise ValueError('expect paddle Variable, got %s' % repr(tensor)) + tensor.persistable = True + default_collection().add(Key.SUMMARY_SCALAR, (name, tensor)) + + +def histogram(name, tensor): + if not isinstance(tensor, F.framework.Variable): + raise ValueError('expect paddle Variable, got %s' % repr(tensor)) + tensor.persistable = True + default_collection().add(Key.SUMMARY_HISTOGRAM, (name, tensor)) diff --git a/propeller/paddle/train/__init__.py b/propeller/paddle/train/__init__.py new file mode 100644 index 0000000..2cc2c5d --- /dev/null +++ b/propeller/paddle/train/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import sys +import logging +from time import time + +log = logging.getLogger(__name__) + +from propeller.paddle.train.monitored_executor import * +from propeller.paddle.train.trainer import * +from propeller.paddle.train.hooks import * +from propeller.train.model import Model +from propeller.paddle.train import exporter +from propeller.paddle.train import distribution +from propeller.paddle.train import metrics diff --git a/propeller/paddle/train/distribution.py b/propeller/paddle/train/distribution.py new file mode 100644 index 0000000..0a7446d --- /dev/null +++ b/propeller/paddle/train/distribution.py @@ -0,0 +1,140 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import functools +import six +import logging +from time import sleep + +import paddle.fluid as F +import paddle.fluid.layers as L + +log = logging.getLogger(__name__) +import propeller.util + +__all__ = ['init_distribuition_env', 'status'] + +status = None + + +class DistributionMode(object): + LOCAL = 0 + NCCL = 1 + + +class DistributionStatus(object): + def __init__(self, config): + if config is None: + self._mode = DistributionMode.LOCAL + self._env = None + self._this = None + else: + try: + self._mode = DistributionMode.NCCL + + cluster = config['cluster'] + task = config['task']['type'] + idx = int(config['task']['index']) + self._this = cluster[task][idx] + + self._env = cluster['chief'] + cluster['worker'] + if len(set(self._env)) != len(self._env): + raise ValueError('duplicate host in dis_config %s' % + config) + + except KeyError as e: + raise ValueError( + 'PROPELLER_DISCONFIG wrong: %s not found in %s' % + (e, repr(dis_config))) + + @property + def mode(self): + return self._mode + + @property + def num_replica(self): + if self._mode == DistributionMode.LOCAL: + return 1 + elif self._mode == DistributionMode.NCCL: + return len(self._env) + else: + raise ValueError('Got unknow distribution mode %s' % + repr(self._mode)) + + @property + def replica_id(self): + if self._mode == DistributionMode.LOCAL: + return 0 + elif self._mode == DistributionMode.NCCL: + return self._env.index(self._this) + else: + raise ValueError('Got unknow distribution mode %s' % + repr(self._mode)) + + @property + def is_master(self): + if self._mode == DistributionMode.LOCAL: + return True + elif self._mode == DistributionMode.NCCL: + return self.replica_id == 0 + else: + raise ValueError('got unknow distribution mode %s' % + repr(self._mode)) + + +dis_config = propeller.util._get_dict_from_environ_or_json_or_file( + None, 'PROPELLER_DISCONFIG') +status = DistributionStatus(dis_config) + + +def run_on_master(func): + """skip function in distribution env""" + + @functools.wraps(func) + def f(*arg, **kwargs): + """f""" + if status is None: + raise ValueError('distribution mode unkown at this point') + if status.mode == DistributionMode.LOCAL: + r = func(*arg, **kwargs) + elif status.mode == DistributionMode.NCCL: + if status.is_master: + r = func(*arg, **kwargs) + else: + r = 0 # skip function + #MPI.COMM_WORLD.Barrier() + return r + + return f + + +def init_distribuition_env(program): + if status.mode == DistributionMode.LOCAL: + log.info('Initializing local training') + elif status.mode == DistributionMode.NCCL: + config = F.DistributeTranspilerConfig() + config.mode = "nccl2" + F.DistributeTranspiler(config=config).transpile( + status.replica_id, + trainers=','.join(status._env), + current_endpoint=status._this, + program=program.train_program, + startup_program=program.startup_program) + log.info('Initializing distribution training with config %s' % + (repr(dis_config))) + if status.is_master: + sleep(30) diff --git a/propeller/paddle/train/exporter.py b/propeller/paddle/train/exporter.py new file mode 100644 index 0000000..9ff8e19 --- /dev/null +++ b/propeller/paddle/train/exporter.py @@ -0,0 +1,104 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys +import os +import itertools +import six +import abc +import logging + +import numpy as np +import paddle.fluid as F +import paddle.fluid.layers as L + +from propeller.paddle.train import Saver +from propeller.types import InferenceSpec + +log = logging.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class Exporter(): + @abc.abstractmethod + def export(self, exe, program, eval_result, state): + raise NotImplementedError() + + +class BestExporter(Exporter): + def __init__(self, export_dir, cmp_fn): + self._export_dir = export_dir + self._best = None + self.cmp_fn = cmp_fn + + def export(self, exe, program, eval_model_spec, eval_result, state): + log.debug('New evaluate result: %s \nold: %s' % + (repr(eval_result), repr(self._best))) + if self._best is None or self.cmp_fn(old=self._best, new=eval_result): + log.debug('[Best Exporter]: export to %s' % self._export_dir) + eval_program = program.train_program + # FIXME: all eval datasets has same name/types/shapes now!!! so every eval program are the smae + + saver = Saver( + self._export_dir, + exe, + program=eval_program, + max_ckpt_to_keep=1) + saver.save(state) + self._best = eval_result + else: + log.debug('[Best Exporter]: skip step %s' % state.gstep) + + +class BestInferenceModelExporter(Exporter): + def __init__(self, export_dir, cmp_fn): + self._export_dir = export_dir + self._best = None + self.cmp_fn = cmp_fn + + def export(self, exe, program, eval_model_spec, eval_result, state): + log.debug('New evaluate result: %s \nold: %s' % + (repr(eval_result), repr(self._best))) + if self._best is None or self.cmp_fn(old=self._best, new=eval_result): + log.debug('[Best Exporter]: export to %s' % self._export_dir) + if eval_model_spec.inference_spec is None: + raise ValueError('model_fn didnt return InferenceSpec') + + inf_sepc_dict = eval_model_spec.inference_spec + if not isinstance(inf_sepc_dict, dict): + inf_sepc_dict = {'inference': inf_sepc_dict} + for inf_sepc_name, inf_sepc in six.iteritems(inf_sepc_dict): + if not isinstance(inf_sepc, InferenceSpec): + raise ValueError('unkonw inference spec type: %s' % v) + + save_dir = os.path.join(self._export_dir, inf_sepc_name) + log.debug('[Best Exporter]: save inference model: "%s" to %s' % + (inf_sepc_name, save_dir)) + feed_var = [i.name for i in inf_sepc.inputs] + fetch_var = inf_sepc.outputs + + eval_program = program.train_program + startup_prog = F.Program() + F.io.save_inference_model( + save_dir, + feed_var, + fetch_var, + exe, + main_program=eval_program) + self._best = eval_result + else: + log.debug('[Best Exporter]: skip step %s' % state.gstep) diff --git a/propeller/paddle/train/hooks.py b/propeller/paddle/train/hooks.py new file mode 100644 index 0000000..24ae1e9 --- /dev/null +++ b/propeller/paddle/train/hooks.py @@ -0,0 +1,295 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys +import six +import os +import itertools + +import numpy as np +import logging +import paddle.fluid as F +import paddle.fluid.layers as L + +from propeller import util +from propeller.paddle.train import distribution +from propeller.paddle.train.metrics import Metrics + +__all__ = [ + 'RunHook', 'TqdmProgressBarHook', 'TqdmNotebookProgressBarHook', + 'CheckpointSaverHook', 'LoggingHook', 'StopAtStepHook', 'EvalHook' +] + +log = logging.getLogger(__name__) + + +class RunHook(object): + def __init__(self): + pass + + def before_train(self): + pass + + def before_run(self, state): + return [] + + def after_run(self, res_list, state): + pass + + def should_stop(self, state): + return False + + def after_train(self): + pass + + +class TqdmProgressBarHook(RunHook): + def __init__(self, max_steps, desc=None): + self.tqdm = None + import tqdm + from propeller import log as main_log + hdl = main_log.handlers[0] + + class TqdmLogginHandler(logging.Handler): + def emit(self, record): + try: + msg = self.format(record) + tqdm.tqdm.write(msg, file=sys.stderr) + self.flush() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + + tqdm_hdl = TqdmLogginHandler() + tqdm_hdl.setFormatter(hdl.formatter) + main_log.removeHandler(hdl) + main_log.addHandler(tqdm_hdl) + self.tqdm = tqdm.tqdm(total=max_steps, desc=None) + + def before_run(self, state): + self.tqdm.n = state.gstep + return [] + + def __del__(self): + if self.tqdm: + self.tqdm.close() + + +class TqdmNotebookProgressBarHook(RunHook): + def __init__(self, max_steps, desc=None): + self.tqdm = None + import tqdm + from propeller import log as main_log + hdl = main_log.handlers[0] + + class TqdmLogginHandler(logging.Handler): + def emit(self, record): + try: + msg = self.format(record) + tqdm.tqdm.write(msg, file=sys.stderr) + self.flush() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + + tqdm_hdl = TqdmLogginHandler() + tqdm_hdl.setFormatter(hdl.formatter) + main_log.removeHandler(hdl) + main_log.addHandler(tqdm_hdl) + self.tqdm = tqdm.tqdm_notebook(total=max_steps, desc=None) + + def before_run(self, state): + self.tqdm.n = state.gstep + self.tqdm.refresh() + return [] + + def __del__(self): + if self.tqdm: + self.tqdm.close() + + +class LoggingHook(RunHook): + def __init__(self, + loss, + per_step=10, + skip_step=100, + summary_writer=None, + summary_record=None): + if per_step is None or skip_step is None: + raise ValueError('wrong step argument, per step: %d skip_step %d' % + (per_step, skip_step)) + self.loss = loss + self.per_step = per_step + self.skip_step = skip_step + self.summary_record = summary_record + self.writer = summary_writer + self.last_state = None + + def before_train(self): + if self.summary_record: + if self.summary_record.scalar: + self.s_name, self.s_tolog = zip(*self.summary_record.scalar) + else: + self.s_name, self.s_tolog = [], [] + + if self.summary_record.histogram: + self.h_name, self.h_tolog = zip(*self.summary_record.histogram) + else: + self.h_name, self.h_tolog = [], [] + + def before_run(self, state): + if state.gstep % self.per_step == 0 and state.step > self.skip_step: + ret = [self.loss] + if self.summary_record: + ret += self.s_tolog + ret += self.h_tolog + return ret + else: + return [] + + def after_run(self, res_list, state): + if state.gstep % self.per_step == 0 and state.step > self.skip_step: + if not self.summary_record: + return + + loss = float(res_list[0]) + s_np = res_list[1:1 + len(self.s_name)] + h_np = res_list[1 + len(self.s_name):1 + len(self.s_name) + len( + self.h_name)] + + if self.last_state is not None: + speed = (state.gstep - self.last_state.gstep) / ( + state.time - self.last_state.time) + else: + speed = -1. + self.last_state = state + + # log to tensorboard + if self.writer is not None: + self.writer.add_scalar('loss', loss, state.gstep) + for name, t in zip(self.s_name, s_np): + if np.isnan(t).any(): + log.warning('Nan summary: %s, skip' % name) + else: + self.writer.add_scalar(name, t, state.gstep) + + for name, t in zip(self.h_name, h_np): + if np.isnan(t).any(): + log.warning('Nan summary: %s, skip' % name) + else: + self.writer.add_histogram(name, t, state.gstep) + + if speed > 0.: + self.writer.add_scalar('global_step', speed, state.gstep) + + # log to stdout + log.debug('\t'.join([ + 'step: %d' % state.gstep, + 'steps/sec: %.5f' % speed, + 'loss: %.5f' % loss, + '' if self.summary_record is None else ' '.join( + map(lambda t: '%s:%s' % t, zip(self.s_name, s_np))), + ])) + + +class StopAtStepHook(RunHook): + def __init__(self, stop_global_step, stop_step): + self._stop_gstep = stop_global_step + self._stop_step = stop_step + + def should_stop(self, state): + if (self._stop_gstep and state.gstep >= self._stop_gstep) or \ + (self._stop_step and state.step >= self._stop_step): + log.info('StopAtStepHook called stop') + return True + else: + return False + + +class EvalHook(RunHook): + """hook this on a eval Executor""" + + def __init__(self, metrics, summary_writer=None): + self.writer = summary_writer + self._result = None + + if not isinstance(metrics, dict): + raise ValueError('metrics should be dict, got %s' % repr(metrics)) + + for k, m in six.iteritems(metrics): + if not isinstance(m, Metrics): + raise ValueError( + 'metrics %s should be instance of propeller.Metrics, got %s' + % (k, repr(m))) + + if len(metrics): + self.names = list(metrics.keys()) + self.metrics = list(metrics.values()) + else: + self.names, self.metrics = [], [] + + def before_train(self): + for m in self.metrics: + m.reset() + + def before_run(self, state): + ls = [m.tensor for m in self.metrics] + for i in ls: + if not (isinstance(i, list) or isinstance(i, tuple)): + raise ValueError( + 'metrics should return tuple or list of tensors, got %s' % + repr(i)) + for ii in i: + if not isinstance(ii, F.framework.Variable): + raise ValueError( + 'metrics tensor be propeller.train.Metrics, got %s of type %s' + % (repr(ii), type(ii))) + ls_flt, self.schema = util.flatten(ls) + #log.debug(ls_flt) + return ls_flt + + def after_run(self, res_list, state): + res = util.unflatten(res_list, self.schema) + for r, m in zip(res, self.metrics): + m.update(r) + + @property + def result(self): + return self._result + + def after_train(self): + printable = [] + self._result = {} + for n, m in zip(self.names, self.metrics): + val = m.eval() + self._result[n] = val + + return self.result + + +class CheckpointSaverHook(RunHook): + def __init__(self, saver, per_step=10, skip_step=100): + self.saver = saver + self.per_step = per_step + self.skip_step = skip_step + + def after_run(self, res_list, state): + if state.gstep % self.per_step == 0 and \ + state.step > self.skip_step: + self.saver.save(state) diff --git a/propeller/paddle/train/metrics.py b/propeller/paddle/train/metrics.py new file mode 100644 index 0000000..3d14659 --- /dev/null +++ b/propeller/paddle/train/metrics.py @@ -0,0 +1,554 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import numpy as np +import itertools +import logging + +import paddle.fluid as F +import paddle.fluid.layers as L +import sklearn.metrics + +log = logging.getLogger(__name__) + +__all__ = [ + 'Metrics', 'F1', 'Recall', 'Precision', 'Mrr', 'Mean', 'Acc', 'ChunkF1', + 'RecallAtPrecision' +] + + +class Metrics(object): + def __init__(self): + self.saver = [] + + @property + def tensor(self): + pass + + def update(self, *args): + pass + + def eval(self): + pass + + +class Mean(Metrics): + def __init__(self, t): + self.t = t + self.reset() + + def reset(self): + self.saver = np.array([]) + + @property + def tensor(self): + self.t.persistable = True + return self.t, + + def update(self, args): + t, = args + t = t.reshape([-1]) + self.saver = np.concatenate([self.saver, t]) + + def eval(self): + return self.saver.mean() + + +class Ppl(Mean): + def eval(self): + return np.exp(self.saver.mean()) + + +class Acc(Mean): + def __init__(self, label, pred): + self.eq = L.equal(pred, label) + self.reset() + + @property + def tensor(self): + self.eq.persistable = True + return self.eq, + + +class MSE(Mean): + def __init__(self, label, pred): + diff = pred - label + self.mse = diff * diff + self.reset() + + @property + def tensor(self): + self.mse.persistable = True + return self.mse, + + +class Cosine(Mean): + def __init__(self, label, pred): + self.cos = L.cos_sim(label, pred) + self.reset() + + @property + def tensor(self): + self.cos.persistable = True + return self.cos, + + +class Precision(Metrics): + def __init__(self, label, pred): + self.label = label + self.pred = pred + self.reset() + + def reset(self): + self.label_saver = np.array([], dtype=np.bool) + self.pred_saver = np.array([], dtype=np.bool) + + @property + def tensor(self): + self.label.persistable = True + self.pred.persistable = True + return self.label, self.pred + + def update(self, args): + label, pred = args + label = label.reshape([-1]).astype(np.bool) + pred = pred.reshape([-1]).astype(np.bool) + if label.shape != pred.shape: + raise ValueError( + 'Metrics precesion: input not match: label:%s pred:%s' % + (label, pred)) + self.label_saver = np.concatenate([self.label_saver, label]) + self.pred_saver = np.concatenate([self.pred_saver, pred]) + + def eval(self): + tp = (self.label_saver & self.pred_saver).astype(np.int64).sum() + t = self.label_saver.astype(np.int64).sum() + return tp / t + + +class Recall(Precision): + def eval(self): + tp = (self.label_saver & self.pred_saver).astype(np.int64).sum() + p = (self.label_saver).astype(np.int64).sum() + return tp / p + + +class F1(Precision): + def eval(self): + tp = (self.label_saver & self.pred_saver).astype(np.int64).sum() + t = self.label_saver.astype(np.int64).sum() + p = self.pred_saver.astype(np.int64).sum() + precision = tp / (t + 1.e-6) + recall = tp / (p + 1.e-6) + return 2 * precision * recall / (precision + recall + 1.e-6) + + +class Auc(Metrics): + def __init__(self, label, pred): + self.pred = pred + self.label = label + self.reset() + + def reset(self): + self.pred_saver = np.array([], dtype=np.float32) + self.label_saver = np.array([], dtype=np.bool) + + @property + def tensor(self): + self.pred.persistable = True + self.label.persistable = True + return [self.pred, self.label] + + def update(self, args): + pred, label = args + pred = pred.reshape([-1]).astype(np.float32) + label = label.reshape([-1]).astype(np.bool) + self.pred_saver = np.concatenate([self.pred_saver, pred]) + self.label_saver = np.concatenate([self.label_saver, label]) + + def eval(self): + fpr, tpr, thresholds = sklearn.metrics.roc_curve( + self.label_saver.astype(np.int64), self.pred_saver) + auc = sklearn.metrics.auc(fpr, tpr) + return auc + + +class RecallAtPrecision(Auc): + def __init__(self, label, pred, precision=0.9): + super(RecallAtPrecision, self).__init__(label, pred) + self.precision = precision + + def eval(self): + self.pred_saver = self.pred_saver.reshape( + [self.label_saver.size, -1])[:, -1] + precision, recall, thresholds = sklearn.metrics.precision_recall_curve( + self.label_saver, self.pred_saver) + for p, r in zip(precision, recall): + if p > self.precision: + return r + + +class PrecisionAtThreshold(Auc): + def __init__(self, label, pred, threshold=0.5): + super().__init__(label, pred) + self.threshold = threshold + + def eval(self): + infered = self.pred_saver > self.threshold + correct_num = np.array(infered & self.label_saver).sum() + infer_num = infered.sum() + return correct_num / (infer_num + 1.e-6) + + +class Mrr(Metrics): + def __init__(self, qid, label, pred): + self.qid = qid + self.label = label + self.pred = pred + self.reset() + + def reset(self): + self.qid_saver = np.array([], dtype=np.int64) + self.label_saver = np.array([], dtype=np.int64) + self.pred_saver = np.array([], dtype=np.float32) + + @property + def tensor(self): + self.qid.persistable = True + self.label.persistable = True + self.pred.persistable = True + return [self.qid, self.label, self.pred] + + def update(self, args): + qid, label, pred = args + if not (qid.shape[0] == label.shape[0] == pred.shape[0]): + raise ValueError( + 'Mrr dimention not match: qid[%s] label[%s], pred[%s]' % + (qid.shape, label.shape, pred.shape)) + self.qid_saver = np.concatenate( + [self.qid_saver, qid.reshape([-1]).astype(np.int64)]) + self.label_saver = np.concatenate( + [self.label_saver, label.reshape([-1]).astype(np.int64)]) + self.pred_saver = np.concatenate( + [self.pred_saver, pred.reshape([-1]).astype(np.float32)]) + + def eval(self): + def key_func(tup): + return tup[0] + + def calc_func(tup): + ranks = [ + 1. / (rank + 1.) + for rank, (_, l, p) in enumerate( + sorted( + tup, key=lambda t: t[2], reverse=True)) if l != 0 + ] + ranks = ranks[0] + return ranks + + mrr_for_qid = [ + calc_func(tup) + for _, tup in itertools.groupby( + sorted( + zip(self.qid_saver, self.label_saver, self.pred_saver), + key=key_func), + key=key_func) + ] + mrr = np.float32(sum(mrr_for_qid) / len(mrr_for_qid)) + return mrr + + +class ChunkF1(Metrics): + def __init__(self, label, pred, seqlen, num_label): + self.label = label + self.pred = pred + self.seqlen = seqlen + self.null_index = num_label - 1 + self.label_cnt = 0 + self.pred_cnt = 0 + self.correct_cnt = 0 + + def _extract_bio_chunk(self, seq): + chunks = [] + cur_chunk = None + + for index in range(len(seq)): + tag = seq[index] + tag_type = tag // 2 + tag_pos = tag % 2 + + if tag == self.null_index: + if cur_chunk is not None: + chunks.append(cur_chunk) + cur_chunk = None + continue + + if tag_pos == 0: + if cur_chunk is not None: + chunks.append(cur_chunk) + cur_chunk = {} + cur_chunk = {"st": index, "en": index + 1, "type": tag_type} + else: + if cur_chunk is None: + cur_chunk = { + "st": index, + "en": index + 1, + "type": tag_type + } + continue + + if cur_chunk["type"] == tag_type: + cur_chunk["en"] = index + 1 + else: + chunks.append(cur_chunk) + cur_chunk = { + "st": index, + "en": index + 1, + "type": tag_type + } + + if cur_chunk is not None: + chunks.append(cur_chunk) + return chunks + + def reset(self): + self.label_cnt = 0 + self.pred_cnt = 0 + self.correct_cnt = 0 + + @property + def tensor(self): + self.pred.persistable = True + self.label.persistable = True + self.seqlen.persistable = True + return [self.pred, self.label, self.seqlen] + + def update(self, args): + pred, label, seqlen = args + pred = pred.reshape([-1]).astype(np.int32).tolist() + label = label.reshape([-1]).astype(np.int32).tolist() + seqlen = seqlen.reshape([-1]).astype(np.int32).tolist() + + max_len = 0 + for l in seqlen: + max_len = max(max_len, l) + + for i in range(len(seqlen)): + seq_st = i * max_len + 1 + seq_en = seq_st + (seqlen[i] - 2) + pred_chunks = self._extract_bio_chunk(pred[seq_st:seq_en]) + label_chunks = self._extract_bio_chunk(label[seq_st:seq_en]) + self.pred_cnt += len(pred_chunks) + self.label_cnt += len(label_chunks) + + pred_index = 0 + label_index = 0 + while label_index < len(label_chunks) and pred_index < len( + pred_chunks): + if pred_chunks[pred_index]['st'] < label_chunks[label_index][ + 'st']: + pred_index += 1 + elif pred_chunks[pred_index]['st'] > label_chunks[label_index][ + 'st']: + label_index += 1 + else: + if pred_chunks[pred_index]['en'] == label_chunks[label_index]['en'] \ + and pred_chunks[pred_index]['type'] == label_chunks[label_index]['type']: + self.correct_cnt += 1 + pred_index += 1 + label_index += 1 + + def eval(self): + if self.pred_cnt == 0: + precision = 0.0 + else: + precision = 1.0 * self.correct_cnt / self.pred_cnt + + if self.label_cnt == 0: + recall = 0.0 + else: + recall = 1.0 * self.correct_cnt / self.label_cnt + + if self.correct_cnt == 0: + f1 = 0.0 + else: + f1 = 2 * precision * recall / (precision + recall) + + return np.float32(f1) + + +class PNRatio(Metrics): + def __init__(self, qid, label, pred): + self.qid = qid + self.label = label + self.pred = pred + self.saver = {} + + def reset(self): + self.saver = {} + + @property + def tensor(self): + self.qid.persistable = True + self.label.persistable = True + self.pred.persistable = True + return [self.qid, self.label, self.pred] + + def update(self, args): + qid, label, pred = args + if not (qid.shape[0] == label.shape[0] == pred.shape[0]): + raise ValueError('dimention not match: qid[%s] label[%s], pred[%s]' + % (qid.shape, label.shape, pred.shape)) + qid = qid.reshape([-1]).tolist() + label = label.reshape([-1]).tolist() + pred = pred.reshape([-1]).tolist() + assert len(qid) == len(label) == len(pred) + for q, l, p in zip(qid, label, pred): + if q not in self.saver: + self.saver[q] = [] + self.saver[q].append((l, p)) + + def eval(self): + p = 0 + n = 0 + for qid, outputs in self.saver.items(): + for i in range(0, len(outputs)): + l1, p1 = outputs[i] + for j in range(i + 1, len(outputs)): + l2, p2 = outputs[j] + if l1 > l2: + if p1 > p2: + p += 1 + elif p1 < p2: + n += 1 + elif l1 < l2: + if p1 < p2: + p += 1 + elif p1 > p2: + n += 1 + pn = p / n if n > 0 else 0.0 + return np.float32(pn) + + +class BinaryPNRatio(PNRatio): + def __init__(self, qid, label, pred): + super(BinaryPNRatio, self).__init__(qid, label, pred) + + def eval(self): + p = 0 + n = 0 + for qid, outputs in self.saver.items(): + pos_set = [] + neg_set = [] + for label, score in outputs: + if label == 1: + pos_set.append(score) + else: + neg_set.append(score) + + for ps in pos_set: + for ns in neg_set: + if ps > ns: + p += 1 + elif ps < ns: + n += 1 + else: + continue + pn = p / n if n > 0 else 0.0 + return np.float32(pn) + + +class PrecisionAtK(Metrics): + def __init__(self, qid, label, pred, k=1): + self.qid = qid + self.label = label + self.pred = pred + self.k = k + self.saver = {} + + def reset(self): + self.saver = {} + + @property + def tensor(self): + self.qid.persistable = True + self.label.persistable = True + self.pred.persistable = True + return [self.qid, self.label, self.pred] + + def update(self, args): + qid, label, pred = args + if not (qid.shape[0] == label.shape[0] == pred.shape[0]): + raise ValueError('dimention not match: qid[%s] label[%s], pred[%s]' + % (qid.shape, label.shape, pred.shape)) + qid = qid.reshape([-1]).tolist() + label = label.reshape([-1]).tolist() + pred = pred.reshape([-1]).tolist() + + assert len(qid) == len(label) == len(pred) + for q, l, p in zip(qid, label, pred): + if q not in self.saver: + self.saver[q] = [] + self.saver[q].append((l, p)) + + def eval(self): + right = 0 + total = 0 + for v in self.saver.values(): + v = sorted(v, key=lambda x: x[1], reverse=True) + k = min(self.k, len(v)) + for i in range(k): + if v[i][0] == 1: + right += 1 + break + total += 1 + + return np.float32(1.0 * right / total) + + +#class SemanticRecallMetrics(Metrics): +# def __init__(self, qid, vec, type_id): +# self.qid = qid +# self.vec = vec +# self.type_id = type_id +# self.reset() +# +# def reset(self): +# self.saver = [] +# +# @property +# def tensor(self): +# return [self.qid, self.vec, self.type_id] +# +# def update(self, args): +# qid, vec, type_id = args +# self.saver.append((qid, vec, type_id)) +# +# def eval(self): +# dic = {} +# for qid, vec, type_id in self.saver(): +# dic.setdefault(i, {}).setdefault(k, []).append(vec) +# +# for qid in dic: +# assert len(dic[qid]) == 3 +# qvec = np.arrray(dic[qid][0]) +# assert len(qvec) == 1 +# ptvec = np.array(dic[qid][1]) +# ntvec = np.array(dic[qid][2]) +# +# np.matmul(qvec, np.transpose(ptvec)) +# np.matmul(qvec, np.transpose(ntvec)) +# diff --git a/propeller/paddle/train/monitored_executor.py b/propeller/paddle/train/monitored_executor.py new file mode 100644 index 0000000..5663697 --- /dev/null +++ b/propeller/paddle/train/monitored_executor.py @@ -0,0 +1,355 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import json +from functools import reduce +import six +from time import time +import shutil + +import logging +import numpy as np +import paddle.fluid as F +import paddle.fluid.layers as L + +from propeller import util +from propeller.types import StopException, ProgramPair +from propeller.paddle.train import hooks +from . import distribution + +log = logging.getLogger(__name__) + +__all__ = ['MonitoredExecutor', 'Saver'] + + +class RunState(object): + @classmethod + def from_str(cls, s): + j = json.loads(s) + ret = RunState() + ret._gstep = j['global_step'] + ret._time = j['time'] + ret._step = 0 + return ret + + def __init__(self): + self._gstep = 0 + self._step = 0 + self._time = time() + + @property + def gstep(self): + return self._gstep + + @property + def step(self): + return self._step + + @property + def time(self): + return self._time + + def __repr__(self): + return repr({'global_step': self._gstep, 'time': self._time}) + + def serialize(self): + return json.dumps({'global_step': self._gstep, 'time': self._time}) + + def next(self): + ret = RunState() + ret._gstep = self._gstep + 1 + ret._step = self._step + 1 + ret._time = time() + return ret + + +class Saver(object): + def __init__(self, + save_dir, + exe, + program, + save_prefix='model', + max_ckpt_to_keep=None): + if exe is not None: + assert isinstance( + exe, F.Executor + ), 'expect normal executor to save, got executor of type %s' % repr( + type(exe)) + self._exe = exe + self._program = program + self._save_dir = save_dir + self._save_prefix = save_prefix + self._max_ckpt_to_keep = 10 if max_ckpt_to_keep is None else max_ckpt_to_keep + + self.ckpt_info_path = os.path.join(save_dir, 'ckpt_info') + + if os.path.exists(self.ckpt_info_path): + self.ckpt_list = [ + p.strip() for p in open(self.ckpt_info_path).readlines() + ] + log.debug('ckpt_list in this Saver: %s' % (self.ckpt_list)) + else: + self.ckpt_list = [] + + @property + def last_ckpt(self): + return self.ckpt_list[-1] if len(self.ckpt_list) else None + + def save(self, state): + save_name = '%s_%d' % (self._save_prefix, state.gstep) + save_dir = os.path.join(self._save_dir, save_name) + tmp_dir = os.path.join(self._save_dir, 'tmp') + try: + shutil.rmtree(save_dir) + shutil.rmtree(tmp_dir) + except OSError: + pass + log.debug('saving step %d to %s' % (state.gstep, save_dir)) + F.io.save_persistables(self._exe, tmp_dir, self._program) + shutil.move(tmp_dir, save_dir) + meta = state.serialize() + open(os.path.join(save_dir, 'meta'), 'w').write(meta) + + self.ckpt_list.append(save_name) + if len(self.ckpt_list) > self._max_ckpt_to_keep: + ckpt_to_keep = self.ckpt_list[-self._max_ckpt_to_keep:] + ckpt_to_remove = set(self.ckpt_list) - set(ckpt_to_keep) + self.ckpt_list = ckpt_to_keep + for ckpt in ckpt_to_remove: + ckpt_dir = os.path.join(self._save_dir, ckpt) + if os.path.exists(ckpt_dir): + shutil.rmtree(ckpt_dir) + log.debug('No. of ckpt exceed %d, clean up: %s' % + (self._max_ckpt_to_keep, ckpt_dir)) + open(self.ckpt_info_path, 'w').write('\n'.join(self.ckpt_list)) + + def restore(self, ckpt=-1): + if not isinstance(ckpt, (int, ) + six.string_types): + raise ValueError('ckpt type not understood %s' % repr(ckpt)) + if isinstance(ckpt, int): + try: + ckpt = self.ckpt_list[ckpt] + except IndexError: + raise ValueError('invalid restore ckpt number %d' % ckpt) + if isinstance(ckpt, six.string_types): + try: + ckpt = self.ckpt_list.index(ckpt) + except ValueError: + raise ValueError('ckpt: %s not in ckpt list: %s' % + (ckpt, self.ckpt_list)) + + path = os.path.join(self._save_dir, self.ckpt_list[ckpt]) + meta_file = os.path.join(path, 'meta') + if not os.path.exists(meta_file): + raise RuntimeError('meta not found in restore dir: %s' % path) + state = RunState.from_str(open(meta_file).read()) + log.info('restore from ckpt %s, ckpt-status: %s' % (path, repr(state))) + + def fn(v): + vpath = os.path.join(path, v.name) + if F.io.is_persistable(v): + if os.path.exists(vpath): + return True + else: + log.warning('var %s not found in checkpoint, ignored' % + v.name) + return False + + F.io.load_vars( + self._exe, path, main_program=self._program, predicate=fn) + return state + + +class MonitoredExecutor(object): + """A wrapper handling the train loop""" + + def __init__( + self, + executor, + program, + loss=None, #must set in train + state=None, + run_config=None, #none if not load + run_hooks=[], + warm_start_setting=None): + if not isinstance(executor, F.Executor): + raise ValueError('PE is no longer supported') + if isinstance(executor, F.ParallelExecutor): + raise ValueError('ParallelExecutor is deprecatd, use Executor') + self._exe = executor + self._hooks = run_hooks + self._state = RunState() # might be overwrite in freeze + self._program = program + self._loss = loss + self._warm_start_setting = warm_start_setting + self._saver = None # will set in prepare + self.result = None # will set after train + if run_config is not None: + self._model_dir = run_config.model_dir + self._save_dir = run_config.model_dir + self._save_steps = run_config.save_steps + self._skip_steps = run_config.skip_steps if run_config.skip_steps else 100 + self._save_prefix = 'model' + self._max_ckpt = run_config.max_ckpt + + @property + def state(self): + return self._state + + def init_or_restore_variables(self): + # The order of this 2 steps really matters + # 1. init train + + F.Executor(F.cuda_places()[0]).run(self._program.startup_program) + # 2. restore param + if self._warm_start_setting is not None: + if not os.path.exists(self._warm_start_setting.from_dir): + raise ValueError('warm start dir not exists: %s' % + self._warm_start_setting.from_dir) + log.info("warm start from %s" % self._warm_start_setting.from_dir) + if self._warm_start_setting.predicate_fn is not None: + + def fn(v): + ret = self._warm_start_setting.predicate_fn(v) + if ret: + log.info('warm start: %s' % v.name) + return ret + + F.io.load_vars( + F.Executor(F.cuda_places()[0]), + self._warm_start_setting.from_dir, + main_program=self._program.train_program, + predicate=fn) + else: + raise NotImplementedError() + + self._saver = Saver( + self._model_dir, + F.Executor(F.cuda_places()[0]), + program=self._program.train_program, + max_ckpt_to_keep=self._max_ckpt) + if self._saver.last_ckpt is not None: + self._state = self._saver.restore() + + def freeze(self): + if self._loss is None: + log.debug('will not freeze a program without loss') + return + if isinstance(self._program.train_program, F.compiler.CompiledProgram): + log.debug('program has already been built') + return + exec_strategy = F.ExecutionStrategy() + exec_strategy.num_threads = 4 #2 for fp32 4 for fp16 + exec_strategy.use_experimental_executor = True + exec_strategy.num_iteration_per_drop_scope = 10 #important shit + + build_strategy = F.BuildStrategy() + build_strategy.remove_unnecessary_lock = False + #build_strategy.fuse_broadcast_ops = True + build_strategy.num_trainers = distribution.status.num_replica + build_strategy.trainer_id = distribution.status.replica_id + build_strategy.memory_optimize = True + + log.info('replica id %d of %d' % (distribution.status.replica_id, + distribution.status.num_replica)) + + program = F.CompiledProgram( + self._program.train_program).with_data_parallel( + loss_name=self._loss.name, + build_strategy=build_strategy, + exec_strategy=exec_strategy) + self._program = ProgramPair( + train_program=program, + startup_program=self._program.startup_program) + + def __enter__(self): + log.debug('freezing program') + self.freeze() + log.debug('done freezing') + log.info('********** Start Loop ************') + # TODO init + + self.result = None + for h in self._hooks: + log.debug('train loop has hook %s' % h) + h.before_train() + return self + + def run(self, fetch_list=[], *args, **kwargs): + #log.debug('Executor running step %d' % self._state.gstep) + if self._hooks: + fetch_list = [fetch_list] + for h in self._hooks: + #log.debug('calling hook.before_run %s' % h) + fetch = h.before_run(self._state) + fetch_list.append(fetch) + fetch_list_len = map(len, fetch_list) + fetch_list, schema = util.flatten(fetch_list) + fetch_list = [ + f.name if not isinstance(f, six.string_types) else f + for f in fetch_list + ] + #if len(set(fetch_list)) != len(fetch_list): + # log.error('strange shit happend when fetch list has idetity tensors %s' % fetch_list) + res = self._exe.run(self._program.train_program, + fetch_list=fetch_list, + *args, + **kwargs) + res = [self.merge_result(r) for r in res] + #log.debug(res) + + res = util.unflatten(res, schema) + ret, res = res[0], res[1:] + for r, h in zip(res, self._hooks): + #log.debug('calling hook.after_run') + h.after_run(r, self._state) + + if any(map(lambda i: i.should_stop(self._state), self._hooks)): + raise StopException('hook call stop') + else: + ret = self._exe.run(self._program.train_program, + fetch_list=fetch_list, + *args, + **kwargs) + self._state = self._state.next() + return ret + + def __exit__(self, err_type, err_value, trace): + if (err_type is None) or isinstance(err_value, ( + F.core.EOFException, StopException, KeyboardInterrupt)): + try: + log.info('********** Stop Loop ************') + self.result = [] + for h in self._hooks: + self.result.append(h.after_train()) + except Exception as e: + log.exception('error occur after loop %s' % repr(e)) + else: + log.info('********** Interupt Loop ************') + log.exception('error occur during loop %s: %s' % + (err_type, err_value)) + + def merge_result(self, ls): + dev_count = len(self._program.train_program._places) if isinstance( + self._program.train_program, F.compiler.CompiledProgram) else 1 + if dev_count == 1: + return ls + else: + shape = (-1, ls.shape[0] // dev_count) + ls.shape[1:] + ret = np.reshape(ls, shape).mean(axis=0) + return ret diff --git a/propeller/paddle/train/trainer.py b/propeller/paddle/train/trainer.py new file mode 100644 index 0000000..46be517 --- /dev/null +++ b/propeller/paddle/train/trainer.py @@ -0,0 +1,510 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import itertools +import six +import inspect +from collections import namedtuple +from contextlib import contextmanager +from six.moves import zip, map +import logging +from time import time + +import paddle.fluid as F +import paddle.fluid.layers as L + +from propeller.types import RunMode, StopException, SummaryRecord, StopException, ModelSpec, InferenceSpec, ProgramPair, RunConfig +from propeller.paddle import summary, collection +from propeller.paddle.data.functional import Dataset +from propeller.paddle.train import distribution +from propeller.train.model import Model +from propeller.paddle.train.monitored_executor import Saver +from propeller.paddle.train import hooks, metrics + +from propeller.paddle.train.monitored_executor import MonitoredExecutor + +log = logging.getLogger(__name__) + +__all__ = ['train_and_eval', 'Learner'] + + +def get_summary_writer(path): + summary_writer = None + try: + from tensorboardX import SummaryWriter + if distribution.status.is_master: + summary_writer = SummaryWriter(os.path.join(path)) + except ImportError: + log.warning('tensorboardX not installed, will not log to tensorboard') + return summary_writer + + +def log_eval_result(name, eval_result, swriter, state): + log.debug(eval_result) + printable = [] + for n, val in six.iteritems(eval_result): + assert val.shape == (), 'metrics eval use float' + printable.append('{}\t{}'.format(n, val)) + if swriter is not None: + swriter.add_scalar(n, val, state.gstep) + log.debug('write to tensorboard %s' % swriter.logdir) + + if len(printable): + log.info('*** eval res: %10s ***' % name) + for p in printable: + log.info(p) + log.info('******************************') + + +def build_net(model_fn, features, mode, params, run_config): + model_spec = model_fn( + features=features, mode=mode, params=params, run_config=run_config) + + if mode == RunMode.TRAIN: + if not isinstance(model_spec.loss, F.framework.Variable): + raise ValueError('model_spec.metrics should be Variable, got %s' % + repr(model_spec.loss)) + if not (model_spec.loss.shape == () or model_spec.loss.shape == (1, )): + raise ValueError('expect scarlar loss, got %s' % + repr(model_spec.loss.shape)) + model_spec.loss.persistable = True + elif mode == RunMode.EVAL: + if not isinstance(model_spec.metrics, dict): + raise ValueError('model_spec.metrics should be dict, got %s' % + repr(model_spec.metrics)) + elif mode == RunMode.PREDICT: + if not isinstance(model_spec.predictions, (list, tuple)): + raise ValueError('model_spec.predictions shuold be list, got %s' % + repr(model_spec.predictions)) + else: + raise ValueError('unkonw mode %s' % mode) + return model_spec + + +class Learner(object): + def __init__(self, + model_class_or_model_fn, + run_config, + params=None, + warm_start_setting=None): + ''' + model_class_or_model_fn(callable|propeller.train.Model): `model_class_or_model_fn` be specified in 2 ways: + 1. subclass of propeller.train.Model which implements: + 1. \_\_init\_\_ (hyper_param, mode, run_config) + 2. forward (features) => (prediction) + 3. backword (loss) => None + 4. loss (predictoin) => (loss) + 5. metrics (optional) (prediction) => (dict of propeller.Metrics) + + 2. a model_fn takes following args: + 1. features + 2. param + 3. mode + 4. run_config(optional) + and returns a `propeller.ModelSpec` + + params: any python object, will pass to your `model_fn` or `propeller.train.Model` + run_config (propeller.RunConfig): run_config.max_steps should not be None. + warm_start_setting (propeller.WarmStartSetting): Optional. warm start variable will overwrite model variable. + ''' + if run_config.model_dir is None: + raise ValueError('model_dir should specified in run_config') + + if issubclass(model_class_or_model_fn, Model): + + def model_fn(features, mode, params, run_config): + if mode != RunMode.PREDICT: + fea, label = features[:-1], features[-1] + else: + fea = features + + model = model_class_or_model_fn( + params, mode, run_config=run_config) + pred = model.forward(fea) + if isinstance(pred, F.framework.Variable): + prediction = [pred] + else: + prediction = pred + if mode == RunMode.TRAIN: + loss = model.loss(pred, label) + model.backward(loss) + return ModelSpec( + loss=loss, predictions=prediction, mode=mode) + elif mode == RunMode.EVAL: + loss = model.loss(pred, label) + me = model.metrics(pred, label) + + inf_spec = InferenceSpec(inputs=fea, outputs=prediction) + if 'loss' not in me: + me['loss'] = metrics.Mean(loss) + return ModelSpec( + loss=loss, + predictions=prediction, + metrics=me, + mode=mode, + inference_spec=inf_spec) + elif mode == RunMode.PREDICT: + return ModelSpec(predictions=prediction, mode=mode) + else: + raise RuntimeError('unknown run mode %s' % mode) + elif inspect.isfunction(model_class_or_model_fn): + model_fn = model_class_or_model_fn + else: + raise ValueError('unknown model %s' % model_class_or_model_fn) + + self.model_fn = model_fn + self.params = params + self.run_config = run_config + self.warm_start_setting = warm_start_setting + + def build_for_train(self, train_dataset): + train_dataset.name = 'train' + train_program = F.Program() + startup_prog = F.Program() + with F.program_guard(train_program, startup_prog): + with F.unique_name.guard(): + with collection.Collections() as collections: + log.info('Building Train Graph...') + fea = train_dataset.features() + model_spec = build_net(self.model_fn, fea, RunMode.TRAIN, + self.params, self.run_config) + log.info('Building Train Graph: Done') + + scalars = collections.get(collection.Key.SUMMARY_SCALAR) + histograms = collections.get(collection.Key.SUMMARY_HISTOGRAM) + skip_optimize_ops = collections.get( + collection.Key.SKIP_OPTIMIZE) + skip_opt = set() + if skip_optimize_ops is not None: + skip_opt |= set(skip_optimize_ops) + if scalars is not None: + skip_opt |= {t for _, t in scalars} + if histograms is not None: + skip_opt |= {t for _, t in histograms} + skip_opt = list(skip_opt) + log.info( + 'Train with: \n> Run_config: %s\n> Params: %s\n> Train_model_spec: %s\n' + % (repr(self.run_config), repr(self.params), repr(model_spec))) + + summary_record = SummaryRecord( + scalar=collections.get(collection.Key.SUMMARY_SCALAR), + histogram=collections.get(collection.Key.SUMMARY_HISTOGRAM), ) + return ProgramPair( + train_program=train_program, + startup_program=startup_prog), model_spec, summary_record + + def build_for_eval(self, ds): + ds.name = 'eval' + program = F.Program() + startup_prog = F.Program() + with F.program_guard(program, startup_prog): + #share var with Train net + with F.unique_name.guard(): + log.info('Building Eval Graph') + fea = ds.features() + model_spec = build_net(self.model_fn, fea, RunMode.EVAL, + self.params, self.run_config) + log.info('Done') + program = program.clone(for_test=True) + log.info( + 'Eval with: \n> Run_config: %s\n> Params: %s\n> Train_model_spec: %s\n' + % (repr(self.run_config), repr(self.params), repr(model_spec))) + return ProgramPair( + train_program=program, startup_program=startup_prog), model_spec + + def build_for_predict(self, ds): + ds.name = 'predict' + program = F.Program() + startup_prog = F.Program() + with F.program_guard(program, startup_prog): + #share var with Train net + with F.unique_name.guard(): + log.info('Building Predict Graph') + fea = ds.features() + model_spec = build_net(self.model_fn, fea, RunMode.PREDICT, + self.params, self.run_config) + log.info('Done') + + program = program.clone(for_test=True) + + log.info( + 'Predict with: \n> Run_config: %s\n> Params: %s\n> Train_model_spec: %s\n' + % (repr(self.run_config), repr(self.params), repr(model_spec))) + return ProgramPair( + train_program=program, startup_program=startup_prog), model_spec + + def train(self, train_ds, train_hooks=[]): + if not isinstance(train_ds, Dataset): + raise ValueError('expect dataset to be instance of Dataset, got %s' + % repr(train_ds)) + + train_program, model_spec, summary_record = self.build_for_train( + train_ds) + train_run_hooks = [ + hooks.StopAtStepHook(self.run_config.max_steps, + self.run_config.run_steps), + hooks.LoggingHook( + model_spec.loss, + summary_record=summary_record, + summary_writer=get_summary_writer( + os.path.join(self.run_config.model_dir, 'train_history')), + per_step=self.run_config.log_steps, + skip_step=self.run_config.skip_steps), + ] + train_run_hooks.extend(train_hooks) + train_executor = F.Executor(F.cuda_places()[0]) + + mon_exe = MonitoredExecutor( + train_executor, + train_program, + loss=model_spec.loss, + run_config=self.run_config, + run_hooks=train_run_hooks, + warm_start_setting=self.warm_start_setting) + + distribution.init_distribuition_env( + train_program) #only initialize distribute training with + mon_exe.init_or_restore_variables() + if distribution.status.is_master: + mon_exe._hooks.append( + hooks.CheckpointSaverHook( + mon_exe._saver, + per_step=mon_exe._save_steps, + skip_step=mon_exe._skip_steps)) + + try: + with mon_exe: + for data in train_ds.start(): + mon_exe.run(feed=data) + except (StopException, F.core.EOFException) as e: + pass + + return mon_exe.result + + def evaluate(self, eval_dataset, eval_hooks=[]): + if not isinstance(eval_dataset, Dataset): + raise ValueError('expect dataset to be instance of Dataset, got %s' + % repr(eval_dataset)) + program, model_spec = self.build_for_eval(eval_dataset) + single_card_place = F.cuda_places()[0] + eval_executor = F.Executor(single_card_place) + + eval_hooks = [ + hooks.StopAtStepHook(self.run_config.eval_max_steps, + self.run_config.eval_max_steps), + hooks.EvalHook(model_spec.metrics, ) + ] + + mon_exe = MonitoredExecutor( + eval_executor, + program, + run_config=self.run_config, + run_hooks=eval_hooks) + mon_exe.init_or_restore_variables() + + try: + with mon_exe: + for data in eval_dataset.start(places=[single_card_place]): + mon_exe.run(feed=data) + except (StopException, F.core.EOFException) as e: + pass + + _, eval_result = mon_exe.result + + summary_writer = get_summary_writer( + os.path.join(self.run_config.model_dir, 'eval_history')) + log_eval_result('eval', eval_result, summary_writer, mon_exe.state) + + return mon_exe.result + + def predict(self, predict_dataset, ckpt=None, steps=-1, split_batch=True): + ''' + Perform predictoin + will call `model_fn` and initiate user-specifed model in `propeller.RunMode.PREDICT` mode + + Args: + infer_dataset (propeller.data.Dataset): should not `shuffle` or `repeat` + steps (int): steps to predict, if -1 is specifed, will stop when `StopException` is raised in `infer_dataset` + split_batch (bool): if True, prediction of each example in a batch is returned. + + Yields: + Evaluated values of predictions tensors. + + ''' + if not isinstance(predict_dataset, Dataset): + raise ValueError('expect dataset to be instance of Dataset, got %s' + % repr(predict_dataset)) + + program, model_spec = self.build_for_predict(predict_dataset) + single_card_place = F.cuda_places()[0] + executor = F.Executor(single_card_place) + pred_run_config = RunConfig( + run_steps=steps if steps == -1 else None, + model_dir=self.run_config.model_dir) + mon_exe = MonitoredExecutor( + executor, + program, + run_config=pred_run_config, ) + mon_exe.init_or_restore_variables() + try: + with mon_exe: + log.info('Runining predict from dir: %s' % repr(mon_exe.state)) + single_card_place = F.cuda_places()[0] + for data in predict_dataset.start(places=[single_card_place]): + res = mon_exe.run(fetch_list=model_spec.predictions, + feed=data) + if split_batch: + res = map(lambda i: i.tolist(), res) + res = zip(*res) # transpose + for r in res: + yield r + else: + yield list(map(lambda i: i.tolist(), res)) + except (StopException, F.core.EOFException) as e: + pass + + +def train_and_eval(_shit=None, + model_class_or_model_fn=None, + params=None, + run_config=None, + train_dataset=None, + eval_dataset=None, + warm_start_setting=None, + train_hooks=[], + eval_hooks=[], + exporters=[]): + ''' + Perform train and evaluate procesure. + will call `model_fn` and initiate user-specifed model in `propeller.RunMode.PREDICT` mode + + Args: + model_class_or_model_fn(callable|propeller.train.Model): `model_class_or_model_fn` be specified in 2 ways: + 1. subclass of propeller.train.Model which implements: + 1. \_\_init\_\_ (hyper_param, mode, run_config) + 2. forward (features) => (prediction) + 3. backword (loss) => None + 4. loss (predictoin) => (loss) + 5. metrics (optional) (prediction) => (dict of propeller.Metrics) + + 2. a model_fn takes following args: + 1. features + 2. param + 3. mode + 4. run_config(optional) + and returns a `propeller.ModelSpec` + + params: any python object, will pass to your `model_fn` or `propeller.train.Model` + run_config (propeller.RunConfig): run_config.max_steps should not be None. + train_dataset (propeller.paddle.data.Dataset): training will stop if global_step > run_config.max_steps. + eval_dataset (propeller.paddle.data.Dataset|dict): Optional, if Dict of propeller.data.Dataset were specified, will perform evluatation on every evaluation sets and report results. + warm_start_setting (propeller.WarmStartSetting): Optional. warm start variable will overwrite model variable. + train_hooks (list of propeller.paddle.train.RunHook): Optional. + eval_hooks (list of propeller.paddle.train.RunHook): Optional. + exporters (list of propeller.paddle.train.Exporter): Optional. + ''' + if _shit is not None: + raise ValueError('specify keyword args to this function') + if model_class_or_model_fn is None or params is None or run_config is None or train_dataset is None: + raise ValueError( + 'some argument is None: model_class_or_model_fn:%s params:%s run_config:%s train_dataset:%s' + % (model_class_or_model_fn, params, run_config, train_dataset)) + + #init distribution env if envvir PROPELLER_DISCONFIG is set + if train_dataset is None: + raise ValueError('train dataset not specified') + + if eval_dataset is None: + raise ValueError('eval dataset not specifed') + + if not isinstance(eval_dataset, (dict, Dataset)): + raise ValueError( + 'Eval dataset should be propeller.Dataset of a list of that, got: %s' + % eval_dataset) + if isinstance(eval_dataset, Dataset): + eval_dataset = {'eval': eval_dataset} + ds_list = list(eval_dataset.values()) + for ds in ds_list: + ds.name = 'eval' + first = ds_list[0] + for d in ds_list[1:]: + if not first.__eq__(d): + raise ValueError( + 'eval dataset has different output_shapes or types: %s' % + repr(ds_list)) + + est = Learner( + model_class_or_model_fn, + run_config, + params, + warm_start_setting=warm_start_setting) + + class EvalHookOnTrainLoop(hooks.RunHook): + def __init__(self): + self.program, self.model_spec = est.build_for_eval( + list(eval_dataset.values())[ + 0]) #eval_datasets must have same output shapes + self.summary_writers = { + ds_name: get_summary_writer( + os.path.join( + os.path.join(run_config.model_dir, 'eval_history'), + ds_name)) + for ds_name in eval_dataset + } + + def after_run(self, _, state): + if state.step > run_config.skip_steps and state.gstep % run_config.eval_steps == 0: + eval_results = {} + for name, ds in six.iteritems(eval_dataset): + ehooks = [ + hooks.StopAtStepHook(est.run_config.eval_max_steps, + est.run_config.eval_max_steps), + hooks.EvalHook( + self.model_spec.metrics, + summary_writer=self.summary_writers[name], ) + ] + single_card_place = F.cuda_places()[0] + eval_executor = F.Executor(single_card_place) + mon_exe = MonitoredExecutor( + eval_executor, + self.program, + run_config=est.run_config, + run_hooks=ehooks + eval_hooks) + try: + with mon_exe: + for data in ds.start(places=[single_card_place]): + mon_exe.run(feed=data) + except (StopException, F.core.EOFException) as e: + pass + hook_results = mon_exe.result + eval_res = hook_results[ + 1] # hook_results: [StopAtStepHook, EvalHook, ...] + eval_results[name] = eval_res + log_eval_result(name, eval_res, self.summary_writers[name], + state) + for exporter in exporters: + exporter.export(eval_executor, self.program, + self.model_spec, eval_results, state) + else: + eval_results = {} + return eval_results + + if distribution.status.is_master: + train_hooks.append(EvalHookOnTrainLoop()) + res = est.train(train_dataset, train_hooks=train_hooks) + return res diff --git a/propeller/service/__init__.py b/propeller/service/__init__.py new file mode 100644 index 0000000..d0c32e2 --- /dev/null +++ b/propeller/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/propeller/service/client.py b/propeller/service/client.py new file mode 100644 index 0000000..3ea23e1 --- /dev/null +++ b/propeller/service/client.py @@ -0,0 +1,104 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals +import asyncio +import threading +import math + +import zmq +import zmq.asyncio +import numpy as np + +from propeller import log +import propeller.service.utils as serv_utils + + +class InferenceBaseClient(object): + def __init__(self, address): + self.context = zmq.Context() + self.address = address + self.socket = self.context.socket(zmq.REQ) + self.socket.connect(address) + log.info("Connecting to server... %s" % address) + + def __call__(self, *args): + for arg in args: + if not isinstance(arg, np.ndarray): + raise ValueError('expect ndarray slot data, got %s' % + repr(arg)) + request = serv_utils.nparray_list_serialize(args) + + self.socket.send(request) + reply = self.socket.recv() + ret = serv_utils.nparray_list_deserialize(reply) + return ret + + +class InferenceClient(InferenceBaseClient): + def __init__(self, address, batch_size=128, num_coroutine=10, timeout=10.): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + context = zmq.asyncio.Context() + self.socket_pool = [ + context.socket(zmq.REQ) for _ in range(num_coroutine) + ] + log.info("Connecting to server... %s" % address) + for socket in self.socket_pool: + socket.connect(address) + self.num_coroutine = num_coroutine + self.batch_size = batch_size + self.timeout = int(timeout * 1000) + + #yapf: disable + def __call__(self, *args): + for arg in args: + if not isinstance(arg, np.ndarray): + raise ValueError('expect ndarray slot data, got %s' % + repr(arg)) + + num_tasks = math.ceil(1. * args[0].shape[0] / self.batch_size) + rets = [None] * num_tasks + + async def get(coroutine_idx=0, num_coroutine=1): + socket = self.socket_pool[coroutine_idx] + while coroutine_idx < num_tasks: + begin = coroutine_idx * self.batch_size + end = (coroutine_idx + 1) * self.batch_size + + arr_list = [arg[begin:end] for arg in args] + request = serv_utils.nparray_list_serialize(arr_list) + try: + await socket.send(request) + await socket.poll(self.timeout, zmq.POLLIN) + reply = await socket.recv(zmq.NOBLOCK) + ret = serv_utils.nparray_list_deserialize(reply) + except Exception as e: + log.exception(e) + ret = None + rets[coroutine_idx] = ret + coroutine_idx += num_coroutine + + futures = [ + get(i, self.num_coroutine) for i in range(self.num_coroutine) + ] + self.loop.run_until_complete(asyncio.wait(futures)) + for r in rets: + if r is None: + raise RuntimeError('Client call failed') + return [np.concatenate(col, 0) for col in zip(*rets)] + #yapf: enable diff --git a/propeller/service/interface.proto b/propeller/service/interface.proto new file mode 100644 index 0000000..e94894c --- /dev/null +++ b/propeller/service/interface.proto @@ -0,0 +1,46 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package interface; + +service Inference { + rpc Infer(Slots) returns (Slots){} +} + +message Slots { + repeated Slot slots = 1; +} + +message Slot { + enum Type { + // Pod Types + BOOL = 0; + INT16 = 1; + INT32 = 2; + INT64 = 3; + FP16 = 4; + FP32 = 5; + FP64 = 6; + // Tensor is used in C++. + SIZE_T = 19; + UINT8 = 20; + INT8 = 21; + } + + Type type = 1; + repeated int64 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480] + bytes data = 3; +} + diff --git a/propeller/service/interface_pb2.py b/propeller/service/interface_pb2.py new file mode 100644 index 0000000..8d0f4f8 --- /dev/null +++ b/propeller/service/interface_pb2.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: interface.proto + +import sys +_b = sys.version_info[0] < 3 and (lambda x: x) or ( + lambda x: x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + +DESCRIPTOR = _descriptor.FileDescriptor( + name='interface.proto', + package='interface', + syntax='proto3', + serialized_options=None, + serialized_pb=_b( + '\n\x0finterface.proto\x12\tinterface\"\'\n\x05Slots\x12\x1e\n\x05slots\x18\x01 \x03(\x0b\x32\x0f.interface.Slot\"\xb8\x01\n\x04Slot\x12\"\n\x04type\x18\x01 \x01(\x0e\x32\x14.interface.Slot.Type\x12\x0c\n\x04\x64ims\x18\x02 \x03(\x03\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"p\n\x04Type\x12\x08\n\x04\x42OOL\x10\x00\x12\t\n\x05INT16\x10\x01\x12\t\n\x05INT32\x10\x02\x12\t\n\x05INT64\x10\x03\x12\x08\n\x04\x46P16\x10\x04\x12\x08\n\x04\x46P32\x10\x05\x12\x08\n\x04\x46P64\x10\x06\x12\n\n\x06SIZE_T\x10\x13\x12\t\n\x05UINT8\x10\x14\x12\x08\n\x04INT8\x10\x15\x32:\n\tInference\x12-\n\x05Infer\x12\x10.interface.Slots\x1a\x10.interface.Slots\"\x00\x62\x06proto3' + )) + +_SLOT_TYPE = _descriptor.EnumDescriptor( + name='Type', + full_name='interface.Slot.Type', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='BOOL', index=0, number=0, serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='INT16', + index=1, + number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='INT32', + index=2, + number=2, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='INT64', + index=3, + number=3, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FP16', index=4, number=4, serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FP32', index=5, number=5, serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FP64', index=6, number=6, serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SIZE_T', + index=7, + number=19, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='UINT8', + index=8, + number=20, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='INT8', + index=9, + number=21, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=144, + serialized_end=256, ) +_sym_db.RegisterEnumDescriptor(_SLOT_TYPE) + +_SLOTS = _descriptor.Descriptor( + name='Slots', + full_name='interface.Slots', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='slots', + full_name='interface.Slots.slots', + index=0, + number=1, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=30, + serialized_end=69, ) + +_SLOT = _descriptor.Descriptor( + name='Slot', + full_name='interface.Slot', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='type', + full_name='interface.Slot.type', + index=0, + number=1, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dims', + full_name='interface.Slot.dims', + index=1, + number=2, + type=3, + cpp_type=2, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='data', + full_name='interface.Slot.data', + index=2, + number=3, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR), + ], + extensions=[], + nested_types=[], + enum_types=[_SLOT_TYPE, ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[], + serialized_start=72, + serialized_end=256, ) + +_SLOTS.fields_by_name['slots'].message_type = _SLOT +_SLOT.fields_by_name['type'].enum_type = _SLOT_TYPE +_SLOT_TYPE.containing_type = _SLOT +DESCRIPTOR.message_types_by_name['Slots'] = _SLOTS +DESCRIPTOR.message_types_by_name['Slot'] = _SLOT +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Slots = _reflection.GeneratedProtocolMessageType( + 'Slots', + (_message.Message, ), + { + 'DESCRIPTOR': _SLOTS, + '__module__': 'interface_pb2' + # @@protoc_insertion_point(class_scope:interface.Slots) + }) +_sym_db.RegisterMessage(Slots) + +Slot = _reflection.GeneratedProtocolMessageType( + 'Slot', + (_message.Message, ), + { + 'DESCRIPTOR': _SLOT, + '__module__': 'interface_pb2' + # @@protoc_insertion_point(class_scope:interface.Slot) + }) +_sym_db.RegisterMessage(Slot) + +_INFERENCE = _descriptor.ServiceDescriptor( + name='Inference', + full_name='interface.Inference', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=258, + serialized_end=316, + methods=[ + _descriptor.MethodDescriptor( + name='Infer', + full_name='interface.Inference.Infer', + index=0, + containing_service=None, + input_type=_SLOTS, + output_type=_SLOTS, + serialized_options=None, ), + ]) +_sym_db.RegisterServiceDescriptor(_INFERENCE) + +DESCRIPTOR.services_by_name['Inference'] = _INFERENCE + +# @@protoc_insertion_point(module_scope) diff --git a/propeller/service/server.py b/propeller/service/server.py new file mode 100644 index 0000000..90ec0ee --- /dev/null +++ b/propeller/service/server.py @@ -0,0 +1,170 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import os +import logging +import six +from time import sleep, time +import multiprocessing + +import zmq +""" Never Never Never import paddle.fluid in main process, or any module would import fluid. +""" + +log = logging.getLogger(__name__) + + +def profile(msg): + def decfn(fn): + def retfn(*args, **kwargs): + start = time() + ret = fn(*args, **kwargs) + end = time() + log.debug('%s timecost: %.5f' % (msg, end - start)) + return ret + + return retfn + + return decfn + + +class Predictor(object): + def __init__(self, model_dir, device_idx=0): + import paddle.fluid as F + log.debug('create predictor on card %d' % device_idx) + config = F.core.AnalysisConfig(model_dir) + config.enable_use_gpu(5000, device_idx) + self._predictor = F.core.create_paddle_predictor(config) + + @profile('paddle') + def __call__(self, args): + for i, a in enumerate(args): + a.name = 'placeholder_%d' % i + res = self._predictor.run(args) + return res + + +def run_worker(model_dir, device_idx, endpoint="ipc://worker.ipc"): + try: + log.debug("run_worker %s" % device_idx) + os.environ["CUDA_VISIBLE_DEVICES"] = os.getenv( + "CUDA_VISIBLE_DEVICES").split(",")[device_idx] + log.debug('cuda_env %s' % os.environ["CUDA_VISIBLE_DEVICES"]) + import paddle.fluid as F + from propeller.service import interface_pb2 + import propeller.service.utils as serv_utils + context = zmq.Context() + socket = context.socket(zmq.REP) + socket.connect(endpoint) + #socket.bind(endpoint) + log.debug("Predictor building %s" % device_idx) + predictor = Predictor(model_dir, 0) + log.debug("Predictor %s" % device_idx) + except Exception as e: + log.exception(e) + + while True: + # Wait for next request from client + try: + message = socket.recv() + log.debug("get message %s" % device_idx) + slots = interface_pb2.Slots() + slots.ParseFromString(message) + pts = [serv_utils.slot_to_paddlearray(s) for s in slots.slots] + ret = predictor(pts) + slots = interface_pb2.Slots( + slots=[serv_utils.paddlearray_to_slot(r) for r in ret]) + socket.send(slots.SerializeToString()) + except Exception as e: + log.exception(e) + socket.send(e.message) + + +class InferencePredictor(object): + def __init__(self, backend_addr, model_dir, n_devices=1): + self.backend_addr = backend_addr + self.model_dir = model_dir + self.n_devices = n_devices + self.children = [] + + def start(self): + for device_idx in range(self.n_devices): + p = multiprocessing.Process( + target=run_worker, + args=(self.model_dir, device_idx, self.backend_addr)) + p.start() + self.children.append(p) + return self + + def join(self): + for p in self.children: + p.join() + + def term(self): + for p in self.children: + log.debug("terminating children %s" % repr(p)) + p.terminate() + + +class InferenceProxy(object): + def __init__(self): + self.backend = None + self.frontend = None + + def listen(self, frontend_addr, backend_addr): + log.info("InferenceProxy starting...") + try: + context = zmq.Context(1) + # Socket facing clients + self.frontend = context.socket(zmq.ROUTER) + self.frontend.bind(frontend_addr) + # Socket facing services + self.backend = context.socket(zmq.DEALER) + self.backend.bind(backend_addr) + log.info("Queue init done") + zmq.device(zmq.QUEUE, self.frontend, self.backend) + except Exception as e: + log.exception(e) + log.info("Bringing down zmq device") + finally: + log.debug('terminating proxy') + if self.frontend is not None: + self.frontend.close() + if self.backend is not None: + self.backend.close() + context.term() + + +class InferenceServer(object): + def __init__(self, model_dir, n_devices): + self.model_dir = model_dir + self.n_devices = n_devices + + def listen(self, port): + frontend_addr = "tcp://*:%s" % port + backend_addr = "ipc://backend.ipc" + predictor = InferencePredictor(backend_addr, self.model_dir, + self.n_devices).start() + try: + proxy = InferenceProxy() + proxy.listen(frontend_addr, backend_addr) + predictor.join() + except KeyboardInterrupt: + log.debug('terminating server') + predictor.term() diff --git a/propeller/service/utils.py b/propeller/service/utils.py new file mode 100644 index 0000000..0acb2c8 --- /dev/null +++ b/propeller/service/utils.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import numpy as np +import struct + +from propeller.service import interface_pb2 + + +def slot_to_numpy(slot): + if slot.type == interface_pb2.Slot.FP32: + dtype = np.float32 + type_str = 'f' + elif slot.type == interface_pb2.Slot.INT32: + type_str = 'i' + dtype = np.int32 + elif slot.type == interface_pb2.Slot.INT64: + dtype = np.int64 + type_str = 'q' + else: + raise RuntimeError('know type %s' % slot.type) + num = len(slot.data) // struct.calcsize(type_str) + arr = struct.unpack('%d%s' % (num, type_str), slot.data) + shape = slot.dims + ret = np.array(arr, dtype=dtype).reshape(shape) + return ret + + +def numpy_to_slot(arr): + if arr.dtype == np.float32: + dtype = interface_pb2.Slot.FP32 + elif arr.dtype == np.int32: + dtype = interface_pb2.Slot.INT32 + elif arr.dtype == np.int64: + dtype = interface_pb2.Slot.INT64 + else: + raise RuntimeError('know type %s' % arr.dtype) + pb = interface_pb2.Slot( + type=dtype, dims=list(arr.shape), data=arr.tobytes()) + return pb + + +def slot_to_paddlearray(slot): + import paddle.fluid.core as core + if slot.type == interface_pb2.Slot.FP32: + type_str = 'f' + dtype = core.PaddleDType.FLOAT32 + elif slot.type == interface_pb2.Slot.INT32: + type_str = 'i' + dtype = core.PaddleDType.INT32 + elif slot.type == interface_pb2.Slot.INT64: + type_str = 'q' + dtype = core.PaddleDType.INT64 + else: + raise RuntimeError('know type %s' % slot.type) + ret = core.PaddleTensor() + ret.shape = slot.dims + ret.dtype = dtype + num = len(slot.data) // struct.calcsize(type_str) + arr = struct.unpack('%d%s' % (num, type_str), slot.data) + ret.data = core.PaddleBuf(arr) + return ret + + +def paddlearray_to_slot(arr): + import paddle.fluid.core as core + if arr.dtype == core.PaddleDType.FLOAT32: + dtype = interface_pb2.Slot.FP32 + type_str = 'f' + arr_data = arr.data.float_data() + elif arr.dtype == core.PaddleDType.INT32: + dtype = interface_pb2.Slot.INT32 + type_str = 'i' + arr_data = arr.data.int32_data() + elif arr.dtype == core.PaddleDType.INT64: + dtype = interface_pb2.Slot.INT64 + type_str = 'q' + arr_data = arr.data.int64_data() + else: + raise RuntimeError('know type %s' % arr.dtype) + data = struct.pack('%d%s' % (len(arr_data), type_str), *arr_data) + pb = interface_pb2.Slot(type=dtype, dims=list(arr.shape), data=data) + return pb + + +def nparray_list_serialize(arr_list): + slot_list = [numpy_to_slot(arr) for arr in arr_list] + slots = interface_pb2.Slots(slots=slot_list) + return slots.SerializeToString() + + +def nparray_list_deserialize(string): + slots = interface_pb2.Slots() + slots.ParseFromString(string) + return [slot_to_numpy(slot) for slot in slots.slots] diff --git a/propeller/tools/__init__.py b/propeller/tools/__init__.py new file mode 100644 index 0000000..d0c32e2 --- /dev/null +++ b/propeller/tools/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/propeller/tools/ckpt_inspector.py b/propeller/tools/ckpt_inspector.py new file mode 100644 index 0000000..3568ada --- /dev/null +++ b/propeller/tools/ckpt_inspector.py @@ -0,0 +1,123 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import os +import struct +import logging +import argparse +import numpy as np +import collections +from distutils import dir_util +import pickle + +#from utils import print_arguments +import paddle.fluid as F +from paddle.fluid.proto import framework_pb2 + +log = logging.getLogger(__name__) +formatter = logging.Formatter( + fmt='[%(levelname)s] %(asctime)s [%(filename)12s:%(lineno)5d]:\t%(message)s' +) +console = logging.StreamHandler() +console.setFormatter(formatter) +log.addHandler(console) +log.setLevel(logging.DEBUG) + + +def gen_arr(data, dtype): + num = len(data) // struct.calcsize(dtype) + arr = struct.unpack('%d%s' % (num, dtype), data) + return arr + + +def parse(filename): + with open(filename, 'rb') as f: + read = lambda fmt: struct.unpack(fmt, f.read(struct.calcsize(fmt))) + _, = read('I') # version + lodsize, = read('Q') + if lodsize != 0: + log.warning('shit, it is LOD tensor!!! skipped!!') + return None + _, = read('I') # version + pbsize, = read('i') + data = f.read(pbsize) + proto = framework_pb2.VarType.TensorDesc() + proto.ParseFromString(data) + log.info('type: [%s] dim %s' % (proto.data_type, proto.dims)) + if proto.data_type == framework_pb2.VarType.FP32: + arr = np.array( + gen_arr(f.read(), 'f'), dtype=np.float32).reshape(proto.dims) + elif proto.data_type == framework_pb2.VarType.INT64: + arr = np.array( + gen_arr(f.read(), 'q'), dtype=np.int64).reshape(proto.dims) + elif proto.data_type == framework_pb2.VarType.INT32: + arr = np.array( + gen_arr(f.read(), 'i'), dtype=np.int32).reshape(proto.dims) + elif proto.data_type == framework_pb2.VarType.INT8: + arr = np.array( + gen_arr(f.read(), 'B'), dtype=np.int8).reshape(proto.dims) + else: + raise RuntimeError('Unknown dtype %s' % proto.data_type) + + return arr + + +def show(arr): + print(repr(arr)) + + +def dump(arr, path): + path = os.path.join(args.to, path) + log.info('dump to %s' % path) + try: + os.makedirs(os.path.dirname(path)) + except FileExistsError: + pass + pickle.dump(arr, open(path, 'wb'), protocol=4) + + +def list_dir(dir_or_file): + if os.path.isfile(dir_or_file): + return [dir_or_file] + else: + return [ + os.path.join(i, kk) for i, _, k in os.walk(dir_or_file) for kk in k + ] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=['show', 'dump'], type=str) + parser.add_argument('file_or_dir', type=str) + parser.add_argument('-t', "--to", type=str, default=None) + parser.add_argument('-v', "--verbose", action='store_true') + args = parser.parse_args() + + files = list_dir(args.file_or_dir) + parsed_arr = map(parse, files) + if args.mode == 'show': + for arr in parsed_arr: + if arr is not None: + show(arr) + elif args.mode == 'dump': + if args.to is None: + raise ValueError('--to dir_name not specified') + for arr, path in zip(parsed_arr, files): + if arr is not None: + dump(arr, path.replace(args.file_or_dir, '')) diff --git a/propeller/tools/start_server.py b/propeller/tools/start_server.py new file mode 100644 index 0000000..23d1cac --- /dev/null +++ b/propeller/tools/start_server.py @@ -0,0 +1,39 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import sys +import os +import argparse +import logging +import logging.handlers +from propeller.service.server import InferenceServer +from propeller import log + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--model_dir', type=str, required=True) + parser.add_argument('-p', '--port', type=int, required=True) + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + + if args.verbose: + log.setLevel(logging.DEBUG) + n_devices = len(os.getenv("CUDA_VISIBLE_DEVICES").split(",")) + server = InferenceServer(args.model_dir, n_devices) + log.info('propeller server listent on port %d' % args.port) + server.listen(args.port) diff --git a/propeller/train/__init__.py b/propeller/train/__init__.py new file mode 100644 index 0000000..d0c32e2 --- /dev/null +++ b/propeller/train/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/propeller/train/model.py b/propeller/train/model.py new file mode 100644 index 0000000..0aca85b --- /dev/null +++ b/propeller/train/model.py @@ -0,0 +1,82 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys +import six +import logging +import os +import itertools +import json +import abc +import numpy as np + + +@six.add_metaclass(abc.ABCMeta) +class Model(): + def __init__(self, config, mode): + """ + Args: + config (dict): hyper param + mode (propeller.RunMode): will creat `TRAIN` and `EVAL` model in propeller.train_and_eval + """ + self.mode = mode + + @abc.abstractmethod + def forward(self, features): + """ + Args: + features (list of Tensor): depends on your Dataset.output_shapes + Returns: + return (Tensor): + """ + pass + + @abc.abstractmethod + def loss(self, predictions, label): + """ + Args: + predictions (Tensor): result of `self.forward` + label (Tensor): depends on your Dataset.output_shapes + Returns: + return (paddle scalar): loss + + + """ + pass + + @abc.abstractmethod + def backward(self, loss): + """ + Call in TRAIN mode + Args: + loss (Tensor): result of `self.loss` + Returns: + None + """ + pass + + @abc.abstractmethod + def metrics(self, predictions, label): + """ + Call in EVAL mode + Args: + predictions (Tensor): result of `self.forward` + label (Tensor): depends on your Dataset.output_shapes + Returns: + (dict): k-v map like: {"metrics_name": propeller.Metrics } + """ + return {} diff --git a/propeller/types.py b/propeller/types.py new file mode 100644 index 0000000..830ce16 --- /dev/null +++ b/propeller/types.py @@ -0,0 +1,103 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import json +from collections import namedtuple + + +class RunMode(object): + TRAIN = 1 + PREDICT = 2 + EVAL = 3 + + +class HParams(object): + def __init__(self, **kwargs): + for k, v in kwargs.items(): + self.__dict__[k] = v + + def __contains__(self, key): + return key in self.__dict__ + + def __getitem__(self, key): + if key not in self.__dict__: + raise ValueError('key(%s) not in HParams.' % key) + return self.__dict__[key] + + def __repr__(self): + return repr(self.to_dict()) + + def __setitem__(self, key, val): + self.__dict__[key] = val + + @staticmethod + def from_json(self, json_str): + d = json.loads(json_str) + if type(d) != dict: + raise ValueError('json object must be dict.') + return HParams.from_dict(d) + + def get(self, key, default=None): + return self.__dict__.get(key, default) + + @staticmethod + def from_dict(self, d): + if type(d) != dict: + raise ValueError('input must be dict.') + hp = HParams(**d) + return hp + + def to_json(self): + return json.dumps(self.__dict__) + + def to_dict(self): + return self.__dict__ + + def join(self, other): + if not isinstance(other, HParams): + raise ValueError('input must be HParams instance.') + self.__dict__.update(**other.__dict__) + return self + + +SummaryRecord = namedtuple('SummaryRecord', ['scalar', 'histogram']) + +WarmStartSetting = namedtuple('WarmStartSetting', ['predicate_fn', 'from_dir']) + +RunConfig = namedtuple('RunConfig', [ + 'model_dir', 'run_steps', 'max_steps', 'save_steps', 'eval_steps', + 'eval_max_steps', 'skip_steps', 'log_steps', 'max_ckpt', 'shit' +]) +RunConfig.__new__.__defaults__ = (None, ) * len(RunConfig._fields) + +ProgramPair = namedtuple('ProgramPair', ['train_program', 'startup_program']) + +InferenceSpec = namedtuple('InferenceSpec', ['inputs', 'outputs']) + +ModelSpec = namedtuple('ModelSpec', [ + 'loss', + 'predictions', + 'metrics', + 'mode', + 'inference_spec', +]) +ModelSpec.__new__.__defaults__ = (None, ) * len(ModelSpec._fields) + + +class StopException(Exception): + pass diff --git a/propeller/util.py b/propeller/util.py new file mode 100644 index 0000000..b73581d --- /dev/null +++ b/propeller/util.py @@ -0,0 +1,119 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import six +import re +import json +import argparse +import itertools +import logging +from functools import reduce + +from propeller.types import RunConfig +from propeller.types import HParams + +log = logging.getLogger(__name__) + + +def ArgumentParser(name): + parser = argparse.ArgumentParser('propeller model') + parser.add_argument('--run_config', type=str, default='') + parser.add_argument( + '--hparam', type=str, nargs='*', action='append', default=[['']]) + return parser + + +def _get_dict_from_environ_or_json_or_file(args, env_name): + if args == '': + return None + if args is None: + s = os.environ.get(env_name) + else: + s = args + if os.path.exists(s): + s = open(s).read() + if isinstance(s, six.string_types): + try: + r = eval(s) + except SyntaxError as e: + raise ValueError('json parse error: %s \n>Got json: %s' % + (repr(e), s)) + return r + else: + return s #None + + +def parse_file(filename): + d = _get_dict_from_environ_or_json_or_file(filename, None) + if d is None: + raise ValueError('file(%s) not found' % filename) + return d + + +def parse_runconfig(args=None): + d = _get_dict_from_environ_or_json_or_file(args.run_config, + 'PROPELLER_RUNCONFIG') + if d is None: + raise ValueError('run_config not found') + return RunConfig(**d) + + +def parse_hparam(args=None): + if args is not None: + hparam_strs = reduce(list.__add__, args.hparam) + else: + hparam_strs = [None] + + hparams = [ + _get_dict_from_environ_or_json_or_file(hp, 'PROPELLER_HPARAMS') + for hp in hparam_strs + ] + hparams = [HParams(**h) for h in hparams if h is not None] + if len(hparams) is None: + raise ValueError('hparam not found') + hparam = reduce(lambda x, y: x.join(y), hparams) + return hparam + + +def flatten(s): + assert is_struture(s) + schema = [len(ss) for ss in s] + flt = list(itertools.chain(*s)) + return flt, schema + + +def unflatten(structure, schema): + start = 0 + res = [] + for _range in schema: + res.append(structure[start:start + _range]) + start += _range + return res + + +def is_struture(s): + return isinstance(s, list) or isinstance(s, tuple) + + +def map_structure(func, s): + if isinstance(s, list) or isinstance(s, tuple): + return [map_structure(func, ss) for ss in s] + elif isinstance(s, dict): + return {k: map_structure(func, v) for k, v in six.iteritems(s)} + else: + return func(s) diff --git a/propeller_xnli_demo.ipynb b/propeller_xnli_demo.ipynb new file mode 100644 index 0000000..9f4cab2 --- /dev/null +++ b/propeller_xnli_demo.ipynb @@ -0,0 +1,10036 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "import numpy as np\n", + "import re\n", + "import logging\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: CUDA_VICIBLE_DEVICES=0\n" + ] + } + ], + "source": [ + "sys.path.append('./')\n", + "%env CUDA_VICIBLE_DEVICES=0\n", + "# if CUDA_VICIBLE_DEVICES is changed, relaunch jupyter kernel to inform paddle" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import propeller.paddle as propeller\n", + "import paddle\n", + "import paddle.fluid as F\n", + "import paddle.fluid.layers as L\n", + "#import model defenition from original ERNIE\n", + "from model.ernie import ErnieModel\n", + "from tokenization import FullTokenizer\n", + "from optimization import optimization\n", + "from propeller import log\n", + "log.setLevel(logging.DEBUG)\n", + "\n", + "if paddle.__version__ not in ['1.5.1', '1.5.2']:\n", + " raise RuntimeError('propeller works in paddle1.5.1')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: cannot create directory ‘ernie1.0_pretrained’: File exists\n" + ] + } + ], + "source": [ + "%%bash\n", + "# download pretrained model&config(ernie1.0) and xnli data\n", + "mkdir ernie1.0_pretrained\n", + "if [ ! -f ernie1.0_pretrained/ERNIE_stable-1.0.1.tar.gz ]\n", + "then\n", + " echo \"download model\"\n", + " wget --no-check-certificate https://baidu-nlp.bj.bcebos.com/ERNIE_stable-1.0.1.tar.gz -P ernie1.0_pretrained\n", + "fi\n", + "\n", + "if [ ! -f task_data_zh.tgz ]\n", + "then\n", + " echo \"download data\"\n", + " wget --no-check-certificate https://ernie.bj.bcebos.com/task_data_zh.tgz\n", + "fi\n", + "\n", + "tar xzf ernie1.0_pretrained/ERNIE_stable-1.0.1.tar.gz -C ernie1.0_pretrained\n", + "tar xzf task_data_zh.tgz" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "#define basic training settings\n", + "EPOCH=3\n", + "BATCH=16\n", + "LR=5e-3\n", + "MAX_SEQLEN=128\n", + "TASK_DATA='./task_data/'\n", + "MODEL='./ernie1.0_pretrained/'\n", + "OUTPUT_DIR='./output'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "!rm -rf {OUTPUT_DIR}" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mkdir: cannot create directory ‘xnli_data’: File exists\n", + "mkdir: cannot create directory ‘xnli_data/train’: File exists\n", + "mkdir: cannot create directory ‘xnli_data/test’: File exists\n", + "mkdir: cannot create directory ‘xnli_data/dev’: File exists\n", + "392702\n", + "2490\n", + "5010\n" + ] + } + ], + "source": [ + "#skip header, and reorganize train data into ./xnli_data \n", + "!mkdir xnli_data\n", + "!mkdir xnli_data/train\n", + "!mkdir xnli_data/test\n", + "!mkdir xnli_data/dev\n", + "\n", + "def remove_header_and_save(fname_in, fname_out):\n", + " with open(fname_out, 'w') as fout:\n", + " buf = open(fname_in).readlines()[1:]\n", + " for i in buf:\n", + " fout.write(i)\n", + " return len(buf)\n", + "train_data_size = remove_header_and_save(TASK_DATA + '/xnli/train.tsv', './xnli_data/train/part.0') \n", + "dev_data_size = remove_header_and_save(TASK_DATA + '/xnli/dev.tsv', './xnli_data/dev/part.0') \n", + "test_data_size = remove_header_and_save(TASK_DATA + '/xnli/test.tsv', './xnli_data/test/part.0') \n", + "print(train_data_size)\n", + "print(dev_data_size)\n", + "print(test_data_size)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['今', '天', '很', '热']\n", + "['cod', '##ing', 'in', 'pad', '##d', '##le', 'is', 'cool']\n", + "['[', 'cl', '##s', '[UNK]', 'i', 'have', 'an', 'pen']\n" + ] + } + ], + "source": [ + "tokenizer = FullTokenizer(MODEL + 'vocab.txt')\n", + "vocab = {j.strip().split('\\t')[0]: i for i, j in enumerate(open(MODEL + 'vocab.txt', encoding='utf8'))}\n", + "\n", + "print(tokenizer.tokenize('今天很热'))\n", + "print(tokenizer.tokenize('coding in paddle is cool'))\n", + "print(tokenizer.tokenize('[CLS]i have an pen')) # note: special token like [CLS], will be segmented, so please add these id after tokenization.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`propeller.data.FeatureColumns` defines the data schema in every data file.\n", + "\n", + "our data consist of 3 columns: seg_a, seg_b, label. with \"\\t\" as delemeter.\n", + "\n", + "`TextColumn` will do 3 things for you: \n", + "\n", + "1. tokenize input sentence with user-defined `tokenizer_func`\n", + "2. vocab lookup\n", + "3. serialize to protobuf bin file (optional)\n", + "\n", + "data file is organized into following patten:\n", + "\n", + "```script\n", + "./xnli_data\n", + "|-- dev\n", + "| `-- part.0\n", + "|-- test\n", + "| `-- part.0\n", + "|-- train\n", + " `-- part.0\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "sep_id = vocab['[SEP]']\n", + "cls_id = vocab['[CLS]']\n", + "unk_id = vocab['[UNK]']\n", + "\n", + "label_map = {\n", + " b\"contradictory\": 0,\n", + " b\"contradiction\": 0,\n", + " b\"entailment\": 1,\n", + " b\"neutral\": 2,\n", + "}\n", + "def tokenizer_func(inputs):\n", + " ret = tokenizer.tokenize(inputs) #`tokenize` will conver bytes to str, so we use a str vocab\n", + " return ret\n", + "\n", + "feature_column = propeller.data.FeatureColumns([\n", + " propeller.data.TextColumn('title', unk_id=unk_id, vocab_dict=vocab, tokenizer=tokenizer_func),\n", + " propeller.data.TextColumn('comment', unk_id=unk_id, vocab_dict=vocab, tokenizer=tokenizer_func),\n", + " propeller.data.LabelColumn('label', vocab_dict=label_map), #be careful, Columns deal with python3 bytes directly.\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## trian model in propeller can be defined in 2 ways:\n", + "1. subclass of `propeller.train.Model` which implements:\n", + " 1. `__init__` (hyper_param, mode, run_config)\n", + " 2. `forward` (features) => (prediction)\n", + " 3. `backword` (loss) => None\n", + " 4. `loss` (predictoin) => (loss)\n", + " 5. `metrics` (optional) (prediction) => (dict of propeller.Metrics)\n", + " \n", + "2. a callable takes following args:\n", + " 1. features\n", + " 2. param\n", + " 3. mode\n", + " 4. run_config(optional)\n", + " \n", + " and returns a propeller.ModelSpec\n", + " \n", + "we use the subclasss approch here" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "class ClassificationErnieModel(propeller.train.Model):\n", + " def __init__(self, hparam, mode, run_config):\n", + " self.hparam = hparam\n", + " self.mode = mode\n", + " self.run_config = run_config\n", + "\n", + " def forward(self, features):\n", + " src_ids, sent_ids = features\n", + " dtype = 'float16' if self.hparam['fp16'] else 'float32'\n", + " zero = L.fill_constant([1], dtype='int64', value=0)\n", + " input_mask = L.cast(L.equal(src_ids, zero), dtype) # assume pad id == 0\n", + " #input_mask = L.unsqueeze(input_mask, axes=[2])\n", + " d_shape = L.shape(src_ids)\n", + " seqlen = d_shape[1]\n", + " batch_size = d_shape[0]\n", + " pos_ids = L.unsqueeze(L.range(0, seqlen, 1, dtype='int32'), axes=[0])\n", + " pos_ids = L.expand(pos_ids, [batch_size, 1])\n", + " pos_ids = L.unsqueeze(pos_ids, axes=[2])\n", + " pos_ids = L.cast(pos_ids, 'int64')\n", + " pos_ids.stop_gradient = True\n", + " input_mask.stop_gradient = True\n", + " task_ids = L.zeros_like(src_ids) + self.hparam.task_id #this shit wont use at the moment\n", + " task_ids.stop_gradient = True\n", + "\n", + " ernie = ErnieModel(\n", + " src_ids=src_ids,\n", + " position_ids=pos_ids,\n", + " sentence_ids=sent_ids,\n", + " task_ids=task_ids,\n", + " input_mask=input_mask,\n", + " config=self.hparam,\n", + " use_fp16=self.hparam['fp16']\n", + " )\n", + "\n", + " cls_feats = ernie.get_pooled_output()\n", + "\n", + " cls_feats = L.dropout(\n", + " x=cls_feats,\n", + " dropout_prob=0.1,\n", + " dropout_implementation=\"upscale_in_train\"\n", + " )\n", + "\n", + " logits = L.fc(\n", + " input=cls_feats,\n", + " size=self.hparam['num_label'],\n", + " param_attr=F.ParamAttr(\n", + " name=\"cls_out_w\",\n", + " initializer=F.initializer.TruncatedNormal(scale=0.02)),\n", + " bias_attr=F.ParamAttr(\n", + " name=\"cls_out_b\", initializer=F.initializer.Constant(0.))\n", + " )\n", + "\n", + " propeller.summary.histogram('pred', logits)\n", + "\n", + " if self.mode is propeller.RunMode.PREDICT:\n", + " probs = L.softmax(logits)\n", + " return probs\n", + " else:\n", + " return logits\n", + "\n", + " def loss(self, predictions, labels):\n", + " ce_loss, probs = L.softmax_with_cross_entropy(\n", + " logits=predictions, label=labels, return_softmax=True)\n", + " #L.Print(ce_loss, message='per_example_loss')\n", + " loss = L.mean(x=ce_loss)\n", + " return loss\n", + "\n", + " def backward(self, loss):\n", + " scheduled_lr, loss_scale = optimization(\n", + " loss=loss,\n", + " warmup_steps=int(self.run_config.max_steps * self.hparam['warmup_proportion']),\n", + " num_train_steps=self.run_config.max_steps,\n", + " learning_rate=self.hparam['learning_rate'],\n", + " train_program=F.default_main_program(),\n", + " startup_prog=F.default_startup_program(),\n", + " weight_decay=self.hparam['weight_decay'],\n", + " scheduler=\"linear_warmup_decay\",)\n", + " propeller.summary.scalar('lr', scheduled_lr)\n", + "\n", + " def metrics(self, predictions, label):\n", + " predictions = L.argmax(predictions, axis=1)\n", + " predictions = L.unsqueeze(predictions, axes=[1])\n", + " acc = propeller.metrics.Acc(label, predictions)\n", + " #auc = propeller.metrics.Auc(label, predictions)\n", + " return {'acc': acc}\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# define some utility function.\n", + "\n", + "def build_2_pair(seg_a, seg_b):\n", + " token_type_a = np.ones_like(seg_a, dtype=np.int64) * 0\n", + " token_type_b = np.ones_like(seg_b, dtype=np.int64) * 1\n", + " sen_emb = np.concatenate([[cls_id], seg_a, [sep_id], seg_b, [sep_id]], 0)\n", + " token_type_emb = np.concatenate([[0], token_type_a, [0], token_type_b, [1]], 0)\n", + " #seqlen = sen_emb.shape[0]\n", + " #deteministic truncate\n", + " sen_emb = sen_emb[0: MAX_SEQLEN]\n", + " token_type_emb = token_type_emb[0: MAX_SEQLEN]\n", + " return sen_emb, token_type_emb\n", + "\n", + "def expand_dims(*args):\n", + " func = lambda i: np.expand_dims(i, -1)\n", + " ret = [func(i) for i in args]\n", + " return ret\n", + "\n", + "def before_pad(seg_a, seg_b, label):\n", + " sentence, segments = build_2_pair(seg_a, seg_b)\n", + " return sentence, segments, label\n", + "\n", + "def after_pad(sentence, segments, label):\n", + " sentence, segments, label = expand_dims(sentence, segments, label)\n", + " return sentence, segments, label" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-18 16:26:51,230 [feature_column.py: 266]:\treading raw files from ./xnli_data/train/part.0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:51,232 [feature_column.py: 266]:\treading raw files from ./xnli_data/dev/part.0\u001b[0m\n" + ] + } + ], + "source": [ + "# a `propeller.paddle.data.Dataset` is built from FeatureColumns\n", + "\n", + "train_ds = feature_column.build_dataset('train', use_gz=False, data_dir='./xnli_data/train', shuffle=True, repeat=True) \\\n", + " .map(before_pad) \\\n", + " .padded_batch(BATCH, (0, 0, 0)) \\\n", + " .map(after_pad)\n", + "\n", + "dev_ds = feature_column.build_dataset('dev', use_gz=False, data_dir='./xnli_data/dev', shuffle=False, repeat=False) \\\n", + " .map(before_pad) \\\n", + " .padded_batch(BATCH, (0, 0, 0)) \\\n", + " .map(after_pad)\n", + "\n", + "shapes = ([-1, MAX_SEQLEN, 1], [-1, MAX_SEQLEN, 1], [-1, 1])\n", + "types = ('int64', 'int64', 'int64')\n", + "train_ds.data_shapes = shapes\n", + "train_ds.data_types = types\n", + "dev_ds.data_shapes = shapes\n", + "dev_ds.data_types = types\n", + "\n", + "warm_start_dir = MODEL + '/params'\n", + "# only the encoder and embedding is loaded from pretrained model\n", + "varname_to_warmstart = re.compile('^encoder.*w_0$|^encoder.*b_0$|^.*embedding$|^.*bias$|^.*scale$')\n", + "ws = propeller.WarmStartSetting(\n", + " predicate_fn=lambda v: varname_to_warmstart.match(v.name) and os.path.exists(os.path.join(warm_start_dir, v.name)),\n", + " from_dir=warm_start_dir\n", + " )\n", + "\n", + "# propeller will export model of highest performance, the criteria is up to you. \n", + "# here we pick the model with maximum evaluatoin accuracy.\n", + "#`BestInferenceModelExporter` is used to export serveable models\n", + "best_inference_exporter = propeller.train.exporter.BestInferenceModelExporter(\n", + " os.path.join(OUTPUT_DIR, 'best'), \n", + " cmp_fn=lambda old, new: new['eval']['acc'] > old['eval']['acc'])\n", + "#`BestExporter` is used to export restartable checkpoint, so that we can restore from it and check test-set accuracy.\n", + "best_exporter = propeller.train.exporter.BestExporter(\n", + " os.path.join(OUTPUT_DIR, 'best_model'), \n", + " cmp_fn=lambda old, new: new['eval']['acc'] > old['eval']['acc'])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "#ERNIE1.0 config \n", + "ernie_config = propeller.HParams(**json.loads(open(MODEL + '/ernie_config.json').read()))\n", + "\n", + "# default term in official config\n", + "ernie_v2_config = propeller.HParams(**{\n", + " \"sent_type_vocab_size\": None, \n", + " \"use_task_id\": False,\n", + " \"task_id\": 0,\n", + "})\n", + "\n", + "# train schema\n", + "train_config = propeller.HParams(**{ \n", + " \"warmup_proportion\": 0.1,\n", + " \"weight_decay\": 0.01,\n", + " \"fp16\": 0,\n", + " \"learning_rate\": 0.00005,\n", + " \"num_label\": 3,\n", + " \"batch_size\": 32\n", + "})\n", + "\n", + "config = ernie_config.join(ernie_v2_config).join(train_config)\n", + "\n", + "run_config = propeller.RunConfig(\n", + " model_dir=OUTPUT_DIR,\n", + " max_steps=EPOCH * train_data_size / BATCH,\n", + " skip_steps=10,\n", + " eval_steps=1000,\n", + " save_steps=1000,\n", + " log_steps=10,\n", + " max_ckpt=3\n", + ")\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Finetune and Eval" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ab9693a920d448fcad8a104c193316b5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=73631), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-18 16:26:51,288 [ trainer.py: 218]:\tBuilding Eval Graph\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:51,889 [ trainer.py: 222]:\tDone\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:51,925 [ trainer.py: 226]:\tEval with: \n", + "> Run_config: RunConfig(model_dir='./output', run_steps=None, max_steps=73631.625, save_steps=1000, eval_steps=1000, eval_max_steps=None, skip_steps=10, log_steps=10, max_ckpt=3, shit=None)\n", + "> Params: {'attention_probs_dropout_prob': 0.1, 'hidden_act': 'relu', 'hidden_dropout_prob': 0.1, 'hidden_size': 768, 'initializer_range': 0.02, 'max_position_embeddings': 513, 'num_attention_heads': 12, 'num_hidden_layers': 12, 'type_vocab_size': 2, 'vocab_size': 18000, 'sent_type_vocab_size': None, 'use_task_id': False, 'task_id': 0, 'warmup_proportion': 0.1, 'weight_decay': 0.01, 'fp16': 0, 'learning_rate': 5e-05, 'num_label': 3, 'batch_size': 32}\n", + "> Train_model_spec: ModelSpec(loss=name: \"mean_0.tmp_0\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: FP32\n", + " dims: 1\n", + " }\n", + " }\n", + "}\n", + "persistable: false\n", + ", predictions=[name: \"fc_73.tmp_1\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: FP32\n", + " dims: -1\n", + " dims: 3\n", + " }\n", + " lod_level: 0\n", + " }\n", + "}\n", + "persistable: true\n", + "], metrics={'acc': , 'loss': }, mode=3, inference_spec=InferenceSpec(inputs=[name: \"eval_placeholder_0\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: INT64\n", + " dims: -1\n", + " dims: 128\n", + " dims: 1\n", + " }\n", + " lod_level: 0\n", + " }\n", + "}\n", + "persistable: false\n", + ", name: \"eval_placeholder_1\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: INT64\n", + " dims: -1\n", + " dims: 128\n", + " dims: 1\n", + " }\n", + " lod_level: 0\n", + " }\n", + "}\n", + "persistable: false\n", + "], outputs=[name: \"fc_73.tmp_1\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: FP32\n", + " dims: -1\n", + " dims: 3\n", + " }\n", + " lod_level: 0\n", + " }\n", + "}\n", + "persistable: true\n", + "]))\n", + "\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:51,943 [ trainer.py: 182]:\tBuilding Train Graph...\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:55,293 [ trainer.py: 186]:\tBuilding Train Graph: Done\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:55,294 [ trainer.py: 202]:\tTrain with: \n", + "> Run_config: RunConfig(model_dir='./output', run_steps=None, max_steps=73631.625, save_steps=1000, eval_steps=1000, eval_max_steps=None, skip_steps=10, log_steps=10, max_ckpt=3, shit=None)\n", + "> Params: {'attention_probs_dropout_prob': 0.1, 'hidden_act': 'relu', 'hidden_dropout_prob': 0.1, 'hidden_size': 768, 'initializer_range': 0.02, 'max_position_embeddings': 513, 'num_attention_heads': 12, 'num_hidden_layers': 12, 'type_vocab_size': 2, 'vocab_size': 18000, 'sent_type_vocab_size': None, 'use_task_id': False, 'task_id': 0, 'warmup_proportion': 0.1, 'weight_decay': 0.01, 'fp16': 0, 'learning_rate': 5e-05, 'num_label': 3, 'batch_size': 32}\n", + "> Train_model_spec: ModelSpec(loss=name: \"mean_0.tmp_0\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: FP32\n", + " dims: 1\n", + " }\n", + " }\n", + "}\n", + "persistable: true\n", + ", predictions=[name: \"fc_73.tmp_1\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: FP32\n", + " dims: -1\n", + " dims: 3\n", + " }\n", + " lod_level: 0\n", + " }\n", + "}\n", + "persistable: true\n", + "], metrics=None, mode=1, inference_spec=None)\n", + "\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:55,297 [distribution.py: 127]:\tInitializing local training\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,305 [monitored_executor.py: 224]:\twarm start from ./ernie1.0_pretrained//params\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,307 [monitored_executor.py: 230]:\twarm start: word_embedding\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,308 [monitored_executor.py: 230]:\twarm start: pos_embedding\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,308 [monitored_executor.py: 230]:\twarm start: sent_embedding\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,308 [monitored_executor.py: 230]:\twarm start: pre_encoder_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,308 [monitored_executor.py: 230]:\twarm start: pre_encoder_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,309 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,309 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,309 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,309 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,309 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,310 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,310 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,310 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,310 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,311 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,311 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,311 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,311 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,312 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,312 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,312 [monitored_executor.py: 230]:\twarm start: encoder_layer_0_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,312 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,312 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,313 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,313 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,313 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,313 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,314 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,314 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,314 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,314 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,315 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,315 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,315 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,315 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,316 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,316 [monitored_executor.py: 230]:\twarm start: encoder_layer_1_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,316 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,316 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,316 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,317 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,317 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,317 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,317 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,318 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,318 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,318 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,318 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,319 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,319 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,319 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,319 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,319 [monitored_executor.py: 230]:\twarm start: encoder_layer_2_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,320 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,320 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,320 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,320 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,320 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,321 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,321 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,321 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,321 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,322 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,322 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,322 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,322 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,323 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,323 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,323 [monitored_executor.py: 230]:\twarm start: encoder_layer_3_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,323 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,323 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,324 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,324 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,324 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,324 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,325 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,325 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,325 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,325 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,326 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,326 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,326 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,326 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,327 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,327 [monitored_executor.py: 230]:\twarm start: encoder_layer_4_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,327 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,327 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,327 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,328 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,328 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,328 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,328 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,329 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,329 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,329 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,329 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,329 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,330 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,330 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,330 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,330 [monitored_executor.py: 230]:\twarm start: encoder_layer_5_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,331 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,331 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,331 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,331 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,331 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,332 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,332 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,332 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,332 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,333 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,333 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,333 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,333 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,334 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,334 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,334 [monitored_executor.py: 230]:\twarm start: encoder_layer_6_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,334 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,334 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,335 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,335 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,335 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,336 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,336 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,337 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,337 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,337 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,337 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,337 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,338 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,338 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,338 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,338 [monitored_executor.py: 230]:\twarm start: encoder_layer_7_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,339 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,339 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,339 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,339 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,339 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,340 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,340 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,340 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,340 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,341 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,341 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,341 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,341 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,342 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,342 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,342 [monitored_executor.py: 230]:\twarm start: encoder_layer_8_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,342 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,342 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,343 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,343 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,343 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,343 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,344 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,344 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,344 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,344 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,345 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,345 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,345 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,345 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,345 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,346 [monitored_executor.py: 230]:\twarm start: encoder_layer_9_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,346 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,346 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,346 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,347 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,347 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,347 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,347 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,348 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,348 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,348 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,348 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,348 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,349 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,349 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,349 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,349 [monitored_executor.py: 230]:\twarm start: encoder_layer_10_post_ffn_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,350 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_query_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,350 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_query_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,350 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_key_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,350 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_key_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,350 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_value_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,351 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_value_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,351 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_output_fc.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,351 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_multi_head_att_output_fc.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,351 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_post_att_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,352 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_post_att_layer_norm_bias\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,352 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_ffn_fc_0.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,352 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_ffn_fc_0.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,352 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_ffn_fc_1.w_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,352 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_ffn_fc_1.b_0\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,353 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_post_ffn_layer_norm_scale\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,353 [monitored_executor.py: 230]:\twarm start: encoder_layer_11_post_ffn_layer_norm_bias\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 16:26:58,766 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,767 [monitored_executor.py: 269]:\treplica id 0 of 1\u001b[0m\n", + "WARNING:root:\n", + " You can try our memory optimize feature to save your memory usage:\n", + " # create a build_strategy variable to set memory optimize option\n", + " build_strategy = compiler.BuildStrategy()\n", + " build_strategy.enable_inplace = True\n", + " build_strategy.memory_optimize = True\n", + " \n", + " # pass the build_strategy to with_data_parallel API\n", + " compiled_prog = compiler.CompiledProgram(main).with_data_parallel(\n", + " loss_name=loss.name, build_strategy=build_strategy)\n", + " \n", + " !!! Memory optimize is our experimental feature !!!\n", + " some variables may be removed/reused internal to save memory usage, \n", + " in order to fetch the right value of the fetch_list, please set the \n", + " persistable property to true for each variable in fetch_list\n", + "\n", + " # Sample\n", + " conv1 = fluid.layers.conv2d(data, 4, 5, 1, act=None) \n", + " # if you need to fetch conv1, then:\n", + " conv1.persistable = True\n", + "\n", + " \n", + "\u001b[37m[DEBUG] 2019-09-18 16:26:58,845 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:26:58,846 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:26:58,846 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:26:58,846 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:26:58,846 [monitored_executor.py: 289]:\ttrain loop has hook .EvalHookOnTrainLoop object at 0x7fbf8645b908>\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:26:58,846 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:26:58,846 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:28:45,878 [ hooks.py: 207]:\tstep: 20\tsteps/sec: -1.00000\tloss: 1.13427\tlr:[1.3581419e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:28:52,271 [ hooks.py: 207]:\tstep: 30\tsteps/sec: 1.40585\tloss: 1.05731\tlr:[2.0372129e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:29:02,782 [ hooks.py: 207]:\tstep: 40\tsteps/sec: 1.03951\tloss: 1.11125\tlr:[2.7162838e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:29:10,110 [ hooks.py: 207]:\tstep: 50\tsteps/sec: 1.32022\tloss: 1.06366\tlr:[3.395355e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:29:19,833 [ hooks.py: 207]:\tstep: 60\tsteps/sec: 0.98553\tloss: 1.10231\tlr:[4.0744257e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:29:27,882 [ hooks.py: 207]:\tstep: 70\tsteps/sec: 1.28016\tloss: 1.11046\tlr:[4.7534968e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:29:35,330 [ hooks.py: 207]:\tstep: 80\tsteps/sec: 1.38325\tloss: 1.07772\tlr:[5.4325676e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:29:42,397 [ hooks.py: 207]:\tstep: 90\tsteps/sec: 1.36822\tloss: 1.11051\tlr:[6.111639e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:29:52,070 [ hooks.py: 207]:\tstep: 100\tsteps/sec: 1.03314\tloss: 1.14308\tlr:[6.79071e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:02,635 [ hooks.py: 207]:\tstep: 110\tsteps/sec: 0.99637\tloss: 1.08845\tlr:[7.469781e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:11,923 [ hooks.py: 207]:\tstep: 120\tsteps/sec: 0.99057\tloss: 1.10118\tlr:[8.1488514e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:21,679 [ hooks.py: 207]:\tstep: 130\tsteps/sec: 1.10114\tloss: 1.06936\tlr:[8.827923e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:29,951 [ hooks.py: 207]:\tstep: 140\tsteps/sec: 1.17615\tloss: 1.11580\tlr:[9.5069936e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:37,408 [ hooks.py: 207]:\tstep: 150\tsteps/sec: 1.33206\tloss: 1.10596\tlr:[1.0186064e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:44,557 [ hooks.py: 207]:\tstep: 160\tsteps/sec: 1.38251\tloss: 1.04888\tlr:[1.0865135e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:52,471 [ hooks.py: 207]:\tstep: 170\tsteps/sec: 1.26193\tloss: 1.07098\tlr:[1.1544207e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:30:59,641 [ hooks.py: 207]:\tstep: 180\tsteps/sec: 1.41183\tloss: 1.07339\tlr:[1.2223278e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:31:10,866 [ hooks.py: 207]:\tstep: 190\tsteps/sec: 0.91393\tloss: 1.08387\tlr:[1.2902349e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:31:18,837 [ hooks.py: 207]:\tstep: 200\tsteps/sec: 1.19959\tloss: 1.20001\tlr:[1.358142e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:31:28,431 [ hooks.py: 207]:\tstep: 210\tsteps/sec: 1.00298\tloss: 1.08696\tlr:[1.4260492e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:31:38,874 [ hooks.py: 207]:\tstep: 220\tsteps/sec: 0.98902\tloss: 1.14416\tlr:[1.4939562e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:31:50,318 [ hooks.py: 207]:\tstep: 230\tsteps/sec: 0.88500\tloss: 1.07068\tlr:[1.5618633e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:31:57,991 [ hooks.py: 207]:\tstep: 240\tsteps/sec: 1.27677\tloss: 1.07384\tlr:[1.6297703e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:32:05,674 [ hooks.py: 207]:\tstep: 250\tsteps/sec: 1.36234\tloss: 1.11945\tlr:[1.6976775e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:32:13,764 [ hooks.py: 207]:\tstep: 260\tsteps/sec: 1.18961\tloss: 1.06923\tlr:[1.7655846e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:32:21,791 [ hooks.py: 207]:\tstep: 270\tsteps/sec: 1.22169\tloss: 1.10070\tlr:[1.8334918e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:32:32,586 [ hooks.py: 207]:\tstep: 280\tsteps/sec: 0.91068\tloss: 1.10118\tlr:[1.9013987e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:32:39,719 [ hooks.py: 207]:\tstep: 290\tsteps/sec: 1.41155\tloss: 1.09875\tlr:[1.969306e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:32:48,036 [ hooks.py: 207]:\tstep: 300\tsteps/sec: 1.24853\tloss: 1.10373\tlr:[2.0372129e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:32:56,140 [ hooks.py: 207]:\tstep: 310\tsteps/sec: 1.27653\tloss: 1.12459\tlr:[2.10512e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:33:03,449 [ hooks.py: 207]:\tstep: 320\tsteps/sec: 1.36484\tloss: 1.07233\tlr:[2.173027e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:33:12,723 [ hooks.py: 207]:\tstep: 330\tsteps/sec: 1.06245\tloss: 1.15019\tlr:[2.2409345e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:33:25,244 [ hooks.py: 207]:\tstep: 340\tsteps/sec: 0.80237\tloss: 1.07442\tlr:[2.3088414e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:33:33,951 [ hooks.py: 207]:\tstep: 350\tsteps/sec: 1.06436\tloss: 1.12987\tlr:[2.3767486e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:33:44,147 [ hooks.py: 207]:\tstep: 360\tsteps/sec: 1.03496\tloss: 1.08704\tlr:[2.4446556e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:33:53,600 [ hooks.py: 207]:\tstep: 370\tsteps/sec: 1.06093\tloss: 1.10018\tlr:[2.5125628e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:34:02,619 [ hooks.py: 207]:\tstep: 380\tsteps/sec: 1.18434\tloss: 1.10820\tlr:[2.5804698e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:34:10,486 [ hooks.py: 207]:\tstep: 390\tsteps/sec: 1.20622\tloss: 1.08344\tlr:[2.648377e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:34:18,326 [ hooks.py: 207]:\tstep: 400\tsteps/sec: 1.23388\tloss: 1.06138\tlr:[2.716284e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:34:27,919 [ hooks.py: 207]:\tstep: 410\tsteps/sec: 1.01997\tloss: 1.10577\tlr:[2.784191e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:34:37,673 [ hooks.py: 207]:\tstep: 420\tsteps/sec: 1.06855\tloss: 1.08179\tlr:[2.8520983e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:34:44,529 [ hooks.py: 207]:\tstep: 430\tsteps/sec: 1.37006\tloss: 1.10540\tlr:[2.9200053e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:34:54,533 [ hooks.py: 207]:\tstep: 440\tsteps/sec: 1.02908\tloss: 1.15470\tlr:[2.9879125e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:35:04,864 [ hooks.py: 207]:\tstep: 450\tsteps/sec: 0.97678\tloss: 0.98555\tlr:[3.0558194e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:35:13,037 [ hooks.py: 207]:\tstep: 460\tsteps/sec: 1.21832\tloss: 0.99871\tlr:[3.1237266e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:35:19,862 [ hooks.py: 207]:\tstep: 470\tsteps/sec: 1.49448\tloss: 1.06754\tlr:[3.1916338e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:35:29,019 [ hooks.py: 207]:\tstep: 480\tsteps/sec: 1.07628\tloss: 1.06732\tlr:[3.2595406e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:35:37,551 [ hooks.py: 207]:\tstep: 490\tsteps/sec: 1.18613\tloss: 1.01875\tlr:[3.3274478e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 16:35:46,824 [ hooks.py: 207]:\tstep: 500\tsteps/sec: 1.09231\tloss: 1.09590\tlr:[3.395355e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:35:56,298 [ hooks.py: 207]:\tstep: 510\tsteps/sec: 1.05217\tloss: 1.05729\tlr:[3.4632621e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:36:05,206 [ hooks.py: 207]:\tstep: 520\tsteps/sec: 1.07546\tloss: 1.08980\tlr:[3.531169e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:36:15,748 [ hooks.py: 207]:\tstep: 530\tsteps/sec: 0.96530\tloss: 1.10138\tlr:[3.5990763e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:36:23,542 [ hooks.py: 207]:\tstep: 540\tsteps/sec: 1.22841\tloss: 1.01145\tlr:[3.6669835e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:36:32,876 [ hooks.py: 207]:\tstep: 550\tsteps/sec: 1.06315\tloss: 1.10759\tlr:[3.7348907e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:36:40,753 [ hooks.py: 207]:\tstep: 560\tsteps/sec: 1.40666\tloss: 0.98144\tlr:[3.8027974e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:36:47,344 [ hooks.py: 207]:\tstep: 570\tsteps/sec: 1.44630\tloss: 1.06504\tlr:[3.870705e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:36:57,775 [ hooks.py: 207]:\tstep: 580\tsteps/sec: 0.99531\tloss: 1.03553\tlr:[3.938612e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:37:06,444 [ hooks.py: 207]:\tstep: 590\tsteps/sec: 1.14197\tloss: 1.01512\tlr:[4.0065192e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:37:14,210 [ hooks.py: 207]:\tstep: 600\tsteps/sec: 1.24784\tloss: 0.96321\tlr:[4.0744258e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:37:21,785 [ hooks.py: 207]:\tstep: 610\tsteps/sec: 1.29718\tloss: 0.93740\tlr:[4.142333e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:37:32,014 [ hooks.py: 207]:\tstep: 620\tsteps/sec: 1.00356\tloss: 1.04023\tlr:[4.21024e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:37:39,367 [ hooks.py: 207]:\tstep: 630\tsteps/sec: 1.32875\tloss: 1.01992\tlr:[4.2781476e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:37:48,018 [ hooks.py: 207]:\tstep: 640\tsteps/sec: 1.15027\tloss: 1.03915\tlr:[4.346054e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:37:54,594 [ hooks.py: 207]:\tstep: 650\tsteps/sec: 1.51911\tloss: 0.98232\tlr:[4.4139615e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:38:03,416 [ hooks.py: 207]:\tstep: 660\tsteps/sec: 1.19868\tloss: 1.14864\tlr:[4.481869e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:38:11,205 [ hooks.py: 207]:\tstep: 670\tsteps/sec: 1.21029\tloss: 0.85062\tlr:[4.5497754e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:38:20,825 [ hooks.py: 207]:\tstep: 680\tsteps/sec: 1.07488\tloss: 1.14913\tlr:[4.617683e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:38:28,502 [ hooks.py: 207]:\tstep: 690\tsteps/sec: 1.25369\tloss: 1.18540\tlr:[4.68559e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:38:36,777 [ hooks.py: 207]:\tstep: 700\tsteps/sec: 1.26367\tloss: 0.92638\tlr:[4.7534973e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:38:45,335 [ hooks.py: 207]:\tstep: 710\tsteps/sec: 1.09970\tloss: 0.98010\tlr:[4.8214038e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:38:53,329 [ hooks.py: 207]:\tstep: 720\tsteps/sec: 1.25679\tloss: 0.91490\tlr:[4.889311e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:39:03,406 [ hooks.py: 207]:\tstep: 730\tsteps/sec: 0.97141\tloss: 0.94612\tlr:[4.957218e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:39:12,473 [ hooks.py: 207]:\tstep: 740\tsteps/sec: 1.15826\tloss: 0.89970\tlr:[5.0251256e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:39:18,985 [ hooks.py: 207]:\tstep: 750\tsteps/sec: 1.46348\tloss: 0.90700\tlr:[5.0930325e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:39:28,353 [ hooks.py: 207]:\tstep: 760\tsteps/sec: 1.09472\tloss: 0.96581\tlr:[5.1609395e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:39:37,841 [ hooks.py: 207]:\tstep: 770\tsteps/sec: 1.05169\tloss: 1.04857\tlr:[5.228847e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:39:46,196 [ hooks.py: 207]:\tstep: 780\tsteps/sec: 1.25017\tloss: 1.03326\tlr:[5.296754e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:39:53,351 [ hooks.py: 207]:\tstep: 790\tsteps/sec: 1.32897\tloss: 0.86965\tlr:[5.364661e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:40:01,244 [ hooks.py: 207]:\tstep: 800\tsteps/sec: 1.35814\tloss: 1.01886\tlr:[5.432568e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:40:08,273 [ hooks.py: 207]:\tstep: 810\tsteps/sec: 1.26767\tloss: 0.83068\tlr:[5.5004753e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:40:16,428 [ hooks.py: 207]:\tstep: 820\tsteps/sec: 1.32353\tloss: 0.92992\tlr:[5.568382e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:40:25,428 [ hooks.py: 207]:\tstep: 830\tsteps/sec: 1.06536\tloss: 1.04981\tlr:[5.636289e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:40:36,150 [ hooks.py: 207]:\tstep: 840\tsteps/sec: 0.97641\tloss: 1.08941\tlr:[5.7041966e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:40:44,110 [ hooks.py: 207]:\tstep: 850\tsteps/sec: 1.20291\tloss: 0.86167\tlr:[5.7721036e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:40:53,327 [ hooks.py: 207]:\tstep: 860\tsteps/sec: 1.07222\tloss: 0.87733\tlr:[5.8400105e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:05,387 [ hooks.py: 207]:\tstep: 870\tsteps/sec: 0.85160\tloss: 0.79774\tlr:[5.9079175e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:11,977 [ hooks.py: 207]:\tstep: 880\tsteps/sec: 1.53583\tloss: 0.80449\tlr:[5.975825e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:21,707 [ hooks.py: 207]:\tstep: 890\tsteps/sec: 1.03382\tloss: 0.96883\tlr:[6.043732e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:28,246 [ hooks.py: 207]:\tstep: 900\tsteps/sec: 1.44946\tloss: 0.87871\tlr:[6.111639e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:36,345 [ hooks.py: 207]:\tstep: 910\tsteps/sec: 1.23927\tloss: 0.88896\tlr:[6.179546e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:45,594 [ hooks.py: 207]:\tstep: 920\tsteps/sec: 1.10507\tloss: 0.73772\tlr:[6.2474533e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:52,283 [ hooks.py: 207]:\tstep: 930\tsteps/sec: 1.49355\tloss: 0.62845\tlr:[6.31536e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:41:59,273 [ hooks.py: 207]:\tstep: 940\tsteps/sec: 1.40420\tloss: 0.85553\tlr:[6.3832676e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:08,733 [ hooks.py: 207]:\tstep: 950\tsteps/sec: 1.04654\tloss: 0.62226\tlr:[6.4511746e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:17,066 [ hooks.py: 207]:\tstep: 960\tsteps/sec: 1.17267\tloss: 1.14116\tlr:[6.519081e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:27,131 [ hooks.py: 207]:\tstep: 970\tsteps/sec: 1.04338\tloss: 0.84316\tlr:[6.586989e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:33,615 [ hooks.py: 207]:\tstep: 980\tsteps/sec: 1.47642\tloss: 0.80039\tlr:[6.6548955e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:42,630 [ hooks.py: 207]:\tstep: 990\tsteps/sec: 1.11598\tloss: 0.86035\tlr:[6.7228034e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:49,330 [ hooks.py: 207]:\tstep: 1000\tsteps/sec: 1.48870\tloss: 0.73376\tlr:[6.79071e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:49,331 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:49,331 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:49,331 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:42:49,331 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:49,332 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:42:49,332 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:50:51,181 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:51,182 [ trainer.py: 58]:\t{'acc': 0.6112449799196787, 'loss': 0.8885895720659158}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:51,183 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:51,184 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:50:51,184 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:50:51,184 [ trainer.py: 70]:\tacc\t0.6112449799196787\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:50:51,184 [ trainer.py: 70]:\tloss\t0.8885895720659158\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 16:50:51,184 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:51,184 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.6112449799196787, 'loss': 0.8885895720659158}} \n", + "old: None\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:51,184 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:51,185 [monitored_executor.py: 122]:\tsaving step 1000 to ./output/best_model/model_1000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 16:50:52,067 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.6112449799196787, 'loss': 0.8885895720659158}} \n", + "old: None\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:52,067 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:52,067 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:50:53,230 [monitored_executor.py: 122]:\tsaving step 1000 to ./output/model_1000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:51:04,658 [ hooks.py: 207]:\tstep: 1010\tsteps/sec: 0.02019\tloss: 1.03070\tlr:[6.858617e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:51:14,038 [ hooks.py: 207]:\tstep: 1020\tsteps/sec: 1.03458\tloss: 0.62447\tlr:[6.9265243e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:51:24,588 [ hooks.py: 207]:\tstep: 1030\tsteps/sec: 0.98896\tloss: 0.89986\tlr:[6.9944313e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:51:31,941 [ hooks.py: 207]:\tstep: 1040\tsteps/sec: 1.32569\tloss: 0.76850\tlr:[7.062338e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:51:39,364 [ hooks.py: 207]:\tstep: 1050\tsteps/sec: 1.30210\tloss: 0.79638\tlr:[7.1302456e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:51:47,096 [ hooks.py: 207]:\tstep: 1060\tsteps/sec: 1.32381\tloss: 1.01125\tlr:[7.1981526e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:51:56,321 [ hooks.py: 207]:\tstep: 1070\tsteps/sec: 1.08305\tloss: 0.76815\tlr:[7.266059e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:52:05,110 [ hooks.py: 207]:\tstep: 1080\tsteps/sec: 1.12023\tloss: 0.66054\tlr:[7.333967e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:52:13,562 [ hooks.py: 207]:\tstep: 1090\tsteps/sec: 1.15966\tloss: 0.61735\tlr:[7.4018735e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:52:21,420 [ hooks.py: 207]:\tstep: 1100\tsteps/sec: 1.33717\tloss: 1.16861\tlr:[7.4697814e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:52:27,391 [ hooks.py: 207]:\tstep: 1110\tsteps/sec: 1.57710\tloss: 0.98887\tlr:[7.537688e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:52:35,338 [ hooks.py: 207]:\tstep: 1120\tsteps/sec: 1.27985\tloss: 0.91038\tlr:[7.605595e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:52:44,201 [ hooks.py: 207]:\tstep: 1130\tsteps/sec: 1.16545\tloss: 0.58769\tlr:[7.673502e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:52:53,057 [ hooks.py: 207]:\tstep: 1140\tsteps/sec: 1.10971\tloss: 0.93298\tlr:[7.74141e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:53:02,008 [ hooks.py: 207]:\tstep: 1150\tsteps/sec: 1.18641\tloss: 1.23474\tlr:[7.809316e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:53:10,108 [ hooks.py: 207]:\tstep: 1160\tsteps/sec: 1.14766\tloss: 0.99367\tlr:[7.877224e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:53:18,872 [ hooks.py: 207]:\tstep: 1170\tsteps/sec: 1.13428\tloss: 0.75282\tlr:[7.94513e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:53:27,896 [ hooks.py: 207]:\tstep: 1180\tsteps/sec: 1.13358\tloss: 1.00549\tlr:[8.0130385e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:53:35,175 [ hooks.py: 207]:\tstep: 1190\tsteps/sec: 1.39854\tloss: 0.57051\tlr:[8.080945e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:53:45,496 [ hooks.py: 207]:\tstep: 1200\tsteps/sec: 0.97294\tloss: 0.89310\tlr:[8.1488515e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:53:52,754 [ hooks.py: 207]:\tstep: 1210\tsteps/sec: 1.35673\tloss: 0.77788\tlr:[8.216759e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:54:01,165 [ hooks.py: 207]:\tstep: 1220\tsteps/sec: 1.20944\tloss: 0.93549\tlr:[8.284666e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:54:09,847 [ hooks.py: 207]:\tstep: 1230\tsteps/sec: 1.14821\tloss: 0.79950\tlr:[8.352573e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:54:18,737 [ hooks.py: 207]:\tstep: 1240\tsteps/sec: 1.10888\tloss: 0.77140\tlr:[8.42048e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:54:26,841 [ hooks.py: 207]:\tstep: 1250\tsteps/sec: 1.20463\tloss: 0.84294\tlr:[8.488388e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:54:36,999 [ hooks.py: 207]:\tstep: 1260\tsteps/sec: 0.99716\tloss: 0.81621\tlr:[8.556295e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:54:45,459 [ hooks.py: 207]:\tstep: 1270\tsteps/sec: 1.20512\tloss: 0.88545\tlr:[8.624202e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:54:53,382 [ hooks.py: 207]:\tstep: 1280\tsteps/sec: 1.22554\tloss: 0.97559\tlr:[8.692108e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:01,846 [ hooks.py: 207]:\tstep: 1290\tsteps/sec: 1.25358\tloss: 0.89620\tlr:[8.7600165e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:08,446 [ hooks.py: 207]:\tstep: 1300\tsteps/sec: 1.42002\tloss: 0.76903\tlr:[8.827923e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:16,119 [ hooks.py: 207]:\tstep: 1310\tsteps/sec: 1.31666\tloss: 1.10289\tlr:[8.8958295e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:22,984 [ hooks.py: 207]:\tstep: 1320\tsteps/sec: 1.44559\tloss: 0.70377\tlr:[8.963738e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:31,188 [ hooks.py: 207]:\tstep: 1330\tsteps/sec: 1.22794\tloss: 0.80531\tlr:[9.031644e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:40,239 [ hooks.py: 207]:\tstep: 1340\tsteps/sec: 1.07422\tloss: 0.70110\tlr:[9.099551e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:49,147 [ hooks.py: 207]:\tstep: 1350\tsteps/sec: 1.16422\tloss: 0.71964\tlr:[9.167458e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:55:57,836 [ hooks.py: 207]:\tstep: 1360\tsteps/sec: 1.14487\tloss: 0.98008\tlr:[9.235366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:56:05,979 [ hooks.py: 207]:\tstep: 1370\tsteps/sec: 1.16856\tloss: 0.76631\tlr:[9.303273e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:56:14,663 [ hooks.py: 207]:\tstep: 1380\tsteps/sec: 1.23539\tloss: 0.86585\tlr:[9.37118e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:56:22,356 [ hooks.py: 207]:\tstep: 1390\tsteps/sec: 1.26902\tloss: 1.02980\tlr:[9.439087e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:56:30,260 [ hooks.py: 207]:\tstep: 1400\tsteps/sec: 1.29943\tloss: 0.74044\tlr:[9.5069945e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:56:36,937 [ hooks.py: 207]:\tstep: 1410\tsteps/sec: 1.41287\tloss: 0.76336\tlr:[9.574901e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:56:46,644 [ hooks.py: 207]:\tstep: 1420\tsteps/sec: 1.05173\tloss: 1.07623\tlr:[9.6428075e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:56:54,864 [ hooks.py: 207]:\tstep: 1430\tsteps/sec: 1.24377\tloss: 0.80735\tlr:[9.710716e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:57:03,615 [ hooks.py: 207]:\tstep: 1440\tsteps/sec: 1.10913\tloss: 0.47975\tlr:[9.778622e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:57:14,014 [ hooks.py: 207]:\tstep: 1450\tsteps/sec: 0.97211\tloss: 0.84913\tlr:[9.84653e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:57:21,968 [ hooks.py: 207]:\tstep: 1460\tsteps/sec: 1.20040\tloss: 0.69793\tlr:[9.914436e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:57:29,861 [ hooks.py: 207]:\tstep: 1470\tsteps/sec: 1.34570\tloss: 0.95934\tlr:[9.982344e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:57:37,421 [ hooks.py: 207]:\tstep: 1480\tsteps/sec: 1.30778\tloss: 0.45159\tlr:[1.0050251e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:57:43,996 [ hooks.py: 207]:\tstep: 1490\tsteps/sec: 1.45872\tloss: 0.63447\tlr:[1.0118158e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:57:53,243 [ hooks.py: 207]:\tstep: 1500\tsteps/sec: 1.09595\tloss: 0.80382\tlr:[1.0186065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:58:01,742 [ hooks.py: 207]:\tstep: 1510\tsteps/sec: 1.19707\tloss: 0.48666\tlr:[1.02539725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:58:11,019 [ hooks.py: 207]:\tstep: 1520\tsteps/sec: 1.08986\tloss: 0.99327\tlr:[1.0321879e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:58:19,010 [ hooks.py: 207]:\tstep: 1530\tsteps/sec: 1.19213\tloss: 0.88962\tlr:[1.03897855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:58:29,262 [ hooks.py: 207]:\tstep: 1540\tsteps/sec: 1.00721\tloss: 0.89425\tlr:[1.0457694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:58:38,539 [ hooks.py: 207]:\tstep: 1550\tsteps/sec: 1.06149\tloss: 0.75126\tlr:[1.05256e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:58:46,113 [ hooks.py: 207]:\tstep: 1560\tsteps/sec: 1.32050\tloss: 0.96468\tlr:[1.0593508e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:58:52,786 [ hooks.py: 207]:\tstep: 1570\tsteps/sec: 1.52017\tloss: 0.69149\tlr:[1.0661414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:59:01,543 [ hooks.py: 207]:\tstep: 1580\tsteps/sec: 1.18065\tloss: 0.68806\tlr:[1.0729322e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:59:12,064 [ hooks.py: 207]:\tstep: 1590\tsteps/sec: 0.92151\tloss: 0.57252\tlr:[1.0797229e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:59:19,261 [ hooks.py: 207]:\tstep: 1600\tsteps/sec: 1.40570\tloss: 0.55106\tlr:[1.0865136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:59:29,569 [ hooks.py: 207]:\tstep: 1610\tsteps/sec: 0.98174\tloss: 0.55142\tlr:[1.0933043e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:59:39,033 [ hooks.py: 207]:\tstep: 1620\tsteps/sec: 1.04721\tloss: 0.94508\tlr:[1.10009505e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 16:59:47,626 [ hooks.py: 207]:\tstep: 1630\tsteps/sec: 1.13904\tloss: 0.64218\tlr:[1.1068857e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 16:59:57,491 [ hooks.py: 207]:\tstep: 1640\tsteps/sec: 1.09333\tloss: 0.90164\tlr:[1.1136764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:00:04,993 [ hooks.py: 207]:\tstep: 1650\tsteps/sec: 1.26889\tloss: 0.59520\tlr:[1.1204672e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:00:13,558 [ hooks.py: 207]:\tstep: 1660\tsteps/sec: 1.09025\tloss: 0.83697\tlr:[1.1272578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:00:22,314 [ hooks.py: 207]:\tstep: 1670\tsteps/sec: 1.14136\tloss: 1.00199\tlr:[1.1340486e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:00:31,719 [ hooks.py: 207]:\tstep: 1680\tsteps/sec: 1.10301\tloss: 0.70694\tlr:[1.1408393e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:00:40,642 [ hooks.py: 207]:\tstep: 1690\tsteps/sec: 1.14856\tloss: 0.69561\tlr:[1.14763e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:00:47,444 [ hooks.py: 207]:\tstep: 1700\tsteps/sec: 1.41233\tloss: 0.66357\tlr:[1.1544207e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:00:57,491 [ hooks.py: 207]:\tstep: 1710\tsteps/sec: 0.99820\tloss: 1.09009\tlr:[1.1612114e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:01:09,147 [ hooks.py: 207]:\tstep: 1720\tsteps/sec: 0.84157\tloss: 0.52738\tlr:[1.1680021e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:01:18,120 [ hooks.py: 207]:\tstep: 1730\tsteps/sec: 1.14678\tloss: 0.96613\tlr:[1.17479285e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:01:25,385 [ hooks.py: 207]:\tstep: 1740\tsteps/sec: 1.39332\tloss: 0.47434\tlr:[1.1815835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:01:35,806 [ hooks.py: 207]:\tstep: 1750\tsteps/sec: 0.95600\tloss: 0.78575\tlr:[1.1883742e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:01:43,330 [ hooks.py: 207]:\tstep: 1760\tsteps/sec: 1.30775\tloss: 0.64152\tlr:[1.195165e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:01:52,220 [ hooks.py: 207]:\tstep: 1770\tsteps/sec: 1.13365\tloss: 0.97550\tlr:[1.2019556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:01,196 [ hooks.py: 207]:\tstep: 1780\tsteps/sec: 1.18270\tloss: 0.60258\tlr:[1.2087464e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:07,831 [ hooks.py: 207]:\tstep: 1790\tsteps/sec: 1.34280\tloss: 0.89306\tlr:[1.2155371e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:16,663 [ hooks.py: 207]:\tstep: 1800\tsteps/sec: 1.16642\tloss: 0.48595\tlr:[1.2223278e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:24,373 [ hooks.py: 207]:\tstep: 1810\tsteps/sec: 1.32130\tloss: 0.52739\tlr:[1.2291185e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:32,515 [ hooks.py: 207]:\tstep: 1820\tsteps/sec: 1.21890\tloss: 1.24288\tlr:[1.2359092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:40,910 [ hooks.py: 207]:\tstep: 1830\tsteps/sec: 1.18991\tloss: 0.91611\tlr:[1.2427e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:47,453 [ hooks.py: 207]:\tstep: 1840\tsteps/sec: 1.50883\tloss: 0.88931\tlr:[1.24949065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:02:56,604 [ hooks.py: 207]:\tstep: 1850\tsteps/sec: 1.06392\tloss: 0.70402\tlr:[1.2562814e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:03:07,752 [ hooks.py: 207]:\tstep: 1860\tsteps/sec: 0.92132\tloss: 0.53732\tlr:[1.263072e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:03:17,302 [ hooks.py: 207]:\tstep: 1870\tsteps/sec: 1.03137\tloss: 0.76623\tlr:[1.2698628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:03:25,813 [ hooks.py: 207]:\tstep: 1880\tsteps/sec: 1.14660\tloss: 0.54416\tlr:[1.2766535e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:03:35,049 [ hooks.py: 207]:\tstep: 1890\tsteps/sec: 1.15845\tloss: 1.02056\tlr:[1.2834441e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:03:44,164 [ hooks.py: 207]:\tstep: 1900\tsteps/sec: 1.05078\tloss: 0.81198\tlr:[1.2902349e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:03:51,987 [ hooks.py: 207]:\tstep: 1910\tsteps/sec: 1.29958\tloss: 1.02639\tlr:[1.2970257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:00,196 [ hooks.py: 207]:\tstep: 1920\tsteps/sec: 1.24326\tloss: 0.73768\tlr:[1.3038162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:08,870 [ hooks.py: 207]:\tstep: 1930\tsteps/sec: 1.14192\tloss: 0.59653\tlr:[1.310607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:17,442 [ hooks.py: 207]:\tstep: 1940\tsteps/sec: 1.17215\tloss: 0.55697\tlr:[1.3173978e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:25,706 [ hooks.py: 207]:\tstep: 1950\tsteps/sec: 1.22412\tloss: 0.86902\tlr:[1.3241884e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:32,181 [ hooks.py: 207]:\tstep: 1960\tsteps/sec: 1.43271\tloss: 0.76441\tlr:[1.3309791e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:40,235 [ hooks.py: 207]:\tstep: 1970\tsteps/sec: 1.28104\tloss: 0.77772\tlr:[1.3377699e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:49,159 [ hooks.py: 207]:\tstep: 1980\tsteps/sec: 1.14885\tloss: 0.74745\tlr:[1.3445607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:04:59,249 [ hooks.py: 207]:\tstep: 1990\tsteps/sec: 0.99035\tloss: 0.81184\tlr:[1.3513512e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:05:08,500 [ hooks.py: 207]:\tstep: 2000\tsteps/sec: 1.05126\tloss: 0.69755\tlr:[1.358142e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:05:08,501 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:05:08,501 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:05:08,501 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:05:08,501 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:05:08,501 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:05:08,501 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:13:07,812 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:07,813 [ trainer.py: 58]:\t{'acc': 0.6819277108433734, 'loss': 0.7712427740677809}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:07,814 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:07,815 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:13:07,815 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:13:07,815 [ trainer.py: 70]:\tacc\t0.6819277108433734\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:13:07,815 [ trainer.py: 70]:\tloss\t0.7712427740677809\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:13:07,815 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:07,816 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.6819277108433734, 'loss': 0.7712427740677809}} \n", + "old: {'eval': {'acc': 0.6112449799196787, 'loss': 0.8885895720659158}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:07,816 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:07,817 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_1000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:07,817 [monitored_executor.py: 122]:\tsaving step 2000 to ./output/best_model/model_2000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:08,809 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_1000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:08,810 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.6819277108433734, 'loss': 0.7712427740677809}} \n", + "old: {'eval': {'acc': 0.6112449799196787, 'loss': 0.8885895720659158}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:08,810 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:08,811 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:10,042 [monitored_executor.py: 122]:\tsaving step 2000 to ./output/model_2000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:20,905 [ hooks.py: 207]:\tstep: 2010\tsteps/sec: 0.02032\tloss: 0.45225\tlr:[1.3649328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:30,436 [ hooks.py: 207]:\tstep: 2020\tsteps/sec: 1.01429\tloss: 0.66881\tlr:[1.3717234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:38,480 [ hooks.py: 207]:\tstep: 2030\tsteps/sec: 1.20082\tloss: 0.61251\tlr:[1.3785141e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:50,826 [ hooks.py: 207]:\tstep: 2040\tsteps/sec: 0.85509\tloss: 0.90623\tlr:[1.3853049e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:13:59,653 [ hooks.py: 207]:\tstep: 2050\tsteps/sec: 1.14205\tloss: 0.83868\tlr:[1.3920955e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 17:14:06,859 [ hooks.py: 207]:\tstep: 2060\tsteps/sec: 1.33695\tloss: 0.81673\tlr:[1.39888625e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:14:16,699 [ hooks.py: 207]:\tstep: 2070\tsteps/sec: 1.02053\tloss: 0.42915\tlr:[1.405677e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:14:23,666 [ hooks.py: 207]:\tstep: 2080\tsteps/sec: 1.41748\tloss: 0.40277\tlr:[1.4124676e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:14:30,700 [ hooks.py: 207]:\tstep: 2090\tsteps/sec: 1.45534\tloss: 0.87661\tlr:[1.4192584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:14:39,669 [ hooks.py: 207]:\tstep: 2100\tsteps/sec: 1.12058\tloss: 0.82266\tlr:[1.4260491e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:14:46,475 [ hooks.py: 207]:\tstep: 2110\tsteps/sec: 1.43890\tloss: 0.97632\tlr:[1.4328398e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:14:53,325 [ hooks.py: 207]:\tstep: 2120\tsteps/sec: 1.44598\tloss: 0.70144\tlr:[1.4396305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:15:02,906 [ hooks.py: 207]:\tstep: 2130\tsteps/sec: 1.10206\tloss: 0.60495\tlr:[1.4464213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:15:10,028 [ hooks.py: 207]:\tstep: 2140\tsteps/sec: 1.34246\tloss: 0.83241\tlr:[1.4532118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:15:18,925 [ hooks.py: 207]:\tstep: 2150\tsteps/sec: 1.10510\tloss: 0.89086\tlr:[1.4600027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:15:26,278 [ hooks.py: 207]:\tstep: 2160\tsteps/sec: 1.38326\tloss: 0.74467\tlr:[1.4667934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:15:34,967 [ hooks.py: 207]:\tstep: 2170\tsteps/sec: 1.16126\tloss: 0.50411\tlr:[1.4735841e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:15:44,325 [ hooks.py: 207]:\tstep: 2180\tsteps/sec: 1.05907\tloss: 1.13197\tlr:[1.4803747e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:15:52,237 [ hooks.py: 207]:\tstep: 2190\tsteps/sec: 1.23299\tloss: 0.40727\tlr:[1.4871655e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:02,140 [ hooks.py: 207]:\tstep: 2200\tsteps/sec: 1.06091\tloss: 0.32300\tlr:[1.4939563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:08,113 [ hooks.py: 207]:\tstep: 2210\tsteps/sec: 1.48952\tloss: 0.54513\tlr:[1.5007468e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:16,792 [ hooks.py: 207]:\tstep: 2220\tsteps/sec: 1.21323\tloss: 0.63435\tlr:[1.5075376e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:25,026 [ hooks.py: 207]:\tstep: 2230\tsteps/sec: 1.22897\tloss: 0.67547\tlr:[1.5143284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:35,373 [ hooks.py: 207]:\tstep: 2240\tsteps/sec: 0.97011\tloss: 0.79481\tlr:[1.521119e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:42,363 [ hooks.py: 207]:\tstep: 2250\tsteps/sec: 1.37268\tloss: 0.59450\tlr:[1.5279098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:50,354 [ hooks.py: 207]:\tstep: 2260\tsteps/sec: 1.30499\tloss: 0.72073\tlr:[1.5347005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:16:57,150 [ hooks.py: 207]:\tstep: 2270\tsteps/sec: 1.35507\tloss: 0.57871\tlr:[1.5414911e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:17:06,791 [ hooks.py: 207]:\tstep: 2280\tsteps/sec: 1.07434\tloss: 0.64274\tlr:[1.548282e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:17:13,690 [ hooks.py: 207]:\tstep: 2290\tsteps/sec: 1.43195\tloss: 0.78511\tlr:[1.5550726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:17:21,166 [ hooks.py: 207]:\tstep: 2300\tsteps/sec: 1.39373\tloss: 0.84730\tlr:[1.5618632e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:17:30,431 [ hooks.py: 207]:\tstep: 2310\tsteps/sec: 1.08371\tloss: 0.99902\tlr:[1.568654e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:17:36,844 [ hooks.py: 207]:\tstep: 2320\tsteps/sec: 1.49766\tloss: 0.56734\tlr:[1.5754447e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:17:47,450 [ hooks.py: 207]:\tstep: 2330\tsteps/sec: 0.94580\tloss: 0.94089\tlr:[1.5822356e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:17:55,037 [ hooks.py: 207]:\tstep: 2340\tsteps/sec: 1.34649\tloss: 0.85595\tlr:[1.589026e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:18:02,857 [ hooks.py: 207]:\tstep: 2350\tsteps/sec: 1.22928\tloss: 0.53463\tlr:[1.5958169e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:18:11,043 [ hooks.py: 207]:\tstep: 2360\tsteps/sec: 1.25128\tloss: 0.63373\tlr:[1.6026077e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:18:20,577 [ hooks.py: 207]:\tstep: 2370\tsteps/sec: 1.06644\tloss: 0.55146\tlr:[1.6093982e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:18:29,244 [ hooks.py: 207]:\tstep: 2380\tsteps/sec: 1.13232\tloss: 0.50471\tlr:[1.616189e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:18:37,920 [ hooks.py: 207]:\tstep: 2390\tsteps/sec: 1.15582\tloss: 0.73062\tlr:[1.6229798e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:18:45,864 [ hooks.py: 207]:\tstep: 2400\tsteps/sec: 1.21909\tloss: 0.70201\tlr:[1.6297703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:18:55,041 [ hooks.py: 207]:\tstep: 2410\tsteps/sec: 1.13654\tloss: 0.95318\tlr:[1.6365611e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:19:03,370 [ hooks.py: 207]:\tstep: 2420\tsteps/sec: 1.15522\tloss: 0.63797\tlr:[1.6433518e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:19:10,678 [ hooks.py: 207]:\tstep: 2430\tsteps/sec: 1.44273\tloss: 0.96334\tlr:[1.6501424e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:19:19,808 [ hooks.py: 207]:\tstep: 2440\tsteps/sec: 1.09006\tloss: 0.95370\tlr:[1.6569333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:19:26,214 [ hooks.py: 207]:\tstep: 2450\tsteps/sec: 1.54233\tloss: 1.03500\tlr:[1.663724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:19:36,155 [ hooks.py: 207]:\tstep: 2460\tsteps/sec: 1.00194\tloss: 0.44112\tlr:[1.6705146e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:19:44,722 [ hooks.py: 207]:\tstep: 2470\tsteps/sec: 1.15644\tloss: 0.64987\tlr:[1.6773054e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:19:52,793 [ hooks.py: 207]:\tstep: 2480\tsteps/sec: 1.23573\tloss: 0.67059\tlr:[1.684096e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:01,507 [ hooks.py: 207]:\tstep: 2490\tsteps/sec: 1.22000\tloss: 0.73762\tlr:[1.6908867e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:09,006 [ hooks.py: 207]:\tstep: 2500\tsteps/sec: 1.23861\tloss: 0.48520\tlr:[1.6976775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:16,185 [ hooks.py: 207]:\tstep: 2510\tsteps/sec: 1.43806\tloss: 0.82395\tlr:[1.7044682e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:24,682 [ hooks.py: 207]:\tstep: 2520\tsteps/sec: 1.16888\tloss: 1.25672\tlr:[1.711259e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:32,952 [ hooks.py: 207]:\tstep: 2530\tsteps/sec: 1.20392\tloss: 0.67219\tlr:[1.7180497e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:39,577 [ hooks.py: 207]:\tstep: 2540\tsteps/sec: 1.52545\tloss: 0.40183\tlr:[1.7248403e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:48,448 [ hooks.py: 207]:\tstep: 2550\tsteps/sec: 1.04881\tloss: 0.61633\tlr:[1.7316312e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:20:57,629 [ hooks.py: 207]:\tstep: 2560\tsteps/sec: 1.16281\tloss: 0.61877\tlr:[1.7384216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:21:05,710 [ hooks.py: 207]:\tstep: 2570\tsteps/sec: 1.22864\tloss: 0.99103\tlr:[1.7452125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:21:15,930 [ hooks.py: 207]:\tstep: 2580\tsteps/sec: 0.99370\tloss: 0.35596\tlr:[1.7520033e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:21:25,187 [ hooks.py: 207]:\tstep: 2590\tsteps/sec: 1.07964\tloss: 0.63254\tlr:[1.7587938e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:21:33,479 [ hooks.py: 207]:\tstep: 2600\tsteps/sec: 1.17883\tloss: 0.56178\tlr:[1.7655846e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:21:41,682 [ hooks.py: 207]:\tstep: 2610\tsteps/sec: 1.24200\tloss: 0.61197\tlr:[1.7723754e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:21:50,083 [ hooks.py: 207]:\tstep: 2620\tsteps/sec: 1.21446\tloss: 0.32744\tlr:[1.7791659e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:21:57,520 [ hooks.py: 207]:\tstep: 2630\tsteps/sec: 1.29916\tloss: 0.34894\tlr:[1.7859567e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:22:06,304 [ hooks.py: 207]:\tstep: 2640\tsteps/sec: 1.14783\tloss: 0.54119\tlr:[1.7927476e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:22:14,646 [ hooks.py: 207]:\tstep: 2650\tsteps/sec: 1.19968\tloss: 0.83114\tlr:[1.799538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:22:24,614 [ hooks.py: 207]:\tstep: 2660\tsteps/sec: 1.00713\tloss: 0.72856\tlr:[1.8063289e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:22:34,240 [ hooks.py: 207]:\tstep: 2670\tsteps/sec: 1.00234\tloss: 0.99080\tlr:[1.8131195e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:22:42,468 [ hooks.py: 207]:\tstep: 2680\tsteps/sec: 1.24702\tloss: 0.39260\tlr:[1.8199102e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:22:51,032 [ hooks.py: 207]:\tstep: 2690\tsteps/sec: 1.18487\tloss: 0.68888\tlr:[1.826701e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:23:00,457 [ hooks.py: 207]:\tstep: 2700\tsteps/sec: 1.08243\tloss: 0.58490\tlr:[1.8334917e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:23:08,649 [ hooks.py: 207]:\tstep: 2710\tsteps/sec: 1.19423\tloss: 0.60291\tlr:[1.8402825e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 17:23:16,649 [ hooks.py: 207]:\tstep: 2720\tsteps/sec: 1.24469\tloss: 0.85060\tlr:[1.8470731e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:23:23,298 [ hooks.py: 207]:\tstep: 2730\tsteps/sec: 1.40201\tloss: 0.76992\tlr:[1.8538638e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:23:31,355 [ hooks.py: 207]:\tstep: 2740\tsteps/sec: 1.26942\tloss: 0.59612\tlr:[1.8606546e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:23:40,441 [ hooks.py: 207]:\tstep: 2750\tsteps/sec: 1.16825\tloss: 0.99226\tlr:[1.8674453e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:23:47,534 [ hooks.py: 207]:\tstep: 2760\tsteps/sec: 1.35349\tloss: 0.71133\tlr:[1.874236e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:23:54,501 [ hooks.py: 207]:\tstep: 2770\tsteps/sec: 1.45980\tloss: 0.82239\tlr:[1.8810268e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:24:04,739 [ hooks.py: 207]:\tstep: 2780\tsteps/sec: 0.97045\tloss: 0.68196\tlr:[1.8878174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:24:13,604 [ hooks.py: 207]:\tstep: 2790\tsteps/sec: 1.11094\tloss: 0.71730\tlr:[1.894608e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:24:23,851 [ hooks.py: 207]:\tstep: 2800\tsteps/sec: 0.97950\tloss: 1.08689\tlr:[1.9013989e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:24:30,716 [ hooks.py: 207]:\tstep: 2810\tsteps/sec: 1.49535\tloss: 0.72136\tlr:[1.9081894e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:24:41,240 [ hooks.py: 207]:\tstep: 2820\tsteps/sec: 0.95503\tloss: 0.49038\tlr:[1.9149802e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:24:50,810 [ hooks.py: 207]:\tstep: 2830\tsteps/sec: 1.04889\tloss: 0.65369\tlr:[1.921771e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:25:00,745 [ hooks.py: 207]:\tstep: 2840\tsteps/sec: 1.01144\tloss: 0.72409\tlr:[1.9285615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:25:10,447 [ hooks.py: 207]:\tstep: 2850\tsteps/sec: 1.02767\tloss: 0.54897\tlr:[1.9353523e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:25:17,893 [ hooks.py: 207]:\tstep: 2860\tsteps/sec: 1.30782\tloss: 0.51579\tlr:[1.9421432e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:25:27,142 [ hooks.py: 207]:\tstep: 2870\tsteps/sec: 1.07879\tloss: 0.76249\tlr:[1.9489336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:25:35,317 [ hooks.py: 207]:\tstep: 2880\tsteps/sec: 1.20241\tloss: 0.67880\tlr:[1.9557245e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:25:42,924 [ hooks.py: 207]:\tstep: 2890\tsteps/sec: 1.33546\tloss: 0.82860\tlr:[1.9625151e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:25:50,822 [ hooks.py: 207]:\tstep: 2900\tsteps/sec: 1.28281\tloss: 0.74806\tlr:[1.969306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:26:00,290 [ hooks.py: 207]:\tstep: 2910\tsteps/sec: 1.05109\tloss: 0.53695\tlr:[1.9760966e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:26:10,165 [ hooks.py: 207]:\tstep: 2920\tsteps/sec: 1.02878\tloss: 0.65183\tlr:[1.9828873e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:26:17,802 [ hooks.py: 207]:\tstep: 2930\tsteps/sec: 1.28009\tloss: 0.94666\tlr:[1.9896781e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:26:26,425 [ hooks.py: 207]:\tstep: 2940\tsteps/sec: 1.16773\tloss: 0.76164\tlr:[1.9964687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:26:34,809 [ hooks.py: 207]:\tstep: 2950\tsteps/sec: 1.19509\tloss: 0.90084\tlr:[2.0032594e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:26:43,146 [ hooks.py: 207]:\tstep: 2960\tsteps/sec: 1.17390\tloss: 0.66876\tlr:[2.0100502e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:26:51,421 [ hooks.py: 207]:\tstep: 2970\tsteps/sec: 1.23692\tloss: 1.03338\tlr:[2.0168409e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:00,217 [ hooks.py: 207]:\tstep: 2980\tsteps/sec: 1.16845\tloss: 0.76820\tlr:[2.0236315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:09,378 [ hooks.py: 207]:\tstep: 2990\tsteps/sec: 1.05772\tloss: 0.73294\tlr:[2.0304224e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:16,390 [ hooks.py: 207]:\tstep: 3000\tsteps/sec: 1.43472\tloss: 0.81748\tlr:[2.037213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:16,394 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:16,394 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:16,394 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:27:16,395 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:16,395 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:27:16,395 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:35:13,283 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:13,285 [ trainer.py: 58]:\t{'acc': 0.7204819277108434, 'loss': 0.7031658165729963}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:13,285 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:13,285 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:35:13,286 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:35:13,286 [ trainer.py: 70]:\tacc\t0.7204819277108434\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:35:13,286 [ trainer.py: 70]:\tloss\t0.7031658165729963\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:35:13,286 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:13,286 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7204819277108434, 'loss': 0.7031658165729963}} \n", + "old: {'eval': {'acc': 0.6819277108433734, 'loss': 0.7712427740677809}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:13,286 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:13,288 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_2000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:13,288 [monitored_executor.py: 122]:\tsaving step 3000 to ./output/best_model/model_3000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:14,253 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_2000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:14,254 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7204819277108434, 'loss': 0.7031658165729963}} \n", + "old: {'eval': {'acc': 0.6819277108433734, 'loss': 0.7712427740677809}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:14,255 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:14,255 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:15,660 [monitored_executor.py: 122]:\tsaving step 3000 to ./output/model_3000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:25,576 [ hooks.py: 207]:\tstep: 3010\tsteps/sec: 0.02043\tloss: 0.96726\tlr:[2.0440037e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:33,825 [ hooks.py: 207]:\tstep: 3020\tsteps/sec: 1.22846\tloss: 1.02525\tlr:[2.0507945e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:41,229 [ hooks.py: 207]:\tstep: 3030\tsteps/sec: 1.30880\tloss: 0.50054\tlr:[2.0575852e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:51,134 [ hooks.py: 207]:\tstep: 3040\tsteps/sec: 1.05395\tloss: 0.86102\tlr:[2.0643758e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:35:59,320 [ hooks.py: 207]:\tstep: 3050\tsteps/sec: 1.20209\tloss: 0.74688\tlr:[2.0711666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:36:06,559 [ hooks.py: 207]:\tstep: 3060\tsteps/sec: 1.39777\tloss: 0.71449\tlr:[2.0779571e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:36:16,077 [ hooks.py: 207]:\tstep: 3070\tsteps/sec: 1.06007\tloss: 0.72904\tlr:[2.084748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:36:23,905 [ hooks.py: 207]:\tstep: 3080\tsteps/sec: 1.23821\tloss: 0.90274\tlr:[2.0915388e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:36:32,308 [ hooks.py: 207]:\tstep: 3090\tsteps/sec: 1.19648\tloss: 0.48141\tlr:[2.0983294e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:36:40,239 [ hooks.py: 207]:\tstep: 3100\tsteps/sec: 1.30098\tloss: 1.24419\tlr:[2.10512e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:36:47,797 [ hooks.py: 207]:\tstep: 3110\tsteps/sec: 1.28449\tloss: 0.64239\tlr:[2.1119109e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:36:55,779 [ hooks.py: 207]:\tstep: 3120\tsteps/sec: 1.28923\tloss: 0.57111\tlr:[2.1187016e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:37:03,083 [ hooks.py: 207]:\tstep: 3130\tsteps/sec: 1.39964\tloss: 0.81586\tlr:[2.1254922e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:37:09,918 [ hooks.py: 207]:\tstep: 3140\tsteps/sec: 1.41796\tloss: 0.77686\tlr:[2.1322829e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 17:37:19,094 [ hooks.py: 207]:\tstep: 3150\tsteps/sec: 1.02361\tloss: 0.84764\tlr:[2.1390737e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:37:28,420 [ hooks.py: 207]:\tstep: 3160\tsteps/sec: 1.11888\tloss: 0.52491\tlr:[2.1458643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:37:35,844 [ hooks.py: 207]:\tstep: 3170\tsteps/sec: 1.38202\tloss: 0.60477\tlr:[2.152655e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:37:42,769 [ hooks.py: 207]:\tstep: 3180\tsteps/sec: 1.41537\tloss: 0.77718\tlr:[2.1594458e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:37:48,997 [ hooks.py: 207]:\tstep: 3190\tsteps/sec: 1.51325\tloss: 0.78872\tlr:[2.1662365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:37:58,160 [ hooks.py: 207]:\tstep: 3200\tsteps/sec: 1.13448\tloss: 0.85237\tlr:[2.1730271e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:38:06,831 [ hooks.py: 207]:\tstep: 3210\tsteps/sec: 1.14700\tloss: 0.79948\tlr:[2.179818e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:38:16,154 [ hooks.py: 207]:\tstep: 3220\tsteps/sec: 1.10489\tloss: 1.12099\tlr:[2.1866086e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:38:22,794 [ hooks.py: 207]:\tstep: 3230\tsteps/sec: 1.45816\tloss: 0.61324\tlr:[2.1933993e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:38:29,223 [ hooks.py: 207]:\tstep: 3240\tsteps/sec: 1.54307\tloss: 0.43002\tlr:[2.2001901e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:38:35,350 [ hooks.py: 207]:\tstep: 3250\tsteps/sec: 1.62323\tloss: 0.48841\tlr:[2.2069808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:38:44,589 [ hooks.py: 207]:\tstep: 3260\tsteps/sec: 1.10111\tloss: 0.57674\tlr:[2.2137714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:38:52,520 [ hooks.py: 207]:\tstep: 3270\tsteps/sec: 1.19677\tloss: 0.40990\tlr:[2.2205622e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:39:00,778 [ hooks.py: 207]:\tstep: 3280\tsteps/sec: 1.25668\tloss: 0.79695\tlr:[2.2273529e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:39:11,079 [ hooks.py: 207]:\tstep: 3290\tsteps/sec: 0.99832\tloss: 1.05640\tlr:[2.2341435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:39:21,134 [ hooks.py: 207]:\tstep: 3300\tsteps/sec: 0.99575\tloss: 0.80878\tlr:[2.2409344e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:39:31,026 [ hooks.py: 207]:\tstep: 3310\tsteps/sec: 0.97932\tloss: 0.61709\tlr:[2.247725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:39:39,792 [ hooks.py: 207]:\tstep: 3320\tsteps/sec: 1.16207\tloss: 0.51374\tlr:[2.2545157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:39:47,511 [ hooks.py: 207]:\tstep: 3330\tsteps/sec: 1.22970\tloss: 0.30863\tlr:[2.2613065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:39:58,024 [ hooks.py: 207]:\tstep: 3340\tsteps/sec: 0.98696\tloss: 0.61598\tlr:[2.2680972e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:40:05,676 [ hooks.py: 207]:\tstep: 3350\tsteps/sec: 1.34668\tloss: 0.54062\tlr:[2.2748878e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:40:13,292 [ hooks.py: 207]:\tstep: 3360\tsteps/sec: 1.25354\tloss: 0.64565\tlr:[2.2816786e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:40:22,404 [ hooks.py: 207]:\tstep: 3370\tsteps/sec: 1.10417\tloss: 0.84200\tlr:[2.2884693e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:40:30,008 [ hooks.py: 207]:\tstep: 3380\tsteps/sec: 1.34439\tloss: 0.40511\tlr:[2.29526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:40:39,373 [ hooks.py: 207]:\tstep: 3390\tsteps/sec: 1.05302\tloss: 0.18678\tlr:[2.3020506e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:40:47,609 [ hooks.py: 207]:\tstep: 3400\tsteps/sec: 1.20633\tloss: 0.59320\tlr:[2.3088414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:40:55,619 [ hooks.py: 207]:\tstep: 3410\tsteps/sec: 1.29414\tloss: 0.78370\tlr:[2.315632e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:41:06,814 [ hooks.py: 207]:\tstep: 3420\tsteps/sec: 0.87723\tloss: 0.69612\tlr:[2.3224227e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:41:16,326 [ hooks.py: 207]:\tstep: 3430\tsteps/sec: 1.06688\tloss: 0.57912\tlr:[2.3292136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:41:24,340 [ hooks.py: 207]:\tstep: 3440\tsteps/sec: 1.20774\tloss: 0.67436\tlr:[2.3360042e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:41:31,019 [ hooks.py: 207]:\tstep: 3450\tsteps/sec: 1.56225\tloss: 0.90509\tlr:[2.3427949e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:41:38,167 [ hooks.py: 207]:\tstep: 3460\tsteps/sec: 1.34828\tloss: 0.65294\tlr:[2.3495857e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:41:46,017 [ hooks.py: 207]:\tstep: 3470\tsteps/sec: 1.31720\tloss: 0.34423\tlr:[2.3563764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:41:54,822 [ hooks.py: 207]:\tstep: 3480\tsteps/sec: 1.12657\tloss: 0.86909\tlr:[2.363167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:02,338 [ hooks.py: 207]:\tstep: 3490\tsteps/sec: 1.39903\tloss: 0.63215\tlr:[2.3699578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:08,340 [ hooks.py: 207]:\tstep: 3500\tsteps/sec: 1.44839\tloss: 0.60273\tlr:[2.3767485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:17,685 [ hooks.py: 207]:\tstep: 3510\tsteps/sec: 1.11925\tloss: 0.69895\tlr:[2.3835391e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:24,853 [ hooks.py: 207]:\tstep: 3520\tsteps/sec: 1.41560\tloss: 0.31981\tlr:[2.39033e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:34,310 [ hooks.py: 207]:\tstep: 3530\tsteps/sec: 1.04500\tloss: 0.67199\tlr:[2.3971206e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:42,646 [ hooks.py: 207]:\tstep: 3540\tsteps/sec: 1.20288\tloss: 0.25074\tlr:[2.4039113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:50,256 [ hooks.py: 207]:\tstep: 3550\tsteps/sec: 1.35472\tloss: 0.50547\tlr:[2.4107021e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:42:58,771 [ hooks.py: 207]:\tstep: 3560\tsteps/sec: 1.15694\tloss: 0.56912\tlr:[2.4174928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:43:08,212 [ hooks.py: 207]:\tstep: 3570\tsteps/sec: 1.06903\tloss: 0.77308\tlr:[2.4242834e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:43:15,548 [ hooks.py: 207]:\tstep: 3580\tsteps/sec: 1.37249\tloss: 0.67827\tlr:[2.4310742e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:43:23,615 [ hooks.py: 207]:\tstep: 3590\tsteps/sec: 1.21377\tloss: 0.95545\tlr:[2.4378649e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:43:31,587 [ hooks.py: 207]:\tstep: 3600\tsteps/sec: 1.28548\tloss: 0.46636\tlr:[2.4446555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:43:40,020 [ hooks.py: 207]:\tstep: 3610\tsteps/sec: 1.16709\tloss: 0.58718\tlr:[2.4514464e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:43:49,118 [ hooks.py: 207]:\tstep: 3620\tsteps/sec: 1.08224\tloss: 0.50563\tlr:[2.458237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:43:57,252 [ hooks.py: 207]:\tstep: 3630\tsteps/sec: 1.23462\tloss: 0.69699\tlr:[2.4650277e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:44:04,659 [ hooks.py: 207]:\tstep: 3640\tsteps/sec: 1.36510\tloss: 0.38347\tlr:[2.4718183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:44:12,699 [ hooks.py: 207]:\tstep: 3650\tsteps/sec: 1.22153\tloss: 0.46326\tlr:[2.4786092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:44:21,596 [ hooks.py: 207]:\tstep: 3660\tsteps/sec: 1.15875\tloss: 0.70848\tlr:[2.4854e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:44:29,558 [ hooks.py: 207]:\tstep: 3670\tsteps/sec: 1.22837\tloss: 0.92610\tlr:[2.4921905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:44:37,388 [ hooks.py: 207]:\tstep: 3680\tsteps/sec: 1.21546\tloss: 0.83359\tlr:[2.4989813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:44:46,981 [ hooks.py: 207]:\tstep: 3690\tsteps/sec: 1.07676\tloss: 0.70789\tlr:[2.5057721e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:44:55,584 [ hooks.py: 207]:\tstep: 3700\tsteps/sec: 1.21284\tloss: 0.75255\tlr:[2.5125628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:04,258 [ hooks.py: 207]:\tstep: 3710\tsteps/sec: 1.11578\tloss: 0.93220\tlr:[2.5193533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:11,329 [ hooks.py: 207]:\tstep: 3720\tsteps/sec: 1.45235\tloss: 0.42322\tlr:[2.526144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:19,075 [ hooks.py: 207]:\tstep: 3730\tsteps/sec: 1.26189\tloss: 0.84318\tlr:[2.5329347e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:28,574 [ hooks.py: 207]:\tstep: 3740\tsteps/sec: 1.04326\tloss: 0.96760\tlr:[2.5397256e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:37,165 [ hooks.py: 207]:\tstep: 3750\tsteps/sec: 1.17672\tloss: 0.57540\tlr:[2.5465162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:44,961 [ hooks.py: 207]:\tstep: 3760\tsteps/sec: 1.29775\tloss: 0.54072\tlr:[2.553307e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:51,585 [ hooks.py: 207]:\tstep: 3770\tsteps/sec: 1.44330\tloss: 0.44460\tlr:[2.5600979e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:45:59,323 [ hooks.py: 207]:\tstep: 3780\tsteps/sec: 1.27624\tloss: 0.83350\tlr:[2.5668882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:46:08,394 [ hooks.py: 207]:\tstep: 3790\tsteps/sec: 1.13960\tloss: 0.41058\tlr:[2.573679e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:46:17,175 [ hooks.py: 207]:\tstep: 3800\tsteps/sec: 1.13918\tloss: 0.84997\tlr:[2.5804698e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 17:46:25,146 [ hooks.py: 207]:\tstep: 3810\tsteps/sec: 1.27202\tloss: 1.02221\tlr:[2.5872605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:46:33,905 [ hooks.py: 207]:\tstep: 3820\tsteps/sec: 1.13075\tloss: 0.87903\tlr:[2.5940513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:46:40,880 [ hooks.py: 207]:\tstep: 3830\tsteps/sec: 1.47251\tloss: 0.81642\tlr:[2.600842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:46:47,480 [ hooks.py: 207]:\tstep: 3840\tsteps/sec: 1.39809\tloss: 0.51748\tlr:[2.6076325e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:46:57,856 [ hooks.py: 207]:\tstep: 3850\tsteps/sec: 0.99859\tloss: 1.11806\tlr:[2.6144233e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:47:07,059 [ hooks.py: 207]:\tstep: 3860\tsteps/sec: 1.08786\tloss: 0.88644\tlr:[2.621214e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:47:13,160 [ hooks.py: 207]:\tstep: 3870\tsteps/sec: 1.55127\tloss: 0.59837\tlr:[2.6280048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:47:24,203 [ hooks.py: 207]:\tstep: 3880\tsteps/sec: 0.92672\tloss: 0.81824\tlr:[2.6347956e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:47:32,303 [ hooks.py: 207]:\tstep: 3890\tsteps/sec: 1.24222\tloss: 0.44804\tlr:[2.6415863e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:47:42,436 [ hooks.py: 207]:\tstep: 3900\tsteps/sec: 0.99207\tloss: 0.70653\tlr:[2.6483767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:47:50,642 [ hooks.py: 207]:\tstep: 3910\tsteps/sec: 1.24348\tloss: 0.49674\tlr:[2.6551676e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:48:01,060 [ hooks.py: 207]:\tstep: 3920\tsteps/sec: 0.96301\tloss: 0.72637\tlr:[2.6619582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:48:10,904 [ hooks.py: 207]:\tstep: 3930\tsteps/sec: 1.01003\tloss: 0.46162\tlr:[2.668749e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:48:20,434 [ hooks.py: 207]:\tstep: 3940\tsteps/sec: 1.05476\tloss: 0.75049\tlr:[2.6755399e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:48:31,067 [ hooks.py: 207]:\tstep: 3950\tsteps/sec: 0.91465\tloss: 0.90155\tlr:[2.6823305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:48:40,241 [ hooks.py: 207]:\tstep: 3960\tsteps/sec: 1.11704\tloss: 0.53043\tlr:[2.6891214e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:48:47,575 [ hooks.py: 207]:\tstep: 3970\tsteps/sec: 1.31165\tloss: 0.78833\tlr:[2.6959118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:48:55,908 [ hooks.py: 207]:\tstep: 3980\tsteps/sec: 1.21067\tloss: 1.01273\tlr:[2.7027025e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:49:06,001 [ hooks.py: 207]:\tstep: 3990\tsteps/sec: 1.00994\tloss: 0.68730\tlr:[2.7094933e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:49:15,241 [ hooks.py: 207]:\tstep: 4000\tsteps/sec: 1.07725\tloss: 0.78724\tlr:[2.716284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:49:15,241 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:49:15,241 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:49:15,242 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:49:15,242 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:49:15,242 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:49:15,242 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:57:14,809 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:14,810 [ trainer.py: 58]:\t{'acc': 0.7389558232931727, 'loss': 0.6419828547499119}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:14,811 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:14,811 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:57:14,811 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:57:14,811 [ trainer.py: 70]:\tacc\t0.7389558232931727\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:57:14,812 [ trainer.py: 70]:\tloss\t0.6419828547499119\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 17:57:14,812 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:14,812 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7389558232931727, 'loss': 0.6419828547499119}} \n", + "old: {'eval': {'acc': 0.7204819277108434, 'loss': 0.7031658165729963}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:14,812 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:14,813 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_3000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:14,813 [monitored_executor.py: 122]:\tsaving step 4000 to ./output/best_model/model_4000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:16,025 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_3000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:16,026 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7389558232931727, 'loss': 0.6419828547499119}} \n", + "old: {'eval': {'acc': 0.7204819277108434, 'loss': 0.7031658165729963}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:16,026 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:16,026 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:17,292 [monitored_executor.py: 122]:\tsaving step 4000 to ./output/model_4000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:20,979 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_1000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:30,222 [ hooks.py: 207]:\tstep: 4010\tsteps/sec: 0.02021\tloss: 0.81292\tlr:[2.7230748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:38,638 [ hooks.py: 207]:\tstep: 4020\tsteps/sec: 1.12929\tloss: 0.60127\tlr:[2.7298656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:50,083 [ hooks.py: 207]:\tstep: 4030\tsteps/sec: 0.89168\tloss: 0.67357\tlr:[2.736656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:57:58,508 [ hooks.py: 207]:\tstep: 4040\tsteps/sec: 1.17824\tloss: 0.84499\tlr:[2.7434467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:58:07,383 [ hooks.py: 207]:\tstep: 4050\tsteps/sec: 1.11200\tloss: 1.30547\tlr:[2.7502376e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:58:14,234 [ hooks.py: 207]:\tstep: 4060\tsteps/sec: 1.45634\tloss: 0.63449\tlr:[2.7570282e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:58:22,145 [ hooks.py: 207]:\tstep: 4070\tsteps/sec: 1.27463\tloss: 0.64581\tlr:[2.763819e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:58:31,256 [ hooks.py: 207]:\tstep: 4080\tsteps/sec: 1.11332\tloss: 0.78673\tlr:[2.7706097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:58:41,369 [ hooks.py: 207]:\tstep: 4090\tsteps/sec: 0.95798\tloss: 0.78595\tlr:[2.7774002e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:58:50,712 [ hooks.py: 207]:\tstep: 4100\tsteps/sec: 1.11816\tloss: 0.73023\tlr:[2.784191e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:58:58,044 [ hooks.py: 207]:\tstep: 4110\tsteps/sec: 1.32994\tloss: 0.74020\tlr:[2.7909817e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:59:09,052 [ hooks.py: 207]:\tstep: 4120\tsteps/sec: 0.90173\tloss: 0.61407\tlr:[2.7977725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:59:19,439 [ hooks.py: 207]:\tstep: 4130\tsteps/sec: 0.96785\tloss: 0.95191\tlr:[2.8045633e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:59:28,174 [ hooks.py: 207]:\tstep: 4140\tsteps/sec: 1.14041\tloss: 0.68095\tlr:[2.811354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:59:35,136 [ hooks.py: 207]:\tstep: 4150\tsteps/sec: 1.46326\tloss: 0.64242\tlr:[2.8181448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:59:44,359 [ hooks.py: 207]:\tstep: 4160\tsteps/sec: 1.05828\tloss: 0.75707\tlr:[2.8249353e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 17:59:52,586 [ hooks.py: 207]:\tstep: 4170\tsteps/sec: 1.22774\tloss: 0.74957\tlr:[2.831726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:00:01,435 [ hooks.py: 207]:\tstep: 4180\tsteps/sec: 1.20874\tloss: 0.63069\tlr:[2.8385168e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:00:09,162 [ hooks.py: 207]:\tstep: 4190\tsteps/sec: 1.19716\tloss: 0.92601\tlr:[2.8453074e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:00:16,335 [ hooks.py: 207]:\tstep: 4200\tsteps/sec: 1.43013\tloss: 0.52847\tlr:[2.8520983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:00:26,270 [ hooks.py: 207]:\tstep: 4210\tsteps/sec: 1.01488\tloss: 0.56884\tlr:[2.8588891e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:00:34,581 [ hooks.py: 207]:\tstep: 4220\tsteps/sec: 1.15893\tloss: 0.74711\tlr:[2.8656796e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 18:00:45,060 [ hooks.py: 207]:\tstep: 4230\tsteps/sec: 0.96756\tloss: 0.92120\tlr:[2.8724702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:00:52,190 [ hooks.py: 207]:\tstep: 4240\tsteps/sec: 1.42643\tloss: 1.07860\tlr:[2.879261e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:01:01,207 [ hooks.py: 207]:\tstep: 4250\tsteps/sec: 1.16854\tloss: 0.57146\tlr:[2.8860517e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:01:11,668 [ hooks.py: 207]:\tstep: 4260\tsteps/sec: 0.92354\tloss: 0.60639\tlr:[2.8928425e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:01:18,407 [ hooks.py: 207]:\tstep: 4270\tsteps/sec: 1.36376\tloss: 0.70603\tlr:[2.8996334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:01:27,039 [ hooks.py: 207]:\tstep: 4280\tsteps/sec: 1.23310\tloss: 0.46914\tlr:[2.9064237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:01:36,318 [ hooks.py: 207]:\tstep: 4290\tsteps/sec: 1.05321\tloss: 0.36282\tlr:[2.9132145e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:01:44,233 [ hooks.py: 207]:\tstep: 4300\tsteps/sec: 1.25486\tloss: 0.79624\tlr:[2.9200053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:01:53,060 [ hooks.py: 207]:\tstep: 4310\tsteps/sec: 1.13080\tloss: 0.41086\tlr:[2.926796e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:02:01,263 [ hooks.py: 207]:\tstep: 4320\tsteps/sec: 1.34050\tloss: 0.65721\tlr:[2.9335868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:02:09,638 [ hooks.py: 207]:\tstep: 4330\tsteps/sec: 1.06078\tloss: 0.36420\tlr:[2.9403775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:02:18,654 [ hooks.py: 207]:\tstep: 4340\tsteps/sec: 1.15604\tloss: 0.51988\tlr:[2.9471683e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:02:27,302 [ hooks.py: 207]:\tstep: 4350\tsteps/sec: 1.17695\tloss: 0.98761\tlr:[2.9539588e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:02:36,017 [ hooks.py: 207]:\tstep: 4360\tsteps/sec: 1.18416\tloss: 0.57713\tlr:[2.9607494e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:02:42,950 [ hooks.py: 207]:\tstep: 4370\tsteps/sec: 1.37675\tloss: 0.44713\tlr:[2.9675402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:02:51,822 [ hooks.py: 207]:\tstep: 4380\tsteps/sec: 1.15020\tloss: 0.72004\tlr:[2.974331e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:03:01,804 [ hooks.py: 207]:\tstep: 4390\tsteps/sec: 1.00745\tloss: 0.46297\tlr:[2.9811217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:03:11,144 [ hooks.py: 207]:\tstep: 4400\tsteps/sec: 1.06236\tloss: 0.87648\tlr:[2.9879126e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:03:20,019 [ hooks.py: 207]:\tstep: 4410\tsteps/sec: 1.09746\tloss: 0.85730\tlr:[2.994703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:03:28,497 [ hooks.py: 207]:\tstep: 4420\tsteps/sec: 1.19017\tloss: 0.78734\tlr:[3.0014937e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:03:37,204 [ hooks.py: 207]:\tstep: 4430\tsteps/sec: 1.16288\tloss: 0.68279\tlr:[3.0082845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:03:44,835 [ hooks.py: 207]:\tstep: 4440\tsteps/sec: 1.23105\tloss: 0.92170\tlr:[3.0150752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:03:53,571 [ hooks.py: 207]:\tstep: 4450\tsteps/sec: 1.20202\tloss: 0.75980\tlr:[3.021866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:04:01,138 [ hooks.py: 207]:\tstep: 4460\tsteps/sec: 1.39882\tloss: 0.80495\tlr:[3.0286568e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:04:09,855 [ hooks.py: 207]:\tstep: 4470\tsteps/sec: 1.08627\tloss: 0.86019\tlr:[3.0354475e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:04:19,081 [ hooks.py: 207]:\tstep: 4480\tsteps/sec: 1.07806\tloss: 0.63740\tlr:[3.042238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:04:29,570 [ hooks.py: 207]:\tstep: 4490\tsteps/sec: 0.97934\tloss: 0.63005\tlr:[3.0490288e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:04:37,294 [ hooks.py: 207]:\tstep: 4500\tsteps/sec: 1.21528\tloss: 0.51669\tlr:[3.0558196e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:04:47,476 [ hooks.py: 207]:\tstep: 4510\tsteps/sec: 1.00460\tloss: 0.49483\tlr:[3.0626103e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:04:58,102 [ hooks.py: 207]:\tstep: 4520\tsteps/sec: 0.94162\tloss: 0.31339\tlr:[3.069401e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:05:06,991 [ hooks.py: 207]:\tstep: 4530\tsteps/sec: 1.09683\tloss: 0.70664\tlr:[3.076192e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:05:15,902 [ hooks.py: 207]:\tstep: 4540\tsteps/sec: 1.18267\tloss: 0.60163\tlr:[3.0829822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:05:26,348 [ hooks.py: 207]:\tstep: 4550\tsteps/sec: 0.95806\tloss: 1.12114\tlr:[3.089773e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:05:33,368 [ hooks.py: 207]:\tstep: 4560\tsteps/sec: 1.36337\tloss: 1.02815\tlr:[3.096564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:05:41,709 [ hooks.py: 207]:\tstep: 4570\tsteps/sec: 1.25949\tloss: 0.54789\tlr:[3.1033545e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:05:50,331 [ hooks.py: 207]:\tstep: 4580\tsteps/sec: 1.15091\tloss: 0.62492\tlr:[3.1101452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:05:57,039 [ hooks.py: 207]:\tstep: 4590\tsteps/sec: 1.45855\tloss: 0.44966\tlr:[3.116936e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:06:04,744 [ hooks.py: 207]:\tstep: 4600\tsteps/sec: 1.28702\tloss: 0.60327\tlr:[3.1237265e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:06:13,502 [ hooks.py: 207]:\tstep: 4610\tsteps/sec: 1.13329\tloss: 0.71050\tlr:[3.130517e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:06:21,430 [ hooks.py: 207]:\tstep: 4620\tsteps/sec: 1.30684\tloss: 0.44577\tlr:[3.137308e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:06:28,705 [ hooks.py: 207]:\tstep: 4630\tsteps/sec: 1.27151\tloss: 0.71178\tlr:[3.1440988e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:06:36,785 [ hooks.py: 207]:\tstep: 4640\tsteps/sec: 1.33390\tloss: 0.69167\tlr:[3.1508895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:06:45,916 [ hooks.py: 207]:\tstep: 4650\tsteps/sec: 1.09123\tloss: 0.95510\tlr:[3.15768e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:06:52,674 [ hooks.py: 207]:\tstep: 4660\tsteps/sec: 1.42249\tloss: 0.61825\tlr:[3.164471e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:02,113 [ hooks.py: 207]:\tstep: 4670\tsteps/sec: 1.13756\tloss: 0.46567\tlr:[3.1712614e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:10,234 [ hooks.py: 207]:\tstep: 4680\tsteps/sec: 1.17661\tloss: 0.44051\tlr:[3.178052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:17,464 [ hooks.py: 207]:\tstep: 4690\tsteps/sec: 1.36702\tloss: 0.64549\tlr:[3.184843e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:24,787 [ hooks.py: 207]:\tstep: 4700\tsteps/sec: 1.33883\tloss: 0.60060\tlr:[3.1916337e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:34,435 [ hooks.py: 207]:\tstep: 4710\tsteps/sec: 1.03599\tloss: 0.82899\tlr:[3.1984244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:44,260 [ hooks.py: 207]:\tstep: 4720\tsteps/sec: 1.02461\tloss: 0.63993\tlr:[3.2052154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:51,786 [ hooks.py: 207]:\tstep: 4730\tsteps/sec: 1.38247\tloss: 0.91002\tlr:[3.2120057e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:07:59,814 [ hooks.py: 207]:\tstep: 4740\tsteps/sec: 1.20857\tloss: 0.78205\tlr:[3.2187963e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:08:08,666 [ hooks.py: 207]:\tstep: 4750\tsteps/sec: 1.12377\tloss: 0.53197\tlr:[3.2255874e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:08:15,571 [ hooks.py: 207]:\tstep: 4760\tsteps/sec: 1.46895\tloss: 0.32240\tlr:[3.232378e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:08:23,806 [ hooks.py: 207]:\tstep: 4770\tsteps/sec: 1.22754\tloss: 0.72660\tlr:[3.2391687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:08:32,451 [ hooks.py: 207]:\tstep: 4780\tsteps/sec: 1.13741\tloss: 0.59761\tlr:[3.2459597e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:08:39,822 [ hooks.py: 207]:\tstep: 4790\tsteps/sec: 1.32637\tloss: 0.54572\tlr:[3.25275e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:08:48,964 [ hooks.py: 207]:\tstep: 4800\tsteps/sec: 1.10650\tloss: 0.70860\tlr:[3.2595406e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:08:57,867 [ hooks.py: 207]:\tstep: 4810\tsteps/sec: 1.11995\tloss: 0.72376\tlr:[3.2663316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:09:07,117 [ hooks.py: 207]:\tstep: 4820\tsteps/sec: 1.11241\tloss: 0.69924\tlr:[3.2731223e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:09:15,121 [ hooks.py: 207]:\tstep: 4830\tsteps/sec: 1.21655\tloss: 0.43033\tlr:[3.279913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:09:22,057 [ hooks.py: 207]:\tstep: 4840\tsteps/sec: 1.48915\tloss: 0.84541\tlr:[3.2867036e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:09:31,514 [ hooks.py: 207]:\tstep: 4850\tsteps/sec: 1.05886\tloss: 0.36816\tlr:[3.2934946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:09:39,519 [ hooks.py: 207]:\tstep: 4860\tsteps/sec: 1.18112\tloss: 0.68076\tlr:[3.300285e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:09:47,905 [ hooks.py: 207]:\tstep: 4870\tsteps/sec: 1.21659\tloss: 0.51373\tlr:[3.307076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:09:56,787 [ hooks.py: 207]:\tstep: 4880\tsteps/sec: 1.16367\tloss: 0.45015\tlr:[3.3138665e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 18:10:03,557 [ hooks.py: 207]:\tstep: 4890\tsteps/sec: 1.41300\tloss: 0.71721\tlr:[3.3206572e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:10:12,282 [ hooks.py: 207]:\tstep: 4900\tsteps/sec: 1.16385\tloss: 0.53736\tlr:[3.327448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:10:22,073 [ hooks.py: 207]:\tstep: 4910\tsteps/sec: 1.04066\tloss: 0.53242\tlr:[3.334239e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:10:29,025 [ hooks.py: 207]:\tstep: 4920\tsteps/sec: 1.38541\tloss: 0.47341\tlr:[3.341029e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:10:38,416 [ hooks.py: 207]:\tstep: 4930\tsteps/sec: 1.05937\tloss: 0.80169\tlr:[3.3478198e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:10:47,385 [ hooks.py: 207]:\tstep: 4940\tsteps/sec: 1.13111\tloss: 0.77217\tlr:[3.3546108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:10:55,582 [ hooks.py: 207]:\tstep: 4950\tsteps/sec: 1.22640\tloss: 1.21280\tlr:[3.3614015e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:03,977 [ hooks.py: 207]:\tstep: 4960\tsteps/sec: 1.18773\tloss: 0.52053\tlr:[3.368192e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:10,818 [ hooks.py: 207]:\tstep: 4970\tsteps/sec: 1.49774\tloss: 0.49776\tlr:[3.374983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:17,598 [ hooks.py: 207]:\tstep: 4980\tsteps/sec: 1.35230\tloss: 0.72446\tlr:[3.3817734e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:26,661 [ hooks.py: 207]:\tstep: 4990\tsteps/sec: 1.18643\tloss: 0.68295\tlr:[3.388564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:36,226 [ hooks.py: 207]:\tstep: 5000\tsteps/sec: 1.06195\tloss: 0.48216\tlr:[3.395355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:36,230 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:36,230 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:36,230 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:11:36,230 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:36,230 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:11:36,230 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:19:29,313 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,314 [ trainer.py: 58]:\t{'acc': 0.7333333333333333, 'loss': 0.6622268511698797}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,315 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,315 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:19:29,316 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:19:29,316 [ trainer.py: 70]:\tacc\t0.7333333333333333\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:19:29,316 [ trainer.py: 70]:\tloss\t0.6622268511698797\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:19:29,316 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,317 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7333333333333333, 'loss': 0.6622268511698797}} \n", + "old: {'eval': {'acc': 0.7389558232931727, 'loss': 0.6419828547499119}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,317 [ exporter.py: 64]:\t[Best Exporter]: skip step 5000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,317 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7333333333333333, 'loss': 0.6622268511698797}} \n", + "old: {'eval': {'acc': 0.7389558232931727, 'loss': 0.6419828547499119}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,317 [ exporter.py: 104]:\t[Best Exporter]: skip step 5000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:29,318 [monitored_executor.py: 122]:\tsaving step 5000 to ./output/model_5000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:33,076 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_2000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:41,515 [ hooks.py: 207]:\tstep: 5010\tsteps/sec: 0.02060\tloss: 0.83475\tlr:[3.4021457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:49,004 [ hooks.py: 207]:\tstep: 5020\tsteps/sec: 1.30142\tloss: 0.39867\tlr:[3.4089364e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:19:57,641 [ hooks.py: 207]:\tstep: 5030\tsteps/sec: 1.15380\tloss: 0.85348\tlr:[3.4157274e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:20:06,512 [ hooks.py: 207]:\tstep: 5040\tsteps/sec: 1.12022\tloss: 0.60167\tlr:[3.422518e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:20:15,123 [ hooks.py: 207]:\tstep: 5050\tsteps/sec: 1.17152\tloss: 0.67409\tlr:[3.4293083e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:20:24,791 [ hooks.py: 207]:\tstep: 5060\tsteps/sec: 1.02821\tloss: 1.00779\tlr:[3.4360994e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:20:33,385 [ hooks.py: 207]:\tstep: 5070\tsteps/sec: 1.15758\tloss: 0.63990\tlr:[3.44289e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:20:42,599 [ hooks.py: 207]:\tstep: 5080\tsteps/sec: 1.09042\tloss: 0.57553\tlr:[3.4496807e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:20:51,109 [ hooks.py: 207]:\tstep: 5090\tsteps/sec: 1.16508\tloss: 1.00275\tlr:[3.4564713e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:21:02,789 [ hooks.py: 207]:\tstep: 5100\tsteps/sec: 0.91095\tloss: 0.77810\tlr:[3.4632623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:21:11,991 [ hooks.py: 207]:\tstep: 5110\tsteps/sec: 1.04553\tloss: 0.68650\tlr:[3.4700526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:21:21,107 [ hooks.py: 207]:\tstep: 5120\tsteps/sec: 1.08844\tloss: 0.62421\tlr:[3.4768433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:21:29,792 [ hooks.py: 207]:\tstep: 5130\tsteps/sec: 1.13689\tloss: 0.65031\tlr:[3.4836343e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:21:39,166 [ hooks.py: 207]:\tstep: 5140\tsteps/sec: 1.05726\tloss: 0.63599\tlr:[3.490425e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:21:50,126 [ hooks.py: 207]:\tstep: 5150\tsteps/sec: 0.91445\tloss: 0.55117\tlr:[3.4972156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:21:57,426 [ hooks.py: 207]:\tstep: 5160\tsteps/sec: 1.40342\tloss: 0.38307\tlr:[3.5040066e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:22:05,906 [ hooks.py: 207]:\tstep: 5170\tsteps/sec: 1.16075\tloss: 0.86935\tlr:[3.510797e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:22:15,983 [ hooks.py: 207]:\tstep: 5180\tsteps/sec: 1.00447\tloss: 0.70414\tlr:[3.5175875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:22:24,437 [ hooks.py: 207]:\tstep: 5190\tsteps/sec: 1.15971\tloss: 1.02249\tlr:[3.5243786e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:22:31,971 [ hooks.py: 207]:\tstep: 5200\tsteps/sec: 1.30116\tloss: 0.65503\tlr:[3.5311692e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:22:39,218 [ hooks.py: 207]:\tstep: 5210\tsteps/sec: 1.35731\tloss: 0.57660\tlr:[3.53796e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:22:48,508 [ hooks.py: 207]:\tstep: 5220\tsteps/sec: 1.10632\tloss: 0.83289\tlr:[3.544751e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:22:55,383 [ hooks.py: 207]:\tstep: 5230\tsteps/sec: 1.45287\tloss: 0.59587\tlr:[3.5515415e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:23:04,369 [ hooks.py: 207]:\tstep: 5240\tsteps/sec: 1.10505\tloss: 0.77432\tlr:[3.5583318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:23:14,612 [ hooks.py: 207]:\tstep: 5250\tsteps/sec: 0.95032\tloss: 0.53692\tlr:[3.565123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:23:23,811 [ hooks.py: 207]:\tstep: 5260\tsteps/sec: 1.12520\tloss: 0.58139\tlr:[3.5719135e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:23:31,291 [ hooks.py: 207]:\tstep: 5270\tsteps/sec: 1.40227\tloss: 0.76323\tlr:[3.578704e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:23:39,396 [ hooks.py: 207]:\tstep: 5280\tsteps/sec: 1.14667\tloss: 0.47634\tlr:[3.585495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:23:51,044 [ hooks.py: 207]:\tstep: 5290\tsteps/sec: 0.89145\tloss: 0.53982\tlr:[3.5922858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:23:59,252 [ hooks.py: 207]:\tstep: 5300\tsteps/sec: 1.17816\tloss: 0.56505\tlr:[3.599076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:24:09,032 [ hooks.py: 207]:\tstep: 5310\tsteps/sec: 1.00629\tloss: 0.88092\tlr:[3.605867e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:24:19,838 [ hooks.py: 207]:\tstep: 5320\tsteps/sec: 0.95201\tloss: 0.43333\tlr:[3.6126577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:24:27,728 [ hooks.py: 207]:\tstep: 5330\tsteps/sec: 1.23981\tloss: 0.69220\tlr:[3.6194484e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:24:35,326 [ hooks.py: 207]:\tstep: 5340\tsteps/sec: 1.35449\tloss: 0.52218\tlr:[3.626239e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:24:42,012 [ hooks.py: 207]:\tstep: 5350\tsteps/sec: 1.54554\tloss: 0.48494\tlr:[3.63303e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 18:24:48,764 [ hooks.py: 207]:\tstep: 5360\tsteps/sec: 1.37174\tloss: 0.35994\tlr:[3.6398204e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:24:57,690 [ hooks.py: 207]:\tstep: 5370\tsteps/sec: 1.17980\tloss: 0.75200\tlr:[3.646611e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:25:04,679 [ hooks.py: 207]:\tstep: 5380\tsteps/sec: 1.41194\tloss: 0.47901\tlr:[3.653402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:25:12,966 [ hooks.py: 207]:\tstep: 5390\tsteps/sec: 1.20947\tloss: 0.44383\tlr:[3.6601927e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:25:19,583 [ hooks.py: 207]:\tstep: 5400\tsteps/sec: 1.50627\tloss: 0.48547\tlr:[3.6669833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:25:27,704 [ hooks.py: 207]:\tstep: 5410\tsteps/sec: 1.24307\tloss: 0.49140\tlr:[3.6737743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:25:36,709 [ hooks.py: 207]:\tstep: 5420\tsteps/sec: 1.15264\tloss: 0.92516\tlr:[3.680565e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:25:44,617 [ hooks.py: 207]:\tstep: 5430\tsteps/sec: 1.19391\tloss: 0.82673\tlr:[3.6873553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:25:56,427 [ hooks.py: 207]:\tstep: 5440\tsteps/sec: 0.86330\tloss: 0.69611\tlr:[3.6941463e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:26:05,935 [ hooks.py: 207]:\tstep: 5450\tsteps/sec: 1.01156\tloss: 0.34817\tlr:[3.700937e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:26:18,674 [ hooks.py: 207]:\tstep: 5460\tsteps/sec: 0.79979\tloss: 0.55611\tlr:[3.7077276e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:26:27,235 [ hooks.py: 207]:\tstep: 5470\tsteps/sec: 1.18958\tloss: 0.86334\tlr:[3.7145186e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:26:37,445 [ hooks.py: 207]:\tstep: 5480\tsteps/sec: 0.99639\tloss: 0.54224\tlr:[3.7213093e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:26:45,751 [ hooks.py: 207]:\tstep: 5490\tsteps/sec: 1.12423\tloss: 0.84588\tlr:[3.7280995e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:26:54,282 [ hooks.py: 207]:\tstep: 5500\tsteps/sec: 1.20461\tloss: 0.54357\tlr:[3.7348906e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:03,045 [ hooks.py: 207]:\tstep: 5510\tsteps/sec: 1.19976\tloss: 1.19404\tlr:[3.7416812e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:12,711 [ hooks.py: 207]:\tstep: 5520\tsteps/sec: 1.00902\tloss: 0.63090\tlr:[3.748472e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:19,623 [ hooks.py: 207]:\tstep: 5530\tsteps/sec: 1.40013\tloss: 0.57662\tlr:[3.755263e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:27,875 [ hooks.py: 207]:\tstep: 5540\tsteps/sec: 1.23270\tloss: 0.72185\tlr:[3.7620535e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:34,906 [ hooks.py: 207]:\tstep: 5550\tsteps/sec: 1.40609\tloss: 0.45619\tlr:[3.7688438e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:41,865 [ hooks.py: 207]:\tstep: 5560\tsteps/sec: 1.51818\tloss: 0.99283\tlr:[3.775635e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:48,560 [ hooks.py: 207]:\tstep: 5570\tsteps/sec: 1.34454\tloss: 0.90589\tlr:[3.7824255e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:27:57,043 [ hooks.py: 207]:\tstep: 5580\tsteps/sec: 1.28757\tloss: 0.98076\tlr:[3.789216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:28:03,987 [ hooks.py: 207]:\tstep: 5590\tsteps/sec: 1.35403\tloss: 0.54466\tlr:[3.7960068e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:28:11,994 [ hooks.py: 207]:\tstep: 5600\tsteps/sec: 1.24533\tloss: 0.91743\tlr:[3.8027978e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:28:20,115 [ hooks.py: 207]:\tstep: 5610\tsteps/sec: 1.24239\tloss: 0.54134\tlr:[3.8095885e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:28:28,475 [ hooks.py: 207]:\tstep: 5620\tsteps/sec: 1.15069\tloss: 0.63579\tlr:[3.8163787e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:28:37,117 [ hooks.py: 207]:\tstep: 5630\tsteps/sec: 1.26160\tloss: 0.95717\tlr:[3.8231698e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:28:46,677 [ hooks.py: 207]:\tstep: 5640\tsteps/sec: 1.02144\tloss: 0.49705\tlr:[3.8299604e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:28:56,416 [ hooks.py: 207]:\tstep: 5650\tsteps/sec: 1.03723\tloss: 0.77437\tlr:[3.836751e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:29:04,846 [ hooks.py: 207]:\tstep: 5660\tsteps/sec: 1.15086\tloss: 0.75966\tlr:[3.843542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:29:11,592 [ hooks.py: 207]:\tstep: 5670\tsteps/sec: 1.46438\tloss: 0.44874\tlr:[3.8503327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:29:19,942 [ hooks.py: 207]:\tstep: 5680\tsteps/sec: 1.21422\tloss: 0.57524\tlr:[3.857123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:29:28,851 [ hooks.py: 207]:\tstep: 5690\tsteps/sec: 1.10642\tloss: 0.85421\tlr:[3.863914e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:29:36,660 [ hooks.py: 207]:\tstep: 5700\tsteps/sec: 1.30797\tloss: 0.62772\tlr:[3.8707047e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:29:44,338 [ hooks.py: 207]:\tstep: 5710\tsteps/sec: 1.30067\tloss: 0.92763\tlr:[3.8774953e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:29:54,857 [ hooks.py: 207]:\tstep: 5720\tsteps/sec: 0.90856\tloss: 0.37506\tlr:[3.8842863e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:03,537 [ hooks.py: 207]:\tstep: 5730\tsteps/sec: 1.21936\tloss: 0.95899\tlr:[3.891077e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:11,256 [ hooks.py: 207]:\tstep: 5740\tsteps/sec: 1.33484\tloss: 0.46097\tlr:[3.8978673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:18,331 [ hooks.py: 207]:\tstep: 5750\tsteps/sec: 1.35795\tloss: 0.59971\tlr:[3.9046583e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:27,446 [ hooks.py: 207]:\tstep: 5760\tsteps/sec: 1.12768\tloss: 0.83315\tlr:[3.911449e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:33,904 [ hooks.py: 207]:\tstep: 5770\tsteps/sec: 1.42769\tloss: 0.40804\tlr:[3.9182396e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:42,895 [ hooks.py: 207]:\tstep: 5780\tsteps/sec: 1.16826\tloss: 0.81320\tlr:[3.9250303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:49,349 [ hooks.py: 207]:\tstep: 5790\tsteps/sec: 1.44261\tloss: 0.77200\tlr:[3.9318213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:30:58,499 [ hooks.py: 207]:\tstep: 5800\tsteps/sec: 1.15213\tloss: 0.57038\tlr:[3.938612e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:31:10,544 [ hooks.py: 207]:\tstep: 5810\tsteps/sec: 0.80795\tloss: 0.66758\tlr:[3.9454026e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:31:19,430 [ hooks.py: 207]:\tstep: 5820\tsteps/sec: 1.16513\tloss: 0.90985\tlr:[3.9521932e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:31:26,562 [ hooks.py: 207]:\tstep: 5830\tsteps/sec: 1.43861\tloss: 0.64213\tlr:[3.958984e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:31:36,229 [ hooks.py: 207]:\tstep: 5840\tsteps/sec: 1.04103\tloss: 0.52080\tlr:[3.9657745e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:31:45,244 [ hooks.py: 207]:\tstep: 5850\tsteps/sec: 1.07577\tloss: 0.86391\tlr:[3.9725655e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:31:52,996 [ hooks.py: 207]:\tstep: 5860\tsteps/sec: 1.23970\tloss: 0.48245\tlr:[3.9793562e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:32:02,507 [ hooks.py: 207]:\tstep: 5870\tsteps/sec: 1.16543\tloss: 0.63327\tlr:[3.9861465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:32:11,404 [ hooks.py: 207]:\tstep: 5880\tsteps/sec: 1.03781\tloss: 0.62123\tlr:[3.9929375e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:32:19,573 [ hooks.py: 207]:\tstep: 5890\tsteps/sec: 1.24153\tloss: 0.53297\tlr:[3.999728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:32:29,243 [ hooks.py: 207]:\tstep: 5900\tsteps/sec: 1.02815\tloss: 0.74031\tlr:[4.0065188e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:32:35,389 [ hooks.py: 207]:\tstep: 5910\tsteps/sec: 1.55565\tloss: 1.12770\tlr:[4.0133098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:32:44,859 [ hooks.py: 207]:\tstep: 5920\tsteps/sec: 1.09001\tloss: 0.45264\tlr:[4.0201005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:32:54,056 [ hooks.py: 207]:\tstep: 5930\tsteps/sec: 1.08573\tloss: 0.36892\tlr:[4.0268907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:02,196 [ hooks.py: 207]:\tstep: 5940\tsteps/sec: 1.33271\tloss: 0.71085\tlr:[4.0336818e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:09,505 [ hooks.py: 207]:\tstep: 5950\tsteps/sec: 1.30195\tloss: 0.74277\tlr:[4.0404724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:18,715 [ hooks.py: 207]:\tstep: 5960\tsteps/sec: 1.02237\tloss: 0.81743\tlr:[4.047263e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:28,187 [ hooks.py: 207]:\tstep: 5970\tsteps/sec: 1.09497\tloss: 0.86931\tlr:[4.054054e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:37,051 [ hooks.py: 207]:\tstep: 5980\tsteps/sec: 1.19558\tloss: 0.75322\tlr:[4.0608447e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:43,933 [ hooks.py: 207]:\tstep: 5990\tsteps/sec: 1.34224\tloss: 0.47086\tlr:[4.0676354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:51,380 [ hooks.py: 207]:\tstep: 6000\tsteps/sec: 1.39134\tloss: 0.93041\tlr:[4.074426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:51,384 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:51,384 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:51,384 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:33:51,384 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:51,384 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:33:51,384 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-18 18:41:44,051 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:44,053 [ trainer.py: 58]:\t{'acc': 0.744578313253012, 'loss': 0.6313066453887866}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:44,053 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:44,054 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:41:44,054 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:41:44,054 [ trainer.py: 70]:\tacc\t0.744578313253012\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:41:44,054 [ trainer.py: 70]:\tloss\t0.6313066453887866\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:41:44,054 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:44,054 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.744578313253012, 'loss': 0.6313066453887866}} \n", + "old: {'eval': {'acc': 0.7389558232931727, 'loss': 0.6419828547499119}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:44,055 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:44,056 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_4000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:44,056 [monitored_executor.py: 122]:\tsaving step 6000 to ./output/best_model/model_6000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:45,182 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_4000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:45,183 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.744578313253012, 'loss': 0.6313066453887866}} \n", + "old: {'eval': {'acc': 0.7389558232931727, 'loss': 0.6419828547499119}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:45,183 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:45,183 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:46,634 [monitored_executor.py: 122]:\tsaving step 6000 to ./output/model_6000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:50,343 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_3000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:41:58,844 [ hooks.py: 207]:\tstep: 6010\tsteps/sec: 0.02051\tloss: 0.75235\tlr:[4.0812167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:42:06,391 [ hooks.py: 207]:\tstep: 6020\tsteps/sec: 1.28525\tloss: 0.30485\tlr:[4.0880073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:42:15,165 [ hooks.py: 207]:\tstep: 6030\tsteps/sec: 1.17361\tloss: 0.51817\tlr:[4.094798e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:42:22,067 [ hooks.py: 207]:\tstep: 6040\tsteps/sec: 1.42043\tloss: 0.49163\tlr:[4.101589e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:42:30,717 [ hooks.py: 207]:\tstep: 6050\tsteps/sec: 1.17052\tloss: 0.69144\tlr:[4.1083797e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:42:39,810 [ hooks.py: 207]:\tstep: 6060\tsteps/sec: 1.10007\tloss: 0.49624\tlr:[4.1151703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:42:48,935 [ hooks.py: 207]:\tstep: 6070\tsteps/sec: 1.09054\tloss: 0.44011\tlr:[4.121961e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:42:57,064 [ hooks.py: 207]:\tstep: 6080\tsteps/sec: 1.26349\tloss: 0.73851\tlr:[4.1287516e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:43:05,203 [ hooks.py: 207]:\tstep: 6090\tsteps/sec: 1.21774\tloss: 0.56757\tlr:[4.1355423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:43:14,618 [ hooks.py: 207]:\tstep: 6100\tsteps/sec: 1.05738\tloss: 0.85838\tlr:[4.1423333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:43:22,982 [ hooks.py: 207]:\tstep: 6110\tsteps/sec: 1.15040\tloss: 0.43440\tlr:[4.149124e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:43:32,901 [ hooks.py: 207]:\tstep: 6120\tsteps/sec: 1.06752\tloss: 0.67968\tlr:[4.1559142e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:43:41,657 [ hooks.py: 207]:\tstep: 6130\tsteps/sec: 1.13778\tloss: 0.97393\tlr:[4.1627052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:43:48,469 [ hooks.py: 207]:\tstep: 6140\tsteps/sec: 1.32035\tloss: 0.42235\tlr:[4.169496e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:43:55,888 [ hooks.py: 207]:\tstep: 6150\tsteps/sec: 1.45270\tloss: 0.38944\tlr:[4.1762865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:44:03,123 [ hooks.py: 207]:\tstep: 6160\tsteps/sec: 1.47528\tloss: 0.99027\tlr:[4.1830775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:44:13,528 [ hooks.py: 207]:\tstep: 6170\tsteps/sec: 0.91514\tloss: 0.40032\tlr:[4.1898682e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:44:22,323 [ hooks.py: 207]:\tstep: 6180\tsteps/sec: 1.18024\tloss: 0.39846\tlr:[4.196659e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:44:29,909 [ hooks.py: 207]:\tstep: 6190\tsteps/sec: 1.25494\tloss: 0.54797\tlr:[4.2034495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:44:38,221 [ hooks.py: 207]:\tstep: 6200\tsteps/sec: 1.21731\tloss: 0.43116\tlr:[4.21024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:44:46,773 [ hooks.py: 207]:\tstep: 6210\tsteps/sec: 1.20398\tloss: 0.60671\tlr:[4.2170308e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:44:54,792 [ hooks.py: 207]:\tstep: 6220\tsteps/sec: 1.20609\tloss: 1.12245\tlr:[4.2238218e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:45:04,148 [ hooks.py: 207]:\tstep: 6230\tsteps/sec: 1.05375\tloss: 0.98207\tlr:[4.2306125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:45:13,469 [ hooks.py: 207]:\tstep: 6240\tsteps/sec: 1.09229\tloss: 0.74313\tlr:[4.237403e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:45:21,599 [ hooks.py: 207]:\tstep: 6250\tsteps/sec: 1.22099\tloss: 0.64488\tlr:[4.2441938e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:45:33,937 [ hooks.py: 207]:\tstep: 6260\tsteps/sec: 0.81023\tloss: 0.82938\tlr:[4.2509844e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:45:43,714 [ hooks.py: 207]:\tstep: 6270\tsteps/sec: 1.03353\tloss: 0.68813\tlr:[4.257775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:45:53,288 [ hooks.py: 207]:\tstep: 6280\tsteps/sec: 1.04550\tloss: 0.76452\tlr:[4.2645657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:45:59,773 [ hooks.py: 207]:\tstep: 6290\tsteps/sec: 1.47887\tloss: 0.94416\tlr:[4.2713567e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:46:09,170 [ hooks.py: 207]:\tstep: 6300\tsteps/sec: 1.08899\tloss: 0.53050\tlr:[4.2781474e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:46:17,313 [ hooks.py: 207]:\tstep: 6310\tsteps/sec: 1.26072\tloss: 0.43387\tlr:[4.284938e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:46:25,720 [ hooks.py: 207]:\tstep: 6320\tsteps/sec: 1.16631\tloss: 0.93158\tlr:[4.2917287e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:46:32,453 [ hooks.py: 207]:\tstep: 6330\tsteps/sec: 1.46662\tloss: 0.55162\tlr:[4.2985193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:46:41,034 [ hooks.py: 207]:\tstep: 6340\tsteps/sec: 1.18430\tloss: 1.21524\tlr:[4.30531e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:46:48,914 [ hooks.py: 207]:\tstep: 6350\tsteps/sec: 1.24915\tloss: 1.06472\tlr:[4.312101e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:46:57,340 [ hooks.py: 207]:\tstep: 6360\tsteps/sec: 1.23096\tloss: 0.77200\tlr:[4.3188917e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:47:06,784 [ hooks.py: 207]:\tstep: 6370\tsteps/sec: 1.04982\tloss: 0.74921\tlr:[4.3256823e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:47:14,919 [ hooks.py: 207]:\tstep: 6380\tsteps/sec: 1.19037\tloss: 0.23453\tlr:[4.332473e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:47:23,462 [ hooks.py: 207]:\tstep: 6390\tsteps/sec: 1.13967\tloss: 0.89921\tlr:[4.3392636e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:47:33,096 [ hooks.py: 207]:\tstep: 6400\tsteps/sec: 1.04205\tloss: 0.52321\tlr:[4.3460543e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:47:44,119 [ hooks.py: 207]:\tstep: 6410\tsteps/sec: 0.92175\tloss: 0.87750\tlr:[4.3528453e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:47:53,244 [ hooks.py: 207]:\tstep: 6420\tsteps/sec: 1.07623\tloss: 0.74365\tlr:[4.359636e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:48:01,773 [ hooks.py: 207]:\tstep: 6430\tsteps/sec: 1.25678\tloss: 0.62018\tlr:[4.3664266e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:48:11,315 [ hooks.py: 207]:\tstep: 6440\tsteps/sec: 1.03768\tloss: 0.27540\tlr:[4.3732172e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:48:18,501 [ hooks.py: 207]:\tstep: 6450\tsteps/sec: 1.36382\tloss: 0.45136\tlr:[4.380008e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:48:24,565 [ hooks.py: 207]:\tstep: 6460\tsteps/sec: 1.56337\tloss: 0.83049\tlr:[4.3867985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:48:32,892 [ hooks.py: 207]:\tstep: 6470\tsteps/sec: 1.27681\tloss: 0.76698\tlr:[4.3935896e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:48:44,076 [ hooks.py: 207]:\tstep: 6480\tsteps/sec: 0.87843\tloss: 0.50343\tlr:[4.4003802e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 18:48:50,828 [ hooks.py: 207]:\tstep: 6490\tsteps/sec: 1.48612\tloss: 0.40847\tlr:[4.407171e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:48:59,805 [ hooks.py: 207]:\tstep: 6500\tsteps/sec: 1.11970\tloss: 0.53601\tlr:[4.4139615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:49:08,858 [ hooks.py: 207]:\tstep: 6510\tsteps/sec: 1.09429\tloss: 0.82867\tlr:[4.420752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:49:17,209 [ hooks.py: 207]:\tstep: 6520\tsteps/sec: 1.23899\tloss: 0.80619\tlr:[4.4275428e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:49:24,678 [ hooks.py: 207]:\tstep: 6530\tsteps/sec: 1.28842\tloss: 1.05485\tlr:[4.4343335e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:49:33,319 [ hooks.py: 207]:\tstep: 6540\tsteps/sec: 1.17691\tloss: 0.75168\tlr:[4.4411245e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:49:42,739 [ hooks.py: 207]:\tstep: 6550\tsteps/sec: 1.06822\tloss: 0.67501\tlr:[4.447915e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:49:51,959 [ hooks.py: 207]:\tstep: 6560\tsteps/sec: 1.04659\tloss: 0.42966\tlr:[4.4547058e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:50:02,259 [ hooks.py: 207]:\tstep: 6570\tsteps/sec: 1.04977\tloss: 0.59474\tlr:[4.4614964e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:50:09,315 [ hooks.py: 207]:\tstep: 6580\tsteps/sec: 1.28698\tloss: 0.81552\tlr:[4.468287e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:50:16,769 [ hooks.py: 207]:\tstep: 6590\tsteps/sec: 1.39705\tloss: 0.62374\tlr:[4.4750777e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:50:24,821 [ hooks.py: 207]:\tstep: 6600\tsteps/sec: 1.20294\tloss: 0.87352\tlr:[4.4818687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:50:33,282 [ hooks.py: 207]:\tstep: 6610\tsteps/sec: 1.21273\tloss: 0.84901\tlr:[4.4886594e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:50:42,025 [ hooks.py: 207]:\tstep: 6620\tsteps/sec: 1.15789\tloss: 0.72417\tlr:[4.49545e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:50:51,152 [ hooks.py: 207]:\tstep: 6630\tsteps/sec: 1.04389\tloss: 0.92393\tlr:[4.5022407e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:02,071 [ hooks.py: 207]:\tstep: 6640\tsteps/sec: 0.98020\tloss: 0.70982\tlr:[4.5090314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:09,175 [ hooks.py: 207]:\tstep: 6650\tsteps/sec: 1.25206\tloss: 0.70588\tlr:[4.515822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:18,720 [ hooks.py: 207]:\tstep: 6660\tsteps/sec: 1.04264\tloss: 0.82352\tlr:[4.522613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:27,249 [ hooks.py: 207]:\tstep: 6670\tsteps/sec: 1.26471\tloss: 0.83521\tlr:[4.5294037e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:34,490 [ hooks.py: 207]:\tstep: 6680\tsteps/sec: 1.33720\tloss: 0.93666\tlr:[4.5361943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:41,758 [ hooks.py: 207]:\tstep: 6690\tsteps/sec: 1.41371\tloss: 0.82614\tlr:[4.542985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:50,080 [ hooks.py: 207]:\tstep: 6700\tsteps/sec: 1.18275\tloss: 0.76069\tlr:[4.5497756e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:51:57,087 [ hooks.py: 207]:\tstep: 6710\tsteps/sec: 1.39896\tloss: 0.56423\tlr:[4.5565663e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:52:07,504 [ hooks.py: 207]:\tstep: 6720\tsteps/sec: 0.99194\tloss: 0.71052\tlr:[4.5633573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:52:16,532 [ hooks.py: 207]:\tstep: 6730\tsteps/sec: 1.09208\tloss: 0.63558\tlr:[4.570148e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:52:25,436 [ hooks.py: 207]:\tstep: 6740\tsteps/sec: 1.09720\tloss: 0.41503\tlr:[4.5769386e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:52:31,973 [ hooks.py: 207]:\tstep: 6750\tsteps/sec: 1.59503\tloss: 0.67409\tlr:[4.5837292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:52:41,106 [ hooks.py: 207]:\tstep: 6760\tsteps/sec: 1.03925\tloss: 0.85700\tlr:[4.59052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:52:50,786 [ hooks.py: 207]:\tstep: 6770\tsteps/sec: 1.06722\tloss: 0.33434\tlr:[4.5973105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:52:58,562 [ hooks.py: 207]:\tstep: 6780\tsteps/sec: 1.29623\tloss: 0.79300\tlr:[4.6041012e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:53:07,041 [ hooks.py: 207]:\tstep: 6790\tsteps/sec: 1.20345\tloss: 0.29918\tlr:[4.6108922e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:53:16,205 [ hooks.py: 207]:\tstep: 6800\tsteps/sec: 1.07523\tloss: 0.53430\tlr:[4.617683e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:53:22,738 [ hooks.py: 207]:\tstep: 6810\tsteps/sec: 1.47952\tloss: 0.58237\tlr:[4.6244735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:53:30,811 [ hooks.py: 207]:\tstep: 6820\tsteps/sec: 1.26462\tloss: 0.63125\tlr:[4.631264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:53:38,677 [ hooks.py: 207]:\tstep: 6830\tsteps/sec: 1.21423\tloss: 0.67619\tlr:[4.6380548e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:53:47,944 [ hooks.py: 207]:\tstep: 6840\tsteps/sec: 1.14170\tloss: 0.56216\tlr:[4.6448455e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:53:55,678 [ hooks.py: 207]:\tstep: 6850\tsteps/sec: 1.27172\tloss: 0.64965\tlr:[4.6516365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:54:03,961 [ hooks.py: 207]:\tstep: 6860\tsteps/sec: 1.14407\tloss: 0.82211\tlr:[4.658427e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:54:12,552 [ hooks.py: 207]:\tstep: 6870\tsteps/sec: 1.27158\tloss: 0.37342\tlr:[4.6652178e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:54:20,640 [ hooks.py: 207]:\tstep: 6880\tsteps/sec: 1.19655\tloss: 0.53102\tlr:[4.6720084e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:54:28,476 [ hooks.py: 207]:\tstep: 6890\tsteps/sec: 1.28046\tloss: 0.59635\tlr:[4.678799e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:54:35,440 [ hooks.py: 207]:\tstep: 6900\tsteps/sec: 1.41670\tloss: 0.62715\tlr:[4.6855897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:54:44,029 [ hooks.py: 207]:\tstep: 6910\tsteps/sec: 1.17503\tloss: 0.64319\tlr:[4.6923808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:54:51,183 [ hooks.py: 207]:\tstep: 6920\tsteps/sec: 1.39568\tloss: 0.80752\tlr:[4.6991714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:01,838 [ hooks.py: 207]:\tstep: 6930\tsteps/sec: 0.97262\tloss: 0.69914\tlr:[4.705962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:09,985 [ hooks.py: 207]:\tstep: 6940\tsteps/sec: 1.16889\tloss: 0.62061\tlr:[4.7127527e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:16,673 [ hooks.py: 207]:\tstep: 6950\tsteps/sec: 1.53602\tloss: 0.48158\tlr:[4.7195434e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:23,760 [ hooks.py: 207]:\tstep: 6960\tsteps/sec: 1.36630\tloss: 0.62752\tlr:[4.726334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:35,283 [ hooks.py: 207]:\tstep: 6970\tsteps/sec: 0.87757\tloss: 0.73712\tlr:[4.733125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:41,835 [ hooks.py: 207]:\tstep: 6980\tsteps/sec: 1.48217\tloss: 0.98409\tlr:[4.7399157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:51,636 [ hooks.py: 207]:\tstep: 6990\tsteps/sec: 1.06053\tloss: 0.52903\tlr:[4.7467063e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:59,276 [ hooks.py: 207]:\tstep: 7000\tsteps/sec: 1.26989\tloss: 0.97993\tlr:[4.753497e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:59,276 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:59,276 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:59,276 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 18:55:59,276 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:59,277 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 18:55:59,277 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:03:52,002 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,003 [ trainer.py: 58]:\t{'acc': 0.7457831325301205, 'loss': 0.6268883449717974}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,004 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,004 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:03:52,005 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:03:52,005 [ trainer.py: 70]:\tacc\t0.7457831325301205\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:03:52,005 [ trainer.py: 70]:\tloss\t0.6268883449717974\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:03:52,005 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,005 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7457831325301205, 'loss': 0.6268883449717974}} \n", + "old: {'eval': {'acc': 0.744578313253012, 'loss': 0.6313066453887866}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,005 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,006 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_6000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,006 [monitored_executor.py: 122]:\tsaving step 7000 to ./output/best_model/model_7000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,969 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_6000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,970 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7457831325301205, 'loss': 0.6268883449717974}} \n", + "old: {'eval': {'acc': 0.744578313253012, 'loss': 0.6313066453887866}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,970 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:52,970 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:54,198 [monitored_executor.py: 122]:\tsaving step 7000 to ./output/model_7000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:03:57,587 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_4000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:04:06,709 [ hooks.py: 207]:\tstep: 7010\tsteps/sec: 0.02052\tloss: 0.55268\tlr:[4.7602876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:04:13,031 [ hooks.py: 207]:\tstep: 7020\tsteps/sec: 1.61737\tloss: 0.65756\tlr:[4.7670783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:04:20,010 [ hooks.py: 207]:\tstep: 7030\tsteps/sec: 1.37042\tloss: 0.74144\tlr:[4.773869e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:04:28,298 [ hooks.py: 207]:\tstep: 7040\tsteps/sec: 1.19314\tloss: 0.65840\tlr:[4.78066e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:04:38,330 [ hooks.py: 207]:\tstep: 7050\tsteps/sec: 1.04432\tloss: 0.41676\tlr:[4.7874506e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:04:45,850 [ hooks.py: 207]:\tstep: 7060\tsteps/sec: 1.28995\tloss: 0.64966\tlr:[4.7942412e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:04:54,270 [ hooks.py: 207]:\tstep: 7070\tsteps/sec: 1.17334\tloss: 0.77999\tlr:[4.801032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:05:02,902 [ hooks.py: 207]:\tstep: 7080\tsteps/sec: 1.26554\tloss: 0.53369\tlr:[4.8078226e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:05:10,858 [ hooks.py: 207]:\tstep: 7090\tsteps/sec: 1.16402\tloss: 0.83984\tlr:[4.8146132e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:05:21,387 [ hooks.py: 207]:\tstep: 7100\tsteps/sec: 0.95160\tloss: 0.47972\tlr:[4.8214042e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:05:30,292 [ hooks.py: 207]:\tstep: 7110\tsteps/sec: 1.12245\tloss: 0.59409\tlr:[4.828195e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:05:39,739 [ hooks.py: 207]:\tstep: 7120\tsteps/sec: 1.01607\tloss: 0.55697\tlr:[4.8349855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:05:47,771 [ hooks.py: 207]:\tstep: 7130\tsteps/sec: 1.28825\tloss: 0.75394\tlr:[4.8417765e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:05:56,511 [ hooks.py: 207]:\tstep: 7140\tsteps/sec: 1.16788\tloss: 0.42212\tlr:[4.848567e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:03,966 [ hooks.py: 207]:\tstep: 7150\tsteps/sec: 1.24349\tloss: 0.28246\tlr:[4.8553575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:12,135 [ hooks.py: 207]:\tstep: 7160\tsteps/sec: 1.34241\tloss: 0.90037\tlr:[4.8621485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:19,705 [ hooks.py: 207]:\tstep: 7170\tsteps/sec: 1.26998\tloss: 0.64349\tlr:[4.868939e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:27,700 [ hooks.py: 207]:\tstep: 7180\tsteps/sec: 1.30862\tloss: 0.84305\tlr:[4.8757298e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:34,479 [ hooks.py: 207]:\tstep: 7190\tsteps/sec: 1.39320\tloss: 0.82862\tlr:[4.8825204e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:43,910 [ hooks.py: 207]:\tstep: 7200\tsteps/sec: 1.07369\tloss: 0.70148\tlr:[4.889311e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:50,336 [ hooks.py: 207]:\tstep: 7210\tsteps/sec: 1.48987\tloss: 0.43377\tlr:[4.8961017e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:06:58,756 [ hooks.py: 207]:\tstep: 7220\tsteps/sec: 1.25949\tloss: 0.61761\tlr:[4.9028928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:07:08,274 [ hooks.py: 207]:\tstep: 7230\tsteps/sec: 1.04764\tloss: 0.91378\tlr:[4.9096834e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:07:15,538 [ hooks.py: 207]:\tstep: 7240\tsteps/sec: 1.32993\tloss: 0.75788\tlr:[4.916474e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:07:23,651 [ hooks.py: 207]:\tstep: 7250\tsteps/sec: 1.26814\tloss: 0.72154\tlr:[4.9232647e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:07:32,988 [ hooks.py: 207]:\tstep: 7260\tsteps/sec: 1.04443\tloss: 0.42178\tlr:[4.9300554e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:07:41,001 [ hooks.py: 207]:\tstep: 7270\tsteps/sec: 1.26487\tloss: 0.98682\tlr:[4.936846e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:07:48,901 [ hooks.py: 207]:\tstep: 7280\tsteps/sec: 1.27605\tloss: 0.75264\tlr:[4.9436367e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:07:57,782 [ hooks.py: 207]:\tstep: 7290\tsteps/sec: 1.13989\tloss: 0.97825\tlr:[4.9504277e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:08:06,525 [ hooks.py: 207]:\tstep: 7300\tsteps/sec: 1.09666\tloss: 0.74725\tlr:[4.9572183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:08:15,360 [ hooks.py: 207]:\tstep: 7310\tsteps/sec: 1.15841\tloss: 1.02772\tlr:[4.964009e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:08:21,903 [ hooks.py: 207]:\tstep: 7320\tsteps/sec: 1.48299\tloss: 0.66469\tlr:[4.9708e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:08:30,753 [ hooks.py: 207]:\tstep: 7330\tsteps/sec: 1.16081\tloss: 0.54722\tlr:[4.9775903e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:08:37,617 [ hooks.py: 207]:\tstep: 7340\tsteps/sec: 1.50365\tloss: 0.92790\tlr:[4.984381e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:08:47,525 [ hooks.py: 207]:\tstep: 7350\tsteps/sec: 0.97449\tloss: 1.05111\tlr:[4.991172e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:08:55,428 [ hooks.py: 207]:\tstep: 7360\tsteps/sec: 1.27387\tloss: 0.81231\tlr:[4.9979626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:09:04,786 [ hooks.py: 207]:\tstep: 7370\tsteps/sec: 1.05259\tloss: 0.44834\tlr:[4.4995355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:09:12,841 [ hooks.py: 207]:\tstep: 7380\tsteps/sec: 1.30015\tloss: 0.80126\tlr:[4.4988567e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:09:20,691 [ hooks.py: 207]:\tstep: 7390\tsteps/sec: 1.25699\tloss: 0.48545\tlr:[4.4981774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:09:28,267 [ hooks.py: 207]:\tstep: 7400\tsteps/sec: 1.26723\tloss: 0.69743\tlr:[4.4974986e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:09:35,966 [ hooks.py: 207]:\tstep: 7410\tsteps/sec: 1.32996\tloss: 0.48998\tlr:[4.496819e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:09:44,696 [ hooks.py: 207]:\tstep: 7420\tsteps/sec: 1.13978\tloss: 0.87304\tlr:[4.4961405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:09:53,260 [ hooks.py: 207]:\tstep: 7430\tsteps/sec: 1.16406\tloss: 0.81057\tlr:[4.4954613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:02,887 [ hooks.py: 207]:\tstep: 7440\tsteps/sec: 1.09552\tloss: 0.45789\tlr:[4.494782e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:10,641 [ hooks.py: 207]:\tstep: 7450\tsteps/sec: 1.24158\tloss: 0.81685\tlr:[4.494103e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:19,020 [ hooks.py: 207]:\tstep: 7460\tsteps/sec: 1.18457\tloss: 0.54925\tlr:[4.493424e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:25,480 [ hooks.py: 207]:\tstep: 7470\tsteps/sec: 1.52440\tloss: 0.41383\tlr:[4.492745e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:32,045 [ hooks.py: 207]:\tstep: 7480\tsteps/sec: 1.58075\tloss: 0.64012\tlr:[4.4920656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:41,116 [ hooks.py: 207]:\tstep: 7490\tsteps/sec: 1.08811\tloss: 0.59480\tlr:[4.4913864e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:49,109 [ hooks.py: 207]:\tstep: 7500\tsteps/sec: 1.25044\tloss: 0.85109\tlr:[4.490708e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:10:58,569 [ hooks.py: 207]:\tstep: 7510\tsteps/sec: 1.07606\tloss: 0.54919\tlr:[4.490029e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:11:05,560 [ hooks.py: 207]:\tstep: 7520\tsteps/sec: 1.39963\tloss: 0.52411\tlr:[4.4893495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:11:13,434 [ hooks.py: 207]:\tstep: 7530\tsteps/sec: 1.22786\tloss: 0.45643\tlr:[4.488671e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:11:22,880 [ hooks.py: 207]:\tstep: 7540\tsteps/sec: 1.07876\tloss: 1.04557\tlr:[4.4879915e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:11:30,529 [ hooks.py: 207]:\tstep: 7550\tsteps/sec: 1.32878\tloss: 0.77283\tlr:[4.487313e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:11:37,149 [ hooks.py: 207]:\tstep: 7560\tsteps/sec: 1.55240\tloss: 1.28436\tlr:[4.4866338e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:11:43,695 [ hooks.py: 207]:\tstep: 7570\tsteps/sec: 1.48251\tloss: 0.83626\tlr:[4.4859546e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:11:53,029 [ hooks.py: 207]:\tstep: 7580\tsteps/sec: 1.09644\tloss: 0.69173\tlr:[4.4852757e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:12:01,171 [ hooks.py: 207]:\tstep: 7590\tsteps/sec: 1.21131\tloss: 0.66986\tlr:[4.484597e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 19:12:10,146 [ hooks.py: 207]:\tstep: 7600\tsteps/sec: 1.09240\tloss: 0.35946\tlr:[4.4839177e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:12:18,933 [ hooks.py: 207]:\tstep: 7610\tsteps/sec: 1.15449\tloss: 0.45321\tlr:[4.4832388e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:12:27,022 [ hooks.py: 207]:\tstep: 7620\tsteps/sec: 1.24995\tloss: 0.55376\tlr:[4.4825596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:12:34,899 [ hooks.py: 207]:\tstep: 7630\tsteps/sec: 1.19737\tloss: 0.36806\tlr:[4.4818804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:12:43,937 [ hooks.py: 207]:\tstep: 7640\tsteps/sec: 1.15845\tloss: 0.36627\tlr:[4.481201e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:12:51,409 [ hooks.py: 207]:\tstep: 7650\tsteps/sec: 1.31510\tloss: 0.90308\tlr:[4.4805223e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:12:59,899 [ hooks.py: 207]:\tstep: 7660\tsteps/sec: 1.19484\tloss: 0.52234\tlr:[4.479843e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:13:08,877 [ hooks.py: 207]:\tstep: 7670\tsteps/sec: 1.11632\tloss: 0.58094\tlr:[4.479164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:13:17,020 [ hooks.py: 207]:\tstep: 7680\tsteps/sec: 1.23954\tloss: 0.83422\tlr:[4.478485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:13:25,427 [ hooks.py: 207]:\tstep: 7690\tsteps/sec: 1.15434\tloss: 0.37835\tlr:[4.477806e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:13:33,119 [ hooks.py: 207]:\tstep: 7700\tsteps/sec: 1.28295\tloss: 0.52806\tlr:[4.477127e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:13:40,987 [ hooks.py: 207]:\tstep: 7710\tsteps/sec: 1.30579\tloss: 0.62331\tlr:[4.476448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:13:48,173 [ hooks.py: 207]:\tstep: 7720\tsteps/sec: 1.43541\tloss: 0.71214\tlr:[4.475769e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:13:55,391 [ hooks.py: 207]:\tstep: 7730\tsteps/sec: 1.33272\tloss: 0.43657\tlr:[4.47509e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:02,633 [ hooks.py: 207]:\tstep: 7740\tsteps/sec: 1.52911\tloss: 0.52746\tlr:[4.474411e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:09,838 [ hooks.py: 207]:\tstep: 7750\tsteps/sec: 1.25068\tloss: 0.91089\tlr:[4.4737317e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:18,331 [ hooks.py: 207]:\tstep: 7760\tsteps/sec: 1.22273\tloss: 0.72249\tlr:[4.473053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:26,292 [ hooks.py: 207]:\tstep: 7770\tsteps/sec: 1.22332\tloss: 0.33743\tlr:[4.472374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:33,710 [ hooks.py: 207]:\tstep: 7780\tsteps/sec: 1.28297\tloss: 0.65643\tlr:[4.471695e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:42,654 [ hooks.py: 207]:\tstep: 7790\tsteps/sec: 1.16137\tloss: 1.04144\tlr:[4.4710156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:50,346 [ hooks.py: 207]:\tstep: 7800\tsteps/sec: 1.25579\tloss: 0.52072\tlr:[4.4703367e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:14:59,276 [ hooks.py: 207]:\tstep: 7810\tsteps/sec: 1.18518\tloss: 0.83242\tlr:[4.469658e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:15:06,728 [ hooks.py: 207]:\tstep: 7820\tsteps/sec: 1.33077\tloss: 0.64371\tlr:[4.4689783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:15:16,256 [ hooks.py: 207]:\tstep: 7830\tsteps/sec: 1.00264\tloss: 0.63122\tlr:[4.4682998e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:15:26,348 [ hooks.py: 207]:\tstep: 7840\tsteps/sec: 0.98930\tloss: 0.55095\tlr:[4.4676202e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:15:35,212 [ hooks.py: 207]:\tstep: 7850\tsteps/sec: 1.13011\tloss: 0.81121\tlr:[4.4669414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:15:45,115 [ hooks.py: 207]:\tstep: 7860\tsteps/sec: 1.04630\tloss: 0.91200\tlr:[4.466262e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:15:51,694 [ hooks.py: 207]:\tstep: 7870\tsteps/sec: 1.54070\tloss: 0.52612\tlr:[4.4655833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:16:00,139 [ hooks.py: 207]:\tstep: 7880\tsteps/sec: 1.18343\tloss: 0.85858\tlr:[4.464904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:16:10,040 [ hooks.py: 207]:\tstep: 7890\tsteps/sec: 0.99306\tloss: 0.64047\tlr:[4.4642253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:16:18,973 [ hooks.py: 207]:\tstep: 7900\tsteps/sec: 1.12061\tloss: 0.44303\tlr:[4.4635457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:16:28,867 [ hooks.py: 207]:\tstep: 7910\tsteps/sec: 1.03189\tloss: 1.08083\tlr:[4.462867e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:16:37,208 [ hooks.py: 207]:\tstep: 7920\tsteps/sec: 1.20355\tloss: 0.36981\tlr:[4.4621876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:16:45,235 [ hooks.py: 207]:\tstep: 7930\tsteps/sec: 1.16933\tloss: 1.04381\tlr:[4.4615088e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:16:55,240 [ hooks.py: 207]:\tstep: 7940\tsteps/sec: 1.03826\tloss: 0.54576\tlr:[4.4608292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:02,689 [ hooks.py: 207]:\tstep: 7950\tsteps/sec: 1.45865\tloss: 1.11190\tlr:[4.4601507e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:13,405 [ hooks.py: 207]:\tstep: 7960\tsteps/sec: 0.90250\tloss: 0.95870\tlr:[4.4594715e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:22,567 [ hooks.py: 207]:\tstep: 7970\tsteps/sec: 1.08536\tloss: 0.42388\tlr:[4.4587923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:29,923 [ hooks.py: 207]:\tstep: 7980\tsteps/sec: 1.27284\tloss: 0.34230\tlr:[4.4581135e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:40,615 [ hooks.py: 207]:\tstep: 7990\tsteps/sec: 0.95560\tloss: 0.57129\tlr:[4.4574343e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:50,509 [ hooks.py: 207]:\tstep: 8000\tsteps/sec: 1.02588\tloss: 0.46767\tlr:[4.4567554e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:50,510 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:50,510 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:50,511 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:17:50,511 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:50,511 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:17:50,515 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:25:43,346 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:43,347 [ trainer.py: 58]:\t{'acc': 0.7566265060240964, 'loss': 0.6208251679363923}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:43,348 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:43,348 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:25:43,349 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:25:43,349 [ trainer.py: 70]:\tacc\t0.7566265060240964\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:25:43,349 [ trainer.py: 70]:\tloss\t0.6208251679363923\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:25:43,349 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:43,349 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7566265060240964, 'loss': 0.6208251679363923}} \n", + "old: {'eval': {'acc': 0.7457831325301205, 'loss': 0.6268883449717974}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:43,349 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:43,350 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_7000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:43,350 [monitored_executor.py: 122]:\tsaving step 8000 to ./output/best_model/model_8000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:44,490 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_7000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:44,491 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7566265060240964, 'loss': 0.6208251679363923}} \n", + "old: {'eval': {'acc': 0.7457831325301205, 'loss': 0.6268883449717974}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:44,491 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:44,492 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:45,868 [monitored_executor.py: 122]:\tsaving step 8000 to ./output/model_8000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:49,284 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_5000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:25:55,691 [ hooks.py: 207]:\tstep: 8010\tsteps/sec: 0.02059\tloss: 0.77926\tlr:[4.456076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:26:05,292 [ hooks.py: 207]:\tstep: 8020\tsteps/sec: 1.06721\tloss: 0.74858\tlr:[4.455397e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 19:26:13,723 [ hooks.py: 207]:\tstep: 8030\tsteps/sec: 1.23276\tloss: 0.73672\tlr:[4.454718e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:26:23,453 [ hooks.py: 207]:\tstep: 8040\tsteps/sec: 1.02207\tloss: 0.83638\tlr:[4.4540393e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:26:32,258 [ hooks.py: 207]:\tstep: 8050\tsteps/sec: 1.14264\tloss: 0.72369\tlr:[4.45336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:26:39,206 [ hooks.py: 207]:\tstep: 8060\tsteps/sec: 1.32338\tloss: 0.55095\tlr:[4.4526812e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:26:46,326 [ hooks.py: 207]:\tstep: 8070\tsteps/sec: 1.48203\tloss: 0.62962\tlr:[4.452002e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:26:54,893 [ hooks.py: 207]:\tstep: 8080\tsteps/sec: 1.16462\tloss: 0.48734\tlr:[4.451323e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:27:03,713 [ hooks.py: 207]:\tstep: 8090\tsteps/sec: 1.17139\tloss: 0.51180\tlr:[4.450644e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:27:10,588 [ hooks.py: 207]:\tstep: 8100\tsteps/sec: 1.42083\tloss: 0.49054\tlr:[4.4499648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:27:19,480 [ hooks.py: 207]:\tstep: 8110\tsteps/sec: 1.12777\tloss: 0.36544\tlr:[4.449286e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:27:26,108 [ hooks.py: 207]:\tstep: 8120\tsteps/sec: 1.46962\tloss: 0.73177\tlr:[4.4486063e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:27:34,462 [ hooks.py: 207]:\tstep: 8130\tsteps/sec: 1.20542\tloss: 0.73783\tlr:[4.4479275e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:27:43,809 [ hooks.py: 207]:\tstep: 8140\tsteps/sec: 1.08209\tloss: 0.82261\tlr:[4.4472483e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:27:54,174 [ hooks.py: 207]:\tstep: 8150\tsteps/sec: 0.95957\tloss: 0.37516\tlr:[4.4465694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:28:02,165 [ hooks.py: 207]:\tstep: 8160\tsteps/sec: 1.35122\tloss: 0.75190\tlr:[4.4458902e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:28:10,752 [ hooks.py: 207]:\tstep: 8170\tsteps/sec: 1.04969\tloss: 1.19885\tlr:[4.4452114e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:28:21,848 [ hooks.py: 207]:\tstep: 8180\tsteps/sec: 0.90626\tloss: 0.77800\tlr:[4.4445318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:28:32,426 [ hooks.py: 207]:\tstep: 8190\tsteps/sec: 0.95283\tloss: 0.52761\tlr:[4.4438533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:28:40,515 [ hooks.py: 207]:\tstep: 8200\tsteps/sec: 1.27408\tloss: 0.58007\tlr:[4.443174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:28:49,946 [ hooks.py: 207]:\tstep: 8210\tsteps/sec: 1.01802\tloss: 0.29825\tlr:[4.442495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:28:58,415 [ hooks.py: 207]:\tstep: 8220\tsteps/sec: 1.28416\tloss: 0.48672\tlr:[4.4418157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:29:06,817 [ hooks.py: 207]:\tstep: 8230\tsteps/sec: 1.11684\tloss: 0.81240\tlr:[4.441137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:29:18,079 [ hooks.py: 207]:\tstep: 8240\tsteps/sec: 0.92934\tloss: 0.44654\tlr:[4.4404576e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:29:24,850 [ hooks.py: 207]:\tstep: 8250\tsteps/sec: 1.40818\tloss: 0.78376\tlr:[4.439779e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:29:33,413 [ hooks.py: 207]:\tstep: 8260\tsteps/sec: 1.16037\tloss: 0.59252\tlr:[4.4390996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:29:40,956 [ hooks.py: 207]:\tstep: 8270\tsteps/sec: 1.35321\tloss: 0.38451\tlr:[4.43842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:29:49,179 [ hooks.py: 207]:\tstep: 8280\tsteps/sec: 1.21413\tloss: 0.66719\tlr:[4.4377415e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:29:57,591 [ hooks.py: 207]:\tstep: 8290\tsteps/sec: 1.16286\tloss: 0.68449\tlr:[4.4370623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:30:07,335 [ hooks.py: 207]:\tstep: 8300\tsteps/sec: 1.02157\tloss: 0.92376\tlr:[4.4363835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:30:16,652 [ hooks.py: 207]:\tstep: 8310\tsteps/sec: 1.05458\tloss: 0.52997\tlr:[4.4357042e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:30:25,280 [ hooks.py: 207]:\tstep: 8320\tsteps/sec: 1.15820\tloss: 0.62105\tlr:[4.435025e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:30:33,494 [ hooks.py: 207]:\tstep: 8330\tsteps/sec: 1.29775\tloss: 0.74129\tlr:[4.4343462e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:30:41,610 [ hooks.py: 207]:\tstep: 8340\tsteps/sec: 1.19491\tloss: 0.27625\tlr:[4.4336673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:30:50,812 [ hooks.py: 207]:\tstep: 8350\tsteps/sec: 1.10467\tloss: 0.73606\tlr:[4.432988e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:30:59,261 [ hooks.py: 207]:\tstep: 8360\tsteps/sec: 1.18929\tloss: 0.63309\tlr:[4.4323093e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:31:08,991 [ hooks.py: 207]:\tstep: 8370\tsteps/sec: 1.03891\tloss: 0.55284\tlr:[4.43163e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:31:15,032 [ hooks.py: 207]:\tstep: 8380\tsteps/sec: 1.51772\tloss: 0.76692\tlr:[4.430951e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:31:25,196 [ hooks.py: 207]:\tstep: 8390\tsteps/sec: 1.01183\tloss: 0.48474\tlr:[4.430272e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:31:34,331 [ hooks.py: 207]:\tstep: 8400\tsteps/sec: 1.09874\tloss: 0.75072\tlr:[4.4295928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:31:42,627 [ hooks.py: 207]:\tstep: 8410\tsteps/sec: 1.23807\tloss: 1.32521\tlr:[4.4289136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:31:49,482 [ hooks.py: 207]:\tstep: 8420\tsteps/sec: 1.42879\tloss: 0.41027\tlr:[4.4282348e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:31:58,824 [ hooks.py: 207]:\tstep: 8430\tsteps/sec: 1.08288\tloss: 0.76494\tlr:[4.4275555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:32:08,082 [ hooks.py: 207]:\tstep: 8440\tsteps/sec: 1.05037\tloss: 0.83835\tlr:[4.4268763e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:32:16,254 [ hooks.py: 207]:\tstep: 8450\tsteps/sec: 1.24595\tloss: 0.74232\tlr:[4.426197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:32:25,689 [ hooks.py: 207]:\tstep: 8460\tsteps/sec: 1.05904\tloss: 0.83521\tlr:[4.4255183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:32:33,716 [ hooks.py: 207]:\tstep: 8470\tsteps/sec: 1.27027\tloss: 0.71744\tlr:[4.424839e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:32:41,205 [ hooks.py: 207]:\tstep: 8480\tsteps/sec: 1.24935\tloss: 0.72344\tlr:[4.4241602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:32:51,310 [ hooks.py: 207]:\tstep: 8490\tsteps/sec: 1.03390\tloss: 0.59581\tlr:[4.423481e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:33:00,162 [ hooks.py: 207]:\tstep: 8500\tsteps/sec: 1.12798\tloss: 0.54908\tlr:[4.422802e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:33:06,953 [ hooks.py: 207]:\tstep: 8510\tsteps/sec: 1.40823\tloss: 0.50592\tlr:[4.422123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:33:17,618 [ hooks.py: 207]:\tstep: 8520\tsteps/sec: 0.96955\tloss: 0.39870\tlr:[4.4214437e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:33:26,666 [ hooks.py: 207]:\tstep: 8530\tsteps/sec: 1.09732\tloss: 0.76729\tlr:[4.420765e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:33:33,820 [ hooks.py: 207]:\tstep: 8540\tsteps/sec: 1.41848\tloss: 0.46725\tlr:[4.420086e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:33:41,815 [ hooks.py: 207]:\tstep: 8550\tsteps/sec: 1.18106\tloss: 0.57098\tlr:[4.4194072e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:33:51,810 [ hooks.py: 207]:\tstep: 8560\tsteps/sec: 1.04077\tloss: 0.49014\tlr:[4.418728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:01,394 [ hooks.py: 207]:\tstep: 8570\tsteps/sec: 1.06942\tloss: 0.65476\tlr:[4.4180488e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:08,176 [ hooks.py: 207]:\tstep: 8580\tsteps/sec: 1.46934\tloss: 1.00917\tlr:[4.4173703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:16,402 [ hooks.py: 207]:\tstep: 8590\tsteps/sec: 1.18085\tloss: 0.79344\tlr:[4.416691e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:22,904 [ hooks.py: 207]:\tstep: 8600\tsteps/sec: 1.58356\tloss: 0.77057\tlr:[4.416012e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:32,637 [ hooks.py: 207]:\tstep: 8610\tsteps/sec: 1.02326\tloss: 0.65456\tlr:[4.4153327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:41,663 [ hooks.py: 207]:\tstep: 8620\tsteps/sec: 1.09554\tloss: 0.45926\tlr:[4.4146538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:49,364 [ hooks.py: 207]:\tstep: 8630\tsteps/sec: 1.29189\tloss: 0.75876\tlr:[4.4139746e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:34:56,373 [ hooks.py: 207]:\tstep: 8640\tsteps/sec: 1.40999\tloss: 0.37735\tlr:[4.4132958e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:35:06,028 [ hooks.py: 207]:\tstep: 8650\tsteps/sec: 1.04499\tloss: 1.05456\tlr:[4.4126165e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:35:15,706 [ hooks.py: 207]:\tstep: 8660\tsteps/sec: 1.02969\tloss: 0.79513\tlr:[4.4119377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:35:22,244 [ hooks.py: 207]:\tstep: 8670\tsteps/sec: 1.57452\tloss: 0.80473\tlr:[4.411258e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:35:31,705 [ hooks.py: 207]:\tstep: 8680\tsteps/sec: 1.02880\tloss: 0.47607\tlr:[4.410579e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 19:35:40,557 [ hooks.py: 207]:\tstep: 8690\tsteps/sec: 1.13676\tloss: 0.60203\tlr:[4.4099e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:35:47,272 [ hooks.py: 207]:\tstep: 8700\tsteps/sec: 1.52234\tloss: 0.57357\tlr:[4.4092216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:35:54,312 [ hooks.py: 207]:\tstep: 8710\tsteps/sec: 1.40790\tloss: 0.64840\tlr:[4.408542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:36:01,417 [ hooks.py: 207]:\tstep: 8720\tsteps/sec: 1.41106\tloss: 0.36531\tlr:[4.407863e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:36:10,024 [ hooks.py: 207]:\tstep: 8730\tsteps/sec: 1.14378\tloss: 0.42598\tlr:[4.407184e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:36:19,659 [ hooks.py: 207]:\tstep: 8740\tsteps/sec: 1.04479\tloss: 0.38462\tlr:[4.4065047e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:36:27,585 [ hooks.py: 207]:\tstep: 8750\tsteps/sec: 1.28824\tloss: 0.61017\tlr:[4.405826e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:36:35,712 [ hooks.py: 207]:\tstep: 8760\tsteps/sec: 1.20228\tloss: 0.37316\tlr:[4.405147e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:36:45,540 [ hooks.py: 207]:\tstep: 8770\tsteps/sec: 1.02364\tloss: 0.54566\tlr:[4.404468e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:36:53,906 [ hooks.py: 207]:\tstep: 8780\tsteps/sec: 1.22053\tloss: 0.36428\tlr:[4.4037886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:37:03,060 [ hooks.py: 207]:\tstep: 8790\tsteps/sec: 1.13535\tloss: 0.52618\tlr:[4.4031098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:37:10,494 [ hooks.py: 207]:\tstep: 8800\tsteps/sec: 1.24601\tloss: 1.00316\tlr:[4.4024306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:37:19,115 [ hooks.py: 207]:\tstep: 8810\tsteps/sec: 1.18325\tloss: 0.49556\tlr:[4.4017517e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:37:29,437 [ hooks.py: 207]:\tstep: 8820\tsteps/sec: 0.94951\tloss: 0.40216\tlr:[4.4010725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:37:36,782 [ hooks.py: 207]:\tstep: 8830\tsteps/sec: 1.36529\tloss: 0.56151\tlr:[4.4003933e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:37:44,909 [ hooks.py: 207]:\tstep: 8840\tsteps/sec: 1.21458\tloss: 0.79051\tlr:[4.3997145e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:37:52,506 [ hooks.py: 207]:\tstep: 8850\tsteps/sec: 1.35940\tloss: 0.76430\tlr:[4.3990356e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:00,065 [ hooks.py: 207]:\tstep: 8860\tsteps/sec: 1.30521\tloss: 0.80766\tlr:[4.3983564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:09,747 [ hooks.py: 207]:\tstep: 8870\tsteps/sec: 0.99017\tloss: 0.68577\tlr:[4.3976772e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:17,993 [ hooks.py: 207]:\tstep: 8880\tsteps/sec: 1.32174\tloss: 0.68805\tlr:[4.3969983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:25,350 [ hooks.py: 207]:\tstep: 8890\tsteps/sec: 1.31033\tloss: 0.32084\tlr:[4.396319e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:33,126 [ hooks.py: 207]:\tstep: 8900\tsteps/sec: 1.31188\tloss: 0.32996\tlr:[4.39564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:41,320 [ hooks.py: 207]:\tstep: 8910\tsteps/sec: 1.20984\tloss: 0.40032\tlr:[4.3949607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:49,875 [ hooks.py: 207]:\tstep: 8920\tsteps/sec: 1.11673\tloss: 1.02890\tlr:[4.394282e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:38:58,818 [ hooks.py: 207]:\tstep: 8930\tsteps/sec: 1.20542\tloss: 1.11270\tlr:[4.3936026e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:05,513 [ hooks.py: 207]:\tstep: 8940\tsteps/sec: 1.39819\tloss: 0.95098\tlr:[4.3929238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:14,279 [ hooks.py: 207]:\tstep: 8950\tsteps/sec: 1.12408\tloss: 0.67010\tlr:[4.392245e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:21,473 [ hooks.py: 207]:\tstep: 8960\tsteps/sec: 1.43262\tloss: 0.78775\tlr:[4.3915657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:30,888 [ hooks.py: 207]:\tstep: 8970\tsteps/sec: 1.06203\tloss: 0.54614\tlr:[4.3908865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:39,779 [ hooks.py: 207]:\tstep: 8980\tsteps/sec: 1.12283\tloss: 0.70469\tlr:[4.3902073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:48,070 [ hooks.py: 207]:\tstep: 8990\tsteps/sec: 1.25034\tloss: 0.70810\tlr:[4.389528e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:56,540 [ hooks.py: 207]:\tstep: 9000\tsteps/sec: 1.15015\tloss: 0.58705\tlr:[4.3888493e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:56,541 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:56,541 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:56,541 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:39:56,545 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:56,545 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:39:56,545 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:47:49,787 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:49,788 [ trainer.py: 58]:\t{'acc': 0.7586345381526104, 'loss': 0.6162728340579913}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:49,789 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:49,789 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:47:49,789 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:47:49,790 [ trainer.py: 70]:\tacc\t0.7586345381526104\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:47:49,790 [ trainer.py: 70]:\tloss\t0.6162728340579913\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 19:47:49,790 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:49,790 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7586345381526104, 'loss': 0.6162728340579913}} \n", + "old: {'eval': {'acc': 0.7566265060240964, 'loss': 0.6208251679363923}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:49,790 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:49,791 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_8000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:49,791 [monitored_executor.py: 122]:\tsaving step 9000 to ./output/best_model/model_9000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:50,953 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_8000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:50,954 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7586345381526104, 'loss': 0.6162728340579913}} \n", + "old: {'eval': {'acc': 0.7566265060240964, 'loss': 0.6208251679363923}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:50,954 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:50,954 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:52,232 [monitored_executor.py: 122]:\tsaving step 9000 to ./output/model_9000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:47:55,955 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_6000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:48:03,369 [ hooks.py: 207]:\tstep: 9010\tsteps/sec: 0.02056\tloss: 0.85667\tlr:[4.38817e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:48:11,435 [ hooks.py: 207]:\tstep: 9020\tsteps/sec: 1.16958\tloss: 0.61951\tlr:[4.3874912e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:48:20,807 [ hooks.py: 207]:\tstep: 9030\tsteps/sec: 1.03672\tloss: 0.62836\tlr:[4.386812e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:48:28,798 [ hooks.py: 207]:\tstep: 9040\tsteps/sec: 1.32725\tloss: 0.88865\tlr:[4.3861328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:48:37,758 [ hooks.py: 207]:\tstep: 9050\tsteps/sec: 1.11051\tloss: 0.43704\tlr:[4.3854543e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:48:44,797 [ hooks.py: 207]:\tstep: 9060\tsteps/sec: 1.31870\tloss: 0.44851\tlr:[4.3847755e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:48:55,550 [ hooks.py: 207]:\tstep: 9070\tsteps/sec: 0.95578\tloss: 0.55036\tlr:[4.3840962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:49:03,971 [ hooks.py: 207]:\tstep: 9080\tsteps/sec: 1.26537\tloss: 0.67757\tlr:[4.3834174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:49:12,499 [ hooks.py: 207]:\tstep: 9090\tsteps/sec: 1.12931\tloss: 0.69717\tlr:[4.3827382e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:49:20,278 [ hooks.py: 207]:\tstep: 9100\tsteps/sec: 1.22161\tloss: 0.42899\tlr:[4.3820593e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:49:29,399 [ hooks.py: 207]:\tstep: 9110\tsteps/sec: 1.10188\tloss: 0.72235\tlr:[4.38138e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 19:49:37,104 [ hooks.py: 207]:\tstep: 9120\tsteps/sec: 1.29096\tloss: 0.87573\tlr:[4.380701e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:49:45,897 [ hooks.py: 207]:\tstep: 9130\tsteps/sec: 1.18105\tloss: 0.51559\tlr:[4.3800217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:49:54,410 [ hooks.py: 207]:\tstep: 9140\tsteps/sec: 1.19204\tloss: 0.42963\tlr:[4.379343e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:50:02,407 [ hooks.py: 207]:\tstep: 9150\tsteps/sec: 1.30848\tloss: 0.82728\tlr:[4.378664e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:50:11,682 [ hooks.py: 207]:\tstep: 9160\tsteps/sec: 1.02869\tloss: 0.65490\tlr:[4.3779844e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:50:19,020 [ hooks.py: 207]:\tstep: 9170\tsteps/sec: 1.38222\tloss: 0.47172\tlr:[4.3773056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:50:27,530 [ hooks.py: 207]:\tstep: 9180\tsteps/sec: 1.17958\tloss: 0.70522\tlr:[4.3766267e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:50:34,659 [ hooks.py: 207]:\tstep: 9190\tsteps/sec: 1.37245\tloss: 0.50989\tlr:[4.375947e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:50:43,667 [ hooks.py: 207]:\tstep: 9200\tsteps/sec: 1.14717\tloss: 0.32402\tlr:[4.3752687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:50:51,049 [ hooks.py: 207]:\tstep: 9210\tsteps/sec: 1.25100\tloss: 0.60552\tlr:[4.3745895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:00,793 [ hooks.py: 207]:\tstep: 9220\tsteps/sec: 1.05638\tloss: 0.49383\tlr:[4.3739103e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:10,629 [ hooks.py: 207]:\tstep: 9230\tsteps/sec: 1.01793\tloss: 0.42699\tlr:[4.3732314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:16,629 [ hooks.py: 207]:\tstep: 9240\tsteps/sec: 1.63830\tloss: 0.46593\tlr:[4.3725522e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:27,795 [ hooks.py: 207]:\tstep: 9250\tsteps/sec: 0.91586\tloss: 0.38237\tlr:[4.371873e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:35,656 [ hooks.py: 207]:\tstep: 9260\tsteps/sec: 1.20678\tloss: 0.60337\tlr:[4.3711945e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:44,579 [ hooks.py: 207]:\tstep: 9270\tsteps/sec: 1.16868\tloss: 0.76347\tlr:[4.370515e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:52,624 [ hooks.py: 207]:\tstep: 9280\tsteps/sec: 1.24206\tloss: 0.63227\tlr:[4.3698357e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:51:59,180 [ hooks.py: 207]:\tstep: 9290\tsteps/sec: 1.48833\tloss: 0.66731\tlr:[4.3691573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:52:08,568 [ hooks.py: 207]:\tstep: 9300\tsteps/sec: 1.10412\tloss: 0.56550\tlr:[4.368478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:52:17,648 [ hooks.py: 207]:\tstep: 9310\tsteps/sec: 1.09101\tloss: 0.79191\tlr:[4.3677992e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:52:26,024 [ hooks.py: 207]:\tstep: 9320\tsteps/sec: 1.17319\tloss: 0.91787\tlr:[4.36712e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:52:34,985 [ hooks.py: 207]:\tstep: 9330\tsteps/sec: 1.11631\tloss: 0.92037\tlr:[4.366441e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:52:41,178 [ hooks.py: 207]:\tstep: 9340\tsteps/sec: 1.53287\tloss: 0.44682\tlr:[4.365762e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:52:49,727 [ hooks.py: 207]:\tstep: 9350\tsteps/sec: 1.22562\tloss: 0.50984\tlr:[4.3650827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:52:56,013 [ hooks.py: 207]:\tstep: 9360\tsteps/sec: 1.57471\tloss: 0.66300\tlr:[4.3644035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:53:05,134 [ hooks.py: 207]:\tstep: 9370\tsteps/sec: 1.09267\tloss: 0.53388\tlr:[4.3637247e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:53:14,015 [ hooks.py: 207]:\tstep: 9380\tsteps/sec: 1.16298\tloss: 0.39558\tlr:[4.3630454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:53:21,133 [ hooks.py: 207]:\tstep: 9390\tsteps/sec: 1.36872\tloss: 0.79181\tlr:[4.3623666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:53:27,638 [ hooks.py: 207]:\tstep: 9400\tsteps/sec: 1.54638\tloss: 0.87344\tlr:[4.3616874e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:53:37,965 [ hooks.py: 207]:\tstep: 9410\tsteps/sec: 0.98132\tloss: 0.79856\tlr:[4.361008e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:53:47,561 [ hooks.py: 207]:\tstep: 9420\tsteps/sec: 1.02697\tloss: 0.41989\tlr:[4.360329e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:53:55,500 [ hooks.py: 207]:\tstep: 9430\tsteps/sec: 1.23826\tloss: 0.79157\tlr:[4.35965e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:54:04,047 [ hooks.py: 207]:\tstep: 9440\tsteps/sec: 1.16038\tloss: 0.41563\tlr:[4.358971e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:54:14,168 [ hooks.py: 207]:\tstep: 9450\tsteps/sec: 1.01768\tloss: 0.74504\tlr:[4.3582924e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:54:20,855 [ hooks.py: 207]:\tstep: 9460\tsteps/sec: 1.39595\tloss: 0.58781\tlr:[4.3576132e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:54:29,181 [ hooks.py: 207]:\tstep: 9470\tsteps/sec: 1.26573\tloss: 0.81747\tlr:[4.356934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:54:36,763 [ hooks.py: 207]:\tstep: 9480\tsteps/sec: 1.29007\tloss: 0.81274\tlr:[4.356255e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:54:45,988 [ hooks.py: 207]:\tstep: 9490\tsteps/sec: 1.08322\tloss: 0.46307\tlr:[4.355576e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:54:54,648 [ hooks.py: 207]:\tstep: 9500\tsteps/sec: 1.10894\tloss: 0.67334\tlr:[4.3548967e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:55:05,123 [ hooks.py: 207]:\tstep: 9510\tsteps/sec: 0.99480\tloss: 0.77151\tlr:[4.354218e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:55:11,617 [ hooks.py: 207]:\tstep: 9520\tsteps/sec: 1.55144\tloss: 0.79791\tlr:[4.353539e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:55:21,023 [ hooks.py: 207]:\tstep: 9530\tsteps/sec: 1.05748\tloss: 0.63624\tlr:[4.3528595e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:55:28,110 [ hooks.py: 207]:\tstep: 9540\tsteps/sec: 1.43987\tloss: 0.93085\tlr:[4.352181e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:55:38,633 [ hooks.py: 207]:\tstep: 9550\tsteps/sec: 0.96322\tloss: 0.66022\tlr:[4.3515018e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:55:46,502 [ hooks.py: 207]:\tstep: 9560\tsteps/sec: 1.22552\tloss: 0.99657\tlr:[4.350823e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:55:55,721 [ hooks.py: 207]:\tstep: 9570\tsteps/sec: 1.08179\tloss: 0.40925\tlr:[4.3501437e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:56:04,428 [ hooks.py: 207]:\tstep: 9580\tsteps/sec: 1.16365\tloss: 0.74317\tlr:[4.3494645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:56:14,573 [ hooks.py: 207]:\tstep: 9590\tsteps/sec: 0.97462\tloss: 0.74526\tlr:[4.3487853e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:56:24,519 [ hooks.py: 207]:\tstep: 9600\tsteps/sec: 1.01588\tloss: 0.60142\tlr:[4.3481064e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:56:32,786 [ hooks.py: 207]:\tstep: 9610\tsteps/sec: 1.16866\tloss: 1.04342\tlr:[4.3474272e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:56:42,513 [ hooks.py: 207]:\tstep: 9620\tsteps/sec: 1.06115\tloss: 0.66002\tlr:[4.346748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:56:49,097 [ hooks.py: 207]:\tstep: 9630\tsteps/sec: 1.45424\tloss: 0.74753\tlr:[4.346069e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:56:56,641 [ hooks.py: 207]:\tstep: 9640\tsteps/sec: 1.35320\tloss: 0.75486\tlr:[4.34539e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:57:04,650 [ hooks.py: 207]:\tstep: 9650\tsteps/sec: 1.26944\tloss: 0.40574\tlr:[4.344711e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:57:14,165 [ hooks.py: 207]:\tstep: 9660\tsteps/sec: 1.06611\tloss: 0.47995\tlr:[4.344032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:57:22,319 [ hooks.py: 207]:\tstep: 9670\tsteps/sec: 1.19094\tloss: 0.75624\tlr:[4.343353e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:57:31,795 [ hooks.py: 207]:\tstep: 9680\tsteps/sec: 1.05526\tloss: 0.46908\tlr:[4.342674e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:57:39,007 [ hooks.py: 207]:\tstep: 9690\tsteps/sec: 1.44225\tloss: 0.55731\tlr:[4.3419946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:57:46,377 [ hooks.py: 207]:\tstep: 9700\tsteps/sec: 1.31201\tloss: 0.56022\tlr:[4.3413154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:57:53,081 [ hooks.py: 207]:\tstep: 9710\tsteps/sec: 1.53356\tloss: 0.73715\tlr:[4.340637e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:58:02,008 [ hooks.py: 207]:\tstep: 9720\tsteps/sec: 1.10240\tloss: 0.27510\tlr:[4.3399577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:58:10,617 [ hooks.py: 207]:\tstep: 9730\tsteps/sec: 1.15490\tloss: 0.45672\tlr:[4.3392785e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:58:19,426 [ hooks.py: 207]:\tstep: 9740\tsteps/sec: 1.15239\tloss: 0.35507\tlr:[4.3385997e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:58:27,915 [ hooks.py: 207]:\tstep: 9750\tsteps/sec: 1.17656\tloss: 0.85436\tlr:[4.337921e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:58:36,371 [ hooks.py: 207]:\tstep: 9760\tsteps/sec: 1.15903\tloss: 0.77788\tlr:[4.3372416e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:58:44,117 [ hooks.py: 207]:\tstep: 9770\tsteps/sec: 1.32994\tloss: 0.53294\tlr:[4.3365624e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 19:58:51,409 [ hooks.py: 207]:\tstep: 9780\tsteps/sec: 1.34407\tloss: 0.53627\tlr:[4.3358832e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:00,912 [ hooks.py: 207]:\tstep: 9790\tsteps/sec: 1.04678\tloss: 0.81699\tlr:[4.3352036e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:07,469 [ hooks.py: 207]:\tstep: 9800\tsteps/sec: 1.53633\tloss: 0.73811\tlr:[4.334525e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:16,457 [ hooks.py: 207]:\tstep: 9810\tsteps/sec: 1.10299\tloss: 0.77395\tlr:[4.3338463e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:24,678 [ hooks.py: 207]:\tstep: 9820\tsteps/sec: 1.17651\tloss: 0.53116\tlr:[4.333167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:32,659 [ hooks.py: 207]:\tstep: 9830\tsteps/sec: 1.32421\tloss: 0.55264\tlr:[4.3324882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:42,104 [ hooks.py: 207]:\tstep: 9840\tsteps/sec: 1.04170\tloss: 0.60916\tlr:[4.331809e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:50,513 [ hooks.py: 207]:\tstep: 9850\tsteps/sec: 1.18567\tloss: 0.63297\tlr:[4.33113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 19:59:59,149 [ hooks.py: 207]:\tstep: 9860\tsteps/sec: 1.21142\tloss: 0.58024\tlr:[4.330451e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:00:08,918 [ hooks.py: 207]:\tstep: 9870\tsteps/sec: 1.01515\tloss: 0.33307\tlr:[4.3297718e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:00:18,095 [ hooks.py: 207]:\tstep: 9880\tsteps/sec: 1.08474\tloss: 0.29051\tlr:[4.3290926e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:00:26,646 [ hooks.py: 207]:\tstep: 9890\tsteps/sec: 1.14324\tloss: 0.70242\tlr:[4.3284137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:00:35,326 [ hooks.py: 207]:\tstep: 9900\tsteps/sec: 1.14707\tloss: 0.80975\tlr:[4.327735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:00:43,209 [ hooks.py: 207]:\tstep: 9910\tsteps/sec: 1.30879\tloss: 1.00460\tlr:[4.3270556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:00:50,759 [ hooks.py: 207]:\tstep: 9920\tsteps/sec: 1.24337\tloss: 0.45933\tlr:[4.326376e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:00:59,609 [ hooks.py: 207]:\tstep: 9930\tsteps/sec: 1.20059\tloss: 0.75574\tlr:[4.3256976e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:07,947 [ hooks.py: 207]:\tstep: 9940\tsteps/sec: 1.14048\tloss: 0.62199\tlr:[4.3250184e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:16,945 [ hooks.py: 207]:\tstep: 9950\tsteps/sec: 1.12534\tloss: 0.80061\tlr:[4.3243395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:23,891 [ hooks.py: 207]:\tstep: 9960\tsteps/sec: 1.49097\tloss: 0.51025\tlr:[4.3236603e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:32,424 [ hooks.py: 207]:\tstep: 9970\tsteps/sec: 1.16200\tloss: 0.48058\tlr:[4.3229815e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:40,765 [ hooks.py: 207]:\tstep: 9980\tsteps/sec: 1.18173\tloss: 0.52338\tlr:[4.3223023e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:48,174 [ hooks.py: 207]:\tstep: 9990\tsteps/sec: 1.39377\tloss: 0.60275\tlr:[4.321623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:55,963 [ hooks.py: 207]:\tstep: 10000\tsteps/sec: 1.20217\tloss: 0.54226\tlr:[4.3209442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:55,964 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:55,964 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:55,964 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:01:55,964 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:55,965 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:01:55,965 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:09:47,800 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:47,802 [ trainer.py: 58]:\t{'acc': 0.7598393574297189, 'loss': 0.5944234219690164}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:47,802 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:47,802 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:09:47,803 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:09:47,803 [ trainer.py: 70]:\tacc\t0.7598393574297189\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:09:47,803 [ trainer.py: 70]:\tloss\t0.5944234219690164\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:09:47,803 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:47,804 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}} \n", + "old: {'eval': {'acc': 0.7586345381526104, 'loss': 0.6162728340579913}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:47,804 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:47,804 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_9000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:47,805 [monitored_executor.py: 122]:\tsaving step 10000 to ./output/best_model/model_10000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:48,802 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_9000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:48,802 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}} \n", + "old: {'eval': {'acc': 0.7586345381526104, 'loss': 0.6162728340579913}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:48,803 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:48,803 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:50,253 [monitored_executor.py: 122]:\tsaving step 10000 to ./output/model_10000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:09:53,668 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_7000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:10:04,437 [ hooks.py: 207]:\tstep: 10010\tsteps/sec: 0.02049\tloss: 0.57604\tlr:[4.3202654e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:10:12,400 [ hooks.py: 207]:\tstep: 10020\tsteps/sec: 1.19407\tloss: 0.93721\tlr:[4.319586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:10:20,998 [ hooks.py: 207]:\tstep: 10030\tsteps/sec: 1.20528\tloss: 0.67139\tlr:[4.318907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:10:31,585 [ hooks.py: 207]:\tstep: 10040\tsteps/sec: 0.94638\tloss: 0.37881\tlr:[4.318228e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:10:40,073 [ hooks.py: 207]:\tstep: 10050\tsteps/sec: 1.18068\tloss: 0.31287\tlr:[4.317549e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:10:47,220 [ hooks.py: 207]:\tstep: 10060\tsteps/sec: 1.38114\tloss: 0.57273\tlr:[4.31687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:10:55,473 [ hooks.py: 207]:\tstep: 10070\tsteps/sec: 1.16978\tloss: 0.96805\tlr:[4.316191e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:11:05,017 [ hooks.py: 207]:\tstep: 10080\tsteps/sec: 1.09459\tloss: 0.60513\tlr:[4.3155116e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:11:11,640 [ hooks.py: 207]:\tstep: 10090\tsteps/sec: 1.42047\tloss: 0.79941\tlr:[4.3148328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:11:20,173 [ hooks.py: 207]:\tstep: 10100\tsteps/sec: 1.16948\tloss: 0.52511\tlr:[4.3141536e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:11:29,562 [ hooks.py: 207]:\tstep: 10110\tsteps/sec: 1.08601\tloss: 0.46332\tlr:[4.3134743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:11:37,733 [ hooks.py: 207]:\tstep: 10120\tsteps/sec: 1.21382\tloss: 0.52352\tlr:[4.3127955e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:11:47,601 [ hooks.py: 207]:\tstep: 10130\tsteps/sec: 1.04473\tloss: 0.91078\tlr:[4.3121163e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:11:55,662 [ hooks.py: 207]:\tstep: 10140\tsteps/sec: 1.22652\tloss: 0.61326\tlr:[4.3114374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:12:04,248 [ hooks.py: 207]:\tstep: 10150\tsteps/sec: 1.14546\tloss: 0.80191\tlr:[4.3107582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:12:13,078 [ hooks.py: 207]:\tstep: 10160\tsteps/sec: 1.16567\tloss: 0.52537\tlr:[4.3100794e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:12:21,352 [ hooks.py: 207]:\tstep: 10170\tsteps/sec: 1.20745\tloss: 0.52301\tlr:[4.3094e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:12:28,893 [ hooks.py: 207]:\tstep: 10180\tsteps/sec: 1.35896\tloss: 0.68534\tlr:[4.308721e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:12:35,855 [ hooks.py: 207]:\tstep: 10190\tsteps/sec: 1.31170\tloss: 0.68020\tlr:[4.308042e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 20:12:47,895 [ hooks.py: 207]:\tstep: 10200\tsteps/sec: 0.87180\tloss: 0.36446\tlr:[4.307363e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:12:56,538 [ hooks.py: 207]:\tstep: 10210\tsteps/sec: 1.15066\tloss: 0.69261\tlr:[4.306684e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:13:06,232 [ hooks.py: 207]:\tstep: 10220\tsteps/sec: 0.98613\tloss: 0.36375\tlr:[4.3060045e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:13:15,060 [ hooks.py: 207]:\tstep: 10230\tsteps/sec: 1.18484\tloss: 0.79959\tlr:[4.305326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:13:24,597 [ hooks.py: 207]:\tstep: 10240\tsteps/sec: 1.02415\tloss: 0.65121\tlr:[4.3046468e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:13:32,479 [ hooks.py: 207]:\tstep: 10250\tsteps/sec: 1.29733\tloss: 0.85551\tlr:[4.303968e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:13:39,739 [ hooks.py: 207]:\tstep: 10260\tsteps/sec: 1.41051\tloss: 0.77619\tlr:[4.3032887e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:13:49,025 [ hooks.py: 207]:\tstep: 10270\tsteps/sec: 1.04239\tloss: 0.59848\tlr:[4.3026095e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:13:57,910 [ hooks.py: 207]:\tstep: 10280\tsteps/sec: 1.11854\tloss: 0.79305\tlr:[4.3019307e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:14:07,840 [ hooks.py: 207]:\tstep: 10290\tsteps/sec: 1.02286\tloss: 0.69166\tlr:[4.3012515e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:14:15,132 [ hooks.py: 207]:\tstep: 10300\tsteps/sec: 1.38684\tloss: 0.71387\tlr:[4.3005726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:14:23,404 [ hooks.py: 207]:\tstep: 10310\tsteps/sec: 1.23403\tloss: 0.88567\tlr:[4.2998934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:14:31,576 [ hooks.py: 207]:\tstep: 10320\tsteps/sec: 1.19953\tloss: 0.64951\tlr:[4.2992142e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:14:40,897 [ hooks.py: 207]:\tstep: 10330\tsteps/sec: 1.06441\tloss: 0.36285\tlr:[4.2985354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:14:50,951 [ hooks.py: 207]:\tstep: 10340\tsteps/sec: 0.96766\tloss: 0.72820\tlr:[4.297856e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:14:59,752 [ hooks.py: 207]:\tstep: 10350\tsteps/sec: 1.22260\tloss: 0.33991\tlr:[4.2971773e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:15:08,034 [ hooks.py: 207]:\tstep: 10360\tsteps/sec: 1.18459\tloss: 0.81691\tlr:[4.2964977e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:15:15,319 [ hooks.py: 207]:\tstep: 10370\tsteps/sec: 1.33950\tloss: 0.86016\tlr:[4.295819e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:15:24,390 [ hooks.py: 207]:\tstep: 10380\tsteps/sec: 1.13709\tloss: 0.56454\tlr:[4.2951397e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:15:31,066 [ hooks.py: 207]:\tstep: 10390\tsteps/sec: 1.44745\tloss: 0.94102\tlr:[4.2944605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:15:39,640 [ hooks.py: 207]:\tstep: 10400\tsteps/sec: 1.20404\tloss: 0.88745\tlr:[4.2937816e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:15:46,607 [ hooks.py: 207]:\tstep: 10410\tsteps/sec: 1.37426\tloss: 0.63680\tlr:[4.2931024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:15:54,477 [ hooks.py: 207]:\tstep: 10420\tsteps/sec: 1.31823\tloss: 0.62739\tlr:[4.2924235e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:16:04,603 [ hooks.py: 207]:\tstep: 10430\tsteps/sec: 0.98508\tloss: 0.66897\tlr:[4.2917443e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:16:13,306 [ hooks.py: 207]:\tstep: 10440\tsteps/sec: 1.13907\tloss: 0.61727\tlr:[4.291065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:16:19,584 [ hooks.py: 207]:\tstep: 10450\tsteps/sec: 1.49614\tloss: 0.86692\tlr:[4.2903863e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:16:28,961 [ hooks.py: 207]:\tstep: 10460\tsteps/sec: 1.12964\tloss: 0.68073\tlr:[4.2897074e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:16:37,179 [ hooks.py: 207]:\tstep: 10470\tsteps/sec: 1.16782\tloss: 0.60697\tlr:[4.2890286e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:16:45,254 [ hooks.py: 207]:\tstep: 10480\tsteps/sec: 1.26059\tloss: 0.52304\tlr:[4.288349e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:16:52,654 [ hooks.py: 207]:\tstep: 10490\tsteps/sec: 1.33639\tloss: 0.99568\tlr:[4.28767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:17:03,159 [ hooks.py: 207]:\tstep: 10500\tsteps/sec: 0.99254\tloss: 0.71917\tlr:[4.286991e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:17:14,223 [ hooks.py: 207]:\tstep: 10510\tsteps/sec: 0.89048\tloss: 1.05475\tlr:[4.286312e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:17:21,648 [ hooks.py: 207]:\tstep: 10520\tsteps/sec: 1.30718\tloss: 0.58907\tlr:[4.2856333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:17:28,186 [ hooks.py: 207]:\tstep: 10530\tsteps/sec: 1.55650\tloss: 0.51577\tlr:[4.284954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:17:35,394 [ hooks.py: 207]:\tstep: 10540\tsteps/sec: 1.36911\tloss: 0.53961\tlr:[4.2842752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:17:44,973 [ hooks.py: 207]:\tstep: 10550\tsteps/sec: 1.04979\tloss: 0.71416\tlr:[4.283596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:17:52,078 [ hooks.py: 207]:\tstep: 10560\tsteps/sec: 1.36715\tloss: 0.80550\tlr:[4.282917e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:18:00,325 [ hooks.py: 207]:\tstep: 10570\tsteps/sec: 1.24549\tloss: 0.64370\tlr:[4.282238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:18:09,297 [ hooks.py: 207]:\tstep: 10580\tsteps/sec: 1.13008\tloss: 0.67500\tlr:[4.2815587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:18:16,560 [ hooks.py: 207]:\tstep: 10590\tsteps/sec: 1.26079\tloss: 0.49325\tlr:[4.2808795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:18:25,968 [ hooks.py: 207]:\tstep: 10600\tsteps/sec: 1.10291\tloss: 0.60049\tlr:[4.2802007e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:18:36,836 [ hooks.py: 207]:\tstep: 10610\tsteps/sec: 0.93108\tloss: 0.48739\tlr:[4.2795218e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:18:43,389 [ hooks.py: 207]:\tstep: 10620\tsteps/sec: 1.48513\tloss: 0.45199\tlr:[4.2788426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:18:51,124 [ hooks.py: 207]:\tstep: 10630\tsteps/sec: 1.32225\tloss: 0.62769\tlr:[4.2781634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:01,136 [ hooks.py: 207]:\tstep: 10640\tsteps/sec: 1.01495\tloss: 0.49411\tlr:[4.2774842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:09,390 [ hooks.py: 207]:\tstep: 10650\tsteps/sec: 1.20921\tloss: 0.38510\tlr:[4.276805e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:18,053 [ hooks.py: 207]:\tstep: 10660\tsteps/sec: 1.11370\tloss: 0.85593\tlr:[4.276126e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:27,641 [ hooks.py: 207]:\tstep: 10670\tsteps/sec: 1.05612\tloss: 0.58590\tlr:[4.2754473e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:34,318 [ hooks.py: 207]:\tstep: 10680\tsteps/sec: 1.49817\tloss: 0.54929\tlr:[4.2747684e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:43,868 [ hooks.py: 207]:\tstep: 10690\tsteps/sec: 1.07816\tloss: 0.60647\tlr:[4.2740892e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:52,520 [ hooks.py: 207]:\tstep: 10700\tsteps/sec: 1.11606\tloss: 0.76229\tlr:[4.27341e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:19:59,563 [ hooks.py: 207]:\tstep: 10710\tsteps/sec: 1.50162\tloss: 0.40810\tlr:[4.272731e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:20:07,898 [ hooks.py: 207]:\tstep: 10720\tsteps/sec: 1.17024\tloss: 0.47941\tlr:[4.2720523e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:20:16,542 [ hooks.py: 207]:\tstep: 10730\tsteps/sec: 1.13815\tloss: 0.36163\tlr:[4.271373e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:20:26,125 [ hooks.py: 207]:\tstep: 10740\tsteps/sec: 1.04707\tloss: 0.83048\tlr:[4.270694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:20:32,703 [ hooks.py: 207]:\tstep: 10750\tsteps/sec: 1.51615\tloss: 0.63835\tlr:[4.2700147e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:20:39,950 [ hooks.py: 207]:\tstep: 10760\tsteps/sec: 1.31289\tloss: 1.02713\tlr:[4.2693362e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:20:48,979 [ hooks.py: 207]:\tstep: 10770\tsteps/sec: 1.19393\tloss: 0.50148\tlr:[4.2686566e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:20:57,045 [ hooks.py: 207]:\tstep: 10780\tsteps/sec: 1.15473\tloss: 0.49550\tlr:[4.2679774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:21:05,335 [ hooks.py: 207]:\tstep: 10790\tsteps/sec: 1.25183\tloss: 0.61192\tlr:[4.2672986e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:21:13,944 [ hooks.py: 207]:\tstep: 10800\tsteps/sec: 1.15427\tloss: 0.95892\tlr:[4.2666197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:21:21,799 [ hooks.py: 207]:\tstep: 10810\tsteps/sec: 1.29306\tloss: 0.68914\tlr:[4.2659405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:21:29,922 [ hooks.py: 207]:\tstep: 10820\tsteps/sec: 1.18098\tloss: 0.60999\tlr:[4.2652613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:21:38,385 [ hooks.py: 207]:\tstep: 10830\tsteps/sec: 1.24962\tloss: 0.92866\tlr:[4.2645825e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:21:48,801 [ hooks.py: 207]:\tstep: 10840\tsteps/sec: 0.96394\tloss: 0.69299\tlr:[4.2639032e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 20:21:56,703 [ hooks.py: 207]:\tstep: 10850\tsteps/sec: 1.22914\tloss: 0.46718\tlr:[4.263224e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:22:05,350 [ hooks.py: 207]:\tstep: 10860\tsteps/sec: 1.16425\tloss: 0.79723\tlr:[4.2625452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:22:12,028 [ hooks.py: 207]:\tstep: 10870\tsteps/sec: 1.42032\tloss: 0.54006\tlr:[4.261866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:22:20,758 [ hooks.py: 207]:\tstep: 10880\tsteps/sec: 1.18813\tloss: 0.51662\tlr:[4.2611868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:22:29,708 [ hooks.py: 207]:\tstep: 10890\tsteps/sec: 1.15186\tloss: 0.86264\tlr:[4.2605076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:22:37,123 [ hooks.py: 207]:\tstep: 10900\tsteps/sec: 1.30593\tloss: 0.84273\tlr:[4.259829e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:22:44,822 [ hooks.py: 207]:\tstep: 10910\tsteps/sec: 1.26421\tloss: 0.87558\tlr:[4.2591502e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:22:53,727 [ hooks.py: 207]:\tstep: 10920\tsteps/sec: 1.18110\tloss: 0.45456\tlr:[4.258471e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:03,285 [ hooks.py: 207]:\tstep: 10930\tsteps/sec: 1.00660\tloss: 1.20816\tlr:[4.2577914e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:11,526 [ hooks.py: 207]:\tstep: 10940\tsteps/sec: 1.24707\tloss: 0.60920\tlr:[4.2571126e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:19,775 [ hooks.py: 207]:\tstep: 10950\tsteps/sec: 1.23858\tloss: 0.58886\tlr:[4.2564334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:28,207 [ hooks.py: 207]:\tstep: 10960\tsteps/sec: 1.16625\tloss: 0.31752\tlr:[4.255755e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:35,395 [ hooks.py: 207]:\tstep: 10970\tsteps/sec: 1.37432\tloss: 0.33283\tlr:[4.2550753e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:43,642 [ hooks.py: 207]:\tstep: 10980\tsteps/sec: 1.23642\tloss: 0.82187\tlr:[4.2543965e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:50,474 [ hooks.py: 207]:\tstep: 10990\tsteps/sec: 1.43540\tloss: 0.55925\tlr:[4.2537176e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:58,771 [ hooks.py: 207]:\tstep: 11000\tsteps/sec: 1.24756\tloss: 1.00092\tlr:[4.2530384e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:58,771 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:58,771 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:58,772 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:23:58,772 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:58,772 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:23:58,772 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:31:50,600 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,602 [ trainer.py: 58]:\t{'acc': 0.7457831325301205, 'loss': 0.6204742030837597}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,602 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,602 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:31:50,603 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:31:50,603 [ trainer.py: 70]:\tacc\t0.7457831325301205\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:31:50,603 [ trainer.py: 70]:\tloss\t0.6204742030837597\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:31:50,603 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,603 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7457831325301205, 'loss': 0.6204742030837597}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,604 [ exporter.py: 64]:\t[Best Exporter]: skip step 11000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,604 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7457831325301205, 'loss': 0.6204742030837597}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,604 [ exporter.py: 104]:\t[Best Exporter]: skip step 11000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:50,605 [monitored_executor.py: 122]:\tsaving step 11000 to ./output/model_11000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:31:53,941 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_8000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:32:02,574 [ hooks.py: 207]:\tstep: 11010\tsteps/sec: 0.02068\tloss: 0.69136\tlr:[4.2523592e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:32:10,769 [ hooks.py: 207]:\tstep: 11020\tsteps/sec: 1.14556\tloss: 0.48441\tlr:[4.2516804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:32:20,586 [ hooks.py: 207]:\tstep: 11030\tsteps/sec: 1.03145\tloss: 0.61891\tlr:[4.2510015e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:32:28,535 [ hooks.py: 207]:\tstep: 11040\tsteps/sec: 1.27173\tloss: 0.61633\tlr:[4.2503223e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:32:36,946 [ hooks.py: 207]:\tstep: 11050\tsteps/sec: 1.16250\tloss: 0.68355\tlr:[4.249643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:32:46,609 [ hooks.py: 207]:\tstep: 11060\tsteps/sec: 1.03172\tloss: 0.74183\tlr:[4.2489643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:32:55,060 [ hooks.py: 207]:\tstep: 11070\tsteps/sec: 1.21981\tloss: 0.48447\tlr:[4.248285e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:33:03,125 [ hooks.py: 207]:\tstep: 11080\tsteps/sec: 1.27670\tloss: 0.70769\tlr:[4.247606e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:33:11,792 [ hooks.py: 207]:\tstep: 11090\tsteps/sec: 1.09908\tloss: 0.79862\tlr:[4.246927e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:33:20,222 [ hooks.py: 207]:\tstep: 11100\tsteps/sec: 1.19963\tloss: 0.57515\tlr:[4.2462478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:33:28,988 [ hooks.py: 207]:\tstep: 11110\tsteps/sec: 1.12078\tloss: 0.94972\tlr:[4.2455686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:33:39,502 [ hooks.py: 207]:\tstep: 11120\tsteps/sec: 0.98343\tloss: 0.73470\tlr:[4.2448897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:33:46,788 [ hooks.py: 207]:\tstep: 11130\tsteps/sec: 1.25262\tloss: 0.53418\tlr:[4.2442105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:33:55,793 [ hooks.py: 207]:\tstep: 11140\tsteps/sec: 1.16803\tloss: 0.49255\tlr:[4.2435313e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:34:05,159 [ hooks.py: 207]:\tstep: 11150\tsteps/sec: 1.08307\tloss: 0.71705\tlr:[4.2428528e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:34:13,904 [ hooks.py: 207]:\tstep: 11160\tsteps/sec: 1.14805\tloss: 0.75446\tlr:[4.2421736e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:34:21,345 [ hooks.py: 207]:\tstep: 11170\tsteps/sec: 1.25407\tloss: 0.96994\tlr:[4.2414948e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:34:30,631 [ hooks.py: 207]:\tstep: 11180\tsteps/sec: 1.12391\tloss: 0.55805\tlr:[4.2408155e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:34:39,178 [ hooks.py: 207]:\tstep: 11190\tsteps/sec: 1.19665\tloss: 1.00766\tlr:[4.240136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:34:48,401 [ hooks.py: 207]:\tstep: 11200\tsteps/sec: 1.03162\tloss: 0.73674\tlr:[4.239457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:34:57,531 [ hooks.py: 207]:\tstep: 11210\tsteps/sec: 1.10985\tloss: 0.31367\tlr:[4.2387783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:35:04,707 [ hooks.py: 207]:\tstep: 11220\tsteps/sec: 1.45430\tloss: 1.01187\tlr:[4.2380994e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:35:13,096 [ hooks.py: 207]:\tstep: 11230\tsteps/sec: 1.15935\tloss: 0.56839\tlr:[4.23742e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:35:20,915 [ hooks.py: 207]:\tstep: 11240\tsteps/sec: 1.29098\tloss: 0.78287\tlr:[4.236741e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:35:29,349 [ hooks.py: 207]:\tstep: 11250\tsteps/sec: 1.20069\tloss: 0.62037\tlr:[4.236062e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:35:39,490 [ hooks.py: 207]:\tstep: 11260\tsteps/sec: 1.00075\tloss: 0.64679\tlr:[4.235383e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:35:45,906 [ hooks.py: 207]:\tstep: 11270\tsteps/sec: 1.39451\tloss: 0.37548\tlr:[4.2347037e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:35:55,927 [ hooks.py: 207]:\tstep: 11280\tsteps/sec: 1.04331\tloss: 0.71697\tlr:[4.2340253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:36:03,489 [ hooks.py: 207]:\tstep: 11290\tsteps/sec: 1.38950\tloss: 1.14564\tlr:[4.2333457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:36:10,668 [ hooks.py: 207]:\tstep: 11300\tsteps/sec: 1.25541\tloss: 0.67460\tlr:[4.232667e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 20:36:21,009 [ hooks.py: 207]:\tstep: 11310\tsteps/sec: 1.01644\tloss: 0.81736\tlr:[4.2319876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:36:30,810 [ hooks.py: 207]:\tstep: 11320\tsteps/sec: 0.98926\tloss: 0.50862\tlr:[4.2313084e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:36:40,472 [ hooks.py: 207]:\tstep: 11330\tsteps/sec: 1.07305\tloss: 0.67146\tlr:[4.2306296e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:36:47,063 [ hooks.py: 207]:\tstep: 11340\tsteps/sec: 1.49689\tloss: 0.61849\tlr:[4.2299504e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:36:56,791 [ hooks.py: 207]:\tstep: 11350\tsteps/sec: 1.02246\tloss: 0.88038\tlr:[4.229271e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:37:04,756 [ hooks.py: 207]:\tstep: 11360\tsteps/sec: 1.28417\tloss: 0.72605\tlr:[4.2285923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:37:13,961 [ hooks.py: 207]:\tstep: 11370\tsteps/sec: 1.09014\tloss: 1.01234\tlr:[4.227913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:37:21,182 [ hooks.py: 207]:\tstep: 11380\tsteps/sec: 1.35643\tloss: 0.43706\tlr:[4.227234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:37:30,544 [ hooks.py: 207]:\tstep: 11390\tsteps/sec: 1.07266\tloss: 0.56217\tlr:[4.226555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:37:40,131 [ hooks.py: 207]:\tstep: 11400\tsteps/sec: 1.00480\tloss: 0.81730\tlr:[4.225876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:37:47,982 [ hooks.py: 207]:\tstep: 11410\tsteps/sec: 1.26085\tloss: 0.41387\tlr:[4.225197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:37:55,723 [ hooks.py: 207]:\tstep: 11420\tsteps/sec: 1.37221\tloss: 0.58080\tlr:[4.2245178e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:02,947 [ hooks.py: 207]:\tstep: 11430\tsteps/sec: 1.37206\tloss: 0.55196\tlr:[4.223839e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:11,579 [ hooks.py: 207]:\tstep: 11440\tsteps/sec: 1.16104\tloss: 0.49029\tlr:[4.2231597e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:19,289 [ hooks.py: 207]:\tstep: 11450\tsteps/sec: 1.33142\tloss: 0.69041\tlr:[4.222481e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:26,867 [ hooks.py: 207]:\tstep: 11460\tsteps/sec: 1.28130\tloss: 0.52614\tlr:[4.2218013e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:35,545 [ hooks.py: 207]:\tstep: 11470\tsteps/sec: 1.16529\tloss: 0.94735\tlr:[4.2211228e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:42,464 [ hooks.py: 207]:\tstep: 11480\tsteps/sec: 1.42800\tloss: 0.39143\tlr:[4.2204436e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:48,507 [ hooks.py: 207]:\tstep: 11490\tsteps/sec: 1.60386\tloss: 0.42153\tlr:[4.2197644e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:38:56,957 [ hooks.py: 207]:\tstep: 11500\tsteps/sec: 1.21155\tloss: 0.47668\tlr:[4.2190855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:39:06,578 [ hooks.py: 207]:\tstep: 11510\tsteps/sec: 1.02655\tloss: 0.51676\tlr:[4.218407e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:39:15,793 [ hooks.py: 207]:\tstep: 11520\tsteps/sec: 1.10271\tloss: 0.84365\tlr:[4.217728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:39:23,621 [ hooks.py: 207]:\tstep: 11530\tsteps/sec: 1.24194\tloss: 0.70848\tlr:[4.2170486e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:39:33,505 [ hooks.py: 207]:\tstep: 11540\tsteps/sec: 1.04491\tloss: 0.44442\tlr:[4.2163694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:39:40,920 [ hooks.py: 207]:\tstep: 11550\tsteps/sec: 1.33471\tloss: 0.71820\tlr:[4.2156902e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:39:50,540 [ hooks.py: 207]:\tstep: 11560\tsteps/sec: 1.00914\tloss: 0.41300\tlr:[4.2150114e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:39:59,875 [ hooks.py: 207]:\tstep: 11570\tsteps/sec: 1.14398\tloss: 0.55599\tlr:[4.214332e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:40:09,742 [ hooks.py: 207]:\tstep: 11580\tsteps/sec: 1.00064\tloss: 0.45522\tlr:[4.2136533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:40:20,147 [ hooks.py: 207]:\tstep: 11590\tsteps/sec: 0.95915\tloss: 0.53278\tlr:[4.212974e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:40:28,826 [ hooks.py: 207]:\tstep: 11600\tsteps/sec: 1.10654\tloss: 0.80340\tlr:[4.2122952e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:40:36,872 [ hooks.py: 207]:\tstep: 11610\tsteps/sec: 1.27336\tloss: 0.38558\tlr:[4.211616e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:40:42,995 [ hooks.py: 207]:\tstep: 11620\tsteps/sec: 1.53794\tloss: 0.84156\tlr:[4.2109372e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:40:52,022 [ hooks.py: 207]:\tstep: 11630\tsteps/sec: 1.11495\tloss: 0.70700\tlr:[4.210258e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:41:01,912 [ hooks.py: 207]:\tstep: 11640\tsteps/sec: 1.08124\tloss: 0.86462\tlr:[4.2095784e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:41:10,136 [ hooks.py: 207]:\tstep: 11650\tsteps/sec: 1.19537\tloss: 0.68652\tlr:[4.2088996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:41:19,968 [ hooks.py: 207]:\tstep: 11660\tsteps/sec: 1.01956\tloss: 0.85774\tlr:[4.208221e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:41:27,809 [ hooks.py: 207]:\tstep: 11670\tsteps/sec: 1.19290\tloss: 0.68096\tlr:[4.207542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:41:36,078 [ hooks.py: 207]:\tstep: 11680\tsteps/sec: 1.20227\tloss: 0.73808\tlr:[4.2068626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:41:45,557 [ hooks.py: 207]:\tstep: 11690\tsteps/sec: 1.06592\tloss: 0.75419\tlr:[4.2061838e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:41:52,967 [ hooks.py: 207]:\tstep: 11700\tsteps/sec: 1.37167\tloss: 0.41641\tlr:[4.2055042e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:00,180 [ hooks.py: 207]:\tstep: 11710\tsteps/sec: 1.46971\tloss: 0.51591\tlr:[4.2048257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:09,254 [ hooks.py: 207]:\tstep: 11720\tsteps/sec: 1.10477\tloss: 0.44315\tlr:[4.2041465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:15,708 [ hooks.py: 207]:\tstep: 11730\tsteps/sec: 1.50024\tloss: 0.43072\tlr:[4.2034673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:22,802 [ hooks.py: 207]:\tstep: 11740\tsteps/sec: 1.37849\tloss: 0.87789\tlr:[4.202788e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:30,458 [ hooks.py: 207]:\tstep: 11750\tsteps/sec: 1.33797\tloss: 1.05720\tlr:[4.2021093e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:38,572 [ hooks.py: 207]:\tstep: 11760\tsteps/sec: 1.24291\tloss: 0.48737\tlr:[4.20143e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:46,246 [ hooks.py: 207]:\tstep: 11770\tsteps/sec: 1.26076\tloss: 0.25200\tlr:[4.2007512e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:42:55,402 [ hooks.py: 207]:\tstep: 11780\tsteps/sec: 1.10768\tloss: 0.72633\tlr:[4.200072e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:43:03,056 [ hooks.py: 207]:\tstep: 11790\tsteps/sec: 1.35778\tloss: 0.48830\tlr:[4.1993928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:43:11,584 [ hooks.py: 207]:\tstep: 11800\tsteps/sec: 1.08885\tloss: 0.54906\tlr:[4.198714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:43:21,718 [ hooks.py: 207]:\tstep: 11810\tsteps/sec: 1.02092\tloss: 0.75210\tlr:[4.1980347e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:43:31,200 [ hooks.py: 207]:\tstep: 11820\tsteps/sec: 1.05645\tloss: 0.38896\tlr:[4.1973555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:43:38,884 [ hooks.py: 207]:\tstep: 11830\tsteps/sec: 1.31826\tloss: 0.57669\tlr:[4.1966767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:43:47,482 [ hooks.py: 207]:\tstep: 11840\tsteps/sec: 1.10214\tloss: 0.93952\tlr:[4.195998e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:43:56,532 [ hooks.py: 207]:\tstep: 11850\tsteps/sec: 1.13468\tloss: 0.76806\tlr:[4.1953186e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:44:05,648 [ hooks.py: 207]:\tstep: 11860\tsteps/sec: 1.10771\tloss: 0.53672\tlr:[4.1946394e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:44:14,242 [ hooks.py: 207]:\tstep: 11870\tsteps/sec: 1.20175\tloss: 0.57710\tlr:[4.1939602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:44:23,478 [ hooks.py: 207]:\tstep: 11880\tsteps/sec: 1.06669\tloss: 0.64459\tlr:[4.1932813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:44:29,676 [ hooks.py: 207]:\tstep: 11890\tsteps/sec: 1.56671\tloss: 0.97598\tlr:[4.1926025e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:44:40,164 [ hooks.py: 207]:\tstep: 11900\tsteps/sec: 0.99143\tloss: 0.77910\tlr:[4.191923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:44:48,129 [ hooks.py: 207]:\tstep: 11910\tsteps/sec: 1.20664\tloss: 0.62273\tlr:[4.191244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:44:56,342 [ hooks.py: 207]:\tstep: 11920\tsteps/sec: 1.19988\tloss: 0.74582\tlr:[4.1905652e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:45:05,059 [ hooks.py: 207]:\tstep: 11930\tsteps/sec: 1.18866\tloss: 1.12799\tlr:[4.1898864e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:45:13,039 [ hooks.py: 207]:\tstep: 11940\tsteps/sec: 1.20679\tloss: 0.67972\tlr:[4.1892068e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:45:23,649 [ hooks.py: 207]:\tstep: 11950\tsteps/sec: 0.97164\tloss: 0.54927\tlr:[4.188528e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 20:45:33,038 [ hooks.py: 207]:\tstep: 11960\tsteps/sec: 1.04028\tloss: 0.46901\tlr:[4.187849e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:45:41,411 [ hooks.py: 207]:\tstep: 11970\tsteps/sec: 1.18666\tloss: 0.54371\tlr:[4.1871703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:45:50,669 [ hooks.py: 207]:\tstep: 11980\tsteps/sec: 1.10195\tloss: 0.71726\tlr:[4.1864907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:45:58,689 [ hooks.py: 207]:\tstep: 11990\tsteps/sec: 1.24109\tloss: 0.43947\tlr:[4.1858122e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:46:09,775 [ hooks.py: 207]:\tstep: 12000\tsteps/sec: 0.92402\tloss: 1.00230\tlr:[4.1851326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:46:09,775 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:46:09,775 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:46:09,775 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:46:09,776 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:46:09,776 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:46:09,776 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:54:02,737 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,738 [ trainer.py: 58]:\t{'acc': 0.7493975903614458, 'loss': 0.6266522885133059}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,739 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,739 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:54:02,739 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:54:02,740 [ trainer.py: 70]:\tacc\t0.7493975903614458\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:54:02,740 [ trainer.py: 70]:\tloss\t0.6266522885133059\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 20:54:02,740 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,740 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7493975903614458, 'loss': 0.6266522885133059}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,740 [ exporter.py: 64]:\t[Best Exporter]: skip step 12000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,740 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7493975903614458, 'loss': 0.6266522885133059}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,740 [ exporter.py: 104]:\t[Best Exporter]: skip step 12000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:02,741 [monitored_executor.py: 122]:\tsaving step 12000 to ./output/model_12000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:05,841 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_9000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:13,334 [ hooks.py: 207]:\tstep: 12010\tsteps/sec: 0.02066\tloss: 0.63636\tlr:[4.1844538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:24,012 [ hooks.py: 207]:\tstep: 12020\tsteps/sec: 0.97808\tloss: 0.63420\tlr:[4.1837746e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:31,057 [ hooks.py: 207]:\tstep: 12030\tsteps/sec: 1.31658\tloss: 0.59465\tlr:[4.1830957e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:39,853 [ hooks.py: 207]:\tstep: 12040\tsteps/sec: 1.23117\tloss: 0.57581\tlr:[4.1824165e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:49,792 [ hooks.py: 207]:\tstep: 12050\tsteps/sec: 1.00242\tloss: 0.38345\tlr:[4.1817373e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:54:58,060 [ hooks.py: 207]:\tstep: 12060\tsteps/sec: 1.15469\tloss: 0.49803\tlr:[4.1810585e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:55:08,616 [ hooks.py: 207]:\tstep: 12070\tsteps/sec: 0.97196\tloss: 0.84754\tlr:[4.1803793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:55:17,274 [ hooks.py: 207]:\tstep: 12080\tsteps/sec: 1.14095\tloss: 0.62704\tlr:[4.1797e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:55:24,548 [ hooks.py: 207]:\tstep: 12090\tsteps/sec: 1.41019\tloss: 0.84073\tlr:[4.179021e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:55:32,484 [ hooks.py: 207]:\tstep: 12100\tsteps/sec: 1.21601\tloss: 0.67026\tlr:[4.178342e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:55:41,964 [ hooks.py: 207]:\tstep: 12110\tsteps/sec: 1.05934\tloss: 0.46717\tlr:[4.177663e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:55:50,447 [ hooks.py: 207]:\tstep: 12120\tsteps/sec: 1.20893\tloss: 0.55042\tlr:[4.1769836e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:55:59,513 [ hooks.py: 207]:\tstep: 12130\tsteps/sec: 1.07619\tloss: 0.75693\tlr:[4.176305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:56:09,790 [ hooks.py: 207]:\tstep: 12140\tsteps/sec: 0.99894\tloss: 0.76308\tlr:[4.175626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:56:19,264 [ hooks.py: 207]:\tstep: 12150\tsteps/sec: 1.04995\tloss: 0.77353\tlr:[4.1749467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:56:27,454 [ hooks.py: 207]:\tstep: 12160\tsteps/sec: 1.17433\tloss: 0.71049\tlr:[4.1742678e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:56:39,247 [ hooks.py: 207]:\tstep: 12170\tsteps/sec: 0.87802\tloss: 0.46214\tlr:[4.1735886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:56:48,912 [ hooks.py: 207]:\tstep: 12180\tsteps/sec: 1.01559\tloss: 0.28141\tlr:[4.1729098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:56:58,381 [ hooks.py: 207]:\tstep: 12190\tsteps/sec: 1.04116\tloss: 0.43156\tlr:[4.1722305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:57:07,210 [ hooks.py: 207]:\tstep: 12200\tsteps/sec: 1.14466\tloss: 0.30310\tlr:[4.1715513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:57:15,395 [ hooks.py: 207]:\tstep: 12210\tsteps/sec: 1.24753\tloss: 0.70441\tlr:[4.1708725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:57:23,529 [ hooks.py: 207]:\tstep: 12220\tsteps/sec: 1.20440\tloss: 0.85715\tlr:[4.1701936e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:57:31,802 [ hooks.py: 207]:\tstep: 12230\tsteps/sec: 1.15090\tloss: 0.64709\tlr:[4.1695144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:57:42,391 [ hooks.py: 207]:\tstep: 12240\tsteps/sec: 0.97128\tloss: 0.46343\tlr:[4.168835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:57:51,672 [ hooks.py: 207]:\tstep: 12250\tsteps/sec: 1.07518\tloss: 0.70311\tlr:[4.168156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:58:00,726 [ hooks.py: 207]:\tstep: 12260\tsteps/sec: 1.07952\tloss: 0.64806\tlr:[4.1674775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:58:08,750 [ hooks.py: 207]:\tstep: 12270\tsteps/sec: 1.30230\tloss: 0.90486\tlr:[4.166798e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:58:16,918 [ hooks.py: 207]:\tstep: 12280\tsteps/sec: 1.20242\tloss: 0.50331\tlr:[4.166119e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:58:25,848 [ hooks.py: 207]:\tstep: 12290\tsteps/sec: 1.08502\tloss: 0.52379\tlr:[4.16544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:58:36,533 [ hooks.py: 207]:\tstep: 12300\tsteps/sec: 0.94321\tloss: 0.73873\tlr:[4.164761e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:58:46,220 [ hooks.py: 207]:\tstep: 12310\tsteps/sec: 1.06866\tloss: 0.34425\tlr:[4.164082e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:58:54,806 [ hooks.py: 207]:\tstep: 12320\tsteps/sec: 1.20558\tloss: 0.47837\tlr:[4.163403e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:59:03,128 [ hooks.py: 207]:\tstep: 12330\tsteps/sec: 1.19966\tloss: 0.74685\tlr:[4.1627238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:59:12,554 [ hooks.py: 207]:\tstep: 12340\tsteps/sec: 1.02649\tloss: 1.08661\tlr:[4.1620446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:59:20,947 [ hooks.py: 207]:\tstep: 12350\tsteps/sec: 1.19550\tloss: 1.04288\tlr:[4.1613654e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:59:30,495 [ hooks.py: 207]:\tstep: 12360\tsteps/sec: 1.07293\tloss: 0.56124\tlr:[4.1606865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:59:40,049 [ hooks.py: 207]:\tstep: 12370\tsteps/sec: 1.04852\tloss: 1.13628\tlr:[4.1600077e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:59:48,071 [ hooks.py: 207]:\tstep: 12380\tsteps/sec: 1.19885\tloss: 0.95043\tlr:[4.1593285e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 20:59:57,145 [ hooks.py: 207]:\tstep: 12390\tsteps/sec: 1.07747\tloss: 0.61905\tlr:[4.158649e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:00:07,413 [ hooks.py: 207]:\tstep: 12400\tsteps/sec: 1.00812\tloss: 1.23940\tlr:[4.1579704e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:00:15,603 [ hooks.py: 207]:\tstep: 12410\tsteps/sec: 1.23028\tloss: 0.60799\tlr:[4.1572912e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 21:00:22,845 [ hooks.py: 207]:\tstep: 12420\tsteps/sec: 1.35775\tloss: 0.60006\tlr:[4.1566123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:00:29,361 [ hooks.py: 207]:\tstep: 12430\tsteps/sec: 1.50868\tloss: 0.64249\tlr:[4.1559328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:00:38,376 [ hooks.py: 207]:\tstep: 12440\tsteps/sec: 1.11703\tloss: 0.72423\tlr:[4.1552543e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:00:47,340 [ hooks.py: 207]:\tstep: 12450\tsteps/sec: 1.08027\tloss: 0.68634\tlr:[4.154575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:00:56,213 [ hooks.py: 207]:\tstep: 12460\tsteps/sec: 1.17607\tloss: 0.54062\tlr:[4.1538962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:01:06,382 [ hooks.py: 207]:\tstep: 12470\tsteps/sec: 0.96404\tloss: 0.70615\tlr:[4.153217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:01:15,977 [ hooks.py: 207]:\tstep: 12480\tsteps/sec: 1.08005\tloss: 0.80939\tlr:[4.152538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:01:22,823 [ hooks.py: 207]:\tstep: 12490\tsteps/sec: 1.42783\tloss: 0.48357\tlr:[4.151859e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:01:31,717 [ hooks.py: 207]:\tstep: 12500\tsteps/sec: 1.12298\tloss: 0.37482\tlr:[4.1511797e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:01:40,267 [ hooks.py: 207]:\tstep: 12510\tsteps/sec: 1.16923\tloss: 0.66271\tlr:[4.150501e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:01:48,202 [ hooks.py: 207]:\tstep: 12520\tsteps/sec: 1.24708\tloss: 0.65945\tlr:[4.149822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:01:57,983 [ hooks.py: 207]:\tstep: 12530\tsteps/sec: 1.03297\tloss: 0.51954\tlr:[4.149143e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:02:07,542 [ hooks.py: 207]:\tstep: 12540\tsteps/sec: 1.05312\tloss: 0.57397\tlr:[4.1484636e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:02:14,681 [ hooks.py: 207]:\tstep: 12550\tsteps/sec: 1.44519\tloss: 0.50383\tlr:[4.1477848e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:02:22,953 [ hooks.py: 207]:\tstep: 12560\tsteps/sec: 1.16965\tloss: 0.51893\tlr:[4.1471056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:02:32,965 [ hooks.py: 207]:\tstep: 12570\tsteps/sec: 1.00061\tloss: 0.25607\tlr:[4.1464264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:02:40,020 [ hooks.py: 207]:\tstep: 12580\tsteps/sec: 1.45837\tloss: 0.69570\tlr:[4.1457475e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:02:48,023 [ hooks.py: 207]:\tstep: 12590\tsteps/sec: 1.22014\tloss: 0.90439\tlr:[4.1450687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:02:55,107 [ hooks.py: 207]:\tstep: 12600\tsteps/sec: 1.43655\tloss: 0.46083\tlr:[4.1443895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:03:03,038 [ hooks.py: 207]:\tstep: 12610\tsteps/sec: 1.21805\tloss: 0.84013\tlr:[4.1437102e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:03:11,575 [ hooks.py: 207]:\tstep: 12620\tsteps/sec: 1.18468\tloss: 0.64062\tlr:[4.143031e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:03:18,363 [ hooks.py: 207]:\tstep: 12630\tsteps/sec: 1.47340\tloss: 0.86876\tlr:[4.1423526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:03:26,584 [ hooks.py: 207]:\tstep: 12640\tsteps/sec: 1.21857\tloss: 0.32869\tlr:[4.1416733e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:03:34,467 [ hooks.py: 207]:\tstep: 12650\tsteps/sec: 1.24989\tloss: 0.91954\tlr:[4.140994e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:03:44,655 [ hooks.py: 207]:\tstep: 12660\tsteps/sec: 0.99174\tloss: 0.44753\tlr:[4.1403153e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:03:52,574 [ hooks.py: 207]:\tstep: 12670\tsteps/sec: 1.21735\tloss: 0.86011\tlr:[4.1396364e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:04:02,496 [ hooks.py: 207]:\tstep: 12680\tsteps/sec: 1.11746\tloss: 0.64714\tlr:[4.1389572e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:04:09,059 [ hooks.py: 207]:\tstep: 12690\tsteps/sec: 1.41507\tloss: 0.51625\tlr:[4.138278e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:04:17,663 [ hooks.py: 207]:\tstep: 12700\tsteps/sec: 1.14080\tloss: 0.60717\tlr:[4.137599e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:04:28,875 [ hooks.py: 207]:\tstep: 12710\tsteps/sec: 0.87802\tloss: 1.01283\tlr:[4.13692e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:04:39,851 [ hooks.py: 207]:\tstep: 12720\tsteps/sec: 0.95476\tloss: 0.93434\tlr:[4.136241e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:04:47,879 [ hooks.py: 207]:\tstep: 12730\tsteps/sec: 1.19859\tloss: 0.48866\tlr:[4.135562e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:04:55,837 [ hooks.py: 207]:\tstep: 12740\tsteps/sec: 1.20672\tloss: 0.48846\tlr:[4.1348827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:05:06,373 [ hooks.py: 207]:\tstep: 12750\tsteps/sec: 0.97015\tloss: 0.76995\tlr:[4.1342035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:05:14,757 [ hooks.py: 207]:\tstep: 12760\tsteps/sec: 1.18973\tloss: 0.64813\tlr:[4.1335246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:05:24,506 [ hooks.py: 207]:\tstep: 12770\tsteps/sec: 1.05557\tloss: 0.43737\tlr:[4.1328454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:05:34,447 [ hooks.py: 207]:\tstep: 12780\tsteps/sec: 1.00447\tloss: 0.68646\tlr:[4.1321666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:05:41,591 [ hooks.py: 207]:\tstep: 12790\tsteps/sec: 1.36725\tloss: 0.78843\tlr:[4.1314874e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:05:49,691 [ hooks.py: 207]:\tstep: 12800\tsteps/sec: 1.22755\tloss: 1.06274\tlr:[4.130808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:06:00,667 [ hooks.py: 207]:\tstep: 12810\tsteps/sec: 0.94449\tloss: 0.59685\tlr:[4.130129e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:06:09,972 [ hooks.py: 207]:\tstep: 12820\tsteps/sec: 1.04532\tloss: 1.02268\tlr:[4.1294505e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:06:18,156 [ hooks.py: 207]:\tstep: 12830\tsteps/sec: 1.18482\tloss: 0.56755\tlr:[4.128771e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:06:26,391 [ hooks.py: 207]:\tstep: 12840\tsteps/sec: 1.23995\tloss: 0.64932\tlr:[4.1280917e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:06:34,888 [ hooks.py: 207]:\tstep: 12850\tsteps/sec: 1.21245\tloss: 0.43100\tlr:[4.1274132e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:06:44,178 [ hooks.py: 207]:\tstep: 12860\tsteps/sec: 1.07292\tloss: 0.56142\tlr:[4.1267336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:06:53,641 [ hooks.py: 207]:\tstep: 12870\tsteps/sec: 1.03661\tloss: 0.69196\tlr:[4.1260548e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:03,672 [ hooks.py: 207]:\tstep: 12880\tsteps/sec: 1.00302\tloss: 0.52239\tlr:[4.125376e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:12,243 [ hooks.py: 207]:\tstep: 12890\tsteps/sec: 1.15829\tloss: 0.64945\tlr:[4.124697e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:19,483 [ hooks.py: 207]:\tstep: 12900\tsteps/sec: 1.42312\tloss: 0.63170\tlr:[4.124018e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:27,029 [ hooks.py: 207]:\tstep: 12910\tsteps/sec: 1.28015\tloss: 0.76854\tlr:[4.1233387e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:33,918 [ hooks.py: 207]:\tstep: 12920\tsteps/sec: 1.42391\tloss: 0.58498\tlr:[4.1226598e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:42,708 [ hooks.py: 207]:\tstep: 12930\tsteps/sec: 1.16705\tloss: 0.54667\tlr:[4.1219802e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:50,236 [ hooks.py: 207]:\tstep: 12940\tsteps/sec: 1.38112\tloss: 0.53895\tlr:[4.1213014e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:07:58,484 [ hooks.py: 207]:\tstep: 12950\tsteps/sec: 1.15707\tloss: 0.58650\tlr:[4.1206225e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:08,634 [ hooks.py: 207]:\tstep: 12960\tsteps/sec: 0.96887\tloss: 0.59842\tlr:[4.119943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:17,609 [ hooks.py: 207]:\tstep: 12970\tsteps/sec: 1.14534\tloss: 0.44092\tlr:[4.1192645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:27,623 [ hooks.py: 207]:\tstep: 12980\tsteps/sec: 1.00268\tloss: 0.59312\tlr:[4.1185853e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:35,961 [ hooks.py: 207]:\tstep: 12990\tsteps/sec: 1.14979\tloss: 0.56273\tlr:[4.1179057e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:45,945 [ hooks.py: 207]:\tstep: 13000\tsteps/sec: 1.04436\tloss: 0.52486\tlr:[4.1172272e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:45,945 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:45,945 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:45,946 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:08:45,946 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:45,946 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:08:45,946 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:16:46,772 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,773 [ trainer.py: 58]:\t{'acc': 0.755421686746988, 'loss': 0.6143025574393761}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,774 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,774 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:16:46,774 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:16:46,774 [ trainer.py: 70]:\tacc\t0.755421686746988\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:16:46,775 [ trainer.py: 70]:\tloss\t0.6143025574393761\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:16:46,775 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,775 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.755421686746988, 'loss': 0.6143025574393761}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,775 [ exporter.py: 64]:\t[Best Exporter]: skip step 13000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,775 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.755421686746988, 'loss': 0.6143025574393761}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,775 [ exporter.py: 104]:\t[Best Exporter]: skip step 13000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:46,776 [monitored_executor.py: 122]:\tsaving step 13000 to ./output/model_13000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 21:16:50,335 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_10000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:16:59,703 [ hooks.py: 207]:\tstep: 13010\tsteps/sec: 0.02026\tloss: 0.51758\tlr:[4.1165476e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:17:07,888 [ hooks.py: 207]:\tstep: 13020\tsteps/sec: 1.17844\tloss: 0.29439\tlr:[4.115869e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:17:16,134 [ hooks.py: 207]:\tstep: 13030\tsteps/sec: 1.17257\tloss: 0.38449\tlr:[4.11519e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:17:26,503 [ hooks.py: 207]:\tstep: 13040\tsteps/sec: 0.95800\tloss: 0.79674\tlr:[4.1145107e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:17:35,255 [ hooks.py: 207]:\tstep: 13050\tsteps/sec: 1.21877\tloss: 0.77170\tlr:[4.113832e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:17:42,836 [ hooks.py: 207]:\tstep: 13060\tsteps/sec: 1.23651\tloss: 0.78980\tlr:[4.1131527e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:17:51,995 [ hooks.py: 207]:\tstep: 13070\tsteps/sec: 1.09132\tloss: 0.53949\tlr:[4.112474e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:18:04,863 [ hooks.py: 207]:\tstep: 13080\tsteps/sec: 0.80245\tloss: 0.63121\tlr:[4.111795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:18:14,843 [ hooks.py: 207]:\tstep: 13090\tsteps/sec: 1.01417\tloss: 0.68024\tlr:[4.1111154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:18:24,184 [ hooks.py: 207]:\tstep: 13100\tsteps/sec: 1.06101\tloss: 0.36549\tlr:[4.1104362e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:18:31,586 [ hooks.py: 207]:\tstep: 13110\tsteps/sec: 1.34330\tloss: 0.57470\tlr:[4.1097574e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:18:41,191 [ hooks.py: 207]:\tstep: 13120\tsteps/sec: 1.05739\tloss: 1.08396\tlr:[4.1090785e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:18:50,801 [ hooks.py: 207]:\tstep: 13130\tsteps/sec: 1.05033\tloss: 0.60753\tlr:[4.1083993e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:18:57,501 [ hooks.py: 207]:\tstep: 13140\tsteps/sec: 1.40240\tloss: 0.50084\tlr:[4.10772e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:19:06,915 [ hooks.py: 207]:\tstep: 13150\tsteps/sec: 1.03472\tloss: 0.94235\tlr:[4.1070416e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:19:15,272 [ hooks.py: 207]:\tstep: 13160\tsteps/sec: 1.29867\tloss: 1.17281\tlr:[4.106362e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:19:22,019 [ hooks.py: 207]:\tstep: 13170\tsteps/sec: 1.34758\tloss: 0.46727\tlr:[4.1056832e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:19:31,913 [ hooks.py: 207]:\tstep: 13180\tsteps/sec: 1.05436\tloss: 0.49326\tlr:[4.105004e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:19:40,959 [ hooks.py: 207]:\tstep: 13190\tsteps/sec: 1.13708\tloss: 0.32875\tlr:[4.104325e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:19:49,364 [ hooks.py: 207]:\tstep: 13200\tsteps/sec: 1.16791\tloss: 0.48099\tlr:[4.1036463e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:19:58,578 [ hooks.py: 207]:\tstep: 13210\tsteps/sec: 1.07166\tloss: 0.25188\tlr:[4.102967e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:20:08,586 [ hooks.py: 207]:\tstep: 13220\tsteps/sec: 0.99228\tloss: 0.32719\tlr:[4.1022882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:20:16,811 [ hooks.py: 207]:\tstep: 13230\tsteps/sec: 1.22202\tloss: 0.56626\tlr:[4.101609e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:20:25,745 [ hooks.py: 207]:\tstep: 13240\tsteps/sec: 1.09798\tloss: 0.53150\tlr:[4.10093e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:20:36,062 [ hooks.py: 207]:\tstep: 13250\tsteps/sec: 1.01992\tloss: 0.46065\tlr:[4.100251e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:20:43,406 [ hooks.py: 207]:\tstep: 13260\tsteps/sec: 1.30272\tloss: 0.89197\tlr:[4.0995717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:20:50,892 [ hooks.py: 207]:\tstep: 13270\tsteps/sec: 1.38882\tloss: 0.37808\tlr:[4.098893e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:20:58,048 [ hooks.py: 207]:\tstep: 13280\tsteps/sec: 1.35159\tloss: 0.63561\tlr:[4.0982137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:21:05,304 [ hooks.py: 207]:\tstep: 13290\tsteps/sec: 1.43444\tloss: 0.77924\tlr:[4.0975345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:21:12,359 [ hooks.py: 207]:\tstep: 13300\tsteps/sec: 1.33685\tloss: 0.31331\tlr:[4.0968553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:21:20,446 [ hooks.py: 207]:\tstep: 13310\tsteps/sec: 1.30311\tloss: 0.53663\tlr:[4.0961764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:21:30,226 [ hooks.py: 207]:\tstep: 13320\tsteps/sec: 0.99752\tloss: 0.74264\tlr:[4.0954976e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:21:40,132 [ hooks.py: 207]:\tstep: 13330\tsteps/sec: 1.04189\tloss: 0.68502\tlr:[4.0948184e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:21:46,982 [ hooks.py: 207]:\tstep: 13340\tsteps/sec: 1.37074\tloss: 0.91626\tlr:[4.0941395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:21:54,876 [ hooks.py: 207]:\tstep: 13350\tsteps/sec: 1.27080\tloss: 0.49381\tlr:[4.09346e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:22:06,352 [ hooks.py: 207]:\tstep: 13360\tsteps/sec: 0.86899\tloss: 0.41580\tlr:[4.0927815e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:22:14,221 [ hooks.py: 207]:\tstep: 13370\tsteps/sec: 1.30354\tloss: 0.66764\tlr:[4.092102e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:22:20,857 [ hooks.py: 207]:\tstep: 13380\tsteps/sec: 1.51809\tloss: 1.05837\tlr:[4.091423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:22:28,770 [ hooks.py: 207]:\tstep: 13390\tsteps/sec: 1.22781\tloss: 0.53163\tlr:[4.090744e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:22:36,757 [ hooks.py: 207]:\tstep: 13400\tsteps/sec: 1.19789\tloss: 1.05973\tlr:[4.0900646e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:22:48,059 [ hooks.py: 207]:\tstep: 13410\tsteps/sec: 0.91428\tloss: 0.62163\tlr:[4.0893858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:22:56,800 [ hooks.py: 207]:\tstep: 13420\tsteps/sec: 1.14560\tloss: 0.82104\tlr:[4.088707e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:23:07,138 [ hooks.py: 207]:\tstep: 13430\tsteps/sec: 0.97654\tloss: 0.57912\tlr:[4.0880273e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:23:14,161 [ hooks.py: 207]:\tstep: 13440\tsteps/sec: 1.42334\tloss: 0.40669\tlr:[4.0873485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:23:23,421 [ hooks.py: 207]:\tstep: 13450\tsteps/sec: 1.07953\tloss: 0.83132\tlr:[4.0866697e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:23:32,686 [ hooks.py: 207]:\tstep: 13460\tsteps/sec: 1.06725\tloss: 0.81357\tlr:[4.0859904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:23:41,108 [ hooks.py: 207]:\tstep: 13470\tsteps/sec: 1.23876\tloss: 0.87183\tlr:[4.0853116e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:23:52,063 [ hooks.py: 207]:\tstep: 13480\tsteps/sec: 0.88699\tloss: 0.96579\tlr:[4.084632e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:23:59,358 [ hooks.py: 207]:\tstep: 13490\tsteps/sec: 1.40837\tloss: 0.48951\tlr:[4.0839535e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:24:07,225 [ hooks.py: 207]:\tstep: 13500\tsteps/sec: 1.18785\tloss: 0.38417\tlr:[4.0832743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:24:17,919 [ hooks.py: 207]:\tstep: 13510\tsteps/sec: 0.97193\tloss: 0.48874\tlr:[4.0825955e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:24:25,492 [ hooks.py: 207]:\tstep: 13520\tsteps/sec: 1.34787\tloss: 0.85370\tlr:[4.0819163e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:24:34,375 [ hooks.py: 207]:\tstep: 13530\tsteps/sec: 1.11950\tloss: 0.32790\tlr:[4.081237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:24:43,402 [ hooks.py: 207]:\tstep: 13540\tsteps/sec: 1.08543\tloss: 0.85621\tlr:[4.080558e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:24:52,039 [ hooks.py: 207]:\tstep: 13550\tsteps/sec: 1.16661\tloss: 0.66211\tlr:[4.0798786e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:25:00,312 [ hooks.py: 207]:\tstep: 13560\tsteps/sec: 1.27767\tloss: 0.52606\tlr:[4.0791998e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:25:08,713 [ hooks.py: 207]:\tstep: 13570\tsteps/sec: 1.13495\tloss: 0.76524\tlr:[4.078521e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:25:17,749 [ hooks.py: 207]:\tstep: 13580\tsteps/sec: 1.10949\tloss: 0.50724\tlr:[4.0778414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:25:26,112 [ hooks.py: 207]:\tstep: 13590\tsteps/sec: 1.22633\tloss: 0.67147\tlr:[4.0771625e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:25:36,294 [ hooks.py: 207]:\tstep: 13600\tsteps/sec: 0.97258\tloss: 0.68801\tlr:[4.0764833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:25:43,180 [ hooks.py: 207]:\tstep: 13610\tsteps/sec: 1.41145\tloss: 0.72823\tlr:[4.075804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:25:53,163 [ hooks.py: 207]:\tstep: 13620\tsteps/sec: 1.00323\tloss: 0.63293\tlr:[4.0751253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:26:02,148 [ hooks.py: 207]:\tstep: 13630\tsteps/sec: 1.14588\tloss: 0.62917\tlr:[4.074446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:26:11,257 [ hooks.py: 207]:\tstep: 13640\tsteps/sec: 1.10183\tloss: 0.61503\tlr:[4.0737676e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 21:26:19,997 [ hooks.py: 207]:\tstep: 13650\tsteps/sec: 1.14922\tloss: 0.50655\tlr:[4.0730883e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:26:27,640 [ hooks.py: 207]:\tstep: 13660\tsteps/sec: 1.25370\tloss: 0.99422\tlr:[4.072409e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:26:37,469 [ hooks.py: 207]:\tstep: 13670\tsteps/sec: 1.01692\tloss: 0.45947\tlr:[4.0717303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:26:47,035 [ hooks.py: 207]:\tstep: 13680\tsteps/sec: 1.04378\tloss: 0.83354\tlr:[4.0710514e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:26:56,507 [ hooks.py: 207]:\tstep: 13690\tsteps/sec: 1.06125\tloss: 0.37543\tlr:[4.0703722e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:27:05,551 [ hooks.py: 207]:\tstep: 13700\tsteps/sec: 1.09293\tloss: 0.70791\tlr:[4.069693e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:27:15,455 [ hooks.py: 207]:\tstep: 13710\tsteps/sec: 1.05202\tloss: 0.23790\tlr:[4.069014e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:27:22,708 [ hooks.py: 207]:\tstep: 13720\tsteps/sec: 1.26473\tloss: 0.49940\tlr:[4.0683353e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:27:33,273 [ hooks.py: 207]:\tstep: 13730\tsteps/sec: 0.98145\tloss: 1.03076\tlr:[4.067656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:27:39,529 [ hooks.py: 207]:\tstep: 13740\tsteps/sec: 1.55978\tloss: 0.72456\tlr:[4.066977e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:27:50,206 [ hooks.py: 207]:\tstep: 13750\tsteps/sec: 0.97490\tloss: 0.99333\tlr:[4.066298e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:27:57,460 [ hooks.py: 207]:\tstep: 13760\tsteps/sec: 1.26675\tloss: 0.53676\tlr:[4.065619e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:28:06,326 [ hooks.py: 207]:\tstep: 13770\tsteps/sec: 1.14205\tloss: 0.65935\tlr:[4.0649396e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:28:15,604 [ hooks.py: 207]:\tstep: 13780\tsteps/sec: 1.14170\tloss: 0.65501\tlr:[4.0642608e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:28:22,912 [ hooks.py: 207]:\tstep: 13790\tsteps/sec: 1.32264\tloss: 0.29741\tlr:[4.0635816e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:28:32,641 [ hooks.py: 207]:\tstep: 13800\tsteps/sec: 1.01659\tloss: 0.25330\tlr:[4.0629024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:28:41,759 [ hooks.py: 207]:\tstep: 13810\tsteps/sec: 1.09551\tloss: 0.66417\tlr:[4.0622235e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:28:53,549 [ hooks.py: 207]:\tstep: 13820\tsteps/sec: 0.85784\tloss: 0.76011\tlr:[4.0615443e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:00,433 [ hooks.py: 207]:\tstep: 13830\tsteps/sec: 1.54239\tloss: 0.23579\tlr:[4.0608655e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:07,509 [ hooks.py: 207]:\tstep: 13840\tsteps/sec: 1.32625\tloss: 0.81826\tlr:[4.0601866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:14,533 [ hooks.py: 207]:\tstep: 13850\tsteps/sec: 1.45680\tloss: 0.70454\tlr:[4.059507e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:22,938 [ hooks.py: 207]:\tstep: 13860\tsteps/sec: 1.16356\tloss: 1.09401\tlr:[4.0588282e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:30,780 [ hooks.py: 207]:\tstep: 13870\tsteps/sec: 1.33151\tloss: 0.84357\tlr:[4.0581497e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:38,905 [ hooks.py: 207]:\tstep: 13880\tsteps/sec: 1.18813\tloss: 0.84296\tlr:[4.05747e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:45,999 [ hooks.py: 207]:\tstep: 13890\tsteps/sec: 1.45384\tloss: 0.63103\tlr:[4.056791e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:29:56,437 [ hooks.py: 207]:\tstep: 13900\tsteps/sec: 0.93034\tloss: 0.58677\tlr:[4.0561124e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:30:06,674 [ hooks.py: 207]:\tstep: 13910\tsteps/sec: 0.97927\tloss: 0.55060\tlr:[4.055433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:30:14,899 [ hooks.py: 207]:\tstep: 13920\tsteps/sec: 1.21195\tloss: 0.52190\tlr:[4.0547544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:30:25,327 [ hooks.py: 207]:\tstep: 13930\tsteps/sec: 0.99550\tloss: 0.79788\tlr:[4.054075e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:30:34,632 [ hooks.py: 207]:\tstep: 13940\tsteps/sec: 1.05646\tloss: 0.60629\tlr:[4.0533963e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:30:42,017 [ hooks.py: 207]:\tstep: 13950\tsteps/sec: 1.25202\tloss: 0.41358\tlr:[4.052717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:30:51,135 [ hooks.py: 207]:\tstep: 13960\tsteps/sec: 1.19234\tloss: 0.33009\tlr:[4.052038e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:30:58,341 [ hooks.py: 207]:\tstep: 13970\tsteps/sec: 1.33747\tloss: 0.34191\tlr:[4.051359e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:06,596 [ hooks.py: 207]:\tstep: 13980\tsteps/sec: 1.21398\tloss: 1.13117\tlr:[4.05068e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:13,649 [ hooks.py: 207]:\tstep: 13990\tsteps/sec: 1.40636\tloss: 0.37883\tlr:[4.0500006e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:20,475 [ hooks.py: 207]:\tstep: 14000\tsteps/sec: 1.47183\tloss: 0.89890\tlr:[4.0493214e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:20,476 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:20,476 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:20,476 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:31:20,476 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:20,476 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:31:20,477 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:39:14,660 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:14,662 [ trainer.py: 58]:\t{'acc': 0.7698795180722892, 'loss': 0.594362390251496}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:14,662 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:14,663 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:39:14,663 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:39:14,663 [ trainer.py: 70]:\tacc\t0.7698795180722892\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:39:14,663 [ trainer.py: 70]:\tloss\t0.594362390251496\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:39:14,663 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:14,663 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:14,663 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:14,664 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_10000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:14,665 [monitored_executor.py: 122]:\tsaving step 14000 to ./output/best_model/model_14000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:15,617 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_10000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:15,618 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}} \n", + "old: {'eval': {'acc': 0.7598393574297189, 'loss': 0.5944234219690164}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:15,618 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:15,618 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:16,843 [monitored_executor.py: 122]:\tsaving step 14000 to ./output/model_14000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:19,999 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_11000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:29,088 [ hooks.py: 207]:\tstep: 14010\tsteps/sec: 0.02046\tloss: 0.39247\tlr:[4.0486426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:37,464 [ hooks.py: 207]:\tstep: 14020\tsteps/sec: 1.19076\tloss: 0.77708\tlr:[4.0479637e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:45,395 [ hooks.py: 207]:\tstep: 14030\tsteps/sec: 1.31615\tloss: 0.72299\tlr:[4.0472845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:39:53,163 [ hooks.py: 207]:\tstep: 14040\tsteps/sec: 1.23851\tloss: 0.62873\tlr:[4.0466053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:40:03,433 [ hooks.py: 207]:\tstep: 14050\tsteps/sec: 0.97015\tloss: 0.43813\tlr:[4.045926e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:40:11,485 [ hooks.py: 207]:\tstep: 14060\tsteps/sec: 1.30361\tloss: 0.48820\tlr:[4.0452473e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 21:40:18,618 [ hooks.py: 207]:\tstep: 14070\tsteps/sec: 1.33885\tloss: 0.90927\tlr:[4.044568e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:40:26,494 [ hooks.py: 207]:\tstep: 14080\tsteps/sec: 1.28332\tloss: 0.76709\tlr:[4.0438892e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:40:34,065 [ hooks.py: 207]:\tstep: 14090\tsteps/sec: 1.29677\tloss: 0.69640\tlr:[4.0432104e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:40:42,171 [ hooks.py: 207]:\tstep: 14100\tsteps/sec: 1.18503\tloss: 0.76730\tlr:[4.042531e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:40:50,753 [ hooks.py: 207]:\tstep: 14110\tsteps/sec: 1.28216\tloss: 0.64339\tlr:[4.0418516e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:40:57,990 [ hooks.py: 207]:\tstep: 14120\tsteps/sec: 1.26815\tloss: 0.54866\tlr:[4.0411727e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:41:08,254 [ hooks.py: 207]:\tstep: 14130\tsteps/sec: 0.97260\tloss: 0.33301\tlr:[4.040494e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:41:16,550 [ hooks.py: 207]:\tstep: 14140\tsteps/sec: 1.20364\tloss: 1.22798\tlr:[4.039815e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:41:26,303 [ hooks.py: 207]:\tstep: 14150\tsteps/sec: 1.08878\tloss: 0.55813\tlr:[4.039136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:41:35,786 [ hooks.py: 207]:\tstep: 14160\tsteps/sec: 1.01784\tloss: 0.41723\tlr:[4.0384566e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:41:45,319 [ hooks.py: 207]:\tstep: 14170\tsteps/sec: 1.08919\tloss: 0.84154\tlr:[4.037778e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:41:52,605 [ hooks.py: 207]:\tstep: 14180\tsteps/sec: 1.30977\tloss: 0.69847\tlr:[4.037099e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:41:59,937 [ hooks.py: 207]:\tstep: 14190\tsteps/sec: 1.44634\tloss: 0.52845\tlr:[4.0364197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:42:08,956 [ hooks.py: 207]:\tstep: 14200\tsteps/sec: 1.07836\tloss: 1.05208\tlr:[4.0357405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:42:16,347 [ hooks.py: 207]:\tstep: 14210\tsteps/sec: 1.38734\tloss: 0.34230\tlr:[4.0350613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:42:24,957 [ hooks.py: 207]:\tstep: 14220\tsteps/sec: 1.14523\tloss: 0.60655\tlr:[4.0343824e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:42:33,117 [ hooks.py: 207]:\tstep: 14230\tsteps/sec: 1.16154\tloss: 0.44978\tlr:[4.0337032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:42:41,527 [ hooks.py: 207]:\tstep: 14240\tsteps/sec: 1.28838\tloss: 0.54713\tlr:[4.0330244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:42:48,520 [ hooks.py: 207]:\tstep: 14250\tsteps/sec: 1.36605\tloss: 0.51008\tlr:[4.032345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:42:56,708 [ hooks.py: 207]:\tstep: 14260\tsteps/sec: 1.22985\tloss: 0.40777\tlr:[4.031666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:43:05,407 [ hooks.py: 207]:\tstep: 14270\tsteps/sec: 1.17027\tloss: 0.72040\tlr:[4.0309867e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:43:13,624 [ hooks.py: 207]:\tstep: 14280\tsteps/sec: 1.19069\tloss: 0.46365\tlr:[4.0303083e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:43:22,192 [ hooks.py: 207]:\tstep: 14290\tsteps/sec: 1.10074\tloss: 0.33753\tlr:[4.0296287e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:43:31,168 [ hooks.py: 207]:\tstep: 14300\tsteps/sec: 1.16030\tloss: 1.02119\tlr:[4.0289495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:43:38,694 [ hooks.py: 207]:\tstep: 14310\tsteps/sec: 1.34080\tloss: 0.78393\tlr:[4.0282706e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:43:47,360 [ hooks.py: 207]:\tstep: 14320\tsteps/sec: 1.15657\tloss: 0.61225\tlr:[4.027592e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:43:56,910 [ hooks.py: 207]:\tstep: 14330\tsteps/sec: 1.05172\tloss: 0.37861\tlr:[4.026913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:44:06,871 [ hooks.py: 207]:\tstep: 14340\tsteps/sec: 0.99473\tloss: 0.52614\tlr:[4.0262334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:44:14,699 [ hooks.py: 207]:\tstep: 14350\tsteps/sec: 1.31244\tloss: 0.61427\tlr:[4.0255545e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:44:23,134 [ hooks.py: 207]:\tstep: 14360\tsteps/sec: 1.16532\tloss: 0.52837\tlr:[4.0248753e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:44:30,006 [ hooks.py: 207]:\tstep: 14370\tsteps/sec: 1.49857\tloss: 0.51319\tlr:[4.024197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:44:41,242 [ hooks.py: 207]:\tstep: 14380\tsteps/sec: 0.90264\tloss: 0.81566\tlr:[4.0235172e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:44:47,896 [ hooks.py: 207]:\tstep: 14390\tsteps/sec: 1.42610\tloss: 0.78103\tlr:[4.0228384e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:44:55,056 [ hooks.py: 207]:\tstep: 14400\tsteps/sec: 1.43190\tloss: 0.59333\tlr:[4.0221592e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:03,412 [ hooks.py: 207]:\tstep: 14410\tsteps/sec: 1.17064\tloss: 0.80331\tlr:[4.02148e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:11,963 [ hooks.py: 207]:\tstep: 14420\tsteps/sec: 1.18407\tloss: 0.56414\tlr:[4.020801e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:20,167 [ hooks.py: 207]:\tstep: 14430\tsteps/sec: 1.24396\tloss: 0.35379\tlr:[4.0201227e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:27,458 [ hooks.py: 207]:\tstep: 14440\tsteps/sec: 1.32332\tloss: 0.63131\tlr:[4.0194434e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:34,600 [ hooks.py: 207]:\tstep: 14450\tsteps/sec: 1.41810\tloss: 0.69437\tlr:[4.0187642e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:42,539 [ hooks.py: 207]:\tstep: 14460\tsteps/sec: 1.23799\tloss: 0.31503\tlr:[4.018085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:49,220 [ hooks.py: 207]:\tstep: 14470\tsteps/sec: 1.51105\tloss: 0.75240\tlr:[4.017406e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:45:57,071 [ hooks.py: 207]:\tstep: 14480\tsteps/sec: 1.24049\tloss: 0.60673\tlr:[4.016727e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:46:05,355 [ hooks.py: 207]:\tstep: 14490\tsteps/sec: 1.27818\tloss: 0.88642\tlr:[4.0160478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:46:15,330 [ hooks.py: 207]:\tstep: 14500\tsteps/sec: 1.00076\tloss: 0.66042\tlr:[4.015369e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:46:23,658 [ hooks.py: 207]:\tstep: 14510\tsteps/sec: 1.16177\tloss: 0.64200\tlr:[4.0146897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:46:32,153 [ hooks.py: 207]:\tstep: 14520\tsteps/sec: 1.17717\tloss: 0.52944\tlr:[4.014011e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:46:39,395 [ hooks.py: 207]:\tstep: 14530\tsteps/sec: 1.33250\tloss: 0.55345\tlr:[4.0133316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:46:48,181 [ hooks.py: 207]:\tstep: 14540\tsteps/sec: 1.18020\tloss: 0.77239\tlr:[4.0126524e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:46:56,657 [ hooks.py: 207]:\tstep: 14550\tsteps/sec: 1.21302\tloss: 0.85609\tlr:[4.0119736e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:47:04,610 [ hooks.py: 207]:\tstep: 14560\tsteps/sec: 1.19469\tloss: 0.65834\tlr:[4.0112944e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:47:13,658 [ hooks.py: 207]:\tstep: 14570\tsteps/sec: 1.12073\tloss: 0.80321\tlr:[4.010616e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:47:21,563 [ hooks.py: 207]:\tstep: 14580\tsteps/sec: 1.31159\tloss: 0.59882\tlr:[4.0099367e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:47:31,259 [ hooks.py: 207]:\tstep: 14590\tsteps/sec: 1.03598\tloss: 0.58451\tlr:[4.0092575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:47:40,704 [ hooks.py: 207]:\tstep: 14600\tsteps/sec: 1.07481\tloss: 0.79887\tlr:[4.0085783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:47:48,389 [ hooks.py: 207]:\tstep: 14610\tsteps/sec: 1.19189\tloss: 0.38635\tlr:[4.0078998e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:47:57,447 [ hooks.py: 207]:\tstep: 14620\tsteps/sec: 1.12959\tloss: 1.02737\tlr:[4.0072206e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:48:05,462 [ hooks.py: 207]:\tstep: 14630\tsteps/sec: 1.27831\tloss: 0.60125\tlr:[4.0065413e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:48:16,299 [ hooks.py: 207]:\tstep: 14640\tsteps/sec: 0.94196\tloss: 0.59786\tlr:[4.005862e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:48:23,168 [ hooks.py: 207]:\tstep: 14650\tsteps/sec: 1.37961\tloss: 0.76765\tlr:[4.0051833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:48:32,930 [ hooks.py: 207]:\tstep: 14660\tsteps/sec: 0.99337\tloss: 1.02819\tlr:[4.0045044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:48:40,957 [ hooks.py: 207]:\tstep: 14670\tsteps/sec: 1.31336\tloss: 0.30499\tlr:[4.0038252e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:48:50,249 [ hooks.py: 207]:\tstep: 14680\tsteps/sec: 1.10383\tloss: 0.72351\tlr:[4.003146e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:48:56,692 [ hooks.py: 207]:\tstep: 14690\tsteps/sec: 1.51071\tloss: 0.96589\tlr:[4.002467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:49:04,906 [ hooks.py: 207]:\tstep: 14700\tsteps/sec: 1.23020\tloss: 0.66189\tlr:[4.001788e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:49:11,564 [ hooks.py: 207]:\tstep: 14710\tsteps/sec: 1.50349\tloss: 0.74929\tlr:[4.0011088e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 21:49:20,667 [ hooks.py: 207]:\tstep: 14720\tsteps/sec: 1.11653\tloss: 0.41403\tlr:[4.0004295e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:49:28,089 [ hooks.py: 207]:\tstep: 14730\tsteps/sec: 1.29138\tloss: 0.73301\tlr:[3.9997507e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:49:35,950 [ hooks.py: 207]:\tstep: 14740\tsteps/sec: 1.29944\tloss: 0.62465\tlr:[3.9990715e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:49:43,590 [ hooks.py: 207]:\tstep: 14750\tsteps/sec: 1.27836\tloss: 0.50009\tlr:[3.9983926e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:49:52,992 [ hooks.py: 207]:\tstep: 14760\tsteps/sec: 1.05723\tloss: 0.67452\tlr:[3.9977138e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:00,878 [ hooks.py: 207]:\tstep: 14770\tsteps/sec: 1.35348\tloss: 1.00488\tlr:[3.9970346e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:09,131 [ hooks.py: 207]:\tstep: 14780\tsteps/sec: 1.14511\tloss: 0.87597\tlr:[3.996355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:16,641 [ hooks.py: 207]:\tstep: 14790\tsteps/sec: 1.37221\tloss: 0.56440\tlr:[3.9956758e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:27,687 [ hooks.py: 207]:\tstep: 14800\tsteps/sec: 0.88607\tloss: 0.52770\tlr:[3.9949973e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:35,524 [ hooks.py: 207]:\tstep: 14810\tsteps/sec: 1.33595\tloss: 0.85798\tlr:[3.994318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:44,542 [ hooks.py: 207]:\tstep: 14820\tsteps/sec: 1.07617\tloss: 0.50311\tlr:[3.9936393e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:53,076 [ hooks.py: 207]:\tstep: 14830\tsteps/sec: 1.16519\tloss: 0.74874\tlr:[3.99296e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:50:59,837 [ hooks.py: 207]:\tstep: 14840\tsteps/sec: 1.53104\tloss: 0.57665\tlr:[3.9922812e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:51:08,325 [ hooks.py: 207]:\tstep: 14850\tsteps/sec: 1.15377\tloss: 0.56157\tlr:[3.991602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:51:15,721 [ hooks.py: 207]:\tstep: 14860\tsteps/sec: 1.39417\tloss: 1.10273\tlr:[3.9909224e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:51:25,198 [ hooks.py: 207]:\tstep: 14870\tsteps/sec: 1.04571\tloss: 0.62778\tlr:[3.990244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:51:31,949 [ hooks.py: 207]:\tstep: 14880\tsteps/sec: 1.44291\tloss: 0.68114\tlr:[3.989565e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:51:38,851 [ hooks.py: 207]:\tstep: 14890\tsteps/sec: 1.45040\tloss: 0.58296\tlr:[3.988886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:51:48,299 [ hooks.py: 207]:\tstep: 14900\tsteps/sec: 1.06560\tloss: 0.49089\tlr:[3.9882067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:51:55,121 [ hooks.py: 207]:\tstep: 14910\tsteps/sec: 1.42558\tloss: 0.64820\tlr:[3.9875278e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:52:03,310 [ hooks.py: 207]:\tstep: 14920\tsteps/sec: 1.28521\tloss: 0.47252\tlr:[3.9868486e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:52:11,349 [ hooks.py: 207]:\tstep: 14930\tsteps/sec: 1.19646\tloss: 0.74085\tlr:[3.9861698e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:52:21,228 [ hooks.py: 207]:\tstep: 14940\tsteps/sec: 1.05277\tloss: 0.44840\tlr:[3.9854905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:52:30,917 [ hooks.py: 207]:\tstep: 14950\tsteps/sec: 1.02722\tloss: 0.81848\tlr:[3.9848113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:52:42,171 [ hooks.py: 207]:\tstep: 14960\tsteps/sec: 0.86237\tloss: 0.51895\tlr:[3.984132e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:52:51,019 [ hooks.py: 207]:\tstep: 14970\tsteps/sec: 1.18770\tloss: 0.91503\tlr:[3.9834533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:52:58,668 [ hooks.py: 207]:\tstep: 14980\tsteps/sec: 1.26352\tloss: 0.63142\tlr:[3.982774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:53:07,194 [ hooks.py: 207]:\tstep: 14990\tsteps/sec: 1.12059\tloss: 0.53707\tlr:[3.982095e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:53:15,812 [ hooks.py: 207]:\tstep: 15000\tsteps/sec: 1.18948\tloss: 0.66576\tlr:[3.9814156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:53:15,816 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:53:15,816 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:53:15,816 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 21:53:15,817 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:53:15,817 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 21:53:15,817 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:01:10,369 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,371 [ trainer.py: 58]:\t{'acc': 0.7578313253012048, 'loss': 0.5867884608033376}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,372 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,373 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:01:10,373 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:01:10,373 [ trainer.py: 70]:\tacc\t0.7578313253012048\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:01:10,373 [ trainer.py: 70]:\tloss\t0.5867884608033376\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:01:10,373 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,373 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7578313253012048, 'loss': 0.5867884608033376}} \n", + "old: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,374 [ exporter.py: 64]:\t[Best Exporter]: skip step 15000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,374 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7578313253012048, 'loss': 0.5867884608033376}} \n", + "old: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,374 [ exporter.py: 104]:\t[Best Exporter]: skip step 15000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:10,375 [monitored_executor.py: 122]:\tsaving step 15000 to ./output/model_15000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:13,546 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_12000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:22,400 [ hooks.py: 207]:\tstep: 15010\tsteps/sec: 0.02055\tloss: 0.38164\tlr:[3.9807368e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:31,364 [ hooks.py: 207]:\tstep: 15020\tsteps/sec: 1.15823\tloss: 0.72279\tlr:[3.9800576e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:38,304 [ hooks.py: 207]:\tstep: 15030\tsteps/sec: 1.32906\tloss: 0.68409\tlr:[3.9793787e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:46,808 [ hooks.py: 207]:\tstep: 15040\tsteps/sec: 1.25780\tloss: 0.61436\tlr:[3.9786995e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:01:53,965 [ hooks.py: 207]:\tstep: 15050\tsteps/sec: 1.35952\tloss: 0.92054\tlr:[3.9780203e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:02:01,626 [ hooks.py: 207]:\tstep: 15060\tsteps/sec: 1.39909\tloss: 0.45607\tlr:[3.9773415e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:02:11,231 [ hooks.py: 207]:\tstep: 15070\tsteps/sec: 1.01686\tloss: 0.54817\tlr:[3.9766623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:02:18,761 [ hooks.py: 207]:\tstep: 15080\tsteps/sec: 1.27552\tloss: 0.67818\tlr:[3.9759834e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:02:26,585 [ hooks.py: 207]:\tstep: 15090\tsteps/sec: 1.26361\tloss: 0.59691\tlr:[3.9753046e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:02:36,832 [ hooks.py: 207]:\tstep: 15100\tsteps/sec: 0.99898\tloss: 0.63913\tlr:[3.9746254e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:02:48,297 [ hooks.py: 207]:\tstep: 15110\tsteps/sec: 0.85472\tloss: 0.28995\tlr:[3.973946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:02:57,823 [ hooks.py: 207]:\tstep: 15120\tsteps/sec: 1.05113\tloss: 0.46050\tlr:[3.9732673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:03:06,857 [ hooks.py: 207]:\tstep: 15130\tsteps/sec: 1.08759\tloss: 0.38840\tlr:[3.972588e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:03:15,981 [ hooks.py: 207]:\tstep: 15140\tsteps/sec: 1.11944\tloss: 0.57203\tlr:[3.9719092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:03:26,962 [ hooks.py: 207]:\tstep: 15150\tsteps/sec: 0.92853\tloss: 0.43667\tlr:[3.97123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:03:36,056 [ hooks.py: 207]:\tstep: 15160\tsteps/sec: 1.12020\tloss: 0.69187\tlr:[3.970551e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:03:43,563 [ hooks.py: 207]:\tstep: 15170\tsteps/sec: 1.21775\tloss: 0.39440\tlr:[3.969872e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 22:03:53,415 [ hooks.py: 207]:\tstep: 15180\tsteps/sec: 1.01642\tloss: 0.49396\tlr:[3.969193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:04:03,275 [ hooks.py: 207]:\tstep: 15190\tsteps/sec: 1.07185\tloss: 0.72234\tlr:[3.968514e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:04:10,043 [ hooks.py: 207]:\tstep: 15200\tsteps/sec: 1.45159\tloss: 0.50849\tlr:[3.9678347e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:04:21,174 [ hooks.py: 207]:\tstep: 15210\tsteps/sec: 0.91162\tloss: 0.54449\tlr:[3.967156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:04:32,681 [ hooks.py: 207]:\tstep: 15220\tsteps/sec: 0.85108\tloss: 0.41004\tlr:[3.9664767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:04:40,463 [ hooks.py: 207]:\tstep: 15230\tsteps/sec: 1.27155\tloss: 0.60571\tlr:[3.9657974e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:04:49,098 [ hooks.py: 207]:\tstep: 15240\tsteps/sec: 1.16685\tloss: 0.56464\tlr:[3.9651186e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:04:58,787 [ hooks.py: 207]:\tstep: 15250\tsteps/sec: 1.05124\tloss: 0.60622\tlr:[3.9644394e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:05:06,412 [ hooks.py: 207]:\tstep: 15260\tsteps/sec: 1.33855\tloss: 0.69238\tlr:[3.96376e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:05:14,158 [ hooks.py: 207]:\tstep: 15270\tsteps/sec: 1.23384\tloss: 0.94864\tlr:[3.9630813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:05:22,587 [ hooks.py: 207]:\tstep: 15280\tsteps/sec: 1.19151\tloss: 0.62996\tlr:[3.962402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:05:32,715 [ hooks.py: 207]:\tstep: 15290\tsteps/sec: 0.99231\tloss: 0.21489\tlr:[3.9617233e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:05:42,042 [ hooks.py: 207]:\tstep: 15300\tsteps/sec: 1.09847\tloss: 0.85556\tlr:[3.961044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:05:48,699 [ hooks.py: 207]:\tstep: 15310\tsteps/sec: 1.44029\tloss: 0.73340\tlr:[3.960365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:05:57,381 [ hooks.py: 207]:\tstep: 15320\tsteps/sec: 1.09299\tloss: 0.45103\tlr:[3.959686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:06:06,850 [ hooks.py: 207]:\tstep: 15330\tsteps/sec: 1.10228\tloss: 0.34555\tlr:[3.959007e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:06:16,776 [ hooks.py: 207]:\tstep: 15340\tsteps/sec: 1.04336\tloss: 0.54800\tlr:[3.958328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:06:23,439 [ hooks.py: 207]:\tstep: 15350\tsteps/sec: 1.43080\tloss: 0.82821\tlr:[3.9576487e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:06:32,789 [ hooks.py: 207]:\tstep: 15360\tsteps/sec: 1.07570\tloss: 0.74277\tlr:[3.95697e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:06:41,070 [ hooks.py: 207]:\tstep: 15370\tsteps/sec: 1.21135\tloss: 0.91706\tlr:[3.9562907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:06:48,331 [ hooks.py: 207]:\tstep: 15380\tsteps/sec: 1.31703\tloss: 0.30068\tlr:[3.955612e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:06:56,987 [ hooks.py: 207]:\tstep: 15390\tsteps/sec: 1.17844\tloss: 0.21533\tlr:[3.9549326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:06,294 [ hooks.py: 207]:\tstep: 15400\tsteps/sec: 1.13798\tloss: 0.63148\tlr:[3.9542538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:13,879 [ hooks.py: 207]:\tstep: 15410\tsteps/sec: 1.25624\tloss: 0.69047\tlr:[3.9535746e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:20,607 [ hooks.py: 207]:\tstep: 15420\tsteps/sec: 1.47580\tloss: 0.96019\tlr:[3.9528957e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:28,261 [ hooks.py: 207]:\tstep: 15430\tsteps/sec: 1.26421\tloss: 0.49372\tlr:[3.9522165e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:37,237 [ hooks.py: 207]:\tstep: 15440\tsteps/sec: 1.17160\tloss: 0.49976\tlr:[3.9515377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:46,173 [ hooks.py: 207]:\tstep: 15450\tsteps/sec: 1.15061\tloss: 0.99383\tlr:[3.9508584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:52,937 [ hooks.py: 207]:\tstep: 15460\tsteps/sec: 1.39681\tloss: 0.88185\tlr:[3.9501792e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:07:59,361 [ hooks.py: 207]:\tstep: 15470\tsteps/sec: 1.56633\tloss: 0.47966\tlr:[3.9495004e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:08:08,973 [ hooks.py: 207]:\tstep: 15480\tsteps/sec: 1.00154\tloss: 0.68748\tlr:[3.9488215e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:08:16,666 [ hooks.py: 207]:\tstep: 15490\tsteps/sec: 1.33465\tloss: 0.47842\tlr:[3.948142e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:08:25,129 [ hooks.py: 207]:\tstep: 15500\tsteps/sec: 1.17887\tloss: 0.61261\tlr:[3.9474628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:08:33,128 [ hooks.py: 207]:\tstep: 15510\tsteps/sec: 1.28237\tloss: 0.78167\tlr:[3.9467843e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:08:39,820 [ hooks.py: 207]:\tstep: 15520\tsteps/sec: 1.42102\tloss: 0.47122\tlr:[3.946105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:08:50,611 [ hooks.py: 207]:\tstep: 15530\tsteps/sec: 0.97814\tloss: 0.39452\tlr:[3.9454262e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:08:57,236 [ hooks.py: 207]:\tstep: 15540\tsteps/sec: 1.46449\tloss: 0.70632\tlr:[3.9447466e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:09:06,762 [ hooks.py: 207]:\tstep: 15550\tsteps/sec: 1.07507\tloss: 0.73443\tlr:[3.9440678e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:09:14,111 [ hooks.py: 207]:\tstep: 15560\tsteps/sec: 1.31485\tloss: 0.52720\tlr:[3.9433886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:09:22,823 [ hooks.py: 207]:\tstep: 15570\tsteps/sec: 1.14054\tloss: 0.44609\tlr:[3.94271e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:09:32,403 [ hooks.py: 207]:\tstep: 15580\tsteps/sec: 1.01400\tloss: 0.23358\tlr:[3.942031e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:09:42,163 [ hooks.py: 207]:\tstep: 15590\tsteps/sec: 1.03116\tloss: 0.63075\tlr:[3.9413517e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:09:51,090 [ hooks.py: 207]:\tstep: 15600\tsteps/sec: 1.19735\tloss: 0.76129\tlr:[3.940673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:09:58,973 [ hooks.py: 207]:\tstep: 15610\tsteps/sec: 1.22738\tloss: 0.61114\tlr:[3.9399936e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:10:05,955 [ hooks.py: 207]:\tstep: 15620\tsteps/sec: 1.40881\tloss: 0.49251\tlr:[3.9393144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:10:16,116 [ hooks.py: 207]:\tstep: 15630\tsteps/sec: 0.99230\tloss: 0.43103\tlr:[3.9386356e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:10:23,808 [ hooks.py: 207]:\tstep: 15640\tsteps/sec: 1.30567\tloss: 0.87987\tlr:[3.9379564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:10:31,954 [ hooks.py: 207]:\tstep: 15650\tsteps/sec: 1.18671\tloss: 0.49621\tlr:[3.9372775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:10:40,603 [ hooks.py: 207]:\tstep: 15660\tsteps/sec: 1.22998\tloss: 0.55395\tlr:[3.9365987e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:10:48,123 [ hooks.py: 207]:\tstep: 15670\tsteps/sec: 1.28238\tloss: 0.50932\tlr:[3.9359194e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:10:54,862 [ hooks.py: 207]:\tstep: 15680\tsteps/sec: 1.48428\tloss: 0.69480\tlr:[3.9352402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:11:01,628 [ hooks.py: 207]:\tstep: 15690\tsteps/sec: 1.57962\tloss: 0.45555\tlr:[3.934561e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:11:12,841 [ hooks.py: 207]:\tstep: 15700\tsteps/sec: 0.83015\tloss: 0.76922\tlr:[3.933882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:11:21,617 [ hooks.py: 207]:\tstep: 15710\tsteps/sec: 1.18246\tloss: 0.90969\tlr:[3.933203e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:11:30,323 [ hooks.py: 207]:\tstep: 15720\tsteps/sec: 1.17400\tloss: 0.59519\tlr:[3.932524e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:11:40,375 [ hooks.py: 207]:\tstep: 15730\tsteps/sec: 0.97181\tloss: 0.65841\tlr:[3.931845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:11:49,850 [ hooks.py: 207]:\tstep: 15740\tsteps/sec: 1.03334\tloss: 0.73815\tlr:[3.931166e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:11:59,235 [ hooks.py: 207]:\tstep: 15750\tsteps/sec: 1.11278\tloss: 0.62150\tlr:[3.930487e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:12:09,055 [ hooks.py: 207]:\tstep: 15760\tsteps/sec: 1.01696\tloss: 0.44827\tlr:[3.9298073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:12:16,098 [ hooks.py: 207]:\tstep: 15770\tsteps/sec: 1.45498\tloss: 0.96375\tlr:[3.9291284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:12:23,691 [ hooks.py: 207]:\tstep: 15780\tsteps/sec: 1.23728\tloss: 0.50191\tlr:[3.9284496e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:12:32,523 [ hooks.py: 207]:\tstep: 15790\tsteps/sec: 1.17011\tloss: 0.56381\tlr:[3.9277707e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:12:42,349 [ hooks.py: 207]:\tstep: 15800\tsteps/sec: 1.02830\tloss: 0.37228\tlr:[3.9270915e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:12:49,773 [ hooks.py: 207]:\tstep: 15810\tsteps/sec: 1.30560\tloss: 0.66664\tlr:[3.9264123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:12:57,621 [ hooks.py: 207]:\tstep: 15820\tsteps/sec: 1.29929\tloss: 0.50963\tlr:[3.9257335e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 22:13:04,825 [ hooks.py: 207]:\tstep: 15830\tsteps/sec: 1.29678\tloss: 0.72096\tlr:[3.9250546e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:13:15,018 [ hooks.py: 207]:\tstep: 15840\tsteps/sec: 0.99793\tloss: 0.73023\tlr:[3.924375e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:13:23,648 [ hooks.py: 207]:\tstep: 15850\tsteps/sec: 1.19742\tloss: 0.51519\tlr:[3.9236962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:13:31,167 [ hooks.py: 207]:\tstep: 15860\tsteps/sec: 1.33207\tloss: 0.40373\tlr:[3.923017e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:13:40,819 [ hooks.py: 207]:\tstep: 15870\tsteps/sec: 1.06412\tloss: 0.57671\tlr:[3.922338e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:13:51,010 [ hooks.py: 207]:\tstep: 15880\tsteps/sec: 0.95239\tloss: 0.48417\tlr:[3.921659e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:02,356 [ hooks.py: 207]:\tstep: 15890\tsteps/sec: 0.90375\tloss: 0.81404\tlr:[3.92098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:10,469 [ hooks.py: 207]:\tstep: 15900\tsteps/sec: 1.22174\tloss: 0.59469\tlr:[3.920301e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:18,258 [ hooks.py: 207]:\tstep: 15910\tsteps/sec: 1.25376\tloss: 0.69631\tlr:[3.9196217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:26,857 [ hooks.py: 207]:\tstep: 15920\tsteps/sec: 1.20139\tloss: 0.76000\tlr:[3.918943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:34,297 [ hooks.py: 207]:\tstep: 15930\tsteps/sec: 1.30665\tloss: 0.72891\tlr:[3.918264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:42,391 [ hooks.py: 207]:\tstep: 15940\tsteps/sec: 1.18064\tloss: 0.24566\tlr:[3.9175848e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:51,728 [ hooks.py: 207]:\tstep: 15950\tsteps/sec: 1.10819\tloss: 0.67531\tlr:[3.9169056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:14:59,486 [ hooks.py: 207]:\tstep: 15960\tsteps/sec: 1.30530\tloss: 1.43362\tlr:[3.9162267e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:10,673 [ hooks.py: 207]:\tstep: 15970\tsteps/sec: 0.90837\tloss: 0.55446\tlr:[3.9155475e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:21,466 [ hooks.py: 207]:\tstep: 15980\tsteps/sec: 0.93549\tloss: 0.49049\tlr:[3.9148686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:29,078 [ hooks.py: 207]:\tstep: 15990\tsteps/sec: 1.20461\tloss: 0.52416\tlr:[3.914189e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:38,162 [ hooks.py: 207]:\tstep: 16000\tsteps/sec: 1.14156\tloss: 0.65041\tlr:[3.9135106e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:38,162 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:38,162 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:38,163 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:15:38,163 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:38,163 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:15:38,163 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:23:38,563 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,564 [ trainer.py: 58]:\t{'acc': 0.755421686746988, 'loss': 0.6132547213481023}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,565 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,566 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:23:38,566 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:23:38,566 [ trainer.py: 70]:\tacc\t0.755421686746988\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:23:38,566 [ trainer.py: 70]:\tloss\t0.6132547213481023\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:23:38,566 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,566 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.755421686746988, 'loss': 0.6132547213481023}} \n", + "old: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,566 [ exporter.py: 64]:\t[Best Exporter]: skip step 16000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,566 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.755421686746988, 'loss': 0.6132547213481023}} \n", + "old: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,567 [ exporter.py: 104]:\t[Best Exporter]: skip step 16000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:38,568 [monitored_executor.py: 122]:\tsaving step 16000 to ./output/model_16000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:41,949 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_13000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:48,300 [ hooks.py: 207]:\tstep: 16010\tsteps/sec: 0.02039\tloss: 0.73939\tlr:[3.912831e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:23:58,901 [ hooks.py: 207]:\tstep: 16020\tsteps/sec: 0.98826\tloss: 1.02932\tlr:[3.912152e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:24:05,894 [ hooks.py: 207]:\tstep: 16030\tsteps/sec: 1.42437\tloss: 0.24446\tlr:[3.911473e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:24:13,910 [ hooks.py: 207]:\tstep: 16040\tsteps/sec: 1.19557\tloss: 0.76561\tlr:[3.910794e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:24:21,904 [ hooks.py: 207]:\tstep: 16050\tsteps/sec: 1.33810\tloss: 0.40212\tlr:[3.910115e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:24:32,336 [ hooks.py: 207]:\tstep: 16060\tsteps/sec: 0.96699\tloss: 0.41306\tlr:[3.9094357e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:24:38,572 [ hooks.py: 207]:\tstep: 16070\tsteps/sec: 1.44912\tloss: 0.51368\tlr:[3.908757e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:24:47,937 [ hooks.py: 207]:\tstep: 16080\tsteps/sec: 1.11646\tloss: 0.68541\tlr:[3.908078e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:24:54,822 [ hooks.py: 207]:\tstep: 16090\tsteps/sec: 1.44584\tloss: 0.69290\tlr:[3.9073988e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:03,657 [ hooks.py: 207]:\tstep: 16100\tsteps/sec: 1.12038\tloss: 0.54830\tlr:[3.90672e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:10,855 [ hooks.py: 207]:\tstep: 16110\tsteps/sec: 1.44819\tloss: 0.53581\tlr:[3.9060404e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:18,159 [ hooks.py: 207]:\tstep: 16120\tsteps/sec: 1.33056\tloss: 0.47423\tlr:[3.9053615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:25,257 [ hooks.py: 207]:\tstep: 16130\tsteps/sec: 1.34243\tloss: 0.31898\tlr:[3.9046827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:33,622 [ hooks.py: 207]:\tstep: 16140\tsteps/sec: 1.23036\tloss: 0.42842\tlr:[3.9040035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:42,750 [ hooks.py: 207]:\tstep: 16150\tsteps/sec: 1.05501\tloss: 0.66664\tlr:[3.9033246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:50,926 [ hooks.py: 207]:\tstep: 16160\tsteps/sec: 1.33619\tloss: 0.56955\tlr:[3.9026454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:25:59,062 [ hooks.py: 207]:\tstep: 16170\tsteps/sec: 1.15928\tloss: 0.63932\tlr:[3.9019666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:26:08,829 [ hooks.py: 207]:\tstep: 16180\tsteps/sec: 1.04500\tloss: 0.58453\tlr:[3.9012873e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:26:17,729 [ hooks.py: 207]:\tstep: 16190\tsteps/sec: 1.14966\tloss: 0.50846\tlr:[3.9006085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:26:27,010 [ hooks.py: 207]:\tstep: 16200\tsteps/sec: 1.09642\tloss: 0.74719\tlr:[3.8999293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:26:34,921 [ hooks.py: 207]:\tstep: 16210\tsteps/sec: 1.20511\tloss: 0.73725\tlr:[3.89925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:26:44,787 [ hooks.py: 207]:\tstep: 16220\tsteps/sec: 1.01832\tloss: 0.51742\tlr:[3.8985712e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:26:52,341 [ hooks.py: 207]:\tstep: 16230\tsteps/sec: 1.38081\tloss: 0.74671\tlr:[3.8978924e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:26:59,685 [ hooks.py: 207]:\tstep: 16240\tsteps/sec: 1.32598\tloss: 0.40644\tlr:[3.897213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:27:08,597 [ hooks.py: 207]:\tstep: 16250\tsteps/sec: 1.10336\tloss: 0.38020\tlr:[3.896534e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:27:17,630 [ hooks.py: 207]:\tstep: 16260\tsteps/sec: 1.07153\tloss: 0.84980\tlr:[3.895855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:27:27,467 [ hooks.py: 207]:\tstep: 16270\tsteps/sec: 1.09117\tloss: 0.76003\tlr:[3.895176e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:27:34,442 [ hooks.py: 207]:\tstep: 16280\tsteps/sec: 1.37265\tloss: 0.78065\tlr:[3.8944974e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 22:27:44,590 [ hooks.py: 207]:\tstep: 16290\tsteps/sec: 0.97855\tloss: 0.51515\tlr:[3.893818e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:27:54,168 [ hooks.py: 207]:\tstep: 16300\tsteps/sec: 1.05519\tloss: 1.04382\tlr:[3.8931394e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:05,103 [ hooks.py: 207]:\tstep: 16310\tsteps/sec: 0.91532\tloss: 0.76825\tlr:[3.8924598e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:13,429 [ hooks.py: 207]:\tstep: 16320\tsteps/sec: 1.19144\tloss: 0.58003\tlr:[3.8917806e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:20,593 [ hooks.py: 207]:\tstep: 16330\tsteps/sec: 1.42591\tloss: 0.41978\tlr:[3.8911017e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:27,534 [ hooks.py: 207]:\tstep: 16340\tsteps/sec: 1.43888\tloss: 0.26009\tlr:[3.890423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:36,665 [ hooks.py: 207]:\tstep: 16350\tsteps/sec: 1.12495\tloss: 0.58837\tlr:[3.8897437e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:44,647 [ hooks.py: 207]:\tstep: 16360\tsteps/sec: 1.20735\tloss: 0.27114\tlr:[3.8890645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:51,483 [ hooks.py: 207]:\tstep: 16370\tsteps/sec: 1.52543\tloss: 0.78783\tlr:[3.8883856e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:28:59,479 [ hooks.py: 207]:\tstep: 16380\tsteps/sec: 1.21695\tloss: 0.43603\tlr:[3.8877068e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:29:06,940 [ hooks.py: 207]:\tstep: 16390\tsteps/sec: 1.37662\tloss: 0.49009\tlr:[3.8870276e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:29:14,011 [ hooks.py: 207]:\tstep: 16400\tsteps/sec: 1.34339\tloss: 1.17249\tlr:[3.8863483e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:29:22,178 [ hooks.py: 207]:\tstep: 16410\tsteps/sec: 1.26400\tloss: 0.65873\tlr:[3.885669e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:29:32,240 [ hooks.py: 207]:\tstep: 16420\tsteps/sec: 0.97102\tloss: 0.53072\tlr:[3.8849903e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:29:39,722 [ hooks.py: 207]:\tstep: 16430\tsteps/sec: 1.34151\tloss: 0.62698\tlr:[3.8843114e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:29:46,660 [ hooks.py: 207]:\tstep: 16440\tsteps/sec: 1.52488\tloss: 0.40174\tlr:[3.883632e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:29:55,238 [ hooks.py: 207]:\tstep: 16450\tsteps/sec: 1.11962\tloss: 0.53633\tlr:[3.882953e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:30:03,185 [ hooks.py: 207]:\tstep: 16460\tsteps/sec: 1.33938\tloss: 0.99060\tlr:[3.8822738e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:30:12,255 [ hooks.py: 207]:\tstep: 16470\tsteps/sec: 1.07581\tloss: 0.60614\tlr:[3.8815946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:30:20,208 [ hooks.py: 207]:\tstep: 16480\tsteps/sec: 1.17812\tloss: 0.73546\tlr:[3.8809158e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:30:27,669 [ hooks.py: 207]:\tstep: 16490\tsteps/sec: 1.33243\tloss: 0.52078\tlr:[3.880237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:30:38,511 [ hooks.py: 207]:\tstep: 16500\tsteps/sec: 0.95500\tloss: 0.44057\tlr:[3.8795577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:30:45,516 [ hooks.py: 207]:\tstep: 16510\tsteps/sec: 1.41450\tloss: 0.76464\tlr:[3.8788785e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:30:54,861 [ hooks.py: 207]:\tstep: 16520\tsteps/sec: 1.07214\tloss: 0.43583\tlr:[3.8781996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:31:04,379 [ hooks.py: 207]:\tstep: 16530\tsteps/sec: 1.01457\tloss: 0.78810\tlr:[3.8775208e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:31:14,016 [ hooks.py: 207]:\tstep: 16540\tsteps/sec: 1.06377\tloss: 0.80871\tlr:[3.8768416e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:31:21,184 [ hooks.py: 207]:\tstep: 16550\tsteps/sec: 1.46069\tloss: 0.35662\tlr:[3.8761624e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:31:29,441 [ hooks.py: 207]:\tstep: 16560\tsteps/sec: 1.17080\tloss: 0.41577\tlr:[3.8754835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:31:38,956 [ hooks.py: 207]:\tstep: 16570\tsteps/sec: 1.05428\tloss: 0.58139\tlr:[3.8748043e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:31:48,817 [ hooks.py: 207]:\tstep: 16580\tsteps/sec: 1.01591\tloss: 0.67929\tlr:[3.8741255e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:31:55,786 [ hooks.py: 207]:\tstep: 16590\tsteps/sec: 1.45766\tloss: 0.73227\tlr:[3.8734463e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:32:06,609 [ hooks.py: 207]:\tstep: 16600\tsteps/sec: 0.93521\tloss: 0.57766\tlr:[3.8727674e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:32:15,426 [ hooks.py: 207]:\tstep: 16610\tsteps/sec: 1.11347\tloss: 0.84392\tlr:[3.8720882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:32:22,535 [ hooks.py: 207]:\tstep: 16620\tsteps/sec: 1.43805\tloss: 0.47774\tlr:[3.8714086e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:32:32,421 [ hooks.py: 207]:\tstep: 16630\tsteps/sec: 1.01053\tloss: 0.66049\tlr:[3.87073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:32:43,014 [ hooks.py: 207]:\tstep: 16640\tsteps/sec: 0.93386\tloss: 0.36299\tlr:[3.870051e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:32:51,337 [ hooks.py: 207]:\tstep: 16650\tsteps/sec: 1.22871\tloss: 0.68331\tlr:[3.869372e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:32:58,528 [ hooks.py: 207]:\tstep: 16660\tsteps/sec: 1.31802\tloss: 0.65886\tlr:[3.8686925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:33:07,215 [ hooks.py: 207]:\tstep: 16670\tsteps/sec: 1.20261\tloss: 0.66704\tlr:[3.8680137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:33:16,260 [ hooks.py: 207]:\tstep: 16680\tsteps/sec: 1.10628\tloss: 0.43810\tlr:[3.8673348e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:33:22,788 [ hooks.py: 207]:\tstep: 16690\tsteps/sec: 1.49160\tloss: 0.31435\tlr:[3.8666556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:33:30,467 [ hooks.py: 207]:\tstep: 16700\tsteps/sec: 1.28702\tloss: 0.57231\tlr:[3.8659764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:33:37,822 [ hooks.py: 207]:\tstep: 16710\tsteps/sec: 1.39227\tloss: 0.58401\tlr:[3.8652975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:33:45,038 [ hooks.py: 207]:\tstep: 16720\tsteps/sec: 1.35637\tloss: 0.34784\tlr:[3.864618e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:33:53,291 [ hooks.py: 207]:\tstep: 16730\tsteps/sec: 1.20700\tloss: 0.68760\tlr:[3.8639395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:01,966 [ hooks.py: 207]:\tstep: 16740\tsteps/sec: 1.25499\tloss: 0.41944\tlr:[3.86326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:08,770 [ hooks.py: 207]:\tstep: 16750\tsteps/sec: 1.33163\tloss: 0.53164\tlr:[3.8625814e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:16,411 [ hooks.py: 207]:\tstep: 16760\tsteps/sec: 1.35998\tloss: 0.53575\tlr:[3.861902e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:24,126 [ hooks.py: 207]:\tstep: 16770\tsteps/sec: 1.24663\tloss: 0.37942\tlr:[3.861223e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:33,352 [ hooks.py: 207]:\tstep: 16780\tsteps/sec: 1.07956\tloss: 0.33761\tlr:[3.860544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:40,859 [ hooks.py: 207]:\tstep: 16790\tsteps/sec: 1.37059\tloss: 0.49264\tlr:[3.859865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:49,007 [ hooks.py: 207]:\tstep: 16800\tsteps/sec: 1.19409\tloss: 0.58966\tlr:[3.8591857e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:34:56,483 [ hooks.py: 207]:\tstep: 16810\tsteps/sec: 1.33265\tloss: 0.64947\tlr:[3.8585065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:35:04,841 [ hooks.py: 207]:\tstep: 16820\tsteps/sec: 1.21080\tloss: 0.54953\tlr:[3.857828e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:35:13,845 [ hooks.py: 207]:\tstep: 16830\tsteps/sec: 1.10501\tloss: 0.79550\tlr:[3.8571492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:35:22,566 [ hooks.py: 207]:\tstep: 16840\tsteps/sec: 1.17336\tloss: 0.45181\tlr:[3.85647e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:35:31,111 [ hooks.py: 207]:\tstep: 16850\tsteps/sec: 1.17130\tloss: 0.42120\tlr:[3.8557904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:35:38,133 [ hooks.py: 207]:\tstep: 16860\tsteps/sec: 1.39865\tloss: 0.72917\tlr:[3.8551116e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:35:47,891 [ hooks.py: 207]:\tstep: 16870\tsteps/sec: 1.03811\tloss: 0.49616\tlr:[3.8544324e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:35:55,906 [ hooks.py: 207]:\tstep: 16880\tsteps/sec: 1.23394\tloss: 0.71337\tlr:[3.853754e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:36:04,830 [ hooks.py: 207]:\tstep: 16890\tsteps/sec: 1.11049\tloss: 0.70868\tlr:[3.8530747e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:36:13,168 [ hooks.py: 207]:\tstep: 16900\tsteps/sec: 1.14179\tloss: 0.85902\tlr:[3.8523955e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:36:23,613 [ hooks.py: 207]:\tstep: 16910\tsteps/sec: 0.98823\tloss: 0.71480\tlr:[3.8517162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:36:31,932 [ hooks.py: 207]:\tstep: 16920\tsteps/sec: 1.26712\tloss: 0.63082\tlr:[3.851037e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:36:39,938 [ hooks.py: 207]:\tstep: 16930\tsteps/sec: 1.19878\tloss: 0.71743\tlr:[3.8503582e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 22:36:48,608 [ hooks.py: 207]:\tstep: 16940\tsteps/sec: 1.15206\tloss: 0.60236\tlr:[3.849679e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:36:56,327 [ hooks.py: 207]:\tstep: 16950\tsteps/sec: 1.27423\tloss: 0.73289\tlr:[3.849e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:06,283 [ hooks.py: 207]:\tstep: 16960\tsteps/sec: 0.99895\tloss: 0.54830\tlr:[3.848321e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:15,260 [ hooks.py: 207]:\tstep: 16970\tsteps/sec: 1.14138\tloss: 0.45528\tlr:[3.847642e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:22,715 [ hooks.py: 207]:\tstep: 16980\tsteps/sec: 1.28625\tloss: 0.45913\tlr:[3.846963e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:30,190 [ hooks.py: 207]:\tstep: 16990\tsteps/sec: 1.37292\tloss: 0.43437\tlr:[3.8462837e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:38,663 [ hooks.py: 207]:\tstep: 17000\tsteps/sec: 1.14276\tloss: 0.59799\tlr:[3.8456044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:38,664 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:38,664 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:38,664 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:37:38,664 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:38,664 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:37:38,665 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:45:36,255 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:36,256 [ trainer.py: 58]:\t{'acc': 0.7751004016064257, 'loss': 0.5636964318079826}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:36,257 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:36,257 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:45:36,258 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:45:36,258 [ trainer.py: 70]:\tacc\t0.7751004016064257\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:45:36,258 [ trainer.py: 70]:\tloss\t0.5636964318079826\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:45:36,258 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:36,258 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7751004016064257, 'loss': 0.5636964318079826}} \n", + "old: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:36,258 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:36,259 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_14000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:36,259 [monitored_executor.py: 122]:\tsaving step 17000 to ./output/best_model/model_17000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:37,201 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_14000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:37,202 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7751004016064257, 'loss': 0.5636964318079826}} \n", + "old: {'eval': {'acc': 0.7698795180722892, 'loss': 0.594362390251496}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:37,202 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:37,202 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:38,451 [monitored_executor.py: 122]:\tsaving step 17000 to ./output/model_17000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:41,965 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_14000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:49,506 [ hooks.py: 207]:\tstep: 17010\tsteps/sec: 0.02039\tloss: 0.52832\tlr:[3.8449256e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:45:56,145 [ hooks.py: 207]:\tstep: 17020\tsteps/sec: 1.53433\tloss: 0.40622\tlr:[3.8442464e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:46:05,675 [ hooks.py: 207]:\tstep: 17030\tsteps/sec: 1.02303\tloss: 0.43095\tlr:[3.843568e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:46:14,753 [ hooks.py: 207]:\tstep: 17040\tsteps/sec: 1.11561\tloss: 0.88180\tlr:[3.8428883e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:46:22,726 [ hooks.py: 207]:\tstep: 17050\tsteps/sec: 1.27538\tloss: 0.86597\tlr:[3.84221e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:46:29,464 [ hooks.py: 207]:\tstep: 17060\tsteps/sec: 1.44315\tloss: 0.41640\tlr:[3.8415303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:46:36,164 [ hooks.py: 207]:\tstep: 17070\tsteps/sec: 1.50978\tloss: 1.06190\tlr:[3.840851e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:46:44,755 [ hooks.py: 207]:\tstep: 17080\tsteps/sec: 1.10898\tloss: 0.95813\tlr:[3.8401722e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:46:53,953 [ hooks.py: 207]:\tstep: 17090\tsteps/sec: 1.10813\tloss: 0.64587\tlr:[3.8394934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:02,222 [ hooks.py: 207]:\tstep: 17100\tsteps/sec: 1.33845\tloss: 0.72770\tlr:[3.838814e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:10,109 [ hooks.py: 207]:\tstep: 17110\tsteps/sec: 1.17801\tloss: 0.54974\tlr:[3.838135e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:18,491 [ hooks.py: 207]:\tstep: 17120\tsteps/sec: 1.19134\tloss: 0.69710\tlr:[3.837456e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:26,255 [ hooks.py: 207]:\tstep: 17130\tsteps/sec: 1.29984\tloss: 0.43807\tlr:[3.836777e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:35,257 [ hooks.py: 207]:\tstep: 17140\tsteps/sec: 1.10185\tloss: 0.54059\tlr:[3.836098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:43,776 [ hooks.py: 207]:\tstep: 17150\tsteps/sec: 1.16671\tloss: 0.43301\tlr:[3.835419e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:50,385 [ hooks.py: 207]:\tstep: 17160\tsteps/sec: 1.44583\tloss: 0.75714\tlr:[3.83474e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:47:59,024 [ hooks.py: 207]:\tstep: 17170\tsteps/sec: 1.17124\tloss: 0.59296\tlr:[3.8340608e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:48:07,627 [ hooks.py: 207]:\tstep: 17180\tsteps/sec: 1.23895\tloss: 0.71119\tlr:[3.8333816e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:48:15,755 [ hooks.py: 207]:\tstep: 17190\tsteps/sec: 1.18672\tloss: 0.45848\tlr:[3.8327027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:48:24,221 [ hooks.py: 207]:\tstep: 17200\tsteps/sec: 1.17302\tloss: 0.80461\tlr:[3.8320235e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:48:33,104 [ hooks.py: 207]:\tstep: 17210\tsteps/sec: 1.14795\tloss: 0.54362\tlr:[3.8313447e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:48:42,286 [ hooks.py: 207]:\tstep: 17220\tsteps/sec: 1.12453\tloss: 0.56724\tlr:[3.830665e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:48:48,632 [ hooks.py: 207]:\tstep: 17230\tsteps/sec: 1.41664\tloss: 0.38220\tlr:[3.8299862e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:48:58,249 [ hooks.py: 207]:\tstep: 17240\tsteps/sec: 1.07660\tloss: 0.31841\tlr:[3.8293078e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:49:08,322 [ hooks.py: 207]:\tstep: 17250\tsteps/sec: 0.99131\tloss: 1.15061\tlr:[3.828628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:49:17,904 [ hooks.py: 207]:\tstep: 17260\tsteps/sec: 1.02270\tloss: 0.53696\tlr:[3.827949e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:49:26,758 [ hooks.py: 207]:\tstep: 17270\tsteps/sec: 1.18715\tloss: 0.39148\tlr:[3.82727e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:49:36,999 [ hooks.py: 207]:\tstep: 17280\tsteps/sec: 0.98584\tloss: 0.75773\tlr:[3.826591e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:49:44,576 [ hooks.py: 207]:\tstep: 17290\tsteps/sec: 1.26521\tloss: 0.53250\tlr:[3.825912e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:49:53,005 [ hooks.py: 207]:\tstep: 17300\tsteps/sec: 1.22154\tloss: 0.41539\tlr:[3.8252332e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:50:00,206 [ hooks.py: 207]:\tstep: 17310\tsteps/sec: 1.36328\tloss: 0.56247\tlr:[3.8245544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:50:08,822 [ hooks.py: 207]:\tstep: 17320\tsteps/sec: 1.14787\tloss: 0.60115\tlr:[3.8238748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:50:15,902 [ hooks.py: 207]:\tstep: 17330\tsteps/sec: 1.35666\tloss: 0.26356\tlr:[3.823196e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:50:23,911 [ hooks.py: 207]:\tstep: 17340\tsteps/sec: 1.28888\tloss: 0.66347\tlr:[3.8225167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:50:31,461 [ hooks.py: 207]:\tstep: 17350\tsteps/sec: 1.37018\tloss: 0.33806\tlr:[3.8218383e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 22:50:41,327 [ hooks.py: 207]:\tstep: 17360\tsteps/sec: 0.98716\tloss: 0.51103\tlr:[3.821159e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:50:49,363 [ hooks.py: 207]:\tstep: 17370\tsteps/sec: 1.24322\tloss: 0.57688\tlr:[3.82048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:50:57,767 [ hooks.py: 207]:\tstep: 17380\tsteps/sec: 1.23445\tloss: 0.89703\tlr:[3.819801e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:51:06,827 [ hooks.py: 207]:\tstep: 17390\tsteps/sec: 1.10848\tloss: 0.85714\tlr:[3.8191218e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:51:14,946 [ hooks.py: 207]:\tstep: 17400\tsteps/sec: 1.19796\tloss: 0.84823\tlr:[3.8184426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:51:23,237 [ hooks.py: 207]:\tstep: 17410\tsteps/sec: 1.15819\tloss: 1.15619\tlr:[3.8177634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:51:32,585 [ hooks.py: 207]:\tstep: 17420\tsteps/sec: 1.14444\tloss: 0.48135\tlr:[3.8170845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:51:42,052 [ hooks.py: 207]:\tstep: 17430\tsteps/sec: 1.05804\tloss: 0.97440\tlr:[3.8164053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:51:48,946 [ hooks.py: 207]:\tstep: 17440\tsteps/sec: 1.36631\tloss: 0.79115\tlr:[3.8157264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:51:58,993 [ hooks.py: 207]:\tstep: 17450\tsteps/sec: 1.01090\tloss: 0.55426\tlr:[3.815047e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:52:06,702 [ hooks.py: 207]:\tstep: 17460\tsteps/sec: 1.32734\tloss: 0.50554\tlr:[3.814368e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:52:14,418 [ hooks.py: 207]:\tstep: 17470\tsteps/sec: 1.24405\tloss: 0.85700\tlr:[3.813689e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:52:22,485 [ hooks.py: 207]:\tstep: 17480\tsteps/sec: 1.29116\tloss: 0.66512\tlr:[3.8130103e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:52:32,631 [ hooks.py: 207]:\tstep: 17490\tsteps/sec: 0.98816\tloss: 0.41446\tlr:[3.812331e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:52:41,918 [ hooks.py: 207]:\tstep: 17500\tsteps/sec: 1.08725\tloss: 0.56249\tlr:[3.811652e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:52:49,260 [ hooks.py: 207]:\tstep: 17510\tsteps/sec: 1.29651\tloss: 0.77680\tlr:[3.810973e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:52:55,462 [ hooks.py: 207]:\tstep: 17520\tsteps/sec: 1.53738\tloss: 0.50433\tlr:[3.810294e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:53:05,046 [ hooks.py: 207]:\tstep: 17530\tsteps/sec: 1.04847\tloss: 0.66270\tlr:[3.809615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:53:13,199 [ hooks.py: 207]:\tstep: 17540\tsteps/sec: 1.29294\tloss: 0.56323\tlr:[3.8089358e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:53:20,111 [ hooks.py: 207]:\tstep: 17550\tsteps/sec: 1.43809\tloss: 0.50144\tlr:[3.808257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:53:27,079 [ hooks.py: 207]:\tstep: 17560\tsteps/sec: 1.41180\tloss: 0.40726\tlr:[3.8075774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:53:34,898 [ hooks.py: 207]:\tstep: 17570\tsteps/sec: 1.27896\tloss: 0.82645\tlr:[3.8068985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:53:42,155 [ hooks.py: 207]:\tstep: 17580\tsteps/sec: 1.44365\tloss: 0.91471\tlr:[3.8062197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:53:49,688 [ hooks.py: 207]:\tstep: 17590\tsteps/sec: 1.22198\tloss: 0.53455\tlr:[3.8055405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:54:02,468 [ hooks.py: 207]:\tstep: 17600\tsteps/sec: 0.82603\tloss: 0.75459\tlr:[3.8048616e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:54:12,083 [ hooks.py: 207]:\tstep: 17610\tsteps/sec: 0.99824\tloss: 0.77662\tlr:[3.8041824e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:54:20,509 [ hooks.py: 207]:\tstep: 17620\tsteps/sec: 1.19833\tloss: 0.56611\tlr:[3.8035032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:54:29,868 [ hooks.py: 207]:\tstep: 17630\tsteps/sec: 1.02878\tloss: 0.85963\tlr:[3.8028244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:54:40,909 [ hooks.py: 207]:\tstep: 17640\tsteps/sec: 0.93280\tloss: 0.48930\tlr:[3.802145e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:54:48,825 [ hooks.py: 207]:\tstep: 17650\tsteps/sec: 1.25259\tloss: 0.62471\tlr:[3.8014663e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:54:57,995 [ hooks.py: 207]:\tstep: 17660\tsteps/sec: 1.12238\tloss: 0.65065\tlr:[3.800787e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:55:06,193 [ hooks.py: 207]:\tstep: 17670\tsteps/sec: 1.19157\tloss: 0.69318\tlr:[3.800108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:55:16,350 [ hooks.py: 207]:\tstep: 17680\tsteps/sec: 1.00031\tloss: 1.00561\tlr:[3.799429e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:55:23,321 [ hooks.py: 207]:\tstep: 17690\tsteps/sec: 1.40597\tloss: 0.37909\tlr:[3.79875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:55:32,240 [ hooks.py: 207]:\tstep: 17700\tsteps/sec: 1.16652\tloss: 0.87828\tlr:[3.7980706e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:55:40,156 [ hooks.py: 207]:\tstep: 17710\tsteps/sec: 1.21165\tloss: 0.33857\tlr:[3.7973918e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:55:49,754 [ hooks.py: 207]:\tstep: 17720\tsteps/sec: 1.04278\tloss: 0.65319\tlr:[3.7967126e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:55:57,676 [ hooks.py: 207]:\tstep: 17730\tsteps/sec: 1.30326\tloss: 0.93131\tlr:[3.7960337e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:56:06,482 [ hooks.py: 207]:\tstep: 17740\tsteps/sec: 1.12753\tloss: 0.40903\tlr:[3.7953545e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:56:14,899 [ hooks.py: 207]:\tstep: 17750\tsteps/sec: 1.15526\tloss: 0.90308\tlr:[3.7946753e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:56:23,033 [ hooks.py: 207]:\tstep: 17760\tsteps/sec: 1.20394\tloss: 0.66463\tlr:[3.7939964e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:56:33,286 [ hooks.py: 207]:\tstep: 17770\tsteps/sec: 1.00377\tloss: 0.63630\tlr:[3.7933172e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:56:40,095 [ hooks.py: 207]:\tstep: 17780\tsteps/sec: 1.46639\tloss: 0.45460\tlr:[3.7926384e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:56:48,655 [ hooks.py: 207]:\tstep: 17790\tsteps/sec: 1.15243\tloss: 0.67783\tlr:[3.791959e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:56:58,049 [ hooks.py: 207]:\tstep: 17800\tsteps/sec: 1.09838\tloss: 0.94660\tlr:[3.7912803e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:57:08,239 [ hooks.py: 207]:\tstep: 17810\tsteps/sec: 0.97329\tloss: 0.77751\tlr:[3.790601e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:57:16,212 [ hooks.py: 207]:\tstep: 17820\tsteps/sec: 1.24429\tloss: 1.11138\tlr:[3.789922e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:57:24,868 [ hooks.py: 207]:\tstep: 17830\tsteps/sec: 1.10102\tloss: 0.97415\tlr:[3.789243e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:57:34,498 [ hooks.py: 207]:\tstep: 17840\tsteps/sec: 1.07524\tloss: 0.53312\tlr:[3.7885642e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:57:41,662 [ hooks.py: 207]:\tstep: 17850\tsteps/sec: 1.44783\tloss: 0.66997\tlr:[3.787885e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:57:50,665 [ hooks.py: 207]:\tstep: 17860\tsteps/sec: 1.03631\tloss: 0.86888\tlr:[3.7872058e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:57:59,509 [ hooks.py: 207]:\tstep: 17870\tsteps/sec: 1.19251\tloss: 0.59165\tlr:[3.786527e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:58:08,884 [ hooks.py: 207]:\tstep: 17880\tsteps/sec: 1.05120\tloss: 0.52308\tlr:[3.7858477e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:58:18,289 [ hooks.py: 207]:\tstep: 17890\tsteps/sec: 1.08374\tloss: 0.34848\tlr:[3.785169e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:58:24,725 [ hooks.py: 207]:\tstep: 17900\tsteps/sec: 1.52717\tloss: 0.38061\tlr:[3.7844897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:58:33,709 [ hooks.py: 207]:\tstep: 17910\tsteps/sec: 1.11188\tloss: 0.49227\tlr:[3.7838105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:58:42,245 [ hooks.py: 207]:\tstep: 17920\tsteps/sec: 1.17257\tloss: 0.33950\tlr:[3.7831316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:58:52,579 [ hooks.py: 207]:\tstep: 17930\tsteps/sec: 0.96396\tloss: 0.54819\tlr:[3.7824528e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:01,653 [ hooks.py: 207]:\tstep: 17940\tsteps/sec: 1.17879\tloss: 0.80227\tlr:[3.7817732e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:10,654 [ hooks.py: 207]:\tstep: 17950\tsteps/sec: 1.02264\tloss: 0.52314\tlr:[3.7810947e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:19,360 [ hooks.py: 207]:\tstep: 17960\tsteps/sec: 1.13383\tloss: 0.59367\tlr:[3.7804155e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:29,328 [ hooks.py: 207]:\tstep: 17970\tsteps/sec: 1.03336\tloss: 0.47744\tlr:[3.7797363e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:35,524 [ hooks.py: 207]:\tstep: 17980\tsteps/sec: 1.61499\tloss: 0.55425\tlr:[3.779057e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:44,220 [ hooks.py: 207]:\tstep: 17990\tsteps/sec: 1.13907\tloss: 0.68014\tlr:[3.7783786e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:53,140 [ hooks.py: 207]:\tstep: 18000\tsteps/sec: 1.15215\tloss: 0.67853\tlr:[3.777699e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:53,140 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:53,140 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:53,141 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 22:59:53,141 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:53,141 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 22:59:53,141 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-18 23:07:50,600 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:50,601 [ trainer.py: 58]:\t{'acc': 0.7855421686746988, 'loss': 0.5396390796089784}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:50,602 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:50,602 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:07:50,603 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:07:50,603 [ trainer.py: 70]:\tacc\t0.7855421686746988\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:07:50,603 [ trainer.py: 70]:\tloss\t0.5396390796089784\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:07:50,603 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:50,603 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}} \n", + "old: {'eval': {'acc': 0.7751004016064257, 'loss': 0.5636964318079826}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:50,603 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:50,604 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_17000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:50,604 [monitored_executor.py: 122]:\tsaving step 18000 to ./output/best_model/model_18000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:51,564 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_17000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:51,564 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}} \n", + "old: {'eval': {'acc': 0.7751004016064257, 'loss': 0.5636964318079826}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:51,565 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:51,565 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:52,806 [monitored_executor.py: 122]:\tsaving step 18000 to ./output/model_18000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:07:56,218 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_15000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:08:05,594 [ hooks.py: 207]:\tstep: 18010\tsteps/sec: 0.02030\tloss: 0.40413\tlr:[3.77702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:08:13,596 [ hooks.py: 207]:\tstep: 18020\tsteps/sec: 1.26193\tloss: 0.51347\tlr:[3.7763413e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:08:19,900 [ hooks.py: 207]:\tstep: 18030\tsteps/sec: 1.51757\tloss: 0.83109\tlr:[3.775662e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:08:28,561 [ hooks.py: 207]:\tstep: 18040\tsteps/sec: 1.18962\tloss: 0.57519\tlr:[3.7749833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:08:37,577 [ hooks.py: 207]:\tstep: 18050\tsteps/sec: 1.16157\tloss: 0.73794\tlr:[3.774304e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:08:45,041 [ hooks.py: 207]:\tstep: 18060\tsteps/sec: 1.21940\tloss: 0.76056\tlr:[3.7736252e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:08:55,511 [ hooks.py: 207]:\tstep: 18070\tsteps/sec: 0.99232\tloss: 0.58680\tlr:[3.772946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:09:03,882 [ hooks.py: 207]:\tstep: 18080\tsteps/sec: 1.21262\tloss: 0.51388\tlr:[3.772267e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:09:12,946 [ hooks.py: 207]:\tstep: 18090\tsteps/sec: 1.12497\tloss: 0.51634\tlr:[3.7715876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:09:22,106 [ hooks.py: 207]:\tstep: 18100\tsteps/sec: 1.09336\tloss: 0.50308\tlr:[3.770909e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:09:31,722 [ hooks.py: 207]:\tstep: 18110\tsteps/sec: 1.01966\tloss: 0.41937\tlr:[3.77023e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:09:42,361 [ hooks.py: 207]:\tstep: 18120\tsteps/sec: 0.96097\tloss: 0.53333\tlr:[3.7695503e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:09:50,222 [ hooks.py: 207]:\tstep: 18130\tsteps/sec: 1.20854\tloss: 0.63309\tlr:[3.7688715e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:10:00,404 [ hooks.py: 207]:\tstep: 18140\tsteps/sec: 0.96514\tloss: 0.53606\tlr:[3.7681923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:10:09,739 [ hooks.py: 207]:\tstep: 18150\tsteps/sec: 1.08786\tloss: 0.38632\tlr:[3.7675134e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:10:17,351 [ hooks.py: 207]:\tstep: 18160\tsteps/sec: 1.37961\tloss: 0.84971\tlr:[3.7668342e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:10:25,388 [ hooks.py: 207]:\tstep: 18170\tsteps/sec: 1.20612\tloss: 0.68929\tlr:[3.766155e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:10:33,299 [ hooks.py: 207]:\tstep: 18180\tsteps/sec: 1.21602\tloss: 0.48472\tlr:[3.765476e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:10:41,422 [ hooks.py: 207]:\tstep: 18190\tsteps/sec: 1.28096\tloss: 0.49644\tlr:[3.7647973e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:10:50,836 [ hooks.py: 207]:\tstep: 18200\tsteps/sec: 1.05843\tloss: 0.33145\tlr:[3.764118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:11:00,368 [ hooks.py: 207]:\tstep: 18210\tsteps/sec: 1.05589\tloss: 0.22756\tlr:[3.7634392e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:11:08,408 [ hooks.py: 207]:\tstep: 18220\tsteps/sec: 1.15163\tloss: 1.01443\tlr:[3.76276e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:11:17,365 [ hooks.py: 207]:\tstep: 18230\tsteps/sec: 1.17993\tloss: 0.49424\tlr:[3.762081e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:11:26,214 [ hooks.py: 207]:\tstep: 18240\tsteps/sec: 1.12958\tloss: 0.74602\tlr:[3.761402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:11:35,987 [ hooks.py: 207]:\tstep: 18250\tsteps/sec: 1.03262\tloss: 0.59587\tlr:[3.7607228e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:11:44,846 [ hooks.py: 207]:\tstep: 18260\tsteps/sec: 1.12329\tloss: 0.68456\tlr:[3.7600435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:11:53,627 [ hooks.py: 207]:\tstep: 18270\tsteps/sec: 1.13710\tloss: 0.36192\tlr:[3.7593643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:05,633 [ hooks.py: 207]:\tstep: 18280\tsteps/sec: 0.83475\tloss: 0.68985\tlr:[3.7586855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:12,592 [ hooks.py: 207]:\tstep: 18290\tsteps/sec: 1.50161\tloss: 0.74034\tlr:[3.7580066e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:20,005 [ hooks.py: 207]:\tstep: 18300\tsteps/sec: 1.29418\tloss: 0.56314\tlr:[3.7573278e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:28,156 [ hooks.py: 207]:\tstep: 18310\tsteps/sec: 1.26687\tloss: 0.77560\tlr:[3.7566482e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:36,559 [ hooks.py: 207]:\tstep: 18320\tsteps/sec: 1.17529\tloss: 0.56661\tlr:[3.7559697e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:43,667 [ hooks.py: 207]:\tstep: 18330\tsteps/sec: 1.39915\tloss: 0.60914\tlr:[3.75529e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:51,810 [ hooks.py: 207]:\tstep: 18340\tsteps/sec: 1.23933\tloss: 1.27647\tlr:[3.7546113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:12:59,202 [ hooks.py: 207]:\tstep: 18350\tsteps/sec: 1.34335\tloss: 0.29268\tlr:[3.753932e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:13:08,071 [ hooks.py: 207]:\tstep: 18360\tsteps/sec: 1.17768\tloss: 0.59240\tlr:[3.7532533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:13:15,220 [ hooks.py: 207]:\tstep: 18370\tsteps/sec: 1.31336\tloss: 0.60793\tlr:[3.752574e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:13:23,915 [ hooks.py: 207]:\tstep: 18380\tsteps/sec: 1.15501\tloss: 0.34228\tlr:[3.7518952e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:13:30,455 [ hooks.py: 207]:\tstep: 18390\tsteps/sec: 1.51755\tloss: 0.62363\tlr:[3.751216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:13:39,804 [ hooks.py: 207]:\tstep: 18400\tsteps/sec: 1.05757\tloss: 0.46644\tlr:[3.7505368e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:13:47,195 [ hooks.py: 207]:\tstep: 18410\tsteps/sec: 1.41945\tloss: 0.53467\tlr:[3.749858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:13:57,753 [ hooks.py: 207]:\tstep: 18420\tsteps/sec: 0.95300\tloss: 1.05674\tlr:[3.7491787e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:14:07,900 [ hooks.py: 207]:\tstep: 18430\tsteps/sec: 0.98237\tloss: 0.33050\tlr:[3.7485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:14:17,328 [ hooks.py: 207]:\tstep: 18440\tsteps/sec: 1.06378\tloss: 1.34126\tlr:[3.747821e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:14:24,737 [ hooks.py: 207]:\tstep: 18450\tsteps/sec: 1.29272\tloss: 0.38057\tlr:[3.7471415e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:14:33,317 [ hooks.py: 207]:\tstep: 18460\tsteps/sec: 1.18930\tloss: 0.78364\tlr:[3.7464622e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:14:43,682 [ hooks.py: 207]:\tstep: 18470\tsteps/sec: 0.96082\tloss: 0.20418\tlr:[3.7457834e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:14:51,946 [ hooks.py: 207]:\tstep: 18480\tsteps/sec: 1.18779\tloss: 0.66847\tlr:[3.745105e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 23:14:59,815 [ hooks.py: 207]:\tstep: 18490\tsteps/sec: 1.27940\tloss: 0.46053\tlr:[3.7444257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:15:07,578 [ hooks.py: 207]:\tstep: 18500\tsteps/sec: 1.32099\tloss: 0.28657\tlr:[3.743746e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:15:15,753 [ hooks.py: 207]:\tstep: 18510\tsteps/sec: 1.20229\tloss: 0.32091\tlr:[3.7430673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:15:23,517 [ hooks.py: 207]:\tstep: 18520\tsteps/sec: 1.32161\tloss: 0.78437\tlr:[3.742388e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:15:31,337 [ hooks.py: 207]:\tstep: 18530\tsteps/sec: 1.23861\tloss: 0.27154\tlr:[3.741709e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:15:39,676 [ hooks.py: 207]:\tstep: 18540\tsteps/sec: 1.18978\tloss: 0.73031\tlr:[3.74103e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:15:48,740 [ hooks.py: 207]:\tstep: 18550\tsteps/sec: 1.07180\tloss: 0.55863\tlr:[3.740351e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:15:58,640 [ hooks.py: 207]:\tstep: 18560\tsteps/sec: 1.06198\tloss: 0.58079\tlr:[3.739672e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:16:09,379 [ hooks.py: 207]:\tstep: 18570\tsteps/sec: 0.91194\tloss: 0.82248\tlr:[3.7389927e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:16:17,982 [ hooks.py: 207]:\tstep: 18580\tsteps/sec: 1.17249\tloss: 0.86802\tlr:[3.738314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:16:26,799 [ hooks.py: 207]:\tstep: 18590\tsteps/sec: 1.14722\tloss: 0.72278\tlr:[3.737635e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:16:36,626 [ hooks.py: 207]:\tstep: 18600\tsteps/sec: 1.01818\tloss: 0.81749\tlr:[3.736956e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:16:45,143 [ hooks.py: 207]:\tstep: 18610\tsteps/sec: 1.16709\tloss: 0.50479\tlr:[3.7362766e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:16:53,013 [ hooks.py: 207]:\tstep: 18620\tsteps/sec: 1.24440\tloss: 0.35200\tlr:[3.7355978e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:17:04,757 [ hooks.py: 207]:\tstep: 18630\tsteps/sec: 0.86138\tloss: 0.59568\tlr:[3.7349186e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:17:12,892 [ hooks.py: 207]:\tstep: 18640\tsteps/sec: 1.25829\tloss: 0.48299\tlr:[3.7342394e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:17:21,849 [ hooks.py: 207]:\tstep: 18650\tsteps/sec: 1.12345\tloss: 0.99927\tlr:[3.7335605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:17:29,669 [ hooks.py: 207]:\tstep: 18660\tsteps/sec: 1.21218\tloss: 0.75587\tlr:[3.7328813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:17:36,743 [ hooks.py: 207]:\tstep: 18670\tsteps/sec: 1.39420\tloss: 0.62747\tlr:[3.732202e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:17:46,398 [ hooks.py: 207]:\tstep: 18680\tsteps/sec: 1.07296\tloss: 0.57835\tlr:[3.731523e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:17:54,513 [ hooks.py: 207]:\tstep: 18690\tsteps/sec: 1.22328\tloss: 0.49546\tlr:[3.730844e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:18:03,232 [ hooks.py: 207]:\tstep: 18700\tsteps/sec: 1.22814\tloss: 0.70247\tlr:[3.7301652e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:18:13,147 [ hooks.py: 207]:\tstep: 18710\tsteps/sec: 0.98519\tloss: 0.50966\tlr:[3.729486e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:18:21,673 [ hooks.py: 207]:\tstep: 18720\tsteps/sec: 1.14091\tloss: 0.91268\tlr:[3.728807e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:18:29,992 [ hooks.py: 207]:\tstep: 18730\tsteps/sec: 1.14555\tloss: 0.73962\tlr:[3.728128e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:18:38,031 [ hooks.py: 207]:\tstep: 18740\tsteps/sec: 1.31941\tloss: 0.61721\tlr:[3.727449e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:18:47,578 [ hooks.py: 207]:\tstep: 18750\tsteps/sec: 1.02219\tloss: 0.31230\tlr:[3.72677e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:18:58,667 [ hooks.py: 207]:\tstep: 18760\tsteps/sec: 0.93471\tloss: 0.23177\tlr:[3.7260907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:19:07,273 [ hooks.py: 207]:\tstep: 18770\tsteps/sec: 1.13337\tloss: 0.68883\tlr:[3.7254118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:19:13,570 [ hooks.py: 207]:\tstep: 18780\tsteps/sec: 1.61496\tloss: 0.25804\tlr:[3.724733e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:19:23,596 [ hooks.py: 207]:\tstep: 18790\tsteps/sec: 0.99974\tloss: 0.53886\tlr:[3.7240534e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:19:31,951 [ hooks.py: 207]:\tstep: 18800\tsteps/sec: 1.19725\tloss: 0.38712\tlr:[3.7233745e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:19:40,263 [ hooks.py: 207]:\tstep: 18810\tsteps/sec: 1.17903\tloss: 0.40533\tlr:[3.7226957e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:19:49,028 [ hooks.py: 207]:\tstep: 18820\tsteps/sec: 1.14473\tloss: 0.78515\tlr:[3.7220165e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:19:56,358 [ hooks.py: 207]:\tstep: 18830\tsteps/sec: 1.35029\tloss: 1.03316\tlr:[3.7213373e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:20:03,871 [ hooks.py: 207]:\tstep: 18840\tsteps/sec: 1.26601\tloss: 0.71201\tlr:[3.7206584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:20:14,386 [ hooks.py: 207]:\tstep: 18850\tsteps/sec: 0.97830\tloss: 0.41573\tlr:[3.7199796e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:20:23,335 [ hooks.py: 207]:\tstep: 18860\tsteps/sec: 1.16897\tloss: 0.63519\tlr:[3.7193004e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:20:30,536 [ hooks.py: 207]:\tstep: 18870\tsteps/sec: 1.34553\tloss: 0.44196\tlr:[3.718621e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:20:38,966 [ hooks.py: 207]:\tstep: 18880\tsteps/sec: 1.18011\tloss: 0.70648\tlr:[3.7179423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:20:49,160 [ hooks.py: 207]:\tstep: 18890\tsteps/sec: 0.94289\tloss: 0.16102\tlr:[3.717263e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:20:57,340 [ hooks.py: 207]:\tstep: 18900\tsteps/sec: 1.32651\tloss: 0.66437\tlr:[3.716584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:21:07,002 [ hooks.py: 207]:\tstep: 18910\tsteps/sec: 1.03236\tloss: 0.45419\tlr:[3.7159047e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:21:15,083 [ hooks.py: 207]:\tstep: 18920\tsteps/sec: 1.16226\tloss: 0.93262\tlr:[3.715226e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:21:25,032 [ hooks.py: 207]:\tstep: 18930\tsteps/sec: 1.03053\tloss: 0.87627\tlr:[3.714547e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:21:33,718 [ hooks.py: 207]:\tstep: 18940\tsteps/sec: 1.17854\tloss: 0.47059\tlr:[3.7138674e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:21:43,755 [ hooks.py: 207]:\tstep: 18950\tsteps/sec: 0.98739\tloss: 0.43946\tlr:[3.7131886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:21:53,325 [ hooks.py: 207]:\tstep: 18960\tsteps/sec: 1.06230\tloss: 0.74166\tlr:[3.7125097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:02,447 [ hooks.py: 207]:\tstep: 18970\tsteps/sec: 1.15579\tloss: 0.30819\tlr:[3.711831e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:10,988 [ hooks.py: 207]:\tstep: 18980\tsteps/sec: 1.09509\tloss: 0.69741\tlr:[3.7111513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:18,003 [ hooks.py: 207]:\tstep: 18990\tsteps/sec: 1.38530\tloss: 0.69754\tlr:[3.7104724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:27,477 [ hooks.py: 207]:\tstep: 19000\tsteps/sec: 1.10175\tloss: 0.45678\tlr:[3.7097936e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:27,478 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:27,478 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:27,478 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:22:27,478 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:27,478 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:22:27,479 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:30:33,636 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,638 [ trainer.py: 58]:\t{'acc': 0.7590361445783133, 'loss': 0.5806995328420248}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,638 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,638 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:30:33,639 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:30:33,639 [ trainer.py: 70]:\tacc\t0.7590361445783133\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:30:33,639 [ trainer.py: 70]:\tloss\t0.5806995328420248\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:30:33,639 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,639 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7590361445783133, 'loss': 0.5806995328420248}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,640 [ exporter.py: 64]:\t[Best Exporter]: skip step 19000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,640 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7590361445783133, 'loss': 0.5806995328420248}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,640 [ exporter.py: 104]:\t[Best Exporter]: skip step 19000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:33,641 [monitored_executor.py: 122]:\tsaving step 19000 to ./output/model_19000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 23:30:37,756 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_16000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:44,772 [ hooks.py: 207]:\tstep: 19010\tsteps/sec: 0.02009\tloss: 0.44450\tlr:[3.7091144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:30:52,851 [ hooks.py: 207]:\tstep: 19020\tsteps/sec: 1.29347\tloss: 0.80372\tlr:[3.708435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:31:01,801 [ hooks.py: 207]:\tstep: 19030\tsteps/sec: 1.14677\tloss: 0.60662\tlr:[3.7077563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:31:10,610 [ hooks.py: 207]:\tstep: 19040\tsteps/sec: 1.08480\tloss: 0.83414\tlr:[3.7070775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:31:20,517 [ hooks.py: 207]:\tstep: 19050\tsteps/sec: 0.97676\tloss: 0.36109\tlr:[3.7063983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:31:29,456 [ hooks.py: 207]:\tstep: 19060\tsteps/sec: 1.14213\tloss: 0.62409\tlr:[3.705719e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:31:39,373 [ hooks.py: 207]:\tstep: 19070\tsteps/sec: 1.02288\tloss: 0.77109\tlr:[3.7050406e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:31:48,920 [ hooks.py: 207]:\tstep: 19080\tsteps/sec: 1.05959\tloss: 0.41437\tlr:[3.7043614e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:31:58,011 [ hooks.py: 207]:\tstep: 19090\tsteps/sec: 1.11869\tloss: 0.74250\tlr:[3.703682e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:32:05,614 [ hooks.py: 207]:\tstep: 19100\tsteps/sec: 1.26022\tloss: 0.68775\tlr:[3.703003e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:32:15,033 [ hooks.py: 207]:\tstep: 19110\tsteps/sec: 1.05545\tloss: 0.91623\tlr:[3.702324e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:32:24,732 [ hooks.py: 207]:\tstep: 19120\tsteps/sec: 1.04070\tloss: 0.67638\tlr:[3.701645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:32:31,128 [ hooks.py: 207]:\tstep: 19130\tsteps/sec: 1.47620\tloss: 0.82508\tlr:[3.7009657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:32:39,245 [ hooks.py: 207]:\tstep: 19140\tsteps/sec: 1.21827\tloss: 1.07009\tlr:[3.700287e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:32:47,767 [ hooks.py: 207]:\tstep: 19150\tsteps/sec: 1.22983\tloss: 0.44909\tlr:[3.6996076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:32:58,503 [ hooks.py: 207]:\tstep: 19160\tsteps/sec: 0.92887\tloss: 0.93880\tlr:[3.6989284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:33:08,136 [ hooks.py: 207]:\tstep: 19170\tsteps/sec: 1.09944\tloss: 0.33843\tlr:[3.6982492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:33:17,395 [ hooks.py: 207]:\tstep: 19180\tsteps/sec: 1.05256\tloss: 0.54413\tlr:[3.6975704e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:33:24,957 [ hooks.py: 207]:\tstep: 19190\tsteps/sec: 1.27183\tloss: 0.62259\tlr:[3.6968915e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:33:33,908 [ hooks.py: 207]:\tstep: 19200\tsteps/sec: 1.15624\tloss: 0.61536\tlr:[3.6962123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:33:44,052 [ hooks.py: 207]:\tstep: 19210\tsteps/sec: 0.99090\tloss: 0.52265\tlr:[3.695533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:33:52,530 [ hooks.py: 207]:\tstep: 19220\tsteps/sec: 1.17506\tloss: 0.56466\tlr:[3.6948546e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:34:01,201 [ hooks.py: 207]:\tstep: 19230\tsteps/sec: 1.15576\tloss: 0.65180\tlr:[3.694175e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:34:10,076 [ hooks.py: 207]:\tstep: 19240\tsteps/sec: 1.09012\tloss: 1.26478\tlr:[3.6934962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:34:21,633 [ hooks.py: 207]:\tstep: 19250\tsteps/sec: 0.87049\tloss: 0.43502\tlr:[3.692817e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:34:30,363 [ hooks.py: 207]:\tstep: 19260\tsteps/sec: 1.13528\tloss: 1.01704\tlr:[3.6921385e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:34:38,863 [ hooks.py: 207]:\tstep: 19270\tsteps/sec: 1.21526\tloss: 0.54168\tlr:[3.691459e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:34:48,222 [ hooks.py: 207]:\tstep: 19280\tsteps/sec: 1.03889\tloss: 0.66657\tlr:[3.6907797e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:34:56,335 [ hooks.py: 207]:\tstep: 19290\tsteps/sec: 1.23493\tloss: 0.36454\tlr:[3.690101e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:35:05,149 [ hooks.py: 207]:\tstep: 19300\tsteps/sec: 1.09788\tloss: 0.71252\tlr:[3.689422e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:35:14,285 [ hooks.py: 207]:\tstep: 19310\tsteps/sec: 1.12893\tloss: 0.27778\tlr:[3.6887428e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:35:22,491 [ hooks.py: 207]:\tstep: 19320\tsteps/sec: 1.20680\tloss: 0.72040\tlr:[3.688064e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:35:34,029 [ hooks.py: 207]:\tstep: 19330\tsteps/sec: 0.88739\tloss: 0.47422\tlr:[3.687385e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:35:42,231 [ hooks.py: 207]:\tstep: 19340\tsteps/sec: 1.17923\tloss: 0.71683\tlr:[3.686706e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:35:53,815 [ hooks.py: 207]:\tstep: 19350\tsteps/sec: 0.89077\tloss: 0.88139\tlr:[3.6860267e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:36:02,919 [ hooks.py: 207]:\tstep: 19360\tsteps/sec: 1.11535\tloss: 0.55384\tlr:[3.6853475e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:36:13,009 [ hooks.py: 207]:\tstep: 19370\tsteps/sec: 0.95291\tloss: 0.40903\tlr:[3.6846686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:36:20,626 [ hooks.py: 207]:\tstep: 19380\tsteps/sec: 1.33298\tloss: 0.90224\tlr:[3.6839894e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:36:29,930 [ hooks.py: 207]:\tstep: 19390\tsteps/sec: 1.05735\tloss: 0.67515\tlr:[3.6833102e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:36:39,792 [ hooks.py: 207]:\tstep: 19400\tsteps/sec: 1.02183\tloss: 0.62360\tlr:[3.682631e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:36:48,022 [ hooks.py: 207]:\tstep: 19410\tsteps/sec: 1.25356\tloss: 0.87208\tlr:[3.6819525e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:36:58,261 [ hooks.py: 207]:\tstep: 19420\tsteps/sec: 0.98580\tloss: 0.32145\tlr:[3.6812733e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:37:06,156 [ hooks.py: 207]:\tstep: 19430\tsteps/sec: 1.22266\tloss: 0.70460\tlr:[3.6805937e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:37:14,839 [ hooks.py: 207]:\tstep: 19440\tsteps/sec: 1.13770\tloss: 0.48451\tlr:[3.6799152e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:37:25,603 [ hooks.py: 207]:\tstep: 19450\tsteps/sec: 0.90645\tloss: 0.49989\tlr:[3.6792364e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:37:34,529 [ hooks.py: 207]:\tstep: 19460\tsteps/sec: 1.11152\tloss: 0.60768\tlr:[3.6785572e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:37:42,789 [ hooks.py: 207]:\tstep: 19470\tsteps/sec: 1.25454\tloss: 1.06187\tlr:[3.677878e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:37:52,241 [ hooks.py: 207]:\tstep: 19480\tsteps/sec: 1.09573\tloss: 0.50125\tlr:[3.677199e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:38:00,319 [ hooks.py: 207]:\tstep: 19490\tsteps/sec: 1.16612\tloss: 0.58357\tlr:[3.67652e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:38:11,472 [ hooks.py: 207]:\tstep: 19500\tsteps/sec: 0.92633\tloss: 0.32914\tlr:[3.675841e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:38:18,189 [ hooks.py: 207]:\tstep: 19510\tsteps/sec: 1.54032\tloss: 0.35650\tlr:[3.675162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:38:27,419 [ hooks.py: 207]:\tstep: 19520\tsteps/sec: 1.03018\tloss: 0.74417\tlr:[3.6744827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:38:38,313 [ hooks.py: 207]:\tstep: 19530\tsteps/sec: 0.96590\tloss: 0.58213\tlr:[3.6738038e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:38:45,289 [ hooks.py: 207]:\tstep: 19540\tsteps/sec: 1.29995\tloss: 0.65977\tlr:[3.6731246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:38:54,637 [ hooks.py: 207]:\tstep: 19550\tsteps/sec: 1.11010\tloss: 0.75577\tlr:[3.6724457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:39:03,492 [ hooks.py: 207]:\tstep: 19560\tsteps/sec: 1.21249\tloss: 0.51909\tlr:[3.671767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:39:12,822 [ hooks.py: 207]:\tstep: 19570\tsteps/sec: 1.03514\tloss: 0.45386\tlr:[3.6710877e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:39:21,937 [ hooks.py: 207]:\tstep: 19580\tsteps/sec: 1.03723\tloss: 0.57582\tlr:[3.6704085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:39:31,204 [ hooks.py: 207]:\tstep: 19590\tsteps/sec: 1.11578\tloss: 0.64307\tlr:[3.6697293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:39:38,266 [ hooks.py: 207]:\tstep: 19600\tsteps/sec: 1.44988\tloss: 0.48607\tlr:[3.6690504e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:39:48,763 [ hooks.py: 207]:\tstep: 19610\tsteps/sec: 0.95183\tloss: 0.40205\tlr:[3.6683712e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:39:59,372 [ hooks.py: 207]:\tstep: 19620\tsteps/sec: 0.94345\tloss: 1.08761\tlr:[3.667692e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:40:09,472 [ hooks.py: 207]:\tstep: 19630\tsteps/sec: 0.96682\tloss: 0.79934\tlr:[3.6670128e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:40:18,656 [ hooks.py: 207]:\tstep: 19640\tsteps/sec: 1.12993\tloss: 0.42428\tlr:[3.666334e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 23:40:26,856 [ hooks.py: 207]:\tstep: 19650\tsteps/sec: 1.17528\tloss: 0.60379\tlr:[3.6656547e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:40:36,462 [ hooks.py: 207]:\tstep: 19660\tsteps/sec: 1.04344\tloss: 0.77414\tlr:[3.664976e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:40:43,821 [ hooks.py: 207]:\tstep: 19670\tsteps/sec: 1.38413\tloss: 0.60152\tlr:[3.664297e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:40:54,435 [ hooks.py: 207]:\tstep: 19680\tsteps/sec: 0.92872\tloss: 0.45781\tlr:[3.6636175e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:41:03,820 [ hooks.py: 207]:\tstep: 19690\tsteps/sec: 1.09081\tloss: 0.53461\tlr:[3.6629386e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:41:10,834 [ hooks.py: 207]:\tstep: 19700\tsteps/sec: 1.39847\tloss: 0.58652\tlr:[3.6622594e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:41:19,886 [ hooks.py: 207]:\tstep: 19710\tsteps/sec: 1.05016\tloss: 0.88609\tlr:[3.661581e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:41:28,636 [ hooks.py: 207]:\tstep: 19720\tsteps/sec: 1.22734\tloss: 0.84512\tlr:[3.6609017e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:41:38,507 [ hooks.py: 207]:\tstep: 19730\tsteps/sec: 1.02526\tloss: 0.50498\tlr:[3.660222e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:41:46,576 [ hooks.py: 207]:\tstep: 19740\tsteps/sec: 1.19607\tloss: 0.42001\tlr:[3.6595433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:41:54,861 [ hooks.py: 207]:\tstep: 19750\tsteps/sec: 1.20015\tloss: 0.68306\tlr:[3.6588648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:42:03,766 [ hooks.py: 207]:\tstep: 19760\tsteps/sec: 1.20101\tloss: 0.89465\tlr:[3.6581852e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:42:10,803 [ hooks.py: 207]:\tstep: 19770\tsteps/sec: 1.33137\tloss: 0.52780\tlr:[3.657506e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:42:18,715 [ hooks.py: 207]:\tstep: 19780\tsteps/sec: 1.26683\tloss: 0.57725\tlr:[3.656827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:42:27,503 [ hooks.py: 207]:\tstep: 19790\tsteps/sec: 1.15007\tloss: 0.47519\tlr:[3.6561483e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:42:34,352 [ hooks.py: 207]:\tstep: 19800\tsteps/sec: 1.43700\tloss: 0.47899\tlr:[3.655469e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:42:44,244 [ hooks.py: 207]:\tstep: 19810\tsteps/sec: 0.97419\tloss: 0.64777\tlr:[3.65479e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:42:54,463 [ hooks.py: 207]:\tstep: 19820\tsteps/sec: 1.01200\tloss: 0.42644\tlr:[3.654111e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:43:04,113 [ hooks.py: 207]:\tstep: 19830\tsteps/sec: 1.08423\tloss: 0.87496\tlr:[3.653432e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:43:11,417 [ hooks.py: 207]:\tstep: 19840\tsteps/sec: 1.30003\tloss: 0.63851\tlr:[3.6527526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:43:21,401 [ hooks.py: 207]:\tstep: 19850\tsteps/sec: 0.99735\tloss: 0.73211\tlr:[3.6520738e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:43:29,452 [ hooks.py: 207]:\tstep: 19860\tsteps/sec: 1.23899\tloss: 0.49152\tlr:[3.651395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:43:39,261 [ hooks.py: 207]:\tstep: 19870\tsteps/sec: 1.03887\tloss: 0.66193\tlr:[3.6507157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:43:47,515 [ hooks.py: 207]:\tstep: 19880\tsteps/sec: 1.16168\tloss: 0.58382\tlr:[3.6500365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:43:57,959 [ hooks.py: 207]:\tstep: 19890\tsteps/sec: 0.99608\tloss: 0.56380\tlr:[3.6493573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:44:06,437 [ hooks.py: 207]:\tstep: 19900\tsteps/sec: 1.10701\tloss: 0.46507\tlr:[3.648679e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:44:15,756 [ hooks.py: 207]:\tstep: 19910\tsteps/sec: 1.10395\tloss: 0.48734\tlr:[3.6479996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:44:23,911 [ hooks.py: 207]:\tstep: 19920\tsteps/sec: 1.25768\tloss: 0.53694\tlr:[3.64732e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:44:30,962 [ hooks.py: 207]:\tstep: 19930\tsteps/sec: 1.39412\tloss: 0.47380\tlr:[3.6466412e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:44:37,536 [ hooks.py: 207]:\tstep: 19940\tsteps/sec: 1.47955\tloss: 0.24807\tlr:[3.6459624e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:44:45,763 [ hooks.py: 207]:\tstep: 19950\tsteps/sec: 1.24111\tloss: 1.09114\tlr:[3.645283e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:44:53,048 [ hooks.py: 207]:\tstep: 19960\tsteps/sec: 1.41815\tloss: 0.84508\tlr:[3.644604e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:00,214 [ hooks.py: 207]:\tstep: 19970\tsteps/sec: 1.34919\tloss: 0.33011\tlr:[3.6439254e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:09,433 [ hooks.py: 207]:\tstep: 19980\tsteps/sec: 1.09853\tloss: 0.83604\tlr:[3.6432462e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:16,818 [ hooks.py: 207]:\tstep: 19990\tsteps/sec: 1.31699\tloss: 0.73680\tlr:[3.6425667e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:26,706 [ hooks.py: 207]:\tstep: 20000\tsteps/sec: 1.02604\tloss: 0.25951\tlr:[3.641888e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:26,711 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:26,711 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:26,711 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:45:26,712 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:26,712 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:45:26,712 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:53:37,147 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,149 [ trainer.py: 58]:\t{'acc': 0.7654618473895582, 'loss': 0.5909705311059952}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,150 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,151 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:53:37,151 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:53:37,151 [ trainer.py: 70]:\tacc\t0.7654618473895582\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:53:37,151 [ trainer.py: 70]:\tloss\t0.5909705311059952\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-18 23:53:37,151 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,151 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7654618473895582, 'loss': 0.5909705311059952}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,151 [ exporter.py: 64]:\t[Best Exporter]: skip step 20000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,152 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7654618473895582, 'loss': 0.5909705311059952}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,152 [ exporter.py: 104]:\t[Best Exporter]: skip step 20000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:37,153 [monitored_executor.py: 122]:\tsaving step 20000 to ./output/model_20000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:40,336 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_17000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:50,108 [ hooks.py: 207]:\tstep: 20010\tsteps/sec: 0.01986\tloss: 0.42230\tlr:[3.641209e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:53:59,380 [ hooks.py: 207]:\tstep: 20020\tsteps/sec: 1.10240\tloss: 0.71595\tlr:[3.64053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:54:08,018 [ hooks.py: 207]:\tstep: 20030\tsteps/sec: 1.16102\tloss: 0.53945\tlr:[3.639851e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:54:15,141 [ hooks.py: 207]:\tstep: 20040\tsteps/sec: 1.34903\tloss: 0.81769\tlr:[3.6391717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:54:23,298 [ hooks.py: 207]:\tstep: 20050\tsteps/sec: 1.26347\tloss: 0.46727\tlr:[3.638493e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:54:33,943 [ hooks.py: 207]:\tstep: 20060\tsteps/sec: 0.94246\tloss: 0.44535\tlr:[3.6378136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:54:41,300 [ hooks.py: 207]:\tstep: 20070\tsteps/sec: 1.32886\tloss: 0.74854\tlr:[3.6371344e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:54:49,401 [ hooks.py: 207]:\tstep: 20080\tsteps/sec: 1.18707\tloss: 0.40298\tlr:[3.6364556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:54:58,414 [ hooks.py: 207]:\tstep: 20090\tsteps/sec: 1.18352\tloss: 0.55172\tlr:[3.6357767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:55:05,649 [ hooks.py: 207]:\tstep: 20100\tsteps/sec: 1.32631\tloss: 0.40170\tlr:[3.6350975e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-18 23:55:13,187 [ hooks.py: 207]:\tstep: 20110\tsteps/sec: 1.36577\tloss: 0.55452\tlr:[3.6344183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:55:21,176 [ hooks.py: 207]:\tstep: 20120\tsteps/sec: 1.22638\tloss: 0.43478\tlr:[3.633739e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:55:30,305 [ hooks.py: 207]:\tstep: 20130\tsteps/sec: 1.03887\tloss: 0.54047\tlr:[3.63306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:55:40,358 [ hooks.py: 207]:\tstep: 20140\tsteps/sec: 1.04125\tloss: 0.46188\tlr:[3.632381e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:55:47,635 [ hooks.py: 207]:\tstep: 20150\tsteps/sec: 1.40413\tloss: 0.84515\tlr:[3.631702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:55:55,637 [ hooks.py: 207]:\tstep: 20160\tsteps/sec: 1.22285\tloss: 0.52260\tlr:[3.6310234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:56:05,593 [ hooks.py: 207]:\tstep: 20170\tsteps/sec: 1.00692\tloss: 0.78184\tlr:[3.630344e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:56:14,237 [ hooks.py: 207]:\tstep: 20180\tsteps/sec: 1.18590\tloss: 0.32020\tlr:[3.6296646e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:56:22,449 [ hooks.py: 207]:\tstep: 20190\tsteps/sec: 1.18811\tloss: 0.31771\tlr:[3.628986e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:56:30,534 [ hooks.py: 207]:\tstep: 20200\tsteps/sec: 1.23950\tloss: 0.42660\tlr:[3.628307e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:56:39,847 [ hooks.py: 207]:\tstep: 20210\tsteps/sec: 1.06348\tloss: 0.26206\tlr:[3.627628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:56:48,737 [ hooks.py: 207]:\tstep: 20220\tsteps/sec: 1.16908\tloss: 0.51230\tlr:[3.626949e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:56:56,869 [ hooks.py: 207]:\tstep: 20230\tsteps/sec: 1.18940\tloss: 0.71299\tlr:[3.6262696e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:57:09,000 [ hooks.py: 207]:\tstep: 20240\tsteps/sec: 0.81906\tloss: 0.65248\tlr:[3.6255908e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:57:18,396 [ hooks.py: 207]:\tstep: 20250\tsteps/sec: 1.06582\tloss: 0.23559\tlr:[3.624912e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:57:26,274 [ hooks.py: 207]:\tstep: 20260\tsteps/sec: 1.27335\tloss: 0.41182\tlr:[3.6242327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:57:34,962 [ hooks.py: 207]:\tstep: 20270\tsteps/sec: 1.16037\tloss: 0.55752\tlr:[3.623554e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:57:41,540 [ hooks.py: 207]:\tstep: 20280\tsteps/sec: 1.53156\tloss: 0.74438\tlr:[3.6228743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:57:49,435 [ hooks.py: 207]:\tstep: 20290\tsteps/sec: 1.29192\tloss: 0.65237\tlr:[3.6221954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:57:56,687 [ hooks.py: 207]:\tstep: 20300\tsteps/sec: 1.34150\tloss: 0.63071\tlr:[3.6215166e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:58:06,775 [ hooks.py: 207]:\tstep: 20310\tsteps/sec: 0.98426\tloss: 0.64556\tlr:[3.6208374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:58:15,420 [ hooks.py: 207]:\tstep: 20320\tsteps/sec: 1.16096\tloss: 0.40890\tlr:[3.620158e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:58:23,982 [ hooks.py: 207]:\tstep: 20330\tsteps/sec: 1.19343\tloss: 0.64932\tlr:[3.6194793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:58:32,924 [ hooks.py: 207]:\tstep: 20340\tsteps/sec: 1.11014\tloss: 0.65461\tlr:[3.6188e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:58:40,661 [ hooks.py: 207]:\tstep: 20350\tsteps/sec: 1.27760\tloss: 0.53012\tlr:[3.6181213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:58:47,902 [ hooks.py: 207]:\tstep: 20360\tsteps/sec: 1.41754\tloss: 0.72457\tlr:[3.617442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:58:54,525 [ hooks.py: 207]:\tstep: 20370\tsteps/sec: 1.41591\tloss: 0.54878\tlr:[3.616763e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:59:06,525 [ hooks.py: 207]:\tstep: 20380\tsteps/sec: 0.85189\tloss: 0.60675\tlr:[3.6160836e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:59:14,412 [ hooks.py: 207]:\tstep: 20390\tsteps/sec: 1.29554\tloss: 0.79986\tlr:[3.615405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:59:22,595 [ hooks.py: 207]:\tstep: 20400\tsteps/sec: 1.21180\tloss: 0.63596\tlr:[3.614726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:59:31,062 [ hooks.py: 207]:\tstep: 20410\tsteps/sec: 1.16467\tloss: 0.64105\tlr:[3.6140467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:59:38,097 [ hooks.py: 207]:\tstep: 20420\tsteps/sec: 1.39809\tloss: 0.38857\tlr:[3.6133675e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:59:46,143 [ hooks.py: 207]:\tstep: 20430\tsteps/sec: 1.24947\tloss: 0.70076\tlr:[3.6126883e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-18 23:59:56,009 [ hooks.py: 207]:\tstep: 20440\tsteps/sec: 1.01366\tloss: 0.68422\tlr:[3.612009e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:00:04,749 [ hooks.py: 207]:\tstep: 20450\tsteps/sec: 1.17670\tloss: 0.33448\tlr:[3.6113302e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:00:12,022 [ hooks.py: 207]:\tstep: 20460\tsteps/sec: 1.34704\tloss: 0.44645\tlr:[3.6106514e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:00:21,085 [ hooks.py: 207]:\tstep: 20470\tsteps/sec: 1.09991\tloss: 0.58245\tlr:[3.6099726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:00:28,966 [ hooks.py: 207]:\tstep: 20480\tsteps/sec: 1.31803\tloss: 0.69471\tlr:[3.6092933e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:00:38,786 [ hooks.py: 207]:\tstep: 20490\tsteps/sec: 1.01435\tloss: 0.49368\tlr:[3.608614e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:00:49,842 [ hooks.py: 207]:\tstep: 20500\tsteps/sec: 0.89299\tloss: 0.66964\tlr:[3.6079357e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:00:58,144 [ hooks.py: 207]:\tstep: 20510\tsteps/sec: 1.21354\tloss: 0.55973\tlr:[3.607256e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:01:08,073 [ hooks.py: 207]:\tstep: 20520\tsteps/sec: 1.00841\tloss: 0.72198\tlr:[3.606577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:01:15,610 [ hooks.py: 207]:\tstep: 20530\tsteps/sec: 1.29180\tloss: 0.51973\tlr:[3.605898e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:01:23,841 [ hooks.py: 207]:\tstep: 20540\tsteps/sec: 1.26390\tloss: 0.55286\tlr:[3.605219e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:01:31,422 [ hooks.py: 207]:\tstep: 20550\tsteps/sec: 1.27316\tloss: 0.93832\tlr:[3.60454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:01:40,503 [ hooks.py: 207]:\tstep: 20560\tsteps/sec: 1.05892\tloss: 0.97352\tlr:[3.603861e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:01:48,299 [ hooks.py: 207]:\tstep: 20570\tsteps/sec: 1.37720\tloss: 0.84766\tlr:[3.603182e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:01:56,460 [ hooks.py: 207]:\tstep: 20580\tsteps/sec: 1.18322\tloss: 0.66731\tlr:[3.6025027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:02:06,379 [ hooks.py: 207]:\tstep: 20590\tsteps/sec: 1.01239\tloss: 0.48213\tlr:[3.6018235e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:02:14,393 [ hooks.py: 207]:\tstep: 20600\tsteps/sec: 1.23625\tloss: 0.27537\tlr:[3.6011446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:02:21,931 [ hooks.py: 207]:\tstep: 20610\tsteps/sec: 1.37211\tloss: 0.42353\tlr:[3.6004654e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:02:29,580 [ hooks.py: 207]:\tstep: 20620\tsteps/sec: 1.31292\tloss: 0.38116\tlr:[3.5997866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:02:55,461 [ hooks.py: 207]:\tstep: 20650\tsteps/sec: 1.03446\tloss: 0.41258\tlr:[3.5977497e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:03:05,503 [ hooks.py: 207]:\tstep: 20660\tsteps/sec: 1.03526\tloss: 0.57388\tlr:[3.5970705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:03:15,328 [ hooks.py: 207]:\tstep: 20670\tsteps/sec: 1.02481\tloss: 0.73025\tlr:[3.596391e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:03:23,184 [ hooks.py: 207]:\tstep: 20680\tsteps/sec: 1.29278\tloss: 0.52448\tlr:[3.5957124e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:03:32,970 [ hooks.py: 207]:\tstep: 20690\tsteps/sec: 1.01959\tloss: 0.63423\tlr:[3.5950336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:03:41,094 [ hooks.py: 207]:\tstep: 20700\tsteps/sec: 1.17195\tloss: 0.70970\tlr:[3.5943543e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:03:49,664 [ hooks.py: 207]:\tstep: 20710\tsteps/sec: 1.23537\tloss: 0.42242\tlr:[3.593675e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:03:56,372 [ hooks.py: 207]:\tstep: 20720\tsteps/sec: 1.44700\tloss: 0.86511\tlr:[3.5929963e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:04:05,751 [ hooks.py: 207]:\tstep: 20730\tsteps/sec: 1.06853\tloss: 0.19942\tlr:[3.592317e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:04:13,063 [ hooks.py: 207]:\tstep: 20740\tsteps/sec: 1.39390\tloss: 0.40484\tlr:[3.5916382e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:04:23,288 [ hooks.py: 207]:\tstep: 20750\tsteps/sec: 0.98867\tloss: 1.01675\tlr:[3.590959e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:04:30,343 [ hooks.py: 207]:\tstep: 20760\tsteps/sec: 1.37451\tloss: 0.59333\tlr:[3.59028e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:04:38,869 [ hooks.py: 207]:\tstep: 20770\tsteps/sec: 1.20366\tloss: 0.76283\tlr:[3.589601e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 00:04:45,035 [ hooks.py: 207]:\tstep: 20780\tsteps/sec: 1.53132\tloss: 0.46215\tlr:[3.5889218e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:04:54,637 [ hooks.py: 207]:\tstep: 20790\tsteps/sec: 1.03175\tloss: 0.61360\tlr:[3.588243e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:05:02,887 [ hooks.py: 207]:\tstep: 20800\tsteps/sec: 1.33340\tloss: 0.82162\tlr:[3.5875637e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:05:10,363 [ hooks.py: 207]:\tstep: 20810\tsteps/sec: 1.25414\tloss: 0.63671\tlr:[3.5868845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:05:20,030 [ hooks.py: 207]:\tstep: 20820\tsteps/sec: 1.03872\tloss: 0.71974\tlr:[3.5862053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:05:26,714 [ hooks.py: 207]:\tstep: 20830\tsteps/sec: 1.41613\tloss: 0.58368\tlr:[3.5855264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:05:36,413 [ hooks.py: 207]:\tstep: 20840\tsteps/sec: 1.02365\tloss: 0.27212\tlr:[3.5848476e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:05:45,245 [ hooks.py: 207]:\tstep: 20850\tsteps/sec: 1.17953\tloss: 0.59898\tlr:[3.584168e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:05:53,533 [ hooks.py: 207]:\tstep: 20860\tsteps/sec: 1.23804\tloss: 0.84128\tlr:[3.583489e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:03,863 [ hooks.py: 207]:\tstep: 20870\tsteps/sec: 0.98375\tloss: 0.56736\tlr:[3.58281e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:11,196 [ hooks.py: 207]:\tstep: 20880\tsteps/sec: 1.29058\tloss: 0.86057\tlr:[3.582131e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:19,682 [ hooks.py: 207]:\tstep: 20890\tsteps/sec: 1.13098\tloss: 0.77776\tlr:[3.581452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:28,403 [ hooks.py: 207]:\tstep: 20900\tsteps/sec: 1.23472\tloss: 0.29839\tlr:[3.580773e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:35,234 [ hooks.py: 207]:\tstep: 20910\tsteps/sec: 1.41326\tloss: 0.85038\tlr:[3.5800942e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:43,622 [ hooks.py: 207]:\tstep: 20920\tsteps/sec: 1.22832\tloss: 0.51506\tlr:[3.579415e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:51,648 [ hooks.py: 207]:\tstep: 20930\tsteps/sec: 1.21819\tloss: 0.79126\tlr:[3.5787358e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:06:58,961 [ hooks.py: 207]:\tstep: 20940\tsteps/sec: 1.40906\tloss: 0.31683\tlr:[3.578057e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:08,831 [ hooks.py: 207]:\tstep: 20950\tsteps/sec: 1.00169\tloss: 0.45514\tlr:[3.577378e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:18,123 [ hooks.py: 207]:\tstep: 20960\tsteps/sec: 1.07474\tloss: 0.71499\tlr:[3.576699e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:25,363 [ hooks.py: 207]:\tstep: 20970\tsteps/sec: 1.28069\tloss: 0.58017\tlr:[3.5760193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:34,758 [ hooks.py: 207]:\tstep: 20980\tsteps/sec: 1.13182\tloss: 0.62866\tlr:[3.5753405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:42,105 [ hooks.py: 207]:\tstep: 20990\tsteps/sec: 1.34614\tloss: 0.53565\tlr:[3.5746616e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:49,074 [ hooks.py: 207]:\tstep: 21000\tsteps/sec: 1.45870\tloss: 0.45347\tlr:[3.573982e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:49,075 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:49,075 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:49,075 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:07:49,075 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:49,076 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:07:49,076 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:15:48,133 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,135 [ trainer.py: 58]:\t{'acc': 0.7694779116465863, 'loss': 0.5693983066922579}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,135 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,136 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:15:48,136 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:15:48,137 [ trainer.py: 70]:\tacc\t0.7694779116465863\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:15:48,137 [ trainer.py: 70]:\tloss\t0.5693983066922579\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:15:48,137 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,137 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7694779116465863, 'loss': 0.5693983066922579}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,137 [ exporter.py: 64]:\t[Best Exporter]: skip step 21000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,137 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7694779116465863, 'loss': 0.5693983066922579}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,137 [ exporter.py: 104]:\t[Best Exporter]: skip step 21000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:48,138 [monitored_executor.py: 122]:\tsaving step 21000 to ./output/model_21000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:15:51,351 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_18000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:00,857 [ hooks.py: 207]:\tstep: 21010\tsteps/sec: 0.02033\tloss: 0.54412\tlr:[3.5733032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:09,500 [ hooks.py: 207]:\tstep: 21020\tsteps/sec: 1.17394\tloss: 0.55177\tlr:[3.5726247e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:18,772 [ hooks.py: 207]:\tstep: 21030\tsteps/sec: 1.08321\tloss: 0.39035\tlr:[3.5719455e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:24,948 [ hooks.py: 207]:\tstep: 21040\tsteps/sec: 1.47611\tloss: 0.43531\tlr:[3.571266e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:33,900 [ hooks.py: 207]:\tstep: 21050\tsteps/sec: 1.14113\tloss: 0.55289\tlr:[3.570587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:43,125 [ hooks.py: 207]:\tstep: 21060\tsteps/sec: 1.12741\tloss: 0.98237\tlr:[3.5699082e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:51,276 [ hooks.py: 207]:\tstep: 21070\tsteps/sec: 1.20077\tloss: 0.70192\tlr:[3.569229e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:16:57,623 [ hooks.py: 207]:\tstep: 21080\tsteps/sec: 1.50706\tloss: 0.62614\tlr:[3.5685498e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:17:05,718 [ hooks.py: 207]:\tstep: 21090\tsteps/sec: 1.26107\tloss: 0.64851\tlr:[3.567871e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:17:15,174 [ hooks.py: 207]:\tstep: 21100\tsteps/sec: 1.07707\tloss: 0.68153\tlr:[3.567192e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:17:21,830 [ hooks.py: 207]:\tstep: 21110\tsteps/sec: 1.48056\tloss: 1.02701\tlr:[3.5665125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:17:31,177 [ hooks.py: 207]:\tstep: 21120\tsteps/sec: 1.06720\tloss: 0.35232\tlr:[3.5658333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:17:40,575 [ hooks.py: 207]:\tstep: 21130\tsteps/sec: 1.07832\tloss: 0.79867\tlr:[3.565155e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:17:47,406 [ hooks.py: 207]:\tstep: 21140\tsteps/sec: 1.44968\tloss: 0.83792\tlr:[3.5644756e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:17:59,516 [ hooks.py: 207]:\tstep: 21150\tsteps/sec: 0.84516\tloss: 0.53849\tlr:[3.5637964e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:18:05,955 [ hooks.py: 207]:\tstep: 21160\tsteps/sec: 1.39295\tloss: 0.26754\tlr:[3.5631172e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:18:16,223 [ hooks.py: 207]:\tstep: 21170\tsteps/sec: 1.01836\tloss: 0.65191\tlr:[3.5624384e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:18:24,531 [ hooks.py: 207]:\tstep: 21180\tsteps/sec: 1.22474\tloss: 0.56714\tlr:[3.561759e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:18:32,049 [ hooks.py: 207]:\tstep: 21190\tsteps/sec: 1.32701\tloss: 0.74777\tlr:[3.56108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:18:40,600 [ hooks.py: 207]:\tstep: 21200\tsteps/sec: 1.15408\tloss: 0.29154\tlr:[3.560401e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:18:50,798 [ hooks.py: 207]:\tstep: 21210\tsteps/sec: 0.98446\tloss: 0.41454\tlr:[3.5597222e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:18:58,218 [ hooks.py: 207]:\tstep: 21220\tsteps/sec: 1.36006\tloss: 0.29533\tlr:[3.559043e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:19:06,582 [ hooks.py: 207]:\tstep: 21230\tsteps/sec: 1.17726\tloss: 0.60516\tlr:[3.558364e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 00:19:15,489 [ hooks.py: 207]:\tstep: 21240\tsteps/sec: 1.11215\tloss: 0.39647\tlr:[3.557685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:19:25,101 [ hooks.py: 207]:\tstep: 21250\tsteps/sec: 1.06262\tloss: 0.34180\tlr:[3.557006e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:19:32,402 [ hooks.py: 207]:\tstep: 21260\tsteps/sec: 1.35336\tloss: 0.58843\tlr:[3.556327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:19:40,941 [ hooks.py: 207]:\tstep: 21270\tsteps/sec: 1.17301\tloss: 0.85062\tlr:[3.555648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:19:48,905 [ hooks.py: 207]:\tstep: 21280\tsteps/sec: 1.28277\tloss: 0.33250\tlr:[3.554969e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:19:56,288 [ hooks.py: 207]:\tstep: 21290\tsteps/sec: 1.30482\tloss: 0.61085\tlr:[3.55429e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:20:03,602 [ hooks.py: 207]:\tstep: 21300\tsteps/sec: 1.44625\tloss: 0.66928\tlr:[3.5536108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:20:12,338 [ hooks.py: 207]:\tstep: 21310\tsteps/sec: 1.11730\tloss: 0.71626\tlr:[3.5529316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:20:20,134 [ hooks.py: 207]:\tstep: 21320\tsteps/sec: 1.25493\tloss: 0.64730\tlr:[3.5522527e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:20:31,045 [ hooks.py: 207]:\tstep: 21330\tsteps/sec: 0.92482\tloss: 0.64272\tlr:[3.5515735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:20:37,512 [ hooks.py: 207]:\tstep: 21340\tsteps/sec: 1.46631\tloss: 0.58150\tlr:[3.5508943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:20:46,231 [ hooks.py: 207]:\tstep: 21350\tsteps/sec: 1.17831\tloss: 0.72036\tlr:[3.550215e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:20:56,956 [ hooks.py: 207]:\tstep: 21360\tsteps/sec: 0.94620\tloss: 0.62671\tlr:[3.5495366e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:21:05,707 [ hooks.py: 207]:\tstep: 21370\tsteps/sec: 1.08640\tloss: 0.40752\tlr:[3.5488574e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:21:16,097 [ hooks.py: 207]:\tstep: 21380\tsteps/sec: 1.00183\tloss: 0.68394\tlr:[3.548178e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:21:24,831 [ hooks.py: 207]:\tstep: 21390\tsteps/sec: 1.10651\tloss: 0.55477\tlr:[3.5474994e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:21:32,769 [ hooks.py: 207]:\tstep: 21400\tsteps/sec: 1.29905\tloss: 0.77635\tlr:[3.54682e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:21:41,867 [ hooks.py: 207]:\tstep: 21410\tsteps/sec: 1.11928\tloss: 0.52041\tlr:[3.5461413e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:21:50,325 [ hooks.py: 207]:\tstep: 21420\tsteps/sec: 1.16242\tloss: 0.55462\tlr:[3.5454617e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:21:59,855 [ hooks.py: 207]:\tstep: 21430\tsteps/sec: 1.08869\tloss: 0.43919\tlr:[3.5447832e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:22:07,926 [ hooks.py: 207]:\tstep: 21440\tsteps/sec: 1.20151\tloss: 0.69184\tlr:[3.544104e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:22:15,582 [ hooks.py: 207]:\tstep: 21450\tsteps/sec: 1.28367\tloss: 0.52251\tlr:[3.5434245e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:22:23,282 [ hooks.py: 207]:\tstep: 21460\tsteps/sec: 1.29763\tloss: 0.53697\tlr:[3.5427456e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:22:34,216 [ hooks.py: 207]:\tstep: 21470\tsteps/sec: 0.95424\tloss: 0.79133\tlr:[3.542067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:22:41,196 [ hooks.py: 207]:\tstep: 21480\tsteps/sec: 1.31217\tloss: 0.29140\tlr:[3.5413876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:22:49,407 [ hooks.py: 207]:\tstep: 21490\tsteps/sec: 1.29341\tloss: 0.40295\tlr:[3.5407083e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:22:58,282 [ hooks.py: 207]:\tstep: 21500\tsteps/sec: 1.12849\tloss: 0.50325\tlr:[3.5400295e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:23:06,923 [ hooks.py: 207]:\tstep: 21510\tsteps/sec: 1.12081\tloss: 0.29463\tlr:[3.539351e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:23:17,095 [ hooks.py: 207]:\tstep: 21520\tsteps/sec: 0.96325\tloss: 0.53216\tlr:[3.5386714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:23:26,102 [ hooks.py: 207]:\tstep: 21530\tsteps/sec: 1.13719\tloss: 0.25229\tlr:[3.5379922e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:23:35,159 [ hooks.py: 207]:\tstep: 21540\tsteps/sec: 1.13155\tloss: 0.39819\tlr:[3.5373134e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:23:41,826 [ hooks.py: 207]:\tstep: 21550\tsteps/sec: 1.47104\tloss: 0.41356\tlr:[3.536634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:23:49,870 [ hooks.py: 207]:\tstep: 21560\tsteps/sec: 1.26599\tloss: 0.79313\tlr:[3.5359553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:23:56,860 [ hooks.py: 207]:\tstep: 21570\tsteps/sec: 1.39546\tloss: 0.70150\tlr:[3.535276e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:24:05,085 [ hooks.py: 207]:\tstep: 21580\tsteps/sec: 1.24670\tloss: 0.35295\tlr:[3.534597e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:24:13,676 [ hooks.py: 207]:\tstep: 21590\tsteps/sec: 1.16750\tloss: 0.96694\tlr:[3.5339177e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:24:23,837 [ hooks.py: 207]:\tstep: 21600\tsteps/sec: 0.98818\tloss: 0.54061\tlr:[3.533239e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:24:33,060 [ hooks.py: 207]:\tstep: 21610\tsteps/sec: 1.06466\tloss: 0.69104\tlr:[3.5325596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:24:41,019 [ hooks.py: 207]:\tstep: 21620\tsteps/sec: 1.24260\tloss: 0.36706\tlr:[3.5318808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:24:49,406 [ hooks.py: 207]:\tstep: 21630\tsteps/sec: 1.22847\tloss: 0.83501\tlr:[3.5312012e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:24:57,931 [ hooks.py: 207]:\tstep: 21640\tsteps/sec: 1.14344\tloss: 0.30217\tlr:[3.5305224e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:25:06,351 [ hooks.py: 207]:\tstep: 21650\tsteps/sec: 1.17276\tloss: 0.59065\tlr:[3.5298435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:25:12,571 [ hooks.py: 207]:\tstep: 21660\tsteps/sec: 1.54421\tloss: 0.48661\tlr:[3.5291647e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:25:22,574 [ hooks.py: 207]:\tstep: 21670\tsteps/sec: 0.99610\tloss: 0.52038\tlr:[3.5284855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:25:31,589 [ hooks.py: 207]:\tstep: 21680\tsteps/sec: 1.16052\tloss: 0.37496\tlr:[3.5278063e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:25:39,365 [ hooks.py: 207]:\tstep: 21690\tsteps/sec: 1.30926\tloss: 0.76213\tlr:[3.5271274e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:25:46,852 [ hooks.py: 207]:\tstep: 21700\tsteps/sec: 1.30859\tloss: 0.58673\tlr:[3.5264486e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:25:54,810 [ hooks.py: 207]:\tstep: 21710\tsteps/sec: 1.23046\tloss: 1.08855\tlr:[3.5257694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:26:05,181 [ hooks.py: 207]:\tstep: 21720\tsteps/sec: 0.99679\tloss: 0.57743\tlr:[3.5250898e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:26:12,644 [ hooks.py: 207]:\tstep: 21730\tsteps/sec: 1.31701\tloss: 0.35038\tlr:[3.5244113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:26:22,708 [ hooks.py: 207]:\tstep: 21740\tsteps/sec: 0.99879\tloss: 0.66112\tlr:[3.523732e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:26:29,888 [ hooks.py: 207]:\tstep: 21750\tsteps/sec: 1.39813\tloss: 0.77235\tlr:[3.523053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:26:38,863 [ hooks.py: 207]:\tstep: 21760\tsteps/sec: 1.13201\tloss: 0.38114\tlr:[3.522374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:26:45,318 [ hooks.py: 207]:\tstep: 21770\tsteps/sec: 1.41673\tloss: 1.00791\tlr:[3.521695e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:26:53,919 [ hooks.py: 207]:\tstep: 21780\tsteps/sec: 1.25201\tloss: 0.23037\tlr:[3.521016e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:27:02,224 [ hooks.py: 207]:\tstep: 21790\tsteps/sec: 1.21991\tloss: 0.61128\tlr:[3.5203368e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:27:11,284 [ hooks.py: 207]:\tstep: 21800\tsteps/sec: 1.05361\tloss: 0.31519\tlr:[3.519658e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:27:20,641 [ hooks.py: 207]:\tstep: 21810\tsteps/sec: 1.08222\tloss: 0.50956\tlr:[3.518979e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:27:29,281 [ hooks.py: 207]:\tstep: 21820\tsteps/sec: 1.17599\tloss: 0.55554\tlr:[3.5183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:27:38,845 [ hooks.py: 207]:\tstep: 21830\tsteps/sec: 1.04921\tloss: 0.59199\tlr:[3.5176206e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:27:46,764 [ hooks.py: 207]:\tstep: 21840\tsteps/sec: 1.22499\tloss: 0.68727\tlr:[3.5169414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:27:54,996 [ hooks.py: 207]:\tstep: 21850\tsteps/sec: 1.24423\tloss: 0.48626\tlr:[3.5162626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:28:02,332 [ hooks.py: 207]:\tstep: 21860\tsteps/sec: 1.39780\tloss: 0.88451\tlr:[3.5155837e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:28:11,774 [ hooks.py: 207]:\tstep: 21870\tsteps/sec: 1.02243\tloss: 0.32481\tlr:[3.5149045e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:28:18,940 [ hooks.py: 207]:\tstep: 21880\tsteps/sec: 1.44672\tloss: 0.58725\tlr:[3.5142257e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 00:28:26,183 [ hooks.py: 207]:\tstep: 21890\tsteps/sec: 1.32636\tloss: 0.28134\tlr:[3.5135465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:28:34,211 [ hooks.py: 207]:\tstep: 21900\tsteps/sec: 1.28523\tloss: 0.87112\tlr:[3.5128673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:28:42,787 [ hooks.py: 207]:\tstep: 21910\tsteps/sec: 1.12983\tloss: 0.40650\tlr:[3.5121884e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:28:52,044 [ hooks.py: 207]:\tstep: 21920\tsteps/sec: 1.07962\tloss: 0.65672\tlr:[3.5115096e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:29:01,832 [ hooks.py: 207]:\tstep: 21930\tsteps/sec: 1.06140\tloss: 0.38458\tlr:[3.5108304e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:29:11,765 [ hooks.py: 207]:\tstep: 21940\tsteps/sec: 0.95016\tloss: 0.95967\tlr:[3.5101508e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:29:20,640 [ hooks.py: 207]:\tstep: 21950\tsteps/sec: 1.16682\tloss: 0.44766\tlr:[3.5094723e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:29:27,072 [ hooks.py: 207]:\tstep: 21960\tsteps/sec: 1.52948\tloss: 0.57792\tlr:[3.5087935e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:29:35,714 [ hooks.py: 207]:\tstep: 21970\tsteps/sec: 1.16612\tloss: 0.53362\tlr:[3.508114e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:29:44,004 [ hooks.py: 207]:\tstep: 21980\tsteps/sec: 1.20204\tloss: 0.53364\tlr:[3.507435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:29:53,650 [ hooks.py: 207]:\tstep: 21990\tsteps/sec: 1.06020\tloss: 0.59340\tlr:[3.506756e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:30:03,375 [ hooks.py: 207]:\tstep: 22000\tsteps/sec: 1.06190\tloss: 0.62311\tlr:[3.506077e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:30:03,376 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:30:03,376 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:30:03,376 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:30:03,376 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:30:03,376 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:30:03,376 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:38:05,445 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,447 [ trainer.py: 58]:\t{'acc': 0.7586345381526104, 'loss': 0.6021387143394886}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,447 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,447 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:38:05,448 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:38:05,448 [ trainer.py: 70]:\tacc\t0.7586345381526104\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:38:05,448 [ trainer.py: 70]:\tloss\t0.6021387143394886\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:38:05,448 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,449 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7586345381526104, 'loss': 0.6021387143394886}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,449 [ exporter.py: 64]:\t[Best Exporter]: skip step 22000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,449 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7586345381526104, 'loss': 0.6021387143394886}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,449 [ exporter.py: 104]:\t[Best Exporter]: skip step 22000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:05,450 [monitored_executor.py: 122]:\tsaving step 22000 to ./output/model_22000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:08,613 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_19000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:17,027 [ hooks.py: 207]:\tstep: 22010\tsteps/sec: 0.02024\tloss: 0.34685\tlr:[3.5053978e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:24,843 [ hooks.py: 207]:\tstep: 22020\tsteps/sec: 1.31133\tloss: 0.99054\tlr:[3.5047186e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:34,356 [ hooks.py: 207]:\tstep: 22030\tsteps/sec: 1.05787\tloss: 0.56103\tlr:[3.5040397e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:41,691 [ hooks.py: 207]:\tstep: 22040\tsteps/sec: 1.30710\tloss: 0.46646\tlr:[3.5033605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:49,642 [ hooks.py: 207]:\tstep: 22050\tsteps/sec: 1.30697\tloss: 0.67053\tlr:[3.5026813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:38:56,449 [ hooks.py: 207]:\tstep: 22060\tsteps/sec: 1.38640\tloss: 0.69450\tlr:[3.5020024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:04,914 [ hooks.py: 207]:\tstep: 22070\tsteps/sec: 1.22627\tloss: 0.54291\tlr:[3.5013236e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:12,052 [ hooks.py: 207]:\tstep: 22080\tsteps/sec: 1.36812\tloss: 0.69908\tlr:[3.500644e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:19,976 [ hooks.py: 207]:\tstep: 22090\tsteps/sec: 1.27713\tloss: 0.60431\tlr:[3.499965e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:27,247 [ hooks.py: 207]:\tstep: 22100\tsteps/sec: 1.29390\tloss: 0.87578\tlr:[3.499286e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:35,638 [ hooks.py: 207]:\tstep: 22110\tsteps/sec: 1.18824\tloss: 0.36463\tlr:[3.4986075e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:43,881 [ hooks.py: 207]:\tstep: 22120\tsteps/sec: 1.30645\tloss: 1.03260\tlr:[3.497928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:52,771 [ hooks.py: 207]:\tstep: 22130\tsteps/sec: 1.09668\tloss: 0.78970\tlr:[3.4972487e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:39:59,964 [ hooks.py: 207]:\tstep: 22140\tsteps/sec: 1.46008\tloss: 0.21822\tlr:[3.49657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:40:08,592 [ hooks.py: 207]:\tstep: 22150\tsteps/sec: 1.13226\tloss: 0.31947\tlr:[3.495891e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:40:16,171 [ hooks.py: 207]:\tstep: 22160\tsteps/sec: 1.29507\tloss: 0.62522\tlr:[3.4952118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:40:23,214 [ hooks.py: 207]:\tstep: 22170\tsteps/sec: 1.43046\tloss: 0.48864\tlr:[3.4945326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:40:32,314 [ hooks.py: 207]:\tstep: 22180\tsteps/sec: 1.10431\tloss: 0.60355\tlr:[3.4938537e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:40:41,200 [ hooks.py: 207]:\tstep: 22190\tsteps/sec: 1.12347\tloss: 0.55668\tlr:[3.493174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:40:48,426 [ hooks.py: 207]:\tstep: 22200\tsteps/sec: 1.33465\tloss: 0.39798\tlr:[3.4924953e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:40:57,429 [ hooks.py: 207]:\tstep: 22210\tsteps/sec: 1.13573\tloss: 0.75875\tlr:[3.491816e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:41:07,412 [ hooks.py: 207]:\tstep: 22220\tsteps/sec: 1.00685\tloss: 0.30868\tlr:[3.4911376e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:41:15,666 [ hooks.py: 207]:\tstep: 22230\tsteps/sec: 1.18114\tloss: 0.96679\tlr:[3.490458e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:41:27,007 [ hooks.py: 207]:\tstep: 22240\tsteps/sec: 0.90268\tloss: 0.67714\tlr:[3.489779e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:41:34,399 [ hooks.py: 207]:\tstep: 22250\tsteps/sec: 1.39452\tloss: 0.63266\tlr:[3.4891003e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:41:41,680 [ hooks.py: 207]:\tstep: 22260\tsteps/sec: 1.30878\tloss: 0.56010\tlr:[3.4884215e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:41:49,192 [ hooks.py: 207]:\tstep: 22270\tsteps/sec: 1.38324\tloss: 0.70360\tlr:[3.4877423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:41:57,452 [ hooks.py: 207]:\tstep: 22280\tsteps/sec: 1.17761\tloss: 0.47167\tlr:[3.487063e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:42:05,890 [ hooks.py: 207]:\tstep: 22290\tsteps/sec: 1.18840\tloss: 0.82680\tlr:[3.4863842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:42:14,267 [ hooks.py: 207]:\tstep: 22300\tsteps/sec: 1.21922\tloss: 0.61921\tlr:[3.485705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:42:22,064 [ hooks.py: 207]:\tstep: 22310\tsteps/sec: 1.26534\tloss: 0.50543\tlr:[3.4850258e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:42:31,997 [ hooks.py: 207]:\tstep: 22320\tsteps/sec: 0.99826\tloss: 0.65681\tlr:[3.4843466e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:42:40,883 [ hooks.py: 207]:\tstep: 22330\tsteps/sec: 1.12874\tloss: 0.72260\tlr:[3.4836678e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:42:47,891 [ hooks.py: 207]:\tstep: 22340\tsteps/sec: 1.42925\tloss: 0.45816\tlr:[3.4829885e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 00:42:56,068 [ hooks.py: 207]:\tstep: 22350\tsteps/sec: 1.20940\tloss: 0.59660\tlr:[3.4823093e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:43:05,398 [ hooks.py: 207]:\tstep: 22360\tsteps/sec: 1.02799\tloss: 0.33175\tlr:[3.4816305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:43:13,609 [ hooks.py: 207]:\tstep: 22370\tsteps/sec: 1.30248\tloss: 0.33381\tlr:[3.4809516e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:43:24,840 [ hooks.py: 207]:\tstep: 22380\tsteps/sec: 0.90740\tloss: 0.60474\tlr:[3.4802724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:43:31,640 [ hooks.py: 207]:\tstep: 22390\tsteps/sec: 1.39956\tloss: 0.27798\tlr:[3.4795932e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:43:41,295 [ hooks.py: 207]:\tstep: 22400\tsteps/sec: 1.04796\tloss: 0.46167\tlr:[3.4789144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:43:48,545 [ hooks.py: 207]:\tstep: 22410\tsteps/sec: 1.38812\tloss: 0.54887\tlr:[3.4782355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:43:55,408 [ hooks.py: 207]:\tstep: 22420\tsteps/sec: 1.46720\tloss: 0.37172\tlr:[3.477556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:44:04,788 [ hooks.py: 207]:\tstep: 22430\tsteps/sec: 1.03226\tloss: 0.80052\tlr:[3.476877e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:44:14,496 [ hooks.py: 207]:\tstep: 22440\tsteps/sec: 1.07646\tloss: 0.62524\tlr:[3.4761983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:44:21,713 [ hooks.py: 207]:\tstep: 22450\tsteps/sec: 1.33713\tloss: 0.68360\tlr:[3.4755194e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:44:31,020 [ hooks.py: 207]:\tstep: 22460\tsteps/sec: 1.08638\tloss: 0.99320\tlr:[3.4748406e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:44:39,272 [ hooks.py: 207]:\tstep: 22470\tsteps/sec: 1.23461\tloss: 0.60759\tlr:[3.474161e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:44:46,805 [ hooks.py: 207]:\tstep: 22480\tsteps/sec: 1.26736\tloss: 0.37814\tlr:[3.473482e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:44:56,025 [ hooks.py: 207]:\tstep: 22490\tsteps/sec: 1.09678\tloss: 0.44731\tlr:[3.472803e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:45:03,840 [ hooks.py: 207]:\tstep: 22500\tsteps/sec: 1.30270\tloss: 0.71131\tlr:[3.4721237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:45:11,620 [ hooks.py: 207]:\tstep: 22510\tsteps/sec: 1.25676\tloss: 0.78766\tlr:[3.4714452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:45:18,182 [ hooks.py: 207]:\tstep: 22520\tsteps/sec: 1.46219\tloss: 0.38883\tlr:[3.470766e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:45:26,979 [ hooks.py: 207]:\tstep: 22530\tsteps/sec: 1.13619\tloss: 0.34186\tlr:[3.4700868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:45:36,735 [ hooks.py: 207]:\tstep: 22540\tsteps/sec: 1.04345\tloss: 0.23851\tlr:[3.4694076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:45:45,458 [ hooks.py: 207]:\tstep: 22550\tsteps/sec: 1.19305\tloss: 0.53636\tlr:[3.4687288e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:45:53,939 [ hooks.py: 207]:\tstep: 22560\tsteps/sec: 1.13099\tloss: 0.82014\tlr:[3.46805e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:46:04,167 [ hooks.py: 207]:\tstep: 22570\tsteps/sec: 1.02646\tloss: 0.45759\tlr:[3.4673703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:46:11,488 [ hooks.py: 207]:\tstep: 22580\tsteps/sec: 1.25561\tloss: 0.82262\tlr:[3.466691e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:46:20,325 [ hooks.py: 207]:\tstep: 22590\tsteps/sec: 1.13661\tloss: 0.68333\tlr:[3.4660123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:46:28,851 [ hooks.py: 207]:\tstep: 22600\tsteps/sec: 1.23025\tloss: 0.83393\tlr:[3.4653334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:46:37,632 [ hooks.py: 207]:\tstep: 22610\tsteps/sec: 1.07917\tloss: 0.82587\tlr:[3.4646542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:46:45,390 [ hooks.py: 207]:\tstep: 22620\tsteps/sec: 1.29664\tloss: 0.76182\tlr:[3.463975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:46:54,900 [ hooks.py: 207]:\tstep: 22630\tsteps/sec: 1.11656\tloss: 0.71959\tlr:[3.463296e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:47:05,170 [ hooks.py: 207]:\tstep: 22640\tsteps/sec: 0.97852\tloss: 0.58658\tlr:[3.4626173e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:47:13,529 [ hooks.py: 207]:\tstep: 22650\tsteps/sec: 1.16235\tloss: 0.60906\tlr:[3.4619377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:47:21,452 [ hooks.py: 207]:\tstep: 22660\tsteps/sec: 1.20746\tloss: 0.50486\tlr:[3.4612593e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:47:32,862 [ hooks.py: 207]:\tstep: 22670\tsteps/sec: 0.90041\tloss: 0.61887\tlr:[3.4605804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:47:40,982 [ hooks.py: 207]:\tstep: 22680\tsteps/sec: 1.23244\tloss: 0.77853\tlr:[3.459901e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:47:47,985 [ hooks.py: 207]:\tstep: 22690\tsteps/sec: 1.42821\tloss: 0.82656\tlr:[3.4592216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:47:57,083 [ hooks.py: 207]:\tstep: 22700\tsteps/sec: 1.10036\tloss: 0.70139\tlr:[3.4585428e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:48:05,493 [ hooks.py: 207]:\tstep: 22710\tsteps/sec: 1.21143\tloss: 0.64307\tlr:[3.457864e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:48:13,714 [ hooks.py: 207]:\tstep: 22720\tsteps/sec: 1.20937\tloss: 0.68108\tlr:[3.457185e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:48:24,811 [ hooks.py: 207]:\tstep: 22730\tsteps/sec: 0.91485\tloss: 0.40862\tlr:[3.4565055e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:48:31,924 [ hooks.py: 207]:\tstep: 22740\tsteps/sec: 1.29764\tloss: 0.33472\tlr:[3.455827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:48:42,395 [ hooks.py: 207]:\tstep: 22750\tsteps/sec: 0.98915\tloss: 0.63557\tlr:[3.4551478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:48:51,453 [ hooks.py: 207]:\tstep: 22760\tsteps/sec: 1.10369\tloss: 0.53007\tlr:[3.4544686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:48:58,205 [ hooks.py: 207]:\tstep: 22770\tsteps/sec: 1.46947\tloss: 0.51016\tlr:[3.4537894e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:49:06,662 [ hooks.py: 207]:\tstep: 22780\tsteps/sec: 1.18046\tloss: 0.53459\tlr:[3.4531105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:49:16,024 [ hooks.py: 207]:\tstep: 22790\tsteps/sec: 1.07095\tloss: 0.38466\tlr:[3.4524313e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:49:24,192 [ hooks.py: 207]:\tstep: 22800\tsteps/sec: 1.24805\tloss: 0.93263\tlr:[3.451752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:49:34,135 [ hooks.py: 207]:\tstep: 22810\tsteps/sec: 0.97789\tloss: 0.63530\tlr:[3.451073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:49:41,585 [ hooks.py: 207]:\tstep: 22820\tsteps/sec: 1.35400\tloss: 0.47594\tlr:[3.450394e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:49:51,051 [ hooks.py: 207]:\tstep: 22830\tsteps/sec: 1.06639\tloss: 0.83618\tlr:[3.449715e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:49:57,459 [ hooks.py: 207]:\tstep: 22840\tsteps/sec: 1.45555\tloss: 0.61287\tlr:[3.4490356e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:50:06,713 [ hooks.py: 207]:\tstep: 22850\tsteps/sec: 1.13106\tloss: 1.03230\tlr:[3.4483564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:50:16,276 [ hooks.py: 207]:\tstep: 22860\tsteps/sec: 1.04669\tloss: 0.55823\tlr:[3.447678e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:50:22,797 [ hooks.py: 207]:\tstep: 22870\tsteps/sec: 1.55705\tloss: 0.59276\tlr:[3.4469987e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:50:32,226 [ hooks.py: 207]:\tstep: 22880\tsteps/sec: 1.05354\tloss: 0.73271\tlr:[3.446319e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:50:40,902 [ hooks.py: 207]:\tstep: 22890\tsteps/sec: 1.14639\tloss: 0.46605\tlr:[3.4456407e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:50:49,201 [ hooks.py: 207]:\tstep: 22900\tsteps/sec: 1.24072\tloss: 0.95249\tlr:[3.444962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:50:55,695 [ hooks.py: 207]:\tstep: 22910\tsteps/sec: 1.49782\tloss: 0.53543\tlr:[3.4442826e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:51:04,745 [ hooks.py: 207]:\tstep: 22920\tsteps/sec: 1.13672\tloss: 0.45927\tlr:[3.443603e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:51:14,952 [ hooks.py: 207]:\tstep: 22930\tsteps/sec: 0.94417\tloss: 0.54043\tlr:[3.4429246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:51:22,975 [ hooks.py: 207]:\tstep: 22940\tsteps/sec: 1.26327\tloss: 0.88196\tlr:[3.4422454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:51:31,675 [ hooks.py: 207]:\tstep: 22950\tsteps/sec: 1.14663\tloss: 0.65256\tlr:[3.441566e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:51:38,342 [ hooks.py: 207]:\tstep: 22960\tsteps/sec: 1.43199\tloss: 0.28703\tlr:[3.4408873e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:51:48,357 [ hooks.py: 207]:\tstep: 22970\tsteps/sec: 1.03226\tloss: 0.60062\tlr:[3.4402085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:51:55,437 [ hooks.py: 207]:\tstep: 22980\tsteps/sec: 1.37450\tloss: 0.53792\tlr:[3.4395292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:52:05,946 [ hooks.py: 207]:\tstep: 22990\tsteps/sec: 0.98379\tloss: 0.68242\tlr:[3.43885e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 00:52:12,291 [ hooks.py: 207]:\tstep: 23000\tsteps/sec: 1.54761\tloss: 0.53244\tlr:[3.4381712e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:52:12,292 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:52:12,292 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:52:12,292 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 00:52:12,292 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:52:12,292 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 00:52:12,293 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:00:12,567 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,569 [ trainer.py: 58]:\t{'acc': 0.7759036144578313, 'loss': 0.5746730091289068}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,569 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,570 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:00:12,570 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:00:12,570 [ trainer.py: 70]:\tacc\t0.7759036144578313\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:00:12,570 [ trainer.py: 70]:\tloss\t0.5746730091289068\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:00:12,570 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,571 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7759036144578313, 'loss': 0.5746730091289068}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,571 [ exporter.py: 64]:\t[Best Exporter]: skip step 23000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,571 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7759036144578313, 'loss': 0.5746730091289068}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,571 [ exporter.py: 104]:\t[Best Exporter]: skip step 23000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:12,572 [monitored_executor.py: 122]:\tsaving step 23000 to ./output/model_23000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:16,600 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_20000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:24,247 [ hooks.py: 207]:\tstep: 23010\tsteps/sec: 0.02033\tloss: 0.30574\tlr:[3.4374923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:30,789 [ hooks.py: 207]:\tstep: 23020\tsteps/sec: 1.51617\tloss: 0.32380\tlr:[3.436813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:39,961 [ hooks.py: 207]:\tstep: 23030\tsteps/sec: 1.05594\tloss: 1.16136\tlr:[3.436134e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:47,567 [ hooks.py: 207]:\tstep: 23040\tsteps/sec: 1.32772\tloss: 0.49861\tlr:[3.4354547e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:00:55,647 [ hooks.py: 207]:\tstep: 23050\tsteps/sec: 1.20731\tloss: 0.55744\tlr:[3.434776e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:01:07,128 [ hooks.py: 207]:\tstep: 23060\tsteps/sec: 0.88113\tloss: 0.59435\tlr:[3.4340967e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:01:15,372 [ hooks.py: 207]:\tstep: 23070\tsteps/sec: 1.21219\tloss: 0.55224\tlr:[3.4334174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:01:23,372 [ hooks.py: 207]:\tstep: 23080\tsteps/sec: 1.26866\tloss: 0.72115\tlr:[3.432739e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:01:30,968 [ hooks.py: 207]:\tstep: 23090\tsteps/sec: 1.28235\tloss: 0.32982\tlr:[3.4320594e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:01:41,101 [ hooks.py: 207]:\tstep: 23100\tsteps/sec: 1.02764\tloss: 0.90887\tlr:[3.4313805e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:01:47,964 [ hooks.py: 207]:\tstep: 23110\tsteps/sec: 1.43468\tloss: 0.93658\tlr:[3.4307013e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:01:56,384 [ hooks.py: 207]:\tstep: 23120\tsteps/sec: 1.18003\tloss: 0.53734\tlr:[3.4300225e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:02:04,065 [ hooks.py: 207]:\tstep: 23130\tsteps/sec: 1.30976\tloss: 0.48714\tlr:[3.4293433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:02:10,833 [ hooks.py: 207]:\tstep: 23140\tsteps/sec: 1.52850\tloss: 0.48178\tlr:[3.428664e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:02:17,624 [ hooks.py: 207]:\tstep: 23150\tsteps/sec: 1.40617\tloss: 0.55583\tlr:[3.4279856e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:02:27,380 [ hooks.py: 207]:\tstep: 23160\tsteps/sec: 1.03150\tloss: 0.58494\tlr:[3.4273067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:02:34,610 [ hooks.py: 207]:\tstep: 23170\tsteps/sec: 1.41299\tloss: 0.89327\tlr:[3.426627e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:02:43,504 [ hooks.py: 207]:\tstep: 23180\tsteps/sec: 1.08857\tloss: 0.26824\tlr:[3.425948e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:02:52,618 [ hooks.py: 207]:\tstep: 23190\tsteps/sec: 1.10086\tloss: 0.68816\tlr:[3.4252695e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:02,380 [ hooks.py: 207]:\tstep: 23200\tsteps/sec: 1.06599\tloss: 0.83788\tlr:[3.4245906e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:09,943 [ hooks.py: 207]:\tstep: 23210\tsteps/sec: 1.29546\tloss: 0.77448\tlr:[3.4239107e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:19,785 [ hooks.py: 207]:\tstep: 23220\tsteps/sec: 1.02722\tloss: 0.46533\tlr:[3.423232e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:28,045 [ hooks.py: 207]:\tstep: 23230\tsteps/sec: 1.18091\tloss: 0.34836\tlr:[3.422553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:34,861 [ hooks.py: 207]:\tstep: 23240\tsteps/sec: 1.52653\tloss: 0.72684\tlr:[3.421874e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:41,748 [ hooks.py: 207]:\tstep: 23250\tsteps/sec: 1.39775\tloss: 0.77181\tlr:[3.421195e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:49,884 [ hooks.py: 207]:\tstep: 23260\tsteps/sec: 1.26674\tloss: 0.45358\tlr:[3.4205157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:03:59,569 [ hooks.py: 207]:\tstep: 23270\tsteps/sec: 1.03521\tloss: 0.48842\tlr:[3.419837e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:04:07,523 [ hooks.py: 207]:\tstep: 23280\tsteps/sec: 1.16758\tloss: 0.55189\tlr:[3.4191577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:04:16,940 [ hooks.py: 207]:\tstep: 23290\tsteps/sec: 1.09362\tloss: 0.77972\tlr:[3.4184784e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:04:24,233 [ hooks.py: 207]:\tstep: 23300\tsteps/sec: 1.41856\tloss: 0.35644\tlr:[3.4177996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:04:34,746 [ hooks.py: 207]:\tstep: 23310\tsteps/sec: 0.95152\tloss: 0.61596\tlr:[3.4171208e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:04:42,583 [ hooks.py: 207]:\tstep: 23320\tsteps/sec: 1.24388\tloss: 0.70256\tlr:[3.4164415e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:04:49,990 [ hooks.py: 207]:\tstep: 23330\tsteps/sec: 1.40919\tloss: 0.78551\tlr:[3.4157623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:04:57,582 [ hooks.py: 207]:\tstep: 23340\tsteps/sec: 1.25277\tloss: 0.77841\tlr:[3.415083e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:05:04,917 [ hooks.py: 207]:\tstep: 23350\tsteps/sec: 1.42711\tloss: 0.44332\tlr:[3.4144043e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:05:13,011 [ hooks.py: 207]:\tstep: 23360\tsteps/sec: 1.20289\tloss: 0.84031\tlr:[3.4137247e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:05:19,788 [ hooks.py: 207]:\tstep: 23370\tsteps/sec: 1.53489\tloss: 0.47603\tlr:[3.413046e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:05:27,801 [ hooks.py: 207]:\tstep: 23380\tsteps/sec: 1.19532\tloss: 0.50187\tlr:[3.4123674e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:05:37,493 [ hooks.py: 207]:\tstep: 23390\tsteps/sec: 1.03957\tloss: 0.39402\tlr:[3.411688e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:05:46,094 [ hooks.py: 207]:\tstep: 23400\tsteps/sec: 1.16846\tloss: 1.10365\tlr:[3.4110086e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:05:57,312 [ hooks.py: 207]:\tstep: 23410\tsteps/sec: 0.87841\tloss: 0.54123\tlr:[3.4103297e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:06:06,890 [ hooks.py: 207]:\tstep: 23420\tsteps/sec: 1.05519\tloss: 0.59502\tlr:[3.4096513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:06:15,339 [ hooks.py: 207]:\tstep: 23430\tsteps/sec: 1.22633\tloss: 0.85565\tlr:[3.408972e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:06:22,599 [ hooks.py: 207]:\tstep: 23440\tsteps/sec: 1.32703\tloss: 0.93211\tlr:[3.4082925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:06:29,640 [ hooks.py: 207]:\tstep: 23450\tsteps/sec: 1.47592\tloss: 0.60936\tlr:[3.4076136e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 01:06:38,988 [ hooks.py: 207]:\tstep: 23460\tsteps/sec: 1.03775\tloss: 0.75504\tlr:[3.4069348e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:06:46,849 [ hooks.py: 207]:\tstep: 23470\tsteps/sec: 1.21567\tloss: 0.33682\tlr:[3.4062556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:06:55,043 [ hooks.py: 207]:\tstep: 23480\tsteps/sec: 1.32795\tloss: 0.49395\tlr:[3.4055764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:07:04,818 [ hooks.py: 207]:\tstep: 23490\tsteps/sec: 0.98733\tloss: 0.68928\tlr:[3.4048975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:07:12,331 [ hooks.py: 207]:\tstep: 23500\tsteps/sec: 1.28437\tloss: 0.58870\tlr:[3.4042187e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:07:22,838 [ hooks.py: 207]:\tstep: 23510\tsteps/sec: 0.98991\tloss: 0.53872\tlr:[3.403539e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:07:29,100 [ hooks.py: 207]:\tstep: 23520\tsteps/sec: 1.54728\tloss: 0.35524\tlr:[3.4028602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:07:39,717 [ hooks.py: 207]:\tstep: 23530\tsteps/sec: 0.97613\tloss: 0.60580\tlr:[3.402181e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:07:48,426 [ hooks.py: 207]:\tstep: 23540\tsteps/sec: 1.12488\tloss: 0.78034\tlr:[3.4015018e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:07:55,832 [ hooks.py: 207]:\tstep: 23550\tsteps/sec: 1.38242\tloss: 0.49875\tlr:[3.400823e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:08:04,961 [ hooks.py: 207]:\tstep: 23560\tsteps/sec: 1.10762\tloss: 0.38990\tlr:[3.4001438e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:08:11,952 [ hooks.py: 207]:\tstep: 23570\tsteps/sec: 1.30765\tloss: 0.82220\tlr:[3.399465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:08:21,543 [ hooks.py: 207]:\tstep: 23580\tsteps/sec: 1.09032\tloss: 0.86932\tlr:[3.3987857e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:08:29,928 [ hooks.py: 207]:\tstep: 23590\tsteps/sec: 1.21827\tloss: 0.72855\tlr:[3.398107e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:08:37,307 [ hooks.py: 207]:\tstep: 23600\tsteps/sec: 1.28831\tloss: 0.74269\tlr:[3.397428e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:08:46,447 [ hooks.py: 207]:\tstep: 23610\tsteps/sec: 1.10815\tloss: 0.68701\tlr:[3.396749e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:08:56,864 [ hooks.py: 207]:\tstep: 23620\tsteps/sec: 0.96211\tloss: 0.58145\tlr:[3.39607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:09:04,820 [ hooks.py: 207]:\tstep: 23630\tsteps/sec: 1.25239\tloss: 0.29100\tlr:[3.3953904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:09:14,216 [ hooks.py: 207]:\tstep: 23640\tsteps/sec: 1.07953\tloss: 1.16965\tlr:[3.3947115e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:09:21,201 [ hooks.py: 207]:\tstep: 23650\tsteps/sec: 1.35692\tloss: 0.95799\tlr:[3.3940327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:09:31,658 [ hooks.py: 207]:\tstep: 23660\tsteps/sec: 0.99281\tloss: 0.82863\tlr:[3.3933535e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:09:38,752 [ hooks.py: 207]:\tstep: 23670\tsteps/sec: 1.38909\tloss: 0.78599\tlr:[3.3926743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:09:47,117 [ hooks.py: 207]:\tstep: 23680\tsteps/sec: 1.19467\tloss: 0.46996\tlr:[3.3919954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:09:56,083 [ hooks.py: 207]:\tstep: 23690\tsteps/sec: 1.12972\tloss: 0.53507\tlr:[3.3913166e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:10:05,438 [ hooks.py: 207]:\tstep: 23700\tsteps/sec: 1.05222\tloss: 0.58980\tlr:[3.3906374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:10:13,472 [ hooks.py: 207]:\tstep: 23710\tsteps/sec: 1.26573\tloss: 0.31520\tlr:[3.389958e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:10:21,896 [ hooks.py: 207]:\tstep: 23720\tsteps/sec: 1.17918\tloss: 0.62095\tlr:[3.3892793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:10:32,680 [ hooks.py: 207]:\tstep: 23730\tsteps/sec: 0.92606\tloss: 0.70187\tlr:[3.3886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:10:39,865 [ hooks.py: 207]:\tstep: 23740\tsteps/sec: 1.43088\tloss: 0.38037\tlr:[3.3879212e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:10:49,811 [ hooks.py: 207]:\tstep: 23750\tsteps/sec: 1.00587\tloss: 0.84122\tlr:[3.387242e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:10:59,317 [ hooks.py: 207]:\tstep: 23760\tsteps/sec: 1.04819\tloss: 0.50882\tlr:[3.386563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:11:06,843 [ hooks.py: 207]:\tstep: 23770\tsteps/sec: 1.28984\tloss: 0.74301\tlr:[3.3858836e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:11:16,379 [ hooks.py: 207]:\tstep: 23780\tsteps/sec: 1.05177\tloss: 0.36135\tlr:[3.3852048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:11:24,286 [ hooks.py: 207]:\tstep: 23790\tsteps/sec: 1.28754\tloss: 0.41585\tlr:[3.384526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:11:30,822 [ hooks.py: 207]:\tstep: 23800\tsteps/sec: 1.45981\tloss: 1.12347\tlr:[3.3838467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:11:39,877 [ hooks.py: 207]:\tstep: 23810\tsteps/sec: 1.14710\tloss: 0.77235\tlr:[3.383167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:11:49,331 [ hooks.py: 207]:\tstep: 23820\tsteps/sec: 1.04922\tloss: 0.65511\tlr:[3.3824886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:11:58,798 [ hooks.py: 207]:\tstep: 23830\tsteps/sec: 1.04918\tloss: 0.59046\tlr:[3.3818094e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:12:07,883 [ hooks.py: 207]:\tstep: 23840\tsteps/sec: 1.05314\tloss: 0.57025\tlr:[3.3811302e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:12:15,777 [ hooks.py: 207]:\tstep: 23850\tsteps/sec: 1.28056\tloss: 0.55530\tlr:[3.380451e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:12:25,350 [ hooks.py: 207]:\tstep: 23860\tsteps/sec: 1.11593\tloss: 0.64787\tlr:[3.379772e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:12:32,161 [ hooks.py: 207]:\tstep: 23870\tsteps/sec: 1.33666\tloss: 0.86924\tlr:[3.3790933e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:12:40,100 [ hooks.py: 207]:\tstep: 23880\tsteps/sec: 1.37909\tloss: 0.61344\tlr:[3.378414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:12:46,795 [ hooks.py: 207]:\tstep: 23890\tsteps/sec: 1.35161\tloss: 0.43764\tlr:[3.3777353e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:12:54,564 [ hooks.py: 207]:\tstep: 23900\tsteps/sec: 1.37751\tloss: 0.47040\tlr:[3.377056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:13:02,495 [ hooks.py: 207]:\tstep: 23910\tsteps/sec: 1.29550\tloss: 0.47646\tlr:[3.3763772e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:13:11,032 [ hooks.py: 207]:\tstep: 23920\tsteps/sec: 1.15262\tloss: 0.37268\tlr:[3.375698e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:13:19,084 [ hooks.py: 207]:\tstep: 23930\tsteps/sec: 1.17387\tloss: 0.39230\tlr:[3.3750188e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:13:27,918 [ hooks.py: 207]:\tstep: 23940\tsteps/sec: 1.11770\tloss: 0.54208\tlr:[3.37434e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:13:38,891 [ hooks.py: 207]:\tstep: 23950\tsteps/sec: 0.94375\tloss: 0.52092\tlr:[3.3736607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:13:45,956 [ hooks.py: 207]:\tstep: 23960\tsteps/sec: 1.37121\tloss: 0.24334\tlr:[3.3729815e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:13:55,743 [ hooks.py: 207]:\tstep: 23970\tsteps/sec: 1.07423\tloss: 0.64352\tlr:[3.3723027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:04,752 [ hooks.py: 207]:\tstep: 23980\tsteps/sec: 1.09698\tloss: 0.71098\tlr:[3.371624e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:13,632 [ hooks.py: 207]:\tstep: 23990\tsteps/sec: 1.11248\tloss: 0.61057\tlr:[3.3709446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:21,966 [ hooks.py: 207]:\tstep: 24000\tsteps/sec: 1.18243\tloss: 0.48590\tlr:[3.3702654e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:21,966 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:21,966 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:21,967 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:14:21,967 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:21,967 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:14:21,967 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:22:30,219 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,220 [ trainer.py: 58]:\t{'acc': 0.7622489959839357, 'loss': 0.5803830635089141}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,221 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,222 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:22:30,222 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:22:30,222 [ trainer.py: 70]:\tacc\t0.7622489959839357\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:22:30,222 [ trainer.py: 70]:\tloss\t0.5803830635089141\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:22:30,222 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,222 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7622489959839357, 'loss': 0.5803830635089141}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,222 [ exporter.py: 64]:\t[Best Exporter]: skip step 24000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,223 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7622489959839357, 'loss': 0.5803830635089141}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,223 [ exporter.py: 104]:\t[Best Exporter]: skip step 24000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:30,224 [monitored_executor.py: 122]:\tsaving step 24000 to ./output/model_24000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 01:22:33,464 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_21000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:41,658 [ hooks.py: 207]:\tstep: 24010\tsteps/sec: 0.02000\tloss: 0.70119\tlr:[3.3695866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:49,911 [ hooks.py: 207]:\tstep: 24020\tsteps/sec: 1.28793\tloss: 0.77579\tlr:[3.3689073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:22:56,688 [ hooks.py: 207]:\tstep: 24030\tsteps/sec: 1.42935\tloss: 0.89557\tlr:[3.368228e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:23:03,429 [ hooks.py: 207]:\tstep: 24040\tsteps/sec: 1.53662\tloss: 0.59259\tlr:[3.367549e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:23:13,418 [ hooks.py: 207]:\tstep: 24050\tsteps/sec: 0.97777\tloss: 0.46549\tlr:[3.36687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:23:24,774 [ hooks.py: 207]:\tstep: 24060\tsteps/sec: 0.90041\tloss: 0.53706\tlr:[3.3661916e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:23:33,386 [ hooks.py: 207]:\tstep: 24070\tsteps/sec: 1.13095\tloss: 0.61295\tlr:[3.3655124e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:23:42,629 [ hooks.py: 207]:\tstep: 24080\tsteps/sec: 1.07457\tloss: 0.56451\tlr:[3.3648328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:23:50,744 [ hooks.py: 207]:\tstep: 24090\tsteps/sec: 1.23414\tloss: 0.41075\tlr:[3.364154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:23:58,878 [ hooks.py: 207]:\tstep: 24100\tsteps/sec: 1.25030\tloss: 0.65191\tlr:[3.3634748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:24:09,965 [ hooks.py: 207]:\tstep: 24110\tsteps/sec: 0.91117\tloss: 0.55182\tlr:[3.3627955e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:24:20,538 [ hooks.py: 207]:\tstep: 24120\tsteps/sec: 0.95682\tloss: 0.72482\tlr:[3.3621167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:24:28,668 [ hooks.py: 207]:\tstep: 24130\tsteps/sec: 1.18580\tloss: 0.58830\tlr:[3.361438e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:24:37,847 [ hooks.py: 207]:\tstep: 24140\tsteps/sec: 1.07659\tloss: 0.60997\tlr:[3.3607586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:24:45,477 [ hooks.py: 207]:\tstep: 24150\tsteps/sec: 1.31018\tloss: 0.39855\tlr:[3.3600794e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:24:54,981 [ hooks.py: 207]:\tstep: 24160\tsteps/sec: 1.08755\tloss: 0.62546\tlr:[3.3594006e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:25:03,499 [ hooks.py: 207]:\tstep: 24170\tsteps/sec: 1.19479\tloss: 0.76179\tlr:[3.3587217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:25:10,872 [ hooks.py: 207]:\tstep: 24180\tsteps/sec: 1.32915\tloss: 0.95524\tlr:[3.358043e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:25:21,238 [ hooks.py: 207]:\tstep: 24190\tsteps/sec: 0.96758\tloss: 0.72079\tlr:[3.3573637e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:25:28,677 [ hooks.py: 207]:\tstep: 24200\tsteps/sec: 1.30312\tloss: 0.49525\tlr:[3.356685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:25:38,161 [ hooks.py: 207]:\tstep: 24210\tsteps/sec: 1.05161\tloss: 0.65488\tlr:[3.3560056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:25:45,158 [ hooks.py: 207]:\tstep: 24220\tsteps/sec: 1.48569\tloss: 0.52852\tlr:[3.3553264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:25:55,788 [ hooks.py: 207]:\tstep: 24230\tsteps/sec: 0.93801\tloss: 0.49243\tlr:[3.3546472e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:26:04,722 [ hooks.py: 207]:\tstep: 24240\tsteps/sec: 1.11743\tloss: 0.52349\tlr:[3.3539683e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:26:12,023 [ hooks.py: 207]:\tstep: 24250\tsteps/sec: 1.32235\tloss: 0.90321\tlr:[3.353289e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:26:21,510 [ hooks.py: 207]:\tstep: 24260\tsteps/sec: 1.08379\tloss: 0.42554\tlr:[3.35261e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:26:30,363 [ hooks.py: 207]:\tstep: 24270\tsteps/sec: 1.13832\tloss: 0.37284\tlr:[3.3519307e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:26:38,127 [ hooks.py: 207]:\tstep: 24280\tsteps/sec: 1.19804\tloss: 0.90121\tlr:[3.3512522e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:26:48,483 [ hooks.py: 207]:\tstep: 24290\tsteps/sec: 0.98623\tloss: 0.25465\tlr:[3.350573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:26:57,077 [ hooks.py: 207]:\tstep: 24300\tsteps/sec: 1.17821\tloss: 0.45134\tlr:[3.3498934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:27:04,019 [ hooks.py: 207]:\tstep: 24310\tsteps/sec: 1.43906\tloss: 0.59684\tlr:[3.3492146e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:27:12,692 [ hooks.py: 207]:\tstep: 24320\tsteps/sec: 1.13858\tloss: 0.34537\tlr:[3.3485358e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:27:22,492 [ hooks.py: 207]:\tstep: 24330\tsteps/sec: 1.02469\tloss: 0.57567\tlr:[3.3478565e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:27:30,649 [ hooks.py: 207]:\tstep: 24340\tsteps/sec: 1.26713\tloss: 0.52602\tlr:[3.3471777e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:27:38,136 [ hooks.py: 207]:\tstep: 24350\tsteps/sec: 1.30084\tloss: 0.42733\tlr:[3.346499e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:27:45,540 [ hooks.py: 207]:\tstep: 24360\tsteps/sec: 1.34160\tloss: 0.69029\tlr:[3.3458196e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:27:54,268 [ hooks.py: 207]:\tstep: 24370\tsteps/sec: 1.15610\tloss: 0.77435\tlr:[3.3451404e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:28:01,887 [ hooks.py: 207]:\tstep: 24380\tsteps/sec: 1.42678\tloss: 0.65731\tlr:[3.3444616e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:28:08,341 [ hooks.py: 207]:\tstep: 24390\tsteps/sec: 1.40651\tloss: 0.75449\tlr:[3.3437827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:28:17,821 [ hooks.py: 207]:\tstep: 24400\tsteps/sec: 1.05201\tloss: 0.33865\tlr:[3.343103e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:28:26,115 [ hooks.py: 207]:\tstep: 24410\tsteps/sec: 1.24124\tloss: 0.64852\tlr:[3.342424e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:28:32,950 [ hooks.py: 207]:\tstep: 24420\tsteps/sec: 1.35312\tloss: 0.53096\tlr:[3.3417455e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:28:41,832 [ hooks.py: 207]:\tstep: 24430\tsteps/sec: 1.21428\tloss: 0.51173\tlr:[3.3410663e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:28:52,388 [ hooks.py: 207]:\tstep: 24440\tsteps/sec: 0.91332\tloss: 0.50811\tlr:[3.340387e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:01,946 [ hooks.py: 207]:\tstep: 24450\tsteps/sec: 1.06638\tloss: 0.34732\tlr:[3.3397082e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:09,782 [ hooks.py: 207]:\tstep: 24460\tsteps/sec: 1.29306\tloss: 0.65291\tlr:[3.339029e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:16,921 [ hooks.py: 207]:\tstep: 24470\tsteps/sec: 1.35401\tloss: 0.26420\tlr:[3.33835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:24,420 [ hooks.py: 207]:\tstep: 24480\tsteps/sec: 1.34655\tloss: 0.58767\tlr:[3.337671e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:33,074 [ hooks.py: 207]:\tstep: 24490\tsteps/sec: 1.15153\tloss: 0.43570\tlr:[3.3369917e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:39,620 [ hooks.py: 207]:\tstep: 24500\tsteps/sec: 1.49520\tloss: 0.60611\tlr:[3.3363125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:49,020 [ hooks.py: 207]:\tstep: 24510\tsteps/sec: 1.08900\tloss: 0.27603\tlr:[3.3356337e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:29:58,181 [ hooks.py: 207]:\tstep: 24520\tsteps/sec: 1.04927\tloss: 0.49789\tlr:[3.3349545e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:30:07,193 [ hooks.py: 207]:\tstep: 24530\tsteps/sec: 1.16533\tloss: 0.51029\tlr:[3.3342752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:30:14,119 [ hooks.py: 207]:\tstep: 24540\tsteps/sec: 1.43002\tloss: 1.02694\tlr:[3.3335964e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:30:22,993 [ hooks.py: 207]:\tstep: 24550\tsteps/sec: 1.11844\tloss: 0.43461\tlr:[3.3329172e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:30:32,549 [ hooks.py: 207]:\tstep: 24560\tsteps/sec: 1.03991\tloss: 0.26126\tlr:[3.332238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:30:40,010 [ hooks.py: 207]:\tstep: 24570\tsteps/sec: 1.42288\tloss: 0.79015\tlr:[3.331559e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:30:49,306 [ hooks.py: 207]:\tstep: 24580\tsteps/sec: 1.00932\tloss: 0.45796\tlr:[3.3308803e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:30:58,752 [ hooks.py: 207]:\tstep: 24590\tsteps/sec: 1.10185\tloss: 0.38197\tlr:[3.330201e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:31:06,455 [ hooks.py: 207]:\tstep: 24600\tsteps/sec: 1.32601\tloss: 0.45444\tlr:[3.329522e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:31:14,933 [ hooks.py: 207]:\tstep: 24610\tsteps/sec: 1.13316\tloss: 0.76654\tlr:[3.328843e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:31:23,199 [ hooks.py: 207]:\tstep: 24620\tsteps/sec: 1.22145\tloss: 0.49871\tlr:[3.328164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:31:33,577 [ hooks.py: 207]:\tstep: 24630\tsteps/sec: 0.96701\tloss: 0.54344\tlr:[3.327485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:31:40,776 [ hooks.py: 207]:\tstep: 24640\tsteps/sec: 1.48945\tloss: 0.47551\tlr:[3.3268057e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 01:31:48,241 [ hooks.py: 207]:\tstep: 24650\tsteps/sec: 1.25010\tloss: 0.33190\tlr:[3.326127e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:31:55,062 [ hooks.py: 207]:\tstep: 24660\tsteps/sec: 1.44703\tloss: 0.48957\tlr:[3.325448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:32:03,239 [ hooks.py: 207]:\tstep: 24670\tsteps/sec: 1.24651\tloss: 0.60293\tlr:[3.324769e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:32:13,824 [ hooks.py: 207]:\tstep: 24680\tsteps/sec: 0.94318\tloss: 0.43282\tlr:[3.3240896e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:32:21,041 [ hooks.py: 207]:\tstep: 24690\tsteps/sec: 1.42971\tloss: 0.40044\tlr:[3.3234108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:32:28,157 [ hooks.py: 207]:\tstep: 24700\tsteps/sec: 1.34560\tloss: 0.54301\tlr:[3.3227316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:32:37,289 [ hooks.py: 207]:\tstep: 24710\tsteps/sec: 1.09583\tloss: 0.46755\tlr:[3.3220524e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:32:48,271 [ hooks.py: 207]:\tstep: 24720\tsteps/sec: 0.91472\tloss: 0.47267\tlr:[3.3213735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:32:55,154 [ hooks.py: 207]:\tstep: 24730\tsteps/sec: 1.49285\tloss: 0.53344\tlr:[3.3206943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:33:06,112 [ hooks.py: 207]:\tstep: 24740\tsteps/sec: 0.91127\tloss: 0.40278\tlr:[3.320015e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:33:14,261 [ hooks.py: 207]:\tstep: 24750\tsteps/sec: 1.15921\tloss: 0.68426\tlr:[3.3193362e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:33:22,975 [ hooks.py: 207]:\tstep: 24760\tsteps/sec: 1.18586\tloss: 1.16661\tlr:[3.318657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:33:29,393 [ hooks.py: 207]:\tstep: 24770\tsteps/sec: 1.57575\tloss: 0.50528\tlr:[3.3179782e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:33:37,259 [ hooks.py: 207]:\tstep: 24780\tsteps/sec: 1.25569\tloss: 0.21749\tlr:[3.317299e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:33:49,512 [ hooks.py: 207]:\tstep: 24790\tsteps/sec: 0.82468\tloss: 0.43100\tlr:[3.3166198e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:33:56,830 [ hooks.py: 207]:\tstep: 24800\tsteps/sec: 1.37402\tloss: 0.32297\tlr:[3.315941e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:34:05,654 [ hooks.py: 207]:\tstep: 24810\tsteps/sec: 1.15105\tloss: 0.49413\tlr:[3.315262e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:34:14,486 [ hooks.py: 207]:\tstep: 24820\tsteps/sec: 1.11248\tloss: 0.62747\tlr:[3.3145832e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:34:20,937 [ hooks.py: 207]:\tstep: 24830\tsteps/sec: 1.60588\tloss: 0.62976\tlr:[3.313904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:34:27,576 [ hooks.py: 207]:\tstep: 24840\tsteps/sec: 1.41778\tloss: 0.70218\tlr:[3.3132248e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:34:36,508 [ hooks.py: 207]:\tstep: 24850\tsteps/sec: 1.16639\tloss: 0.71051\tlr:[3.3125456e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:34:43,631 [ hooks.py: 207]:\tstep: 24860\tsteps/sec: 1.35304\tloss: 0.55008\tlr:[3.3118664e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:34:51,372 [ hooks.py: 207]:\tstep: 24870\tsteps/sec: 1.34015\tloss: 0.58918\tlr:[3.3111875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:35:01,052 [ hooks.py: 207]:\tstep: 24880\tsteps/sec: 1.05372\tloss: 0.37685\tlr:[3.310509e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:35:09,556 [ hooks.py: 207]:\tstep: 24890\tsteps/sec: 1.08070\tloss: 0.47087\tlr:[3.30983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:35:18,438 [ hooks.py: 207]:\tstep: 24900\tsteps/sec: 1.15350\tloss: 0.85524\tlr:[3.3091506e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:35:28,831 [ hooks.py: 207]:\tstep: 24910\tsteps/sec: 0.96408\tloss: 0.38655\tlr:[3.3084718e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:35:37,321 [ hooks.py: 207]:\tstep: 24920\tsteps/sec: 1.18420\tloss: 0.56074\tlr:[3.307793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:35:47,166 [ hooks.py: 207]:\tstep: 24930\tsteps/sec: 1.02240\tloss: 1.03498\tlr:[3.3071137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:35:55,167 [ hooks.py: 207]:\tstep: 24940\tsteps/sec: 1.27440\tloss: 0.45786\tlr:[3.3064345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:02,536 [ hooks.py: 207]:\tstep: 24950\tsteps/sec: 1.38542\tloss: 0.31975\tlr:[3.3057553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:11,481 [ hooks.py: 207]:\tstep: 24960\tsteps/sec: 1.10573\tloss: 0.35770\tlr:[3.3050765e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:19,234 [ hooks.py: 207]:\tstep: 24970\tsteps/sec: 1.18978\tloss: 0.52545\tlr:[3.3043973e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:28,226 [ hooks.py: 207]:\tstep: 24980\tsteps/sec: 1.16373\tloss: 0.35260\tlr:[3.303718e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:37,492 [ hooks.py: 207]:\tstep: 24990\tsteps/sec: 1.02283\tloss: 0.23369\tlr:[3.3030392e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:47,101 [ hooks.py: 207]:\tstep: 25000\tsteps/sec: 1.05410\tloss: 0.28566\tlr:[3.30236e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:47,102 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:47,102 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:47,102 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:36:47,102 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:47,102 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:36:47,103 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:44:50,010 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,011 [ trainer.py: 58]:\t{'acc': 0.7819277108433735, 'loss': 0.5502896990913612}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,012 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,012 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:44:50,013 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:44:50,013 [ trainer.py: 70]:\tacc\t0.7819277108433735\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:44:50,013 [ trainer.py: 70]:\tloss\t0.5502896990913612\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:44:50,013 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,013 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7819277108433735, 'loss': 0.5502896990913612}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,013 [ exporter.py: 64]:\t[Best Exporter]: skip step 25000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,013 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7819277108433735, 'loss': 0.5502896990913612}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,013 [ exporter.py: 104]:\t[Best Exporter]: skip step 25000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:50,014 [monitored_executor.py: 122]:\tsaving step 25000 to ./output/model_25000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:44:53,141 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_22000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:45:01,749 [ hooks.py: 207]:\tstep: 25010\tsteps/sec: 0.02024\tloss: 0.34160\tlr:[3.3016808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:45:12,837 [ hooks.py: 207]:\tstep: 25020\tsteps/sec: 0.88776\tloss: 0.63466\tlr:[3.301002e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:45:21,008 [ hooks.py: 207]:\tstep: 25030\tsteps/sec: 1.26351\tloss: 0.56526\tlr:[3.300323e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:45:29,625 [ hooks.py: 207]:\tstep: 25040\tsteps/sec: 1.12748\tloss: 0.26646\tlr:[3.299644e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:45:36,568 [ hooks.py: 207]:\tstep: 25050\tsteps/sec: 1.49357\tloss: 0.94711\tlr:[3.2989647e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:45:43,835 [ hooks.py: 207]:\tstep: 25060\tsteps/sec: 1.33669\tloss: 0.41647\tlr:[3.2982858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:45:52,962 [ hooks.py: 207]:\tstep: 25070\tsteps/sec: 1.08598\tloss: 0.59476\tlr:[3.297607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:46:01,561 [ hooks.py: 207]:\tstep: 25080\tsteps/sec: 1.25706\tloss: 0.47127\tlr:[3.2969278e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:46:08,924 [ hooks.py: 207]:\tstep: 25090\tsteps/sec: 1.19129\tloss: 0.61633\tlr:[3.2962485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:46:18,598 [ hooks.py: 207]:\tstep: 25100\tsteps/sec: 1.05061\tloss: 0.27473\tlr:[3.2955697e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 01:46:27,177 [ hooks.py: 207]:\tstep: 25110\tsteps/sec: 1.20204\tloss: 0.94583\tlr:[3.294891e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:46:36,653 [ hooks.py: 207]:\tstep: 25120\tsteps/sec: 1.07952\tloss: 0.45268\tlr:[3.2942113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:46:43,245 [ hooks.py: 207]:\tstep: 25130\tsteps/sec: 1.40011\tloss: 0.48377\tlr:[3.2935324e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:46:51,562 [ hooks.py: 207]:\tstep: 25140\tsteps/sec: 1.31970\tloss: 0.60189\tlr:[3.2928536e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:46:58,477 [ hooks.py: 207]:\tstep: 25150\tsteps/sec: 1.32271\tloss: 0.16763\tlr:[3.2921744e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:47:06,775 [ hooks.py: 207]:\tstep: 25160\tsteps/sec: 1.27766\tloss: 0.35198\tlr:[3.2914948e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:47:13,982 [ hooks.py: 207]:\tstep: 25170\tsteps/sec: 1.34135\tloss: 0.46510\tlr:[3.2908163e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:47:23,493 [ hooks.py: 207]:\tstep: 25180\tsteps/sec: 1.04697\tloss: 0.23400\tlr:[3.290137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:47:31,929 [ hooks.py: 207]:\tstep: 25190\tsteps/sec: 1.21889\tloss: 0.28900\tlr:[3.289458e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:47:43,524 [ hooks.py: 207]:\tstep: 25200\tsteps/sec: 0.84734\tloss: 0.60028\tlr:[3.2887787e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:47:53,099 [ hooks.py: 207]:\tstep: 25210\tsteps/sec: 1.03942\tloss: 0.54476\tlr:[3.2881e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:48:00,829 [ hooks.py: 207]:\tstep: 25220\tsteps/sec: 1.38960\tloss: 0.53790\tlr:[3.287421e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:48:08,764 [ hooks.py: 207]:\tstep: 25230\tsteps/sec: 1.14704\tloss: 0.64446\tlr:[3.2867414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:48:16,970 [ hooks.py: 207]:\tstep: 25240\tsteps/sec: 1.23767\tloss: 0.61705\tlr:[3.2860626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:48:26,255 [ hooks.py: 207]:\tstep: 25250\tsteps/sec: 1.08824\tloss: 0.36238\tlr:[3.2853837e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:48:34,647 [ hooks.py: 207]:\tstep: 25260\tsteps/sec: 1.20861\tloss: 0.20898\tlr:[3.284705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:48:45,056 [ hooks.py: 207]:\tstep: 25270\tsteps/sec: 0.94396\tloss: 0.44175\tlr:[3.2840253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:48:55,086 [ hooks.py: 207]:\tstep: 25280\tsteps/sec: 1.02659\tloss: 0.50507\tlr:[3.2833465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:49:04,565 [ hooks.py: 207]:\tstep: 25290\tsteps/sec: 1.04395\tloss: 0.29242\tlr:[3.2826676e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:49:12,674 [ hooks.py: 207]:\tstep: 25300\tsteps/sec: 1.23538\tloss: 0.60432\tlr:[3.2819884e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:49:21,215 [ hooks.py: 207]:\tstep: 25310\tsteps/sec: 1.21297\tloss: 0.69551\tlr:[3.2813092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:49:28,194 [ hooks.py: 207]:\tstep: 25320\tsteps/sec: 1.35351\tloss: 0.35624\tlr:[3.2806303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:49:35,363 [ hooks.py: 207]:\tstep: 25330\tsteps/sec: 1.44093\tloss: 1.12582\tlr:[3.279951e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:49:43,970 [ hooks.py: 207]:\tstep: 25340\tsteps/sec: 1.14190\tloss: 0.40779\tlr:[3.2792723e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:49:53,911 [ hooks.py: 207]:\tstep: 25350\tsteps/sec: 1.00662\tloss: 0.43083\tlr:[3.278593e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:50:03,149 [ hooks.py: 207]:\tstep: 25360\tsteps/sec: 1.13971\tloss: 1.00342\tlr:[3.2779142e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:50:09,135 [ hooks.py: 207]:\tstep: 25370\tsteps/sec: 1.46118\tloss: 0.47555\tlr:[3.2772354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:50:18,556 [ hooks.py: 207]:\tstep: 25380\tsteps/sec: 1.06544\tloss: 0.49937\tlr:[3.2765558e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:50:26,823 [ hooks.py: 207]:\tstep: 25390\tsteps/sec: 1.23458\tloss: 0.56480\tlr:[3.2758766e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:50:35,448 [ hooks.py: 207]:\tstep: 25400\tsteps/sec: 1.21857\tloss: 0.65559\tlr:[3.2751977e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:50:43,155 [ hooks.py: 207]:\tstep: 25410\tsteps/sec: 1.20153\tloss: 1.02196\tlr:[3.274519e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:50:51,309 [ hooks.py: 207]:\tstep: 25420\tsteps/sec: 1.29689\tloss: 0.43379\tlr:[3.2738397e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:00,477 [ hooks.py: 207]:\tstep: 25430\tsteps/sec: 1.10865\tloss: 0.67903\tlr:[3.273161e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:09,196 [ hooks.py: 207]:\tstep: 25440\tsteps/sec: 1.12247\tloss: 0.66572\tlr:[3.2724816e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:18,398 [ hooks.py: 207]:\tstep: 25450\tsteps/sec: 1.08296\tloss: 0.61122\tlr:[3.2718024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:25,917 [ hooks.py: 207]:\tstep: 25460\tsteps/sec: 1.37482\tloss: 0.49258\tlr:[3.2711232e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:35,002 [ hooks.py: 207]:\tstep: 25470\tsteps/sec: 1.07699\tloss: 0.38676\tlr:[3.2704444e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:43,042 [ hooks.py: 207]:\tstep: 25480\tsteps/sec: 1.23036\tloss: 0.74958\tlr:[3.269765e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:51,148 [ hooks.py: 207]:\tstep: 25490\tsteps/sec: 1.24637\tloss: 0.51586\tlr:[3.269086e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:51:59,734 [ hooks.py: 207]:\tstep: 25500\tsteps/sec: 1.13050\tloss: 0.30389\tlr:[3.2684067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:52:08,192 [ hooks.py: 207]:\tstep: 25510\tsteps/sec: 1.16908\tloss: 0.78160\tlr:[3.267728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:52:17,647 [ hooks.py: 207]:\tstep: 25520\tsteps/sec: 1.08946\tloss: 0.43087\tlr:[3.2670494e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:52:26,160 [ hooks.py: 207]:\tstep: 25530\tsteps/sec: 1.23278\tloss: 0.68602\tlr:[3.26637e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:52:33,861 [ hooks.py: 207]:\tstep: 25540\tsteps/sec: 1.23841\tloss: 0.33286\tlr:[3.265691e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:52:42,589 [ hooks.py: 207]:\tstep: 25550\tsteps/sec: 1.17032\tloss: 0.70501\tlr:[3.2650118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:52:50,753 [ hooks.py: 207]:\tstep: 25560\tsteps/sec: 1.19572\tloss: 0.53175\tlr:[3.264333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:02,679 [ hooks.py: 207]:\tstep: 25570\tsteps/sec: 0.88220\tloss: 0.27998\tlr:[3.2636533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:09,801 [ hooks.py: 207]:\tstep: 25580\tsteps/sec: 1.29810\tloss: 0.46079\tlr:[3.2629745e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:17,961 [ hooks.py: 207]:\tstep: 25590\tsteps/sec: 1.22396\tloss: 0.42557\tlr:[3.2622956e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:27,083 [ hooks.py: 207]:\tstep: 25600\tsteps/sec: 1.12458\tloss: 0.56362\tlr:[3.2616164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:33,315 [ hooks.py: 207]:\tstep: 25610\tsteps/sec: 1.47758\tloss: 0.45598\tlr:[3.2609372e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:41,611 [ hooks.py: 207]:\tstep: 25620\tsteps/sec: 1.34716\tloss: 0.70615\tlr:[3.2602584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:49,304 [ hooks.py: 207]:\tstep: 25630\tsteps/sec: 1.21961\tloss: 0.54587\tlr:[3.2595795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:53:58,775 [ hooks.py: 207]:\tstep: 25640\tsteps/sec: 1.05366\tloss: 0.45655\tlr:[3.2589003e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:54:07,762 [ hooks.py: 207]:\tstep: 25650\tsteps/sec: 1.11963\tloss: 0.52073\tlr:[3.258221e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:54:15,057 [ hooks.py: 207]:\tstep: 25660\tsteps/sec: 1.37412\tloss: 0.30315\tlr:[3.2575423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:54:23,384 [ hooks.py: 207]:\tstep: 25670\tsteps/sec: 1.18737\tloss: 0.85565\tlr:[3.2568634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:54:32,239 [ hooks.py: 207]:\tstep: 25680\tsteps/sec: 1.16482\tloss: 0.69329\tlr:[3.2561842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:54:40,776 [ hooks.py: 207]:\tstep: 25690\tsteps/sec: 1.14514\tloss: 0.67505\tlr:[3.255505e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:54:51,843 [ hooks.py: 207]:\tstep: 25700\tsteps/sec: 0.91709\tloss: 0.99017\tlr:[3.254826e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:55:02,355 [ hooks.py: 207]:\tstep: 25710\tsteps/sec: 0.99572\tloss: 0.27863\tlr:[3.254147e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:55:10,759 [ hooks.py: 207]:\tstep: 25720\tsteps/sec: 1.14211\tloss: 0.41507\tlr:[3.2534677e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:55:17,134 [ hooks.py: 207]:\tstep: 25730\tsteps/sec: 1.44325\tloss: 0.54814\tlr:[3.2527885e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:55:25,162 [ hooks.py: 207]:\tstep: 25740\tsteps/sec: 1.29240\tloss: 0.60942\tlr:[3.2521097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:55:34,936 [ hooks.py: 207]:\tstep: 25750\tsteps/sec: 1.00885\tloss: 0.66227\tlr:[3.2514305e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 01:55:43,166 [ hooks.py: 207]:\tstep: 25760\tsteps/sec: 1.22747\tloss: 0.46362\tlr:[3.2507513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:55:49,677 [ hooks.py: 207]:\tstep: 25770\tsteps/sec: 1.54836\tloss: 0.62752\tlr:[3.2500724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:55:57,563 [ hooks.py: 207]:\tstep: 25780\tsteps/sec: 1.20669\tloss: 0.65737\tlr:[3.2493936e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:56:07,844 [ hooks.py: 207]:\tstep: 25790\tsteps/sec: 1.01260\tloss: 0.81890\tlr:[3.248714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:56:14,451 [ hooks.py: 207]:\tstep: 25800\tsteps/sec: 1.49799\tloss: 0.71972\tlr:[3.248035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:56:23,321 [ hooks.py: 207]:\tstep: 25810\tsteps/sec: 1.12073\tloss: 0.68494\tlr:[3.2473563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:56:31,954 [ hooks.py: 207]:\tstep: 25820\tsteps/sec: 1.19663\tloss: 0.64856\tlr:[3.2466774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:56:39,270 [ hooks.py: 207]:\tstep: 25830\tsteps/sec: 1.33177\tloss: 0.86378\tlr:[3.2459982e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:56:47,404 [ hooks.py: 207]:\tstep: 25840\tsteps/sec: 1.16955\tloss: 0.57929\tlr:[3.245319e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:56:56,368 [ hooks.py: 207]:\tstep: 25850\tsteps/sec: 1.21621\tloss: 0.66619\tlr:[3.2446405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:57:04,443 [ hooks.py: 207]:\tstep: 25860\tsteps/sec: 1.14533\tloss: 0.22672\tlr:[3.243961e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:57:14,207 [ hooks.py: 207]:\tstep: 25870\tsteps/sec: 1.06231\tloss: 0.67027\tlr:[3.2432818e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:57:20,928 [ hooks.py: 207]:\tstep: 25880\tsteps/sec: 1.47309\tloss: 0.44438\tlr:[3.2426033e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:57:33,346 [ hooks.py: 207]:\tstep: 25890\tsteps/sec: 0.80138\tloss: 0.52032\tlr:[3.241924e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:57:41,507 [ hooks.py: 207]:\tstep: 25900\tsteps/sec: 1.30151\tloss: 0.73697\tlr:[3.241245e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:57:48,363 [ hooks.py: 207]:\tstep: 25910\tsteps/sec: 1.32163\tloss: 0.44426\tlr:[3.2405656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:57:57,487 [ hooks.py: 207]:\tstep: 25920\tsteps/sec: 1.15240\tloss: 0.28915\tlr:[3.2398868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:58:05,091 [ hooks.py: 207]:\tstep: 25930\tsteps/sec: 1.24824\tloss: 0.26464\tlr:[3.239208e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:58:15,105 [ hooks.py: 207]:\tstep: 25940\tsteps/sec: 0.99286\tloss: 0.62891\tlr:[3.2385287e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:58:24,325 [ hooks.py: 207]:\tstep: 25950\tsteps/sec: 1.12656\tloss: 0.57061\tlr:[3.2378495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:58:30,859 [ hooks.py: 207]:\tstep: 25960\tsteps/sec: 1.58708\tloss: 0.82533\tlr:[3.2371707e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:58:40,261 [ hooks.py: 207]:\tstep: 25970\tsteps/sec: 1.01674\tloss: 0.76937\tlr:[3.2364915e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:58:49,204 [ hooks.py: 207]:\tstep: 25980\tsteps/sec: 1.14604\tloss: 0.61815\tlr:[3.2358123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:58:57,642 [ hooks.py: 207]:\tstep: 25990\tsteps/sec: 1.18116\tloss: 0.93114\tlr:[3.2351334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:59:08,805 [ hooks.py: 207]:\tstep: 26000\tsteps/sec: 0.89655\tloss: 0.48431\tlr:[3.2344546e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:59:08,806 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:59:08,806 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:59:08,807 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 01:59:08,807 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:59:08,807 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 01:59:08,807 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:07:13,341 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,342 [ trainer.py: 58]:\t{'acc': 0.7602409638554217, 'loss': 0.5758169206480185}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,343 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,343 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:07:13,343 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:07:13,344 [ trainer.py: 70]:\tacc\t0.7602409638554217\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:07:13,344 [ trainer.py: 70]:\tloss\t0.5758169206480185\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:07:13,344 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,344 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7602409638554217, 'loss': 0.5758169206480185}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,344 [ exporter.py: 64]:\t[Best Exporter]: skip step 26000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,344 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7602409638554217, 'loss': 0.5758169206480185}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,344 [ exporter.py: 104]:\t[Best Exporter]: skip step 26000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:13,345 [monitored_executor.py: 122]:\tsaving step 26000 to ./output/model_26000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:16,701 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_23000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:24,293 [ hooks.py: 207]:\tstep: 26010\tsteps/sec: 0.02018\tloss: 0.79764\tlr:[3.2337754e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:33,478 [ hooks.py: 207]:\tstep: 26020\tsteps/sec: 1.04841\tloss: 0.53082\tlr:[3.233096e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:43,048 [ hooks.py: 207]:\tstep: 26030\tsteps/sec: 1.08079\tloss: 0.26725\tlr:[3.2324173e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:07:51,901 [ hooks.py: 207]:\tstep: 26040\tsteps/sec: 1.12746\tloss: 0.68664\tlr:[3.2317384e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:00,115 [ hooks.py: 207]:\tstep: 26050\tsteps/sec: 1.23992\tloss: 0.73801\tlr:[3.2310592e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:10,785 [ hooks.py: 207]:\tstep: 26060\tsteps/sec: 0.94754\tloss: 0.81593\tlr:[3.23038e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:19,737 [ hooks.py: 207]:\tstep: 26070\tsteps/sec: 1.09750\tloss: 0.49632\tlr:[3.229701e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:27,697 [ hooks.py: 207]:\tstep: 26080\tsteps/sec: 1.24428\tloss: 0.81137\tlr:[3.2290223e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:35,973 [ hooks.py: 207]:\tstep: 26090\tsteps/sec: 1.19602\tloss: 0.75491\tlr:[3.228343e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:42,954 [ hooks.py: 207]:\tstep: 26100\tsteps/sec: 1.37414\tloss: 0.57066\tlr:[3.227664e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:51,888 [ hooks.py: 207]:\tstep: 26110\tsteps/sec: 1.22148\tloss: 0.85552\tlr:[3.226985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:08:58,646 [ hooks.py: 207]:\tstep: 26120\tsteps/sec: 1.41378\tloss: 0.30015\tlr:[3.2263062e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:09:04,761 [ hooks.py: 207]:\tstep: 26130\tsteps/sec: 1.53417\tloss: 0.63745\tlr:[3.225627e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:09:15,644 [ hooks.py: 207]:\tstep: 26140\tsteps/sec: 0.95864\tloss: 0.41535\tlr:[3.2249478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:09:24,925 [ hooks.py: 207]:\tstep: 26150\tsteps/sec: 1.02988\tloss: 0.39002\tlr:[3.224269e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:09:33,449 [ hooks.py: 207]:\tstep: 26160\tsteps/sec: 1.21712\tloss: 0.37068\tlr:[3.2235897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:09:41,986 [ hooks.py: 207]:\tstep: 26170\tsteps/sec: 1.23678\tloss: 0.42453\tlr:[3.2229105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:09:49,366 [ hooks.py: 207]:\tstep: 26180\tsteps/sec: 1.23135\tloss: 0.46404\tlr:[3.2222313e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:09:59,528 [ hooks.py: 207]:\tstep: 26190\tsteps/sec: 0.99584\tloss: 0.51753\tlr:[3.2215525e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:10:10,203 [ hooks.py: 207]:\tstep: 26200\tsteps/sec: 0.96276\tloss: 0.49466\tlr:[3.2208733e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:10:18,250 [ hooks.py: 207]:\tstep: 26210\tsteps/sec: 1.22300\tloss: 0.74672\tlr:[3.220194e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 02:10:27,353 [ hooks.py: 207]:\tstep: 26220\tsteps/sec: 1.12441\tloss: 0.44321\tlr:[3.2195152e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:10:35,469 [ hooks.py: 207]:\tstep: 26230\tsteps/sec: 1.17386\tloss: 0.95622\tlr:[3.2188364e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:10:44,991 [ hooks.py: 207]:\tstep: 26240\tsteps/sec: 1.08097\tloss: 0.86377\tlr:[3.218157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:10:56,119 [ hooks.py: 207]:\tstep: 26250\tsteps/sec: 0.91152\tloss: 0.55302\tlr:[3.2174776e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:11:06,131 [ hooks.py: 207]:\tstep: 26260\tsteps/sec: 1.00467\tloss: 0.64060\tlr:[3.2167987e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:11:14,015 [ hooks.py: 207]:\tstep: 26270\tsteps/sec: 1.23078\tloss: 0.50213\tlr:[3.2161202e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:11:21,195 [ hooks.py: 207]:\tstep: 26280\tsteps/sec: 1.46506\tloss: 0.47918\tlr:[3.215441e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:11:29,641 [ hooks.py: 207]:\tstep: 26290\tsteps/sec: 1.14201\tloss: 0.38244\tlr:[3.2147615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:11:37,756 [ hooks.py: 207]:\tstep: 26300\tsteps/sec: 1.18045\tloss: 0.45319\tlr:[3.2140826e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:11:47,470 [ hooks.py: 207]:\tstep: 26310\tsteps/sec: 1.08145\tloss: 0.41004\tlr:[3.2134034e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:11:57,552 [ hooks.py: 207]:\tstep: 26320\tsteps/sec: 0.99242\tloss: 1.15539\tlr:[3.2127242e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:12:04,797 [ hooks.py: 207]:\tstep: 26330\tsteps/sec: 1.35387\tloss: 0.27132\tlr:[3.2120457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:12:12,529 [ hooks.py: 207]:\tstep: 26340\tsteps/sec: 1.33484\tloss: 0.43689\tlr:[3.2113665e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:12:20,185 [ hooks.py: 207]:\tstep: 26350\tsteps/sec: 1.27257\tloss: 0.75883\tlr:[3.2106876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:12:27,419 [ hooks.py: 207]:\tstep: 26360\tsteps/sec: 1.39999\tloss: 0.50509\tlr:[3.210008e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:12:36,219 [ hooks.py: 207]:\tstep: 26370\tsteps/sec: 1.13914\tloss: 0.47641\tlr:[3.2093292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:12:45,271 [ hooks.py: 207]:\tstep: 26380\tsteps/sec: 1.08923\tloss: 0.21939\tlr:[3.2086504e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:12:54,588 [ hooks.py: 207]:\tstep: 26390\tsteps/sec: 1.07545\tloss: 0.65974\tlr:[3.207971e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:13:03,095 [ hooks.py: 207]:\tstep: 26400\tsteps/sec: 1.25155\tloss: 0.45666\tlr:[3.2072923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:13:13,473 [ hooks.py: 207]:\tstep: 26410\tsteps/sec: 0.91727\tloss: 0.69612\tlr:[3.206613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:13:21,891 [ hooks.py: 207]:\tstep: 26420\tsteps/sec: 1.22457\tloss: 1.07701\tlr:[3.2059343e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:13:31,309 [ hooks.py: 207]:\tstep: 26430\tsteps/sec: 1.05976\tloss: 0.59058\tlr:[3.205255e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:13:39,860 [ hooks.py: 207]:\tstep: 26440\tsteps/sec: 1.13940\tloss: 0.76544\tlr:[3.204576e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:13:49,889 [ hooks.py: 207]:\tstep: 26450\tsteps/sec: 0.99653\tloss: 0.41863\tlr:[3.203897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:13:56,780 [ hooks.py: 207]:\tstep: 26460\tsteps/sec: 1.49446\tloss: 0.85930\tlr:[3.2032178e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:14:08,172 [ hooks.py: 207]:\tstep: 26470\tsteps/sec: 0.85823\tloss: 0.72335\tlr:[3.2025386e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:14:15,569 [ hooks.py: 207]:\tstep: 26480\tsteps/sec: 1.36185\tloss: 0.60259\tlr:[3.2018594e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:14:22,001 [ hooks.py: 207]:\tstep: 26490\tsteps/sec: 1.53164\tloss: 0.32044\tlr:[3.2011805e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:14:30,182 [ hooks.py: 207]:\tstep: 26500\tsteps/sec: 1.23319\tloss: 0.61986\tlr:[3.2005013e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:14:39,915 [ hooks.py: 207]:\tstep: 26510\tsteps/sec: 1.03106\tloss: 0.40362\tlr:[3.199822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:14:50,410 [ hooks.py: 207]:\tstep: 26520\tsteps/sec: 0.95294\tloss: 0.56625\tlr:[3.1991432e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:14:57,806 [ hooks.py: 207]:\tstep: 26530\tsteps/sec: 1.35758\tloss: 0.92877\tlr:[3.1984644e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:15:08,601 [ hooks.py: 207]:\tstep: 26540\tsteps/sec: 0.91927\tloss: 0.94640\tlr:[3.1977856e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:15:17,957 [ hooks.py: 207]:\tstep: 26550\tsteps/sec: 1.07827\tloss: 0.57322\tlr:[3.197106e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:15:25,225 [ hooks.py: 207]:\tstep: 26560\tsteps/sec: 1.37047\tloss: 1.05069\tlr:[3.196427e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:15:34,755 [ hooks.py: 207]:\tstep: 26570\tsteps/sec: 1.01238\tloss: 0.34340\tlr:[3.1957483e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:15:42,825 [ hooks.py: 207]:\tstep: 26580\tsteps/sec: 1.31708\tloss: 0.47945\tlr:[3.195069e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:15:51,981 [ hooks.py: 207]:\tstep: 26590\tsteps/sec: 1.10371\tloss: 0.39142\tlr:[3.19439e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:16:04,083 [ hooks.py: 207]:\tstep: 26600\tsteps/sec: 0.79676\tloss: 0.70974\tlr:[3.1937114e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:16:13,112 [ hooks.py: 207]:\tstep: 26610\tsteps/sec: 1.12518\tloss: 0.38368\tlr:[3.193032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:16:22,059 [ hooks.py: 207]:\tstep: 26620\tsteps/sec: 1.14748\tloss: 0.30333\tlr:[3.1923526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:16:30,450 [ hooks.py: 207]:\tstep: 26630\tsteps/sec: 1.18199\tloss: 0.90498\tlr:[3.1916737e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:16:38,080 [ hooks.py: 207]:\tstep: 26640\tsteps/sec: 1.28833\tloss: 0.89036\tlr:[3.190995e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:16:45,015 [ hooks.py: 207]:\tstep: 26650\tsteps/sec: 1.44552\tloss: 0.45799\tlr:[3.190316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:16:54,110 [ hooks.py: 207]:\tstep: 26660\tsteps/sec: 1.09975\tloss: 0.40625\tlr:[3.1896365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:17:03,105 [ hooks.py: 207]:\tstep: 26670\tsteps/sec: 1.18960\tloss: 0.81241\tlr:[3.1889576e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:17:13,039 [ hooks.py: 207]:\tstep: 26680\tsteps/sec: 0.94679\tloss: 0.65347\tlr:[3.1882784e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:17:21,837 [ hooks.py: 207]:\tstep: 26690\tsteps/sec: 1.17932\tloss: 0.18489\tlr:[3.1875996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:17:31,375 [ hooks.py: 207]:\tstep: 26700\tsteps/sec: 1.04766\tloss: 0.89973\tlr:[3.1869204e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:17:38,720 [ hooks.py: 207]:\tstep: 26710\tsteps/sec: 1.30105\tloss: 0.40166\tlr:[3.186241e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:17:45,913 [ hooks.py: 207]:\tstep: 26720\tsteps/sec: 1.41153\tloss: 0.32420\tlr:[3.1855623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:17:54,732 [ hooks.py: 207]:\tstep: 26730\tsteps/sec: 1.13607\tloss: 0.64654\tlr:[3.184883e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:18:03,299 [ hooks.py: 207]:\tstep: 26740\tsteps/sec: 1.22242\tloss: 0.35215\tlr:[3.184204e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:18:13,833 [ hooks.py: 207]:\tstep: 26750\tsteps/sec: 0.88087\tloss: 0.67182\tlr:[3.183525e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:18:22,924 [ hooks.py: 207]:\tstep: 26760\tsteps/sec: 1.10569\tloss: 0.38123\tlr:[3.182846e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:18:31,375 [ hooks.py: 207]:\tstep: 26770\tsteps/sec: 1.27752\tloss: 0.62140\tlr:[3.1821666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:18:40,366 [ hooks.py: 207]:\tstep: 26780\tsteps/sec: 1.08100\tloss: 0.31371\tlr:[3.1814878e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:18:47,621 [ hooks.py: 207]:\tstep: 26790\tsteps/sec: 1.40324\tloss: 0.51652\tlr:[3.180809e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:18:58,246 [ hooks.py: 207]:\tstep: 26800\tsteps/sec: 0.92357\tloss: 0.55239\tlr:[3.1801297e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:19:07,792 [ hooks.py: 207]:\tstep: 26810\tsteps/sec: 1.07515\tloss: 0.35803\tlr:[3.1794505e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:19:15,115 [ hooks.py: 207]:\tstep: 26820\tsteps/sec: 1.33713\tloss: 0.42344\tlr:[3.1787717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:19:23,985 [ hooks.py: 207]:\tstep: 26830\tsteps/sec: 1.11420\tloss: 0.34035\tlr:[3.1780928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:19:32,501 [ hooks.py: 207]:\tstep: 26840\tsteps/sec: 1.21884\tloss: 0.95569\tlr:[3.1774136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:19:40,094 [ hooks.py: 207]:\tstep: 26850\tsteps/sec: 1.26944\tloss: 0.39037\tlr:[3.1767344e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:19:47,761 [ hooks.py: 207]:\tstep: 26860\tsteps/sec: 1.34387\tloss: 0.60307\tlr:[3.1760555e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 02:19:54,261 [ hooks.py: 207]:\tstep: 26870\tsteps/sec: 1.42483\tloss: 0.31716\tlr:[3.1753767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:20:03,801 [ hooks.py: 207]:\tstep: 26880\tsteps/sec: 1.07880\tloss: 0.08361\tlr:[3.1746975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:20:12,822 [ hooks.py: 207]:\tstep: 26890\tsteps/sec: 1.07592\tloss: 0.80588\tlr:[3.1740183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:20:24,285 [ hooks.py: 207]:\tstep: 26900\tsteps/sec: 0.89874\tloss: 0.32268\tlr:[3.1733394e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:20:31,514 [ hooks.py: 207]:\tstep: 26910\tsteps/sec: 1.43434\tloss: 0.63437\tlr:[3.1726602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:20:38,705 [ hooks.py: 207]:\tstep: 26920\tsteps/sec: 1.33881\tloss: 0.34671\tlr:[3.1719814e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:20:47,068 [ hooks.py: 207]:\tstep: 26930\tsteps/sec: 1.17823\tloss: 0.32666\tlr:[3.171302e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:20:57,564 [ hooks.py: 207]:\tstep: 26940\tsteps/sec: 0.97270\tloss: 0.80124\tlr:[3.170623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:06,306 [ hooks.py: 207]:\tstep: 26950\tsteps/sec: 1.11511\tloss: 0.65783\tlr:[3.1699437e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:16,093 [ hooks.py: 207]:\tstep: 26960\tsteps/sec: 1.05507\tloss: 0.74915\tlr:[3.169265e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:24,312 [ hooks.py: 207]:\tstep: 26970\tsteps/sec: 1.17767\tloss: 0.83729\tlr:[3.168586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:33,228 [ hooks.py: 207]:\tstep: 26980\tsteps/sec: 1.12834\tloss: 0.58304\tlr:[3.167907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:42,732 [ hooks.py: 207]:\tstep: 26990\tsteps/sec: 1.08065\tloss: 0.35786\tlr:[3.1672276e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:50,123 [ hooks.py: 207]:\tstep: 27000\tsteps/sec: 1.33144\tloss: 0.68673\tlr:[3.1665488e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:50,124 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:50,124 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:50,124 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:21:50,124 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:50,125 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:21:50,125 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:29:53,023 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,024 [ trainer.py: 58]:\t{'acc': 0.7895582329317269, 'loss': 0.531344682455827}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,025 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,025 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:29:53,025 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:29:53,025 [ trainer.py: 70]:\tacc\t0.7895582329317269\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:29:53,026 [ trainer.py: 70]:\tloss\t0.531344682455827\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:29:53,026 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,026 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,026 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,027 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_18000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,027 [monitored_executor.py: 122]:\tsaving step 27000 to ./output/best_model/model_27000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,992 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_18000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,992 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}} \n", + "old: {'eval': {'acc': 0.7855421686746988, 'loss': 0.5396390796089784}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,993 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:53,993 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:55,250 [monitored_executor.py: 122]:\tsaving step 27000 to ./output/model_27000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:29:58,394 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_24000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:30:07,378 [ hooks.py: 207]:\tstep: 27010\tsteps/sec: 0.02012\tloss: 0.34173\tlr:[3.16587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:30:17,368 [ hooks.py: 207]:\tstep: 27020\tsteps/sec: 0.99382\tloss: 0.76980\tlr:[3.165191e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:30:26,603 [ hooks.py: 207]:\tstep: 27030\tsteps/sec: 1.07758\tloss: 0.39129\tlr:[3.164512e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:30:35,640 [ hooks.py: 207]:\tstep: 27040\tsteps/sec: 1.09406\tloss: 0.96188\tlr:[3.1638327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:30:43,216 [ hooks.py: 207]:\tstep: 27050\tsteps/sec: 1.31880\tloss: 0.44637\tlr:[3.1631538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:30:54,066 [ hooks.py: 207]:\tstep: 27060\tsteps/sec: 0.90701\tloss: 0.96414\tlr:[3.1624746e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:03,009 [ hooks.py: 207]:\tstep: 27070\tsteps/sec: 1.23444\tloss: 0.86814\tlr:[3.1617954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:10,221 [ hooks.py: 207]:\tstep: 27080\tsteps/sec: 1.27171\tloss: 0.56059\tlr:[3.1611162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:18,560 [ hooks.py: 207]:\tstep: 27090\tsteps/sec: 1.18855\tloss: 0.78674\tlr:[3.1604377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:25,099 [ hooks.py: 207]:\tstep: 27100\tsteps/sec: 1.45627\tloss: 0.33275\tlr:[3.1597585e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:34,813 [ hooks.py: 207]:\tstep: 27110\tsteps/sec: 1.06309\tloss: 0.39680\tlr:[3.1590793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:42,835 [ hooks.py: 207]:\tstep: 27120\tsteps/sec: 1.27564\tloss: 0.32720\tlr:[3.1584004e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:52,495 [ hooks.py: 207]:\tstep: 27130\tsteps/sec: 1.05298\tloss: 0.68898\tlr:[3.1577212e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:31:59,621 [ hooks.py: 207]:\tstep: 27140\tsteps/sec: 1.35039\tloss: 0.24103\tlr:[3.157042e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:32:06,350 [ hooks.py: 207]:\tstep: 27150\tsteps/sec: 1.50480\tloss: 0.40853\tlr:[3.1563628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:32:14,801 [ hooks.py: 207]:\tstep: 27160\tsteps/sec: 1.15945\tloss: 0.26101\tlr:[3.155684e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:32:21,820 [ hooks.py: 207]:\tstep: 27170\tsteps/sec: 1.40642\tloss: 1.01862\tlr:[3.155005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:32:29,781 [ hooks.py: 207]:\tstep: 27180\tsteps/sec: 1.22332\tloss: 0.40608\tlr:[3.1543255e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:32:38,555 [ hooks.py: 207]:\tstep: 27190\tsteps/sec: 1.11046\tloss: 0.79228\tlr:[3.1536467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:32:46,706 [ hooks.py: 207]:\tstep: 27200\tsteps/sec: 1.29957\tloss: 0.77671\tlr:[3.152968e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:32:54,722 [ hooks.py: 207]:\tstep: 27210\tsteps/sec: 1.25980\tloss: 0.71222\tlr:[3.1522883e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:33:03,490 [ hooks.py: 207]:\tstep: 27220\tsteps/sec: 1.20394\tloss: 0.50120\tlr:[3.151609e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:33:11,190 [ hooks.py: 207]:\tstep: 27230\tsteps/sec: 1.25031\tloss: 0.39564\tlr:[3.1509302e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:33:19,251 [ hooks.py: 207]:\tstep: 27240\tsteps/sec: 1.20916\tloss: 0.64219\tlr:[3.1502514e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:33:27,550 [ hooks.py: 207]:\tstep: 27250\tsteps/sec: 1.18718\tloss: 0.40880\tlr:[3.1495725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:33:36,492 [ hooks.py: 207]:\tstep: 27260\tsteps/sec: 1.17315\tloss: 0.46858\tlr:[3.1488933e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:33:47,657 [ hooks.py: 207]:\tstep: 27270\tsteps/sec: 0.89794\tloss: 0.55657\tlr:[3.148214e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:33:54,962 [ hooks.py: 207]:\tstep: 27280\tsteps/sec: 1.32267\tloss: 0.68545\tlr:[3.1475352e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 02:34:03,114 [ hooks.py: 207]:\tstep: 27290\tsteps/sec: 1.20382\tloss: 0.76198\tlr:[3.146856e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:34:11,490 [ hooks.py: 207]:\tstep: 27300\tsteps/sec: 1.24775\tloss: 0.59368\tlr:[3.146177e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:34:19,416 [ hooks.py: 207]:\tstep: 27310\tsteps/sec: 1.22329\tloss: 0.38765\tlr:[3.145498e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:34:27,437 [ hooks.py: 207]:\tstep: 27320\tsteps/sec: 1.28719\tloss: 0.69208\tlr:[3.144819e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:34:34,285 [ hooks.py: 207]:\tstep: 27330\tsteps/sec: 1.34281\tloss: 0.65986\tlr:[3.14414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:34:41,999 [ hooks.py: 207]:\tstep: 27340\tsteps/sec: 1.34396\tloss: 0.79296\tlr:[3.1434607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:34:49,817 [ hooks.py: 207]:\tstep: 27350\tsteps/sec: 1.29363\tloss: 0.63375\tlr:[3.142782e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:34:59,510 [ hooks.py: 207]:\tstep: 27360\tsteps/sec: 1.03487\tloss: 0.71512\tlr:[3.1421027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:35:09,240 [ hooks.py: 207]:\tstep: 27370\tsteps/sec: 1.01571\tloss: 0.38916\tlr:[3.1414234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:35:19,820 [ hooks.py: 207]:\tstep: 27380\tsteps/sec: 0.95576\tloss: 0.46929\tlr:[3.1407446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:35:29,480 [ hooks.py: 207]:\tstep: 27390\tsteps/sec: 1.02233\tloss: 0.34871\tlr:[3.1400657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:35:36,209 [ hooks.py: 207]:\tstep: 27400\tsteps/sec: 1.53588\tloss: 0.48085\tlr:[3.1393865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:35:43,408 [ hooks.py: 207]:\tstep: 27410\tsteps/sec: 1.28907\tloss: 0.48260\tlr:[3.1387073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:35:52,606 [ hooks.py: 207]:\tstep: 27420\tsteps/sec: 1.17156\tloss: 0.63995\tlr:[3.1380285e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:01,481 [ hooks.py: 207]:\tstep: 27430\tsteps/sec: 1.13087\tloss: 0.39862\tlr:[3.1373493e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:09,177 [ hooks.py: 207]:\tstep: 27440\tsteps/sec: 1.21021\tloss: 1.11719\tlr:[3.13667e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:19,556 [ hooks.py: 207]:\tstep: 27450\tsteps/sec: 0.99182\tloss: 0.21360\tlr:[3.135991e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:28,401 [ hooks.py: 207]:\tstep: 27460\tsteps/sec: 1.13685\tloss: 0.62699\tlr:[3.135312e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:36,905 [ hooks.py: 207]:\tstep: 27470\tsteps/sec: 1.21119\tloss: 0.79786\tlr:[3.134633e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:43,889 [ hooks.py: 207]:\tstep: 27480\tsteps/sec: 1.29388\tloss: 0.60065\tlr:[3.1339536e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:51,234 [ hooks.py: 207]:\tstep: 27490\tsteps/sec: 1.41332\tloss: 0.49354\tlr:[3.1332747e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:36:59,985 [ hooks.py: 207]:\tstep: 27500\tsteps/sec: 1.19006\tloss: 0.93846\tlr:[3.132596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:37:09,302 [ hooks.py: 207]:\tstep: 27510\tsteps/sec: 1.05303\tloss: 0.54938\tlr:[3.1319167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:37:18,906 [ hooks.py: 207]:\tstep: 27520\tsteps/sec: 1.04427\tloss: 0.82527\tlr:[3.1312375e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:37:27,024 [ hooks.py: 207]:\tstep: 27530\tsteps/sec: 1.27955\tloss: 0.49724\tlr:[3.1305586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:37:35,203 [ hooks.py: 207]:\tstep: 27540\tsteps/sec: 1.18709\tloss: 0.76317\tlr:[3.1298798e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:37:43,094 [ hooks.py: 207]:\tstep: 27550\tsteps/sec: 1.30217\tloss: 0.61595\tlr:[3.1292006e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:37:50,593 [ hooks.py: 207]:\tstep: 27560\tsteps/sec: 1.27324\tloss: 0.67225\tlr:[3.1285213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:37:59,772 [ hooks.py: 207]:\tstep: 27570\tsteps/sec: 1.07326\tloss: 0.49223\tlr:[3.1278425e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:38:08,683 [ hooks.py: 207]:\tstep: 27580\tsteps/sec: 1.15838\tloss: 0.50486\tlr:[3.127164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:38:16,726 [ hooks.py: 207]:\tstep: 27590\tsteps/sec: 1.26111\tloss: 0.92739\tlr:[3.1264844e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:38:28,716 [ hooks.py: 207]:\tstep: 27600\tsteps/sec: 0.82372\tloss: 0.80870\tlr:[3.1258056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:38:37,397 [ hooks.py: 207]:\tstep: 27610\tsteps/sec: 1.19691\tloss: 0.64225\tlr:[3.1251264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:38:46,816 [ hooks.py: 207]:\tstep: 27620\tsteps/sec: 1.01490\tloss: 0.70227\tlr:[3.1244475e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:38:58,946 [ hooks.py: 207]:\tstep: 27630\tsteps/sec: 0.83875\tloss: 0.93504\tlr:[3.123768e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:39:07,380 [ hooks.py: 207]:\tstep: 27640\tsteps/sec: 1.16267\tloss: 0.46257\tlr:[3.123089e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:39:15,236 [ hooks.py: 207]:\tstep: 27650\tsteps/sec: 1.23003\tloss: 0.42745\tlr:[3.1224103e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:39:23,439 [ hooks.py: 207]:\tstep: 27660\tsteps/sec: 1.27161\tloss: 0.49422\tlr:[3.121731e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:39:32,447 [ hooks.py: 207]:\tstep: 27670\tsteps/sec: 1.14464\tloss: 0.76657\tlr:[3.121052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:39:40,644 [ hooks.py: 207]:\tstep: 27680\tsteps/sec: 1.19016\tloss: 0.43014\tlr:[3.1203726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:39:48,429 [ hooks.py: 207]:\tstep: 27690\tsteps/sec: 1.30008\tloss: 0.68936\tlr:[3.119694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:39:58,737 [ hooks.py: 207]:\tstep: 27700\tsteps/sec: 0.93146\tloss: 0.65585\tlr:[3.119015e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:40:09,421 [ hooks.py: 207]:\tstep: 27710\tsteps/sec: 0.93006\tloss: 0.23629\tlr:[3.1183354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:40:17,172 [ hooks.py: 207]:\tstep: 27720\tsteps/sec: 1.34075\tloss: 0.56583\tlr:[3.1176565e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:40:27,019 [ hooks.py: 207]:\tstep: 27730\tsteps/sec: 1.03753\tloss: 0.39803\tlr:[3.1169777e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:40:36,130 [ hooks.py: 207]:\tstep: 27740\tsteps/sec: 1.08374\tloss: 0.72024\tlr:[3.1162985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:40:44,907 [ hooks.py: 207]:\tstep: 27750\tsteps/sec: 1.13523\tloss: 0.38867\tlr:[3.1156193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:40:53,767 [ hooks.py: 207]:\tstep: 27760\tsteps/sec: 1.12337\tloss: 0.73421\tlr:[3.1149408e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:41:02,383 [ hooks.py: 207]:\tstep: 27770\tsteps/sec: 1.26834\tloss: 0.89711\tlr:[3.114262e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:41:09,017 [ hooks.py: 207]:\tstep: 27780\tsteps/sec: 1.34934\tloss: 0.77913\tlr:[3.113582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:41:18,055 [ hooks.py: 207]:\tstep: 27790\tsteps/sec: 1.12971\tloss: 0.73956\tlr:[3.1129035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:41:27,215 [ hooks.py: 207]:\tstep: 27800\tsteps/sec: 1.10292\tloss: 0.55757\tlr:[3.1122243e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:41:36,532 [ hooks.py: 207]:\tstep: 27810\tsteps/sec: 1.07330\tloss: 0.45449\tlr:[3.1115454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:41:44,733 [ hooks.py: 207]:\tstep: 27820\tsteps/sec: 1.17383\tloss: 0.69706\tlr:[3.1108662e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:41:52,221 [ hooks.py: 207]:\tstep: 27830\tsteps/sec: 1.32895\tloss: 0.43665\tlr:[3.110187e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:42:04,060 [ hooks.py: 207]:\tstep: 27840\tsteps/sec: 0.85723\tloss: 0.32694\tlr:[3.109508e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:42:12,405 [ hooks.py: 207]:\tstep: 27850\tsteps/sec: 1.21743\tloss: 0.47095\tlr:[3.1088293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:42:22,579 [ hooks.py: 207]:\tstep: 27860\tsteps/sec: 0.99588\tloss: 0.44461\tlr:[3.1081498e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:42:30,158 [ hooks.py: 207]:\tstep: 27870\tsteps/sec: 1.27363\tloss: 0.30917\tlr:[3.107471e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:42:38,628 [ hooks.py: 207]:\tstep: 27880\tsteps/sec: 1.13252\tloss: 1.01496\tlr:[3.106792e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:42:47,572 [ hooks.py: 207]:\tstep: 27890\tsteps/sec: 1.20955\tloss: 0.82094\tlr:[3.106113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:42:54,674 [ hooks.py: 207]:\tstep: 27900\tsteps/sec: 1.33721\tloss: 0.35990\tlr:[3.1054336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:43:04,189 [ hooks.py: 207]:\tstep: 27910\tsteps/sec: 1.05579\tloss: 0.77581\tlr:[3.1047548e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:43:11,050 [ hooks.py: 207]:\tstep: 27920\tsteps/sec: 1.47144\tloss: 0.27765\tlr:[3.104076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:43:18,740 [ hooks.py: 207]:\tstep: 27930\tsteps/sec: 1.29082\tloss: 0.82282\tlr:[3.1033964e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 02:43:28,348 [ hooks.py: 207]:\tstep: 27940\tsteps/sec: 1.05970\tloss: 0.64870\tlr:[3.1027175e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:43:37,125 [ hooks.py: 207]:\tstep: 27950\tsteps/sec: 1.15298\tloss: 0.49788\tlr:[3.1020387e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:43:43,883 [ hooks.py: 207]:\tstep: 27960\tsteps/sec: 1.32323\tloss: 0.40450\tlr:[3.1013595e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:43:52,016 [ hooks.py: 207]:\tstep: 27970\tsteps/sec: 1.35810\tloss: 0.75004\tlr:[3.10068e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:00,537 [ hooks.py: 207]:\tstep: 27980\tsteps/sec: 1.14611\tloss: 0.27821\tlr:[3.100001e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:09,793 [ hooks.py: 207]:\tstep: 27990\tsteps/sec: 1.04137\tloss: 0.24722\tlr:[3.0993226e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:18,749 [ hooks.py: 207]:\tstep: 28000\tsteps/sec: 1.09327\tloss: 0.41522\tlr:[3.098643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:18,749 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:18,749 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:18,749 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:44:18,750 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:18,750 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:44:18,750 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:52:19,908 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,909 [ trainer.py: 58]:\t{'acc': 0.7759036144578313, 'loss': 0.5532791945987787}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,910 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,910 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:52:19,911 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:52:19,911 [ trainer.py: 70]:\tacc\t0.7759036144578313\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:52:19,911 [ trainer.py: 70]:\tloss\t0.5532791945987787\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 02:52:19,911 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,911 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7759036144578313, 'loss': 0.5532791945987787}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,911 [ exporter.py: 64]:\t[Best Exporter]: skip step 28000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,911 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7759036144578313, 'loss': 0.5532791945987787}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,912 [ exporter.py: 104]:\t[Best Exporter]: skip step 28000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:19,912 [monitored_executor.py: 122]:\tsaving step 28000 to ./output/model_28000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:23,496 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_25000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:31,808 [ hooks.py: 207]:\tstep: 28010\tsteps/sec: 0.02031\tloss: 0.43872\tlr:[3.0979638e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:39,588 [ hooks.py: 207]:\tstep: 28020\tsteps/sec: 1.24239\tloss: 0.33088\tlr:[3.0972853e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:47,712 [ hooks.py: 207]:\tstep: 28030\tsteps/sec: 1.26653\tloss: 0.49012\tlr:[3.0966065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:52:55,840 [ hooks.py: 207]:\tstep: 28040\tsteps/sec: 1.21506\tloss: 0.62937\tlr:[3.095927e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:02,054 [ hooks.py: 207]:\tstep: 28050\tsteps/sec: 1.63838\tloss: 0.57502\tlr:[3.095248e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:12,317 [ hooks.py: 207]:\tstep: 28060\tsteps/sec: 0.98131\tloss: 0.60647\tlr:[3.094569e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:19,029 [ hooks.py: 207]:\tstep: 28070\tsteps/sec: 1.43801\tloss: 0.49068\tlr:[3.09389e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:25,381 [ hooks.py: 207]:\tstep: 28080\tsteps/sec: 1.51640\tloss: 0.68450\tlr:[3.0932108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:36,509 [ hooks.py: 207]:\tstep: 28090\tsteps/sec: 0.92446\tloss: 0.46906\tlr:[3.0925316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:43,942 [ hooks.py: 207]:\tstep: 28100\tsteps/sec: 1.33594\tloss: 0.73868\tlr:[3.0918527e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:50,181 [ hooks.py: 207]:\tstep: 28110\tsteps/sec: 1.58829\tloss: 0.22303\tlr:[3.0911735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:53:57,748 [ hooks.py: 207]:\tstep: 28120\tsteps/sec: 1.37188\tloss: 0.33513\tlr:[3.0904946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:54:05,399 [ hooks.py: 207]:\tstep: 28130\tsteps/sec: 1.26755\tloss: 0.19564\tlr:[3.0898154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:54:13,951 [ hooks.py: 207]:\tstep: 28140\tsteps/sec: 1.17420\tloss: 0.75045\tlr:[3.0891366e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:54:20,588 [ hooks.py: 207]:\tstep: 28150\tsteps/sec: 1.50291\tloss: 0.22581\tlr:[3.0884574e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:54:28,073 [ hooks.py: 207]:\tstep: 28160\tsteps/sec: 1.30048\tloss: 0.55206\tlr:[3.087778e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:54:36,390 [ hooks.py: 207]:\tstep: 28170\tsteps/sec: 1.18080\tloss: 0.68589\tlr:[3.0870993e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:54:44,378 [ hooks.py: 207]:\tstep: 28180\tsteps/sec: 1.23774\tloss: 0.51656\tlr:[3.08642e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:54:52,723 [ hooks.py: 207]:\tstep: 28190\tsteps/sec: 1.30073\tloss: 0.45818\tlr:[3.085741e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:00,340 [ hooks.py: 207]:\tstep: 28200\tsteps/sec: 1.28946\tloss: 0.91999\tlr:[3.0850617e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:09,523 [ hooks.py: 207]:\tstep: 28210\tsteps/sec: 1.07165\tloss: 0.49523\tlr:[3.084383e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:18,227 [ hooks.py: 207]:\tstep: 28220\tsteps/sec: 1.17828\tloss: 0.38659\tlr:[3.083704e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:24,961 [ hooks.py: 207]:\tstep: 28230\tsteps/sec: 1.43314\tloss: 0.69692\tlr:[3.0830244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:31,278 [ hooks.py: 207]:\tstep: 28240\tsteps/sec: 1.61547\tloss: 0.48698\tlr:[3.082346e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:38,474 [ hooks.py: 207]:\tstep: 28250\tsteps/sec: 1.41437\tloss: 0.50736\tlr:[3.081667e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:48,113 [ hooks.py: 207]:\tstep: 28260\tsteps/sec: 1.04005\tloss: 0.91542\tlr:[3.0809875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:55:56,516 [ hooks.py: 207]:\tstep: 28270\tsteps/sec: 1.15830\tloss: 0.40422\tlr:[3.0803083e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:56:04,840 [ hooks.py: 207]:\tstep: 28280\tsteps/sec: 1.15429\tloss: 0.56459\tlr:[3.0796295e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:56:14,195 [ hooks.py: 207]:\tstep: 28290\tsteps/sec: 1.09567\tloss: 0.66771\tlr:[3.0789506e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:56:23,471 [ hooks.py: 207]:\tstep: 28300\tsteps/sec: 1.10521\tloss: 0.66544\tlr:[3.0782714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:56:30,348 [ hooks.py: 207]:\tstep: 28310\tsteps/sec: 1.43758\tloss: 0.72499\tlr:[3.0775926e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:56:39,025 [ hooks.py: 207]:\tstep: 28320\tsteps/sec: 1.14226\tloss: 0.66637\tlr:[3.0769137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:56:49,901 [ hooks.py: 207]:\tstep: 28330\tsteps/sec: 0.91286\tloss: 0.60724\tlr:[3.0762345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:56:58,854 [ hooks.py: 207]:\tstep: 28340\tsteps/sec: 1.09656\tloss: 0.70143\tlr:[3.0755557e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:57:07,362 [ hooks.py: 207]:\tstep: 28350\tsteps/sec: 1.24957\tloss: 0.61755\tlr:[3.0748764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:57:18,386 [ hooks.py: 207]:\tstep: 28360\tsteps/sec: 0.90540\tloss: 0.35776\tlr:[3.0741972e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:57:27,249 [ hooks.py: 207]:\tstep: 28370\tsteps/sec: 1.13525\tloss: 0.42906\tlr:[3.073518e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:57:35,701 [ hooks.py: 207]:\tstep: 28380\tsteps/sec: 1.14421\tloss: 0.24340\tlr:[3.072839e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:57:46,311 [ hooks.py: 207]:\tstep: 28390\tsteps/sec: 0.94523\tloss: 0.84740\tlr:[3.07216e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 02:57:57,341 [ hooks.py: 207]:\tstep: 28400\tsteps/sec: 0.90081\tloss: 0.30937\tlr:[3.071481e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:58:08,063 [ hooks.py: 207]:\tstep: 28410\tsteps/sec: 0.93409\tloss: 0.48274\tlr:[3.070802e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:58:17,720 [ hooks.py: 207]:\tstep: 28420\tsteps/sec: 1.06522\tloss: 0.47538\tlr:[3.0701227e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:58:26,321 [ hooks.py: 207]:\tstep: 28430\tsteps/sec: 1.14322\tloss: 0.55994\tlr:[3.0694435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:58:33,304 [ hooks.py: 207]:\tstep: 28440\tsteps/sec: 1.45689\tloss: 0.58311\tlr:[3.0687646e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:58:41,316 [ hooks.py: 207]:\tstep: 28450\tsteps/sec: 1.23487\tloss: 1.09967\tlr:[3.0680854e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:58:49,833 [ hooks.py: 207]:\tstep: 28460\tsteps/sec: 1.15247\tloss: 0.58962\tlr:[3.0674066e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:58:59,339 [ hooks.py: 207]:\tstep: 28470\tsteps/sec: 1.03144\tloss: 0.73088\tlr:[3.0667277e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:59:10,675 [ hooks.py: 207]:\tstep: 28480\tsteps/sec: 0.91062\tloss: 0.84430\tlr:[3.0660485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:59:17,297 [ hooks.py: 207]:\tstep: 28490\tsteps/sec: 1.45797\tloss: 0.97338\tlr:[3.0653693e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:59:26,068 [ hooks.py: 207]:\tstep: 28500\tsteps/sec: 1.10660\tloss: 0.63411\tlr:[3.06469e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:59:36,026 [ hooks.py: 207]:\tstep: 28510\tsteps/sec: 1.05531\tloss: 0.27817\tlr:[3.0640113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:59:47,656 [ hooks.py: 207]:\tstep: 28520\tsteps/sec: 0.86961\tloss: 0.48638\tlr:[3.063332e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 02:59:55,791 [ hooks.py: 207]:\tstep: 28530\tsteps/sec: 1.19550\tloss: 0.83893\tlr:[3.0626532e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:00:04,464 [ hooks.py: 207]:\tstep: 28540\tsteps/sec: 1.11579\tloss: 0.41463\tlr:[3.0619743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:00:14,463 [ hooks.py: 207]:\tstep: 28550\tsteps/sec: 1.03602\tloss: 0.33643\tlr:[3.061295e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:00:21,630 [ hooks.py: 207]:\tstep: 28560\tsteps/sec: 1.39950\tloss: 0.59602\tlr:[3.0606163e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:00:28,941 [ hooks.py: 207]:\tstep: 28570\tsteps/sec: 1.36260\tloss: 0.49002\tlr:[3.059937e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:00:36,500 [ hooks.py: 207]:\tstep: 28580\tsteps/sec: 1.27317\tloss: 0.43987\tlr:[3.059258e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:00:47,096 [ hooks.py: 207]:\tstep: 28590\tsteps/sec: 0.99396\tloss: 0.82048\tlr:[3.058579e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:00:53,790 [ hooks.py: 207]:\tstep: 28600\tsteps/sec: 1.46070\tloss: 0.45122\tlr:[3.0578998e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:01:04,272 [ hooks.py: 207]:\tstep: 28610\tsteps/sec: 0.93687\tloss: 0.37701\tlr:[3.0572206e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:01:13,303 [ hooks.py: 207]:\tstep: 28620\tsteps/sec: 1.15831\tloss: 0.66431\tlr:[3.0565418e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:01:20,580 [ hooks.py: 207]:\tstep: 28630\tsteps/sec: 1.32165\tloss: 0.38259\tlr:[3.0558625e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:01:29,028 [ hooks.py: 207]:\tstep: 28640\tsteps/sec: 1.18589\tloss: 0.38677\tlr:[3.0551833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:01:35,785 [ hooks.py: 207]:\tstep: 28650\tsteps/sec: 1.46804\tloss: 0.42461\tlr:[3.0545045e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:01:44,498 [ hooks.py: 207]:\tstep: 28660\tsteps/sec: 1.09493\tloss: 0.27330\tlr:[3.0538253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:01:52,658 [ hooks.py: 207]:\tstep: 28670\tsteps/sec: 1.35706\tloss: 0.29623\tlr:[3.053146e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:02:00,861 [ hooks.py: 207]:\tstep: 28680\tsteps/sec: 1.18713\tloss: 0.58238\tlr:[3.0524672e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:02:08,316 [ hooks.py: 207]:\tstep: 28690\tsteps/sec: 1.34729\tloss: 0.79392\tlr:[3.051788e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:02:15,275 [ hooks.py: 207]:\tstep: 28700\tsteps/sec: 1.33534\tloss: 0.35648\tlr:[3.0511092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:02:24,921 [ hooks.py: 207]:\tstep: 28710\tsteps/sec: 1.06791\tloss: 0.57378\tlr:[3.0504301e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:02:34,602 [ hooks.py: 207]:\tstep: 28720\tsteps/sec: 1.03692\tloss: 0.38309\tlr:[3.049751e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:02:43,096 [ hooks.py: 207]:\tstep: 28730\tsteps/sec: 1.22146\tloss: 0.62055\tlr:[3.0490723e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:02:51,135 [ hooks.py: 207]:\tstep: 28740\tsteps/sec: 1.16332\tloss: 0.59320\tlr:[3.048393e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:00,028 [ hooks.py: 207]:\tstep: 28750\tsteps/sec: 1.12124\tloss: 0.45942\tlr:[3.0477138e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:09,390 [ hooks.py: 207]:\tstep: 28760\tsteps/sec: 1.10362\tloss: 0.53877\tlr:[3.0470346e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:18,254 [ hooks.py: 207]:\tstep: 28770\tsteps/sec: 1.16334\tloss: 0.83995\tlr:[3.0463558e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:26,336 [ hooks.py: 207]:\tstep: 28780\tsteps/sec: 1.19958\tloss: 0.66464\tlr:[3.0456771e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:34,064 [ hooks.py: 207]:\tstep: 28790\tsteps/sec: 1.23506\tloss: 0.66509\tlr:[3.0449979e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:43,507 [ hooks.py: 207]:\tstep: 28800\tsteps/sec: 1.12388\tloss: 0.30757\tlr:[3.0443185e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:51,641 [ hooks.py: 207]:\tstep: 28810\tsteps/sec: 1.21691\tloss: 0.35835\tlr:[3.0436397e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:03:59,007 [ hooks.py: 207]:\tstep: 28820\tsteps/sec: 1.38322\tloss: 0.61575\tlr:[3.0429605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:04:08,749 [ hooks.py: 207]:\tstep: 28830\tsteps/sec: 1.01864\tloss: 0.41572\tlr:[3.0422814e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:04:15,563 [ hooks.py: 207]:\tstep: 28840\tsteps/sec: 1.44747\tloss: 0.29537\tlr:[3.0416026e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:04:25,312 [ hooks.py: 207]:\tstep: 28850\tsteps/sec: 1.01531\tloss: 0.31983\tlr:[3.0409234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:04:33,997 [ hooks.py: 207]:\tstep: 28860\tsteps/sec: 1.11790\tloss: 0.95822\tlr:[3.0402445e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:04:44,494 [ hooks.py: 207]:\tstep: 28870\tsteps/sec: 0.98310\tloss: 0.41256\tlr:[3.0395653e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:04:52,594 [ hooks.py: 207]:\tstep: 28880\tsteps/sec: 1.27237\tloss: 0.62750\tlr:[3.0388865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:05:05,032 [ hooks.py: 207]:\tstep: 28890\tsteps/sec: 0.78856\tloss: 0.42562\tlr:[3.0382074e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:05:12,596 [ hooks.py: 207]:\tstep: 28900\tsteps/sec: 1.36989\tloss: 0.85804\tlr:[3.0375284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:05:23,804 [ hooks.py: 207]:\tstep: 28910\tsteps/sec: 0.88686\tloss: 0.41796\tlr:[3.0368492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:05:32,468 [ hooks.py: 207]:\tstep: 28920\tsteps/sec: 1.16115\tloss: 0.52523\tlr:[3.0361702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:05:40,569 [ hooks.py: 207]:\tstep: 28930\tsteps/sec: 1.20797\tloss: 0.61914\tlr:[3.0354913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:05:49,791 [ hooks.py: 207]:\tstep: 28940\tsteps/sec: 1.02477\tloss: 0.64205\tlr:[3.0348121e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:05:58,522 [ hooks.py: 207]:\tstep: 28950\tsteps/sec: 1.24054\tloss: 0.69646\tlr:[3.0341329e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:05,362 [ hooks.py: 207]:\tstep: 28960\tsteps/sec: 1.41908\tloss: 0.60196\tlr:[3.033454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:13,799 [ hooks.py: 207]:\tstep: 28970\tsteps/sec: 1.13800\tloss: 0.59917\tlr:[3.0327748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:23,755 [ hooks.py: 207]:\tstep: 28980\tsteps/sec: 1.06241\tloss: 0.53246\tlr:[3.0320954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:32,431 [ hooks.py: 207]:\tstep: 28990\tsteps/sec: 1.15692\tloss: 0.72068\tlr:[3.0314164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:42,411 [ hooks.py: 207]:\tstep: 29000\tsteps/sec: 0.99705\tloss: 0.62705\tlr:[3.030738e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:42,412 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:42,412 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:42,412 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:06:42,412 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:42,412 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:06:42,412 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-19 03:14:45,144 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,145 [ trainer.py: 58]:\t{'acc': 0.7767068273092369, 'loss': 0.5575891977701432}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,146 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,146 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:14:45,147 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:14:45,147 [ trainer.py: 70]:\tacc\t0.7767068273092369\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:14:45,147 [ trainer.py: 70]:\tloss\t0.5575891977701432\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:14:45,147 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,147 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7767068273092369, 'loss': 0.5575891977701432}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,147 [ exporter.py: 64]:\t[Best Exporter]: skip step 29000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,147 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7767068273092369, 'loss': 0.5575891977701432}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,148 [ exporter.py: 104]:\t[Best Exporter]: skip step 29000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:45,148 [monitored_executor.py: 122]:\tsaving step 29000 to ./output/model_29000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:48,326 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_26000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:14:55,640 [ hooks.py: 207]:\tstep: 29010\tsteps/sec: 0.02026\tloss: 0.46941\tlr:[3.0300584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:15:05,511 [ hooks.py: 207]:\tstep: 29020\tsteps/sec: 1.04389\tloss: 0.34429\tlr:[3.0293791e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:15:12,699 [ hooks.py: 207]:\tstep: 29030\tsteps/sec: 1.43964\tloss: 0.71162\tlr:[3.0287003e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:15:19,771 [ hooks.py: 207]:\tstep: 29040\tsteps/sec: 1.34879\tloss: 0.70954\tlr:[3.0280215e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:15:31,117 [ hooks.py: 207]:\tstep: 29050\tsteps/sec: 0.89379\tloss: 0.90326\tlr:[3.0273424e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:15:38,539 [ hooks.py: 207]:\tstep: 29060\tsteps/sec: 1.36450\tloss: 0.17694\tlr:[3.026663e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:15:45,657 [ hooks.py: 207]:\tstep: 29070\tsteps/sec: 1.36655\tloss: 0.60702\tlr:[3.025984e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:15:54,711 [ hooks.py: 207]:\tstep: 29080\tsteps/sec: 1.10127\tloss: 0.30545\tlr:[3.0253052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:16:03,733 [ hooks.py: 207]:\tstep: 29090\tsteps/sec: 1.20171\tloss: 0.76505\tlr:[3.024626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:16:11,375 [ hooks.py: 207]:\tstep: 29100\tsteps/sec: 1.20801\tloss: 0.66106\tlr:[3.0239467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:16:20,908 [ hooks.py: 207]:\tstep: 29110\tsteps/sec: 1.05823\tloss: 0.66898\tlr:[3.0232679e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:16:28,934 [ hooks.py: 207]:\tstep: 29120\tsteps/sec: 1.26402\tloss: 0.51710\tlr:[3.0225887e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:16:37,494 [ hooks.py: 207]:\tstep: 29130\tsteps/sec: 1.16875\tloss: 0.65942\tlr:[3.0219095e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:16:44,385 [ hooks.py: 207]:\tstep: 29140\tsteps/sec: 1.42115\tloss: 0.66728\tlr:[3.0212306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:16:54,196 [ hooks.py: 207]:\tstep: 29150\tsteps/sec: 1.02264\tloss: 0.97010\tlr:[3.0205518e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:17:03,626 [ hooks.py: 207]:\tstep: 29160\tsteps/sec: 1.08502\tloss: 0.47033\tlr:[3.0198726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:17:13,567 [ hooks.py: 207]:\tstep: 29170\tsteps/sec: 0.99977\tloss: 0.27516\tlr:[3.0191934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:17:20,765 [ hooks.py: 207]:\tstep: 29180\tsteps/sec: 1.32954\tloss: 0.78798\tlr:[3.0185141e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:17:29,658 [ hooks.py: 207]:\tstep: 29190\tsteps/sec: 1.11873\tloss: 0.20835\tlr:[3.0178355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:17:37,954 [ hooks.py: 207]:\tstep: 29200\tsteps/sec: 1.24482\tloss: 0.58021\tlr:[3.0171563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:17:45,087 [ hooks.py: 207]:\tstep: 29210\tsteps/sec: 1.30296\tloss: 0.28449\tlr:[3.0164769e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:17:53,311 [ hooks.py: 207]:\tstep: 29220\tsteps/sec: 1.31491\tloss: 0.58524\tlr:[3.015798e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:01,901 [ hooks.py: 207]:\tstep: 29230\tsteps/sec: 1.13919\tloss: 0.61660\tlr:[3.0151194e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:10,827 [ hooks.py: 207]:\tstep: 29240\tsteps/sec: 1.13043\tloss: 0.84246\tlr:[3.0144402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:18,594 [ hooks.py: 207]:\tstep: 29250\tsteps/sec: 1.29742\tloss: 0.22321\tlr:[3.013761e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:26,530 [ hooks.py: 207]:\tstep: 29260\tsteps/sec: 1.23973\tloss: 0.77169\tlr:[3.0130817e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:33,575 [ hooks.py: 207]:\tstep: 29270\tsteps/sec: 1.39089\tloss: 0.60738\tlr:[3.0124029e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:43,424 [ hooks.py: 207]:\tstep: 29280\tsteps/sec: 1.06022\tloss: 0.27530\tlr:[3.0117237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:51,462 [ hooks.py: 207]:\tstep: 29290\tsteps/sec: 1.21592\tloss: 0.27341\tlr:[3.0110448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:18:58,145 [ hooks.py: 207]:\tstep: 29300\tsteps/sec: 1.52743\tloss: 0.56543\tlr:[3.0103658e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:19:08,458 [ hooks.py: 207]:\tstep: 29310\tsteps/sec: 0.96830\tloss: 0.70714\tlr:[3.0096868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:19:15,651 [ hooks.py: 207]:\tstep: 29320\tsteps/sec: 1.33597\tloss: 0.52132\tlr:[3.0090076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:19:25,037 [ hooks.py: 207]:\tstep: 29330\tsteps/sec: 1.06322\tloss: 0.49029\tlr:[3.0083285e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:19:33,364 [ hooks.py: 207]:\tstep: 29340\tsteps/sec: 1.23761\tloss: 0.36909\tlr:[3.0076497e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:19:41,682 [ hooks.py: 207]:\tstep: 29350\tsteps/sec: 1.17931\tloss: 0.44765\tlr:[3.0069705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:19:49,669 [ hooks.py: 207]:\tstep: 29360\tsteps/sec: 1.18607\tloss: 0.44833\tlr:[3.0062913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:19:59,148 [ hooks.py: 207]:\tstep: 29370\tsteps/sec: 1.08993\tloss: 0.39890\tlr:[3.0056124e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:20:09,686 [ hooks.py: 207]:\tstep: 29380\tsteps/sec: 0.95425\tloss: 0.75460\tlr:[3.0049336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:20:18,396 [ hooks.py: 207]:\tstep: 29390\tsteps/sec: 1.13846\tloss: 0.49670\tlr:[3.0042544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:20:26,448 [ hooks.py: 207]:\tstep: 29400\tsteps/sec: 1.27567\tloss: 0.36354\tlr:[3.0035755e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:20:34,402 [ hooks.py: 207]:\tstep: 29410\tsteps/sec: 1.24695\tloss: 0.45385\tlr:[3.0028965e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:20:41,777 [ hooks.py: 207]:\tstep: 29420\tsteps/sec: 1.35111\tloss: 0.63301\tlr:[3.002217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:20:52,046 [ hooks.py: 207]:\tstep: 29430\tsteps/sec: 0.98974\tloss: 0.70752\tlr:[3.0015379e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:20:58,962 [ hooks.py: 207]:\tstep: 29440\tsteps/sec: 1.47321\tloss: 0.31827\tlr:[3.0008592e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:21:08,640 [ hooks.py: 207]:\tstep: 29450\tsteps/sec: 1.01556\tloss: 0.48980\tlr:[3.0001804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:21:18,041 [ hooks.py: 207]:\tstep: 29460\tsteps/sec: 1.06542\tloss: 0.76520\tlr:[2.9995008e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:21:26,322 [ hooks.py: 207]:\tstep: 29470\tsteps/sec: 1.18715\tloss: 1.11983\tlr:[2.998822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:21:34,953 [ hooks.py: 207]:\tstep: 29480\tsteps/sec: 1.10696\tloss: 0.42905\tlr:[2.9981431e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:21:42,921 [ hooks.py: 207]:\tstep: 29490\tsteps/sec: 1.35289\tloss: 0.69315\tlr:[2.9974643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:21:52,165 [ hooks.py: 207]:\tstep: 29500\tsteps/sec: 1.07535\tloss: 0.62912\tlr:[2.996785e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:21:59,437 [ hooks.py: 207]:\tstep: 29510\tsteps/sec: 1.38869\tloss: 0.75438\tlr:[2.9961055e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:22:07,255 [ hooks.py: 207]:\tstep: 29520\tsteps/sec: 1.20798\tloss: 0.45804\tlr:[2.9954268e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 03:22:15,263 [ hooks.py: 207]:\tstep: 29530\tsteps/sec: 1.23369\tloss: 0.25487\tlr:[2.994748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:22:23,281 [ hooks.py: 207]:\tstep: 29540\tsteps/sec: 1.35721\tloss: 0.65171\tlr:[2.9940687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:22:32,635 [ hooks.py: 207]:\tstep: 29550\tsteps/sec: 1.01962\tloss: 0.42784\tlr:[2.9933895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:22:42,219 [ hooks.py: 207]:\tstep: 29560\tsteps/sec: 1.03719\tloss: 0.48769\tlr:[2.9927107e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:22:51,086 [ hooks.py: 207]:\tstep: 29570\tsteps/sec: 1.17350\tloss: 0.69879\tlr:[2.9920315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:00,864 [ hooks.py: 207]:\tstep: 29580\tsteps/sec: 1.01491\tloss: 0.51024\tlr:[2.991352e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:09,219 [ hooks.py: 207]:\tstep: 29590\tsteps/sec: 1.15078\tloss: 0.67245\tlr:[2.990673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:17,693 [ hooks.py: 207]:\tstep: 29600\tsteps/sec: 1.26476\tloss: 0.44831\tlr:[2.9899946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:25,654 [ hooks.py: 207]:\tstep: 29610\tsteps/sec: 1.20813\tloss: 0.59648\tlr:[2.989315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:33,739 [ hooks.py: 207]:\tstep: 29620\tsteps/sec: 1.27011\tloss: 0.56422\tlr:[2.9886358e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:42,680 [ hooks.py: 207]:\tstep: 29630\tsteps/sec: 1.11334\tloss: 0.11468\tlr:[2.9879573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:50,773 [ hooks.py: 207]:\tstep: 29640\tsteps/sec: 1.20634\tloss: 0.19684\tlr:[2.9872783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:23:58,932 [ hooks.py: 207]:\tstep: 29650\tsteps/sec: 1.28227\tloss: 0.80976\tlr:[2.986599e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:24:09,738 [ hooks.py: 207]:\tstep: 29660\tsteps/sec: 0.90333\tloss: 0.49030\tlr:[2.9859197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:24:18,314 [ hooks.py: 207]:\tstep: 29670\tsteps/sec: 1.20570\tloss: 0.51438\tlr:[2.9852408e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:24:26,859 [ hooks.py: 207]:\tstep: 29680\tsteps/sec: 1.13775\tloss: 0.68352\tlr:[2.9845622e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:24:36,810 [ hooks.py: 207]:\tstep: 29690\tsteps/sec: 1.00707\tloss: 0.47385\tlr:[2.9838824e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:24:43,597 [ hooks.py: 207]:\tstep: 29700\tsteps/sec: 1.52113\tloss: 0.14816\tlr:[2.9832034e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:24:50,961 [ hooks.py: 207]:\tstep: 29710\tsteps/sec: 1.32099\tloss: 0.40848\tlr:[2.9825249e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:24:59,654 [ hooks.py: 207]:\tstep: 29720\tsteps/sec: 1.16488\tloss: 0.45888\tlr:[2.9818453e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:25:06,835 [ hooks.py: 207]:\tstep: 29730\tsteps/sec: 1.35395\tloss: 0.76980\tlr:[2.9811665e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:25:15,098 [ hooks.py: 207]:\tstep: 29740\tsteps/sec: 1.20289\tloss: 0.82521\tlr:[2.9804873e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:25:22,874 [ hooks.py: 207]:\tstep: 29750\tsteps/sec: 1.26681\tloss: 0.46244\tlr:[2.9798084e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:25:31,972 [ hooks.py: 207]:\tstep: 29760\tsteps/sec: 1.13064\tloss: 0.52531\tlr:[2.9791292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:25:41,670 [ hooks.py: 207]:\tstep: 29770\tsteps/sec: 1.02719\tloss: 0.59098\tlr:[2.97845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:25:51,092 [ hooks.py: 207]:\tstep: 29780\tsteps/sec: 1.02849\tloss: 0.36957\tlr:[2.9777711e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:26:01,777 [ hooks.py: 207]:\tstep: 29790\tsteps/sec: 0.98846\tloss: 0.33001\tlr:[2.9770925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:26:10,123 [ hooks.py: 207]:\tstep: 29800\tsteps/sec: 1.15906\tloss: 0.64833\tlr:[2.9764129e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:26:16,229 [ hooks.py: 207]:\tstep: 29810\tsteps/sec: 1.53107\tloss: 0.38326\tlr:[2.9757339e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:26:25,784 [ hooks.py: 207]:\tstep: 29820\tsteps/sec: 1.09141\tloss: 0.36220\tlr:[2.9750552e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:26:34,793 [ hooks.py: 207]:\tstep: 29830\tsteps/sec: 1.10665\tloss: 0.34815\tlr:[2.9743764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:26:42,763 [ hooks.py: 207]:\tstep: 29840\tsteps/sec: 1.23226\tloss: 0.67820\tlr:[2.9736968e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:26:52,270 [ hooks.py: 207]:\tstep: 29850\tsteps/sec: 1.05054\tloss: 0.38862\tlr:[2.973018e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:27:02,191 [ hooks.py: 207]:\tstep: 29860\tsteps/sec: 1.10332\tloss: 0.87696\tlr:[2.9723391e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:27:10,575 [ hooks.py: 207]:\tstep: 29870\tsteps/sec: 1.09955\tloss: 1.08198\tlr:[2.9716599e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:27:19,092 [ hooks.py: 207]:\tstep: 29880\tsteps/sec: 1.19964\tloss: 0.51646\tlr:[2.9709807e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:27:28,435 [ hooks.py: 207]:\tstep: 29890\tsteps/sec: 1.07823\tloss: 0.58662\tlr:[2.9703016e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:27:36,168 [ hooks.py: 207]:\tstep: 29900\tsteps/sec: 1.26984\tloss: 0.32558\tlr:[2.9696228e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:27:44,594 [ hooks.py: 207]:\tstep: 29910\tsteps/sec: 1.19188\tloss: 0.48806\tlr:[2.9689434e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:27:53,964 [ hooks.py: 207]:\tstep: 29920\tsteps/sec: 1.07551\tloss: 1.01627\tlr:[2.9682642e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:28:00,957 [ hooks.py: 207]:\tstep: 29930\tsteps/sec: 1.41014\tloss: 0.85969\tlr:[2.9675855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:28:09,151 [ hooks.py: 207]:\tstep: 29940\tsteps/sec: 1.23536\tloss: 0.51604\tlr:[2.9669067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:28:19,570 [ hooks.py: 207]:\tstep: 29950\tsteps/sec: 0.91100\tloss: 0.44143\tlr:[2.9662275e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:28:29,690 [ hooks.py: 207]:\tstep: 29960\tsteps/sec: 0.97787\tloss: 0.61537\tlr:[2.9655483e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:28:39,721 [ hooks.py: 207]:\tstep: 29970\tsteps/sec: 1.05399\tloss: 0.46816\tlr:[2.9648694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:28:47,183 [ hooks.py: 207]:\tstep: 29980\tsteps/sec: 1.27621\tloss: 0.78641\tlr:[2.9641904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:28:56,903 [ hooks.py: 207]:\tstep: 29990\tsteps/sec: 1.05988\tloss: 0.29378\tlr:[2.963511e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:29:05,779 [ hooks.py: 207]:\tstep: 30000\tsteps/sec: 1.12737\tloss: 0.61388\tlr:[2.9628318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:29:05,781 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:29:05,781 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:29:05,781 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:29:05,781 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:29:05,781 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:29:05,781 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:37:07,347 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,349 [ trainer.py: 58]:\t{'acc': 0.7779116465863454, 'loss': 0.5618297452918994}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,349 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,350 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:37:07,350 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:37:07,350 [ trainer.py: 70]:\tacc\t0.7779116465863454\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:37:07,350 [ trainer.py: 70]:\tloss\t0.5618297452918994\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:37:07,351 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,351 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.5618297452918994}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,351 [ exporter.py: 64]:\t[Best Exporter]: skip step 30000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,351 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.5618297452918994}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,351 [ exporter.py: 104]:\t[Best Exporter]: skip step 30000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:07,352 [monitored_executor.py: 122]:\tsaving step 30000 to ./output/model_30000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 03:37:10,516 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_27000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:21,179 [ hooks.py: 207]:\tstep: 30010\tsteps/sec: 0.02019\tloss: 0.66746\tlr:[2.9621531e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:29,470 [ hooks.py: 207]:\tstep: 30020\tsteps/sec: 1.22756\tloss: 0.06228\tlr:[2.961474e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:36,749 [ hooks.py: 207]:\tstep: 30030\tsteps/sec: 1.35505\tloss: 0.59172\tlr:[2.9607949e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:44,691 [ hooks.py: 207]:\tstep: 30040\tsteps/sec: 1.19151\tloss: 0.31618\tlr:[2.9601159e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:37:55,272 [ hooks.py: 207]:\tstep: 30050\tsteps/sec: 0.97846\tloss: 0.76368\tlr:[2.959437e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:38:02,871 [ hooks.py: 207]:\tstep: 30060\tsteps/sec: 1.41874\tloss: 0.21906\tlr:[2.9587578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:38:11,464 [ hooks.py: 207]:\tstep: 30070\tsteps/sec: 1.10612\tloss: 0.52597\tlr:[2.9580782e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:38:20,285 [ hooks.py: 207]:\tstep: 30080\tsteps/sec: 1.07578\tloss: 0.25151\tlr:[2.9573997e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:38:28,380 [ hooks.py: 207]:\tstep: 30090\tsteps/sec: 1.31798\tloss: 0.73780\tlr:[2.9567205e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:38:37,937 [ hooks.py: 207]:\tstep: 30100\tsteps/sec: 1.04670\tloss: 0.39103\tlr:[2.9560417e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:38:46,547 [ hooks.py: 207]:\tstep: 30110\tsteps/sec: 1.10429\tloss: 0.89876\tlr:[2.9553625e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:38:56,707 [ hooks.py: 207]:\tstep: 30120\tsteps/sec: 1.01749\tloss: 0.54993\tlr:[2.9546833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:39:05,139 [ hooks.py: 207]:\tstep: 30130\tsteps/sec: 1.18106\tloss: 0.15941\tlr:[2.9540044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:39:14,276 [ hooks.py: 207]:\tstep: 30140\tsteps/sec: 1.07309\tloss: 1.25947\tlr:[2.9533248e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:39:22,205 [ hooks.py: 207]:\tstep: 30150\tsteps/sec: 1.29937\tloss: 0.59859\tlr:[2.9526462e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:39:30,134 [ hooks.py: 207]:\tstep: 30160\tsteps/sec: 1.26677\tloss: 0.43015\tlr:[2.9519673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:39:41,365 [ hooks.py: 207]:\tstep: 30170\tsteps/sec: 0.89433\tloss: 0.38035\tlr:[2.951288e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:39:47,897 [ hooks.py: 207]:\tstep: 30180\tsteps/sec: 1.47077\tloss: 0.45548\tlr:[2.9506087e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:39:56,621 [ hooks.py: 207]:\tstep: 30190\tsteps/sec: 1.17042\tloss: 0.16103\tlr:[2.9499297e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:40:06,606 [ hooks.py: 207]:\tstep: 30200\tsteps/sec: 0.99953\tloss: 0.41063\tlr:[2.9492508e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:40:13,688 [ hooks.py: 207]:\tstep: 30210\tsteps/sec: 1.37675\tloss: 0.33061\tlr:[2.9485716e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:40:22,709 [ hooks.py: 207]:\tstep: 30220\tsteps/sec: 1.09791\tloss: 0.39494\tlr:[2.9478924e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:40:32,394 [ hooks.py: 207]:\tstep: 30230\tsteps/sec: 1.05617\tloss: 0.36857\tlr:[2.9472136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:40:39,358 [ hooks.py: 207]:\tstep: 30240\tsteps/sec: 1.48648\tloss: 0.70731\tlr:[2.9465347e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:40:48,671 [ hooks.py: 207]:\tstep: 30250\tsteps/sec: 1.07701\tloss: 0.28083\tlr:[2.9458555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:40:56,423 [ hooks.py: 207]:\tstep: 30260\tsteps/sec: 1.25734\tloss: 0.58015\tlr:[2.9451763e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:41:05,330 [ hooks.py: 207]:\tstep: 30270\tsteps/sec: 1.11225\tloss: 0.19596\tlr:[2.9444975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:41:13,754 [ hooks.py: 207]:\tstep: 30280\tsteps/sec: 1.18121\tloss: 0.52030\tlr:[2.9438183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:41:23,940 [ hooks.py: 207]:\tstep: 30290\tsteps/sec: 1.02087\tloss: 0.24231\tlr:[2.9431392e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:41:31,314 [ hooks.py: 207]:\tstep: 30300\tsteps/sec: 1.31456\tloss: 0.56719\tlr:[2.9424604e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:41:39,729 [ hooks.py: 207]:\tstep: 30310\tsteps/sec: 1.20230\tloss: 0.77532\tlr:[2.9417812e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:41:47,978 [ hooks.py: 207]:\tstep: 30320\tsteps/sec: 1.21247\tloss: 0.32128\tlr:[2.9411023e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:41:56,543 [ hooks.py: 207]:\tstep: 30330\tsteps/sec: 1.15162\tloss: 0.25663\tlr:[2.9404226e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:42:04,852 [ hooks.py: 207]:\tstep: 30340\tsteps/sec: 1.21052\tloss: 0.73756\tlr:[2.9397439e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:42:14,021 [ hooks.py: 207]:\tstep: 30350\tsteps/sec: 1.11174\tloss: 0.54776\tlr:[2.939065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:42:21,146 [ hooks.py: 207]:\tstep: 30360\tsteps/sec: 1.34775\tloss: 0.24967\tlr:[2.9383858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:42:30,115 [ hooks.py: 207]:\tstep: 30370\tsteps/sec: 1.13824\tloss: 0.85254\tlr:[2.9377066e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:42:38,559 [ hooks.py: 207]:\tstep: 30380\tsteps/sec: 1.22312\tloss: 0.55018\tlr:[2.9370278e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:42:46,151 [ hooks.py: 207]:\tstep: 30390\tsteps/sec: 1.25046\tloss: 0.55255\tlr:[2.9363488e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:42:54,991 [ hooks.py: 207]:\tstep: 30400\tsteps/sec: 1.12566\tloss: 0.43302\tlr:[2.9356697e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:43:03,250 [ hooks.py: 207]:\tstep: 30410\tsteps/sec: 1.30463\tloss: 0.18895\tlr:[2.9349905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:43:11,097 [ hooks.py: 207]:\tstep: 30420\tsteps/sec: 1.19696\tloss: 0.46708\tlr:[2.9343115e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:43:19,650 [ hooks.py: 207]:\tstep: 30430\tsteps/sec: 1.19073\tloss: 0.34730\tlr:[2.9336323e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:43:30,423 [ hooks.py: 207]:\tstep: 30440\tsteps/sec: 0.91199\tloss: 0.19754\tlr:[2.9329532e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:43:37,131 [ hooks.py: 207]:\tstep: 30450\tsteps/sec: 1.42772\tloss: 0.31337\tlr:[2.9322746e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:43:45,863 [ hooks.py: 207]:\tstep: 30460\tsteps/sec: 1.13374\tloss: 0.46471\tlr:[2.9315954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:43:53,880 [ hooks.py: 207]:\tstep: 30470\tsteps/sec: 1.35316\tloss: 0.37307\tlr:[2.9309162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:44:05,039 [ hooks.py: 207]:\tstep: 30480\tsteps/sec: 0.84440\tloss: 0.74843\tlr:[2.9302373e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:44:12,938 [ hooks.py: 207]:\tstep: 30490\tsteps/sec: 1.36463\tloss: 0.36295\tlr:[2.9295581e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:44:20,443 [ hooks.py: 207]:\tstep: 30500\tsteps/sec: 1.30918\tloss: 0.22315\tlr:[2.9288793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:44:29,743 [ hooks.py: 207]:\tstep: 30510\tsteps/sec: 1.08705\tloss: 0.42697\tlr:[2.9282002e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:44:37,996 [ hooks.py: 207]:\tstep: 30520\tsteps/sec: 1.17223\tloss: 0.39721\tlr:[2.9275208e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:44:49,122 [ hooks.py: 207]:\tstep: 30530\tsteps/sec: 0.89521\tloss: 0.28195\tlr:[2.9268422e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:44:57,223 [ hooks.py: 207]:\tstep: 30540\tsteps/sec: 1.21022\tloss: 0.21623\tlr:[2.9261633e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:45:06,684 [ hooks.py: 207]:\tstep: 30550\tsteps/sec: 1.06617\tloss: 0.21938\tlr:[2.9254841e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:45:15,710 [ hooks.py: 207]:\tstep: 30560\tsteps/sec: 1.12577\tloss: 0.51809\tlr:[2.9248049e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:45:26,508 [ hooks.py: 207]:\tstep: 30570\tsteps/sec: 0.94185\tloss: 0.97407\tlr:[2.9241257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:45:33,858 [ hooks.py: 207]:\tstep: 30580\tsteps/sec: 1.38986\tloss: 0.83249\tlr:[2.9234468e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:45:42,664 [ hooks.py: 207]:\tstep: 30590\tsteps/sec: 1.10330\tloss: 0.60096\tlr:[2.9227676e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:45:50,674 [ hooks.py: 207]:\tstep: 30600\tsteps/sec: 1.25956\tloss: 0.39689\tlr:[2.9220884e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:45:59,444 [ hooks.py: 207]:\tstep: 30610\tsteps/sec: 1.17512\tloss: 0.42512\tlr:[2.9214096e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:46:07,940 [ hooks.py: 207]:\tstep: 30620\tsteps/sec: 1.14255\tloss: 0.35304\tlr:[2.9207304e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:46:16,123 [ hooks.py: 207]:\tstep: 30630\tsteps/sec: 1.22736\tloss: 0.53685\tlr:[2.9200515e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:46:25,880 [ hooks.py: 207]:\tstep: 30640\tsteps/sec: 1.01360\tloss: 0.71593\tlr:[2.9193725e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 03:46:34,373 [ hooks.py: 207]:\tstep: 30650\tsteps/sec: 1.21328\tloss: 0.44871\tlr:[2.9186936e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:46:40,635 [ hooks.py: 207]:\tstep: 30660\tsteps/sec: 1.55952\tloss: 0.55772\tlr:[2.918014e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:46:46,853 [ hooks.py: 207]:\tstep: 30670\tsteps/sec: 1.51670\tloss: 0.22288\tlr:[2.917335e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:46:55,677 [ hooks.py: 207]:\tstep: 30680\tsteps/sec: 1.16979\tloss: 0.72031\tlr:[2.916656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:47:07,463 [ hooks.py: 207]:\tstep: 30690\tsteps/sec: 0.85878\tloss: 0.45963\tlr:[2.9159772e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:47:16,414 [ hooks.py: 207]:\tstep: 30700\tsteps/sec: 1.11570\tloss: 0.26739\tlr:[2.9152983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:47:24,987 [ hooks.py: 207]:\tstep: 30710\tsteps/sec: 1.16452\tloss: 0.72071\tlr:[2.9146191e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:47:34,153 [ hooks.py: 207]:\tstep: 30720\tsteps/sec: 1.10663\tloss: 0.48793\tlr:[2.9139403e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:47:42,461 [ hooks.py: 207]:\tstep: 30730\tsteps/sec: 1.17189\tloss: 0.17913\tlr:[2.9132607e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:47:51,695 [ hooks.py: 207]:\tstep: 30740\tsteps/sec: 1.09805\tloss: 0.70297\tlr:[2.9125818e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:00,285 [ hooks.py: 207]:\tstep: 30750\tsteps/sec: 1.16904\tloss: 0.71419\tlr:[2.9119026e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:09,510 [ hooks.py: 207]:\tstep: 30760\tsteps/sec: 1.09925\tloss: 0.30022\tlr:[2.9112238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:15,888 [ hooks.py: 207]:\tstep: 30770\tsteps/sec: 1.43641\tloss: 0.26502\tlr:[2.9105446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:25,447 [ hooks.py: 207]:\tstep: 30780\tsteps/sec: 1.03392\tloss: 0.53999\tlr:[2.9098655e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:34,532 [ hooks.py: 207]:\tstep: 30790\tsteps/sec: 1.17619\tloss: 0.81701\tlr:[2.9091863e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:41,639 [ hooks.py: 207]:\tstep: 30800\tsteps/sec: 1.39154\tloss: 0.52859\tlr:[2.9085073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:49,699 [ hooks.py: 207]:\tstep: 30810\tsteps/sec: 1.25603\tloss: 0.55311\tlr:[2.9078283e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:48:56,767 [ hooks.py: 207]:\tstep: 30820\tsteps/sec: 1.40014\tloss: 0.38351\tlr:[2.907149e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:49:05,219 [ hooks.py: 207]:\tstep: 30830\tsteps/sec: 1.15970\tloss: 0.52328\tlr:[2.9064702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:49:13,848 [ hooks.py: 207]:\tstep: 30840\tsteps/sec: 1.15275\tloss: 0.78546\tlr:[2.9057914e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:49:23,622 [ hooks.py: 207]:\tstep: 30850\tsteps/sec: 1.06001\tloss: 0.54213\tlr:[2.9051122e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:49:33,145 [ hooks.py: 207]:\tstep: 30860\tsteps/sec: 1.04127\tloss: 0.63627\tlr:[2.904433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:49:40,654 [ hooks.py: 207]:\tstep: 30870\tsteps/sec: 1.24795\tloss: 0.37825\tlr:[2.9037541e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:49:51,917 [ hooks.py: 207]:\tstep: 30880\tsteps/sec: 0.92066\tloss: 0.45503\tlr:[2.9030749e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:02,085 [ hooks.py: 207]:\tstep: 30890\tsteps/sec: 1.02368\tloss: 0.29478\tlr:[2.9023957e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:11,474 [ hooks.py: 207]:\tstep: 30900\tsteps/sec: 1.02682\tloss: 0.35255\tlr:[2.9017168e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:18,350 [ hooks.py: 207]:\tstep: 30910\tsteps/sec: 1.47768\tloss: 0.58097\tlr:[2.9010378e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:26,200 [ hooks.py: 207]:\tstep: 30920\tsteps/sec: 1.23178\tloss: 0.29286\tlr:[2.9003586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:34,087 [ hooks.py: 207]:\tstep: 30930\tsteps/sec: 1.31739\tloss: 0.39332\tlr:[2.8996794e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:41,786 [ hooks.py: 207]:\tstep: 30940\tsteps/sec: 1.27520\tloss: 0.46493\tlr:[2.8990005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:49,780 [ hooks.py: 207]:\tstep: 30950\tsteps/sec: 1.27835\tloss: 0.19431\tlr:[2.8983217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:50:56,773 [ hooks.py: 207]:\tstep: 30960\tsteps/sec: 1.33292\tloss: 0.40729\tlr:[2.8976428e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:06,256 [ hooks.py: 207]:\tstep: 30970\tsteps/sec: 1.05731\tloss: 0.34673\tlr:[2.8969636e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:15,222 [ hooks.py: 207]:\tstep: 30980\tsteps/sec: 1.16479\tloss: 0.29308\tlr:[2.8962844e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:23,294 [ hooks.py: 207]:\tstep: 30990\tsteps/sec: 1.24573\tloss: 0.22439\tlr:[2.8956056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:30,162 [ hooks.py: 207]:\tstep: 31000\tsteps/sec: 1.44008\tloss: 0.51030\tlr:[2.8949264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:30,163 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:30,163 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:30,163 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:51:30,164 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:30,164 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:51:30,164 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:59:32,383 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,385 [ trainer.py: 58]:\t{'acc': 0.7779116465863454, 'loss': 0.5712448086780615}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,385 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,386 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:59:32,386 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:59:32,386 [ trainer.py: 70]:\tacc\t0.7779116465863454\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:59:32,386 [ trainer.py: 70]:\tloss\t0.5712448086780615\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 03:59:32,386 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,387 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.5712448086780615}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,387 [ exporter.py: 64]:\t[Best Exporter]: skip step 31000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,387 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.5712448086780615}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,387 [ exporter.py: 104]:\t[Best Exporter]: skip step 31000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:32,388 [monitored_executor.py: 122]:\tsaving step 31000 to ./output/model_31000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:35,855 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_28000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:43,646 [ hooks.py: 207]:\tstep: 31010\tsteps/sec: 0.02027\tloss: 0.38724\tlr:[2.8942473e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:50,715 [ hooks.py: 207]:\tstep: 31020\tsteps/sec: 1.36655\tloss: 0.58914\tlr:[2.8935685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 03:59:58,752 [ hooks.py: 207]:\tstep: 31030\tsteps/sec: 1.26524\tloss: 0.30504\tlr:[2.8928893e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:00:07,122 [ hooks.py: 207]:\tstep: 31040\tsteps/sec: 1.18189\tloss: 0.74855\tlr:[2.89221e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:00:14,033 [ hooks.py: 207]:\tstep: 31050\tsteps/sec: 1.49058\tloss: 0.40797\tlr:[2.8915312e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:00:21,056 [ hooks.py: 207]:\tstep: 31060\tsteps/sec: 1.31378\tloss: 0.40123\tlr:[2.8908524e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:00:31,179 [ hooks.py: 207]:\tstep: 31070\tsteps/sec: 0.99373\tloss: 0.49709\tlr:[2.8901728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:00:40,096 [ hooks.py: 207]:\tstep: 31080\tsteps/sec: 1.19135\tloss: 0.73998\tlr:[2.889494e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:00:49,533 [ hooks.py: 207]:\tstep: 31090\tsteps/sec: 1.05461\tloss: 0.33481\tlr:[2.8888151e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:00:56,311 [ hooks.py: 207]:\tstep: 31100\tsteps/sec: 1.37022\tloss: 0.59458\tlr:[2.888136e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 04:01:04,916 [ hooks.py: 207]:\tstep: 31110\tsteps/sec: 1.16974\tloss: 0.52829\tlr:[2.8874567e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:01:14,752 [ hooks.py: 207]:\tstep: 31120\tsteps/sec: 1.07215\tloss: 0.24035\tlr:[2.8867775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:01:23,686 [ hooks.py: 207]:\tstep: 31130\tsteps/sec: 1.10965\tloss: 0.56440\tlr:[2.8860988e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:01:30,878 [ hooks.py: 207]:\tstep: 31140\tsteps/sec: 1.29659\tloss: 0.26064\tlr:[2.88542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:01:40,433 [ hooks.py: 207]:\tstep: 31150\tsteps/sec: 1.10705\tloss: 0.30641\tlr:[2.8847408e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:01:47,339 [ hooks.py: 207]:\tstep: 31160\tsteps/sec: 1.43599\tloss: 0.77338\tlr:[2.8840615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:01:54,452 [ hooks.py: 207]:\tstep: 31170\tsteps/sec: 1.35632\tloss: 0.29251\tlr:[2.8833827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:02:02,986 [ hooks.py: 207]:\tstep: 31180\tsteps/sec: 1.26056\tloss: 0.48954\tlr:[2.8827035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:02:10,988 [ hooks.py: 207]:\tstep: 31190\tsteps/sec: 1.19098\tloss: 0.69256\tlr:[2.8820243e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:02:18,643 [ hooks.py: 207]:\tstep: 31200\tsteps/sec: 1.31886\tloss: 0.60175\tlr:[2.8813454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:02:27,919 [ hooks.py: 207]:\tstep: 31210\tsteps/sec: 1.03672\tloss: 0.61100\tlr:[2.8806664e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:02:37,069 [ hooks.py: 207]:\tstep: 31220\tsteps/sec: 1.13007\tloss: 0.63022\tlr:[2.879987e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:02:46,663 [ hooks.py: 207]:\tstep: 31230\tsteps/sec: 1.04051\tloss: 0.52916\tlr:[2.8793082e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:02:53,694 [ hooks.py: 207]:\tstep: 31240\tsteps/sec: 1.45525\tloss: 0.51342\tlr:[2.8786291e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:03:01,797 [ hooks.py: 207]:\tstep: 31250\tsteps/sec: 1.21134\tloss: 0.50171\tlr:[2.87795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:03:11,403 [ hooks.py: 207]:\tstep: 31260\tsteps/sec: 1.04791\tloss: 0.64915\tlr:[2.877271e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:03:19,887 [ hooks.py: 207]:\tstep: 31270\tsteps/sec: 1.19629\tloss: 0.92491\tlr:[2.8765919e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:03:26,529 [ hooks.py: 207]:\tstep: 31280\tsteps/sec: 1.39508\tloss: 0.56084\tlr:[2.875913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:03:34,570 [ hooks.py: 207]:\tstep: 31290\tsteps/sec: 1.31505\tloss: 0.24550\tlr:[2.8752342e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:03:44,370 [ hooks.py: 207]:\tstep: 31300\tsteps/sec: 0.99846\tloss: 0.47859\tlr:[2.874555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:03:55,355 [ hooks.py: 207]:\tstep: 31310\tsteps/sec: 0.88856\tloss: 0.81283\tlr:[2.8738757e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:04:04,509 [ hooks.py: 207]:\tstep: 31320\tsteps/sec: 1.11776\tloss: 0.60130\tlr:[2.8731969e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:04:12,117 [ hooks.py: 207]:\tstep: 31330\tsteps/sec: 1.36151\tloss: 0.70592\tlr:[2.8725173e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:04:20,221 [ hooks.py: 207]:\tstep: 31340\tsteps/sec: 1.24594\tloss: 0.65817\tlr:[2.8718381e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:04:29,476 [ hooks.py: 207]:\tstep: 31350\tsteps/sec: 1.08609\tloss: 0.45828\tlr:[2.8711593e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:04:36,218 [ hooks.py: 207]:\tstep: 31360\tsteps/sec: 1.36937\tloss: 0.32549\tlr:[2.8704804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:04:45,090 [ hooks.py: 207]:\tstep: 31370\tsteps/sec: 1.19998\tloss: 0.56036\tlr:[2.8698012e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:04:52,013 [ hooks.py: 207]:\tstep: 31380\tsteps/sec: 1.41302\tloss: 0.45545\tlr:[2.8691222e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:02,396 [ hooks.py: 207]:\tstep: 31390\tsteps/sec: 1.00048\tloss: 0.56306\tlr:[2.8684433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:09,735 [ hooks.py: 207]:\tstep: 31400\tsteps/sec: 1.31833\tloss: 0.62098\tlr:[2.8677641e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:18,578 [ hooks.py: 207]:\tstep: 31410\tsteps/sec: 1.12845\tloss: 0.49259\tlr:[2.867085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:27,818 [ hooks.py: 207]:\tstep: 31420\tsteps/sec: 1.05187\tloss: 0.80420\tlr:[2.8664057e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:34,882 [ hooks.py: 207]:\tstep: 31430\tsteps/sec: 1.47209\tloss: 0.59985\tlr:[2.8657269e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:43,070 [ hooks.py: 207]:\tstep: 31440\tsteps/sec: 1.19794\tloss: 0.71565\tlr:[2.865048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:49,804 [ hooks.py: 207]:\tstep: 31450\tsteps/sec: 1.50654\tloss: 0.46872\tlr:[2.8643688e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:05:57,960 [ hooks.py: 207]:\tstep: 31460\tsteps/sec: 1.20191\tloss: 0.54606\tlr:[2.8636896e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:06:08,741 [ hooks.py: 207]:\tstep: 31470\tsteps/sec: 0.94737\tloss: 0.25131\tlr:[2.8630107e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:06:15,972 [ hooks.py: 207]:\tstep: 31480\tsteps/sec: 1.26202\tloss: 0.57821\tlr:[2.8623315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:06:23,595 [ hooks.py: 207]:\tstep: 31490\tsteps/sec: 1.35913\tloss: 0.74675\tlr:[2.8616523e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:06:33,493 [ hooks.py: 207]:\tstep: 31500\tsteps/sec: 1.04921\tloss: 0.40899\tlr:[2.8609735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:06:43,126 [ hooks.py: 207]:\tstep: 31510\tsteps/sec: 1.03479\tloss: 0.33866\tlr:[2.8602944e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:06:51,243 [ hooks.py: 207]:\tstep: 31520\tsteps/sec: 1.20646\tloss: 0.32635\tlr:[2.8596152e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:06:59,588 [ hooks.py: 207]:\tstep: 31530\tsteps/sec: 1.26178\tloss: 0.54578\tlr:[2.8589362e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:07:06,944 [ hooks.py: 207]:\tstep: 31540\tsteps/sec: 1.30740\tloss: 0.26077\tlr:[2.8582572e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:07:17,137 [ hooks.py: 207]:\tstep: 31550\tsteps/sec: 0.95535\tloss: 0.44724\tlr:[2.8575783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:07:25,960 [ hooks.py: 207]:\tstep: 31560\tsteps/sec: 1.16426\tloss: 0.24031\tlr:[2.8568991e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:07:34,009 [ hooks.py: 207]:\tstep: 31570\tsteps/sec: 1.21413\tloss: 0.51061\tlr:[2.8562199e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:07:43,853 [ hooks.py: 207]:\tstep: 31580\tsteps/sec: 1.05667\tloss: 0.46040\tlr:[2.855541e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:07:50,832 [ hooks.py: 207]:\tstep: 31590\tsteps/sec: 1.41094\tloss: 0.29057\tlr:[2.8548622e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:08:00,959 [ hooks.py: 207]:\tstep: 31600\tsteps/sec: 0.99398\tloss: 0.52025\tlr:[2.854183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:08:08,476 [ hooks.py: 207]:\tstep: 31610\tsteps/sec: 1.27342\tloss: 0.46335\tlr:[2.8535038e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:08:17,125 [ hooks.py: 207]:\tstep: 31620\tsteps/sec: 1.19100\tloss: 0.83770\tlr:[2.8528251e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:08:25,644 [ hooks.py: 207]:\tstep: 31630\tsteps/sec: 1.16190\tloss: 0.52147\tlr:[2.8521457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:08:33,787 [ hooks.py: 207]:\tstep: 31640\tsteps/sec: 1.25347\tloss: 0.40023\tlr:[2.8514665e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:08:42,941 [ hooks.py: 207]:\tstep: 31650\tsteps/sec: 1.06669\tloss: 0.36133\tlr:[2.8507875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:08:52,540 [ hooks.py: 207]:\tstep: 31660\tsteps/sec: 1.04987\tloss: 0.67894\tlr:[2.8501086e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:09:02,497 [ hooks.py: 207]:\tstep: 31670\tsteps/sec: 1.04465\tloss: 0.45045\tlr:[2.8494294e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:09:10,354 [ hooks.py: 207]:\tstep: 31680\tsteps/sec: 1.23020\tloss: 0.33990\tlr:[2.8487502e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:09:18,567 [ hooks.py: 207]:\tstep: 31690\tsteps/sec: 1.20893\tloss: 0.52302\tlr:[2.8480714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:09:27,883 [ hooks.py: 207]:\tstep: 31700\tsteps/sec: 1.06313\tloss: 0.31299\tlr:[2.8473925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:09:34,726 [ hooks.py: 207]:\tstep: 31710\tsteps/sec: 1.50389\tloss: 0.33511\tlr:[2.8467133e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:09:45,299 [ hooks.py: 207]:\tstep: 31720\tsteps/sec: 0.95422\tloss: 0.63916\tlr:[2.8460341e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:09:55,725 [ hooks.py: 207]:\tstep: 31730\tsteps/sec: 0.90464\tloss: 0.25270\tlr:[2.8453553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:10:07,435 [ hooks.py: 207]:\tstep: 31740\tsteps/sec: 0.88848\tloss: 0.94885\tlr:[2.8446762e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:10:16,085 [ hooks.py: 207]:\tstep: 31750\tsteps/sec: 1.13752\tloss: 0.29164\tlr:[2.8439974e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 04:10:24,582 [ hooks.py: 207]:\tstep: 31760\tsteps/sec: 1.21561\tloss: 0.38547\tlr:[2.843318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:10:32,099 [ hooks.py: 207]:\tstep: 31770\tsteps/sec: 1.28582\tloss: 0.46822\tlr:[2.842639e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:10:39,362 [ hooks.py: 207]:\tstep: 31780\tsteps/sec: 1.43916\tloss: 0.27509\tlr:[2.8419598e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:10:47,441 [ hooks.py: 207]:\tstep: 31790\tsteps/sec: 1.20974\tloss: 0.15678\tlr:[2.841281e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:10:55,219 [ hooks.py: 207]:\tstep: 31800\tsteps/sec: 1.23133\tloss: 0.61360\tlr:[2.840602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:11:04,914 [ hooks.py: 207]:\tstep: 31810\tsteps/sec: 1.07030\tloss: 0.48880\tlr:[2.8399232e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:11:12,960 [ hooks.py: 207]:\tstep: 31820\tsteps/sec: 1.17950\tloss: 1.07818\tlr:[2.8392436e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:11:23,404 [ hooks.py: 207]:\tstep: 31830\tsteps/sec: 1.00003\tloss: 0.28631\tlr:[2.8385648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:11:33,542 [ hooks.py: 207]:\tstep: 31840\tsteps/sec: 0.98232\tloss: 0.41460\tlr:[2.8378858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:11:42,024 [ hooks.py: 207]:\tstep: 31850\tsteps/sec: 1.12767\tloss: 0.28665\tlr:[2.8372067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:11:51,723 [ hooks.py: 207]:\tstep: 31860\tsteps/sec: 1.06873\tloss: 0.53299\tlr:[2.8365275e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:12:00,457 [ hooks.py: 207]:\tstep: 31870\tsteps/sec: 1.18158\tloss: 0.85771\tlr:[2.8358485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:12:09,235 [ hooks.py: 207]:\tstep: 31880\tsteps/sec: 1.09017\tloss: 0.59374\tlr:[2.8351693e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:12:17,395 [ hooks.py: 207]:\tstep: 31890\tsteps/sec: 1.23889\tloss: 0.66056\tlr:[2.8344904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:12:27,502 [ hooks.py: 207]:\tstep: 31900\tsteps/sec: 0.99650\tloss: 0.49315\tlr:[2.833811e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:12:37,374 [ hooks.py: 207]:\tstep: 31910\tsteps/sec: 1.00752\tloss: 0.32216\tlr:[2.8331324e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:12:45,744 [ hooks.py: 207]:\tstep: 31920\tsteps/sec: 1.18876\tloss: 0.44932\tlr:[2.8324532e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:12:57,287 [ hooks.py: 207]:\tstep: 31930\tsteps/sec: 0.84772\tloss: 0.96523\tlr:[2.831774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:07,037 [ hooks.py: 207]:\tstep: 31940\tsteps/sec: 1.05656\tloss: 0.44661\tlr:[2.8310948e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:16,481 [ hooks.py: 207]:\tstep: 31950\tsteps/sec: 1.02347\tloss: 0.53809\tlr:[2.8304159e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:25,827 [ hooks.py: 207]:\tstep: 31960\tsteps/sec: 1.11137\tloss: 0.46220\tlr:[2.829737e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:33,297 [ hooks.py: 207]:\tstep: 31970\tsteps/sec: 1.34074\tloss: 0.56554\tlr:[2.8290578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:44,233 [ hooks.py: 207]:\tstep: 31980\tsteps/sec: 0.93555\tloss: 0.44015\tlr:[2.8283786e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:51,557 [ hooks.py: 207]:\tstep: 31990\tsteps/sec: 1.33428\tloss: 0.82801\tlr:[2.8276998e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:59,619 [ hooks.py: 207]:\tstep: 32000\tsteps/sec: 1.28027\tloss: 0.47942\tlr:[2.8270208e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:59,620 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:59,620 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:59,620 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:13:59,620 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:59,620 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:13:59,620 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:22:02,045 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,045 [ trainer.py: 58]:\t{'acc': 0.7746987951807229, 'loss': 0.5631154232586806}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,046 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,046 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:22:02,046 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:22:02,047 [ trainer.py: 70]:\tacc\t0.7746987951807229\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:22:02,047 [ trainer.py: 70]:\tloss\t0.5631154232586806\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:22:02,047 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,047 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7746987951807229, 'loss': 0.5631154232586806}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,048 [ exporter.py: 64]:\t[Best Exporter]: skip step 32000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,048 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7746987951807229, 'loss': 0.5631154232586806}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,048 [ exporter.py: 104]:\t[Best Exporter]: skip step 32000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:02,049 [monitored_executor.py: 122]:\tsaving step 32000 to ./output/model_32000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:05,264 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_29000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:13,353 [ hooks.py: 207]:\tstep: 32010\tsteps/sec: 0.02024\tloss: 0.60851\tlr:[2.8263416e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:20,680 [ hooks.py: 207]:\tstep: 32020\tsteps/sec: 1.31541\tloss: 0.26227\tlr:[2.8256623e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:29,813 [ hooks.py: 207]:\tstep: 32030\tsteps/sec: 1.16299\tloss: 0.55639\tlr:[2.8249835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:36,901 [ hooks.py: 207]:\tstep: 32040\tsteps/sec: 1.35741\tloss: 0.70651\tlr:[2.8243046e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:46,453 [ hooks.py: 207]:\tstep: 32050\tsteps/sec: 1.04184\tloss: 0.23074\tlr:[2.823625e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:22:56,094 [ hooks.py: 207]:\tstep: 32060\tsteps/sec: 1.04984\tloss: 0.65993\tlr:[2.8229462e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:23:06,964 [ hooks.py: 207]:\tstep: 32070\tsteps/sec: 0.91510\tloss: 0.54625\tlr:[2.8222674e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:23:14,628 [ hooks.py: 207]:\tstep: 32080\tsteps/sec: 1.28986\tloss: 0.44784\tlr:[2.8215882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:23:23,267 [ hooks.py: 207]:\tstep: 32090\tsteps/sec: 1.17564\tloss: 0.70433\tlr:[2.820909e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:23:30,507 [ hooks.py: 207]:\tstep: 32100\tsteps/sec: 1.42078\tloss: 0.39865\tlr:[2.8202301e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:23:38,315 [ hooks.py: 207]:\tstep: 32110\tsteps/sec: 1.19035\tloss: 0.50802\tlr:[2.8195514e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:23:47,386 [ hooks.py: 207]:\tstep: 32120\tsteps/sec: 1.16146\tloss: 0.52583\tlr:[2.818872e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:23:55,724 [ hooks.py: 207]:\tstep: 32130\tsteps/sec: 1.22024\tloss: 0.29792\tlr:[2.818193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:24:03,158 [ hooks.py: 207]:\tstep: 32140\tsteps/sec: 1.40343\tloss: 0.62913\tlr:[2.8175142e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:24:11,589 [ hooks.py: 207]:\tstep: 32150\tsteps/sec: 1.10894\tloss: 0.50692\tlr:[2.8168353e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:24:20,692 [ hooks.py: 207]:\tstep: 32160\tsteps/sec: 1.12816\tloss: 0.62524\tlr:[2.8161561e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:24:30,400 [ hooks.py: 207]:\tstep: 32170\tsteps/sec: 1.03002\tloss: 0.50329\tlr:[2.8154765e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:24:38,172 [ hooks.py: 207]:\tstep: 32180\tsteps/sec: 1.26198\tloss: 0.20295\tlr:[2.814798e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:24:45,816 [ hooks.py: 207]:\tstep: 32190\tsteps/sec: 1.30242\tloss: 0.61066\tlr:[2.814119e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:24:54,406 [ hooks.py: 207]:\tstep: 32200\tsteps/sec: 1.18571\tloss: 0.70851\tlr:[2.8134396e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:25:04,051 [ hooks.py: 207]:\tstep: 32210\tsteps/sec: 1.02573\tloss: 0.62908\tlr:[2.8127606e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 04:25:11,249 [ hooks.py: 207]:\tstep: 32220\tsteps/sec: 1.37149\tloss: 0.68371\tlr:[2.8120816e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:25:17,874 [ hooks.py: 207]:\tstep: 32230\tsteps/sec: 1.51951\tloss: 1.04070\tlr:[2.8114024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:25:26,910 [ hooks.py: 207]:\tstep: 32240\tsteps/sec: 1.09376\tloss: 0.43336\tlr:[2.8107232e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:25:34,301 [ hooks.py: 207]:\tstep: 32250\tsteps/sec: 1.32013\tloss: 0.13046\tlr:[2.8100445e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:25:43,965 [ hooks.py: 207]:\tstep: 32260\tsteps/sec: 1.04943\tloss: 0.42402\tlr:[2.8093657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:25:53,730 [ hooks.py: 207]:\tstep: 32270\tsteps/sec: 1.00857\tloss: 0.57150\tlr:[2.808686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:26:02,335 [ hooks.py: 207]:\tstep: 32280\tsteps/sec: 1.26167\tloss: 0.63108\tlr:[2.8080072e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:26:11,978 [ hooks.py: 207]:\tstep: 32290\tsteps/sec: 1.00011\tloss: 0.48239\tlr:[2.807328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:26:20,599 [ hooks.py: 207]:\tstep: 32300\tsteps/sec: 1.19737\tloss: 0.61412\tlr:[2.8066492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:26:29,068 [ hooks.py: 207]:\tstep: 32310\tsteps/sec: 1.15597\tloss: 0.54881\tlr:[2.80597e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:26:36,229 [ hooks.py: 207]:\tstep: 32320\tsteps/sec: 1.36275\tloss: 0.57760\tlr:[2.8052911e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:26:45,918 [ hooks.py: 207]:\tstep: 32330\tsteps/sec: 1.06050\tloss: 0.28958\tlr:[2.8046119e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:26:52,742 [ hooks.py: 207]:\tstep: 32340\tsteps/sec: 1.43384\tloss: 0.55720\tlr:[2.8039332e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:01,100 [ hooks.py: 207]:\tstep: 32350\tsteps/sec: 1.25350\tloss: 0.46133\tlr:[2.8032538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:09,729 [ hooks.py: 207]:\tstep: 32360\tsteps/sec: 1.09444\tloss: 0.21415\tlr:[2.8025748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:19,140 [ hooks.py: 207]:\tstep: 32370\tsteps/sec: 1.07232\tloss: 0.52390\tlr:[2.801896e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:26,342 [ hooks.py: 207]:\tstep: 32380\tsteps/sec: 1.36021\tloss: 0.66808\tlr:[2.8012164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:34,470 [ hooks.py: 207]:\tstep: 32390\tsteps/sec: 1.28992\tloss: 0.43942\tlr:[2.8005374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:41,349 [ hooks.py: 207]:\tstep: 32400\tsteps/sec: 1.35033\tloss: 0.97944\tlr:[2.7998583e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:50,871 [ hooks.py: 207]:\tstep: 32410\tsteps/sec: 1.11043\tloss: 0.49024\tlr:[2.7991795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:27:58,413 [ hooks.py: 207]:\tstep: 32420\tsteps/sec: 1.31849\tloss: 0.37745\tlr:[2.7985006e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:28:06,840 [ hooks.py: 207]:\tstep: 32430\tsteps/sec: 1.15609\tloss: 0.52936\tlr:[2.797821e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:28:15,289 [ hooks.py: 207]:\tstep: 32440\tsteps/sec: 1.21873\tloss: 0.42246\tlr:[2.7971422e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:28:24,447 [ hooks.py: 207]:\tstep: 32450\tsteps/sec: 1.09263\tloss: 0.51352\tlr:[2.7964634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:28:35,209 [ hooks.py: 207]:\tstep: 32460\tsteps/sec: 0.92851\tloss: 0.29145\tlr:[2.7957843e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:28:44,579 [ hooks.py: 207]:\tstep: 32470\tsteps/sec: 1.07412\tloss: 0.41192\tlr:[2.795105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:28:52,087 [ hooks.py: 207]:\tstep: 32480\tsteps/sec: 1.28753\tloss: 0.34988\tlr:[2.794426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:29:00,511 [ hooks.py: 207]:\tstep: 32490\tsteps/sec: 1.23085\tloss: 0.34119\tlr:[2.7937469e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:29:09,957 [ hooks.py: 207]:\tstep: 32500\tsteps/sec: 1.05037\tloss: 0.35381\tlr:[2.7930677e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:29:20,187 [ hooks.py: 207]:\tstep: 32510\tsteps/sec: 0.94282\tloss: 0.46002\tlr:[2.7923887e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:29:29,100 [ hooks.py: 207]:\tstep: 32520\tsteps/sec: 1.16319\tloss: 0.27978\tlr:[2.7917098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:29:36,587 [ hooks.py: 207]:\tstep: 32530\tsteps/sec: 1.29497\tloss: 0.94924\tlr:[2.7910306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:29:45,340 [ hooks.py: 207]:\tstep: 32540\tsteps/sec: 1.19547\tloss: 0.39344\tlr:[2.7903514e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:29:54,135 [ hooks.py: 207]:\tstep: 32550\tsteps/sec: 1.07279\tloss: 0.47581\tlr:[2.7896725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:30:02,654 [ hooks.py: 207]:\tstep: 32560\tsteps/sec: 1.27154\tloss: 0.23805\tlr:[2.7889937e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:30:11,320 [ hooks.py: 207]:\tstep: 32570\tsteps/sec: 1.10706\tloss: 0.77636\tlr:[2.7883145e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:30:19,003 [ hooks.py: 207]:\tstep: 32580\tsteps/sec: 1.29677\tloss: 0.40850\tlr:[2.7876353e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:30:29,542 [ hooks.py: 207]:\tstep: 32590\tsteps/sec: 0.95760\tloss: 0.62589\tlr:[2.7869564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:30:37,138 [ hooks.py: 207]:\tstep: 32600\tsteps/sec: 1.30490\tloss: 0.33675\tlr:[2.7862774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:30:45,997 [ hooks.py: 207]:\tstep: 32610\tsteps/sec: 1.08831\tloss: 0.44739\tlr:[2.7855982e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:30:54,661 [ hooks.py: 207]:\tstep: 32620\tsteps/sec: 1.23080\tloss: 0.67133\tlr:[2.7849192e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:31:01,393 [ hooks.py: 207]:\tstep: 32630\tsteps/sec: 1.40299\tloss: 0.65504\tlr:[2.7842401e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:31:11,118 [ hooks.py: 207]:\tstep: 32640\tsteps/sec: 1.00095\tloss: 0.25436\tlr:[2.783561e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:31:19,597 [ hooks.py: 207]:\tstep: 32650\tsteps/sec: 1.26093\tloss: 0.22185\tlr:[2.7828817e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:31:30,882 [ hooks.py: 207]:\tstep: 32660\tsteps/sec: 0.88840\tloss: 0.35042\tlr:[2.7822029e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:31:39,220 [ hooks.py: 207]:\tstep: 32670\tsteps/sec: 1.14950\tloss: 0.75077\tlr:[2.781524e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:31:47,623 [ hooks.py: 207]:\tstep: 32680\tsteps/sec: 1.21877\tloss: 0.65151\tlr:[2.7808448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:31:55,375 [ hooks.py: 207]:\tstep: 32690\tsteps/sec: 1.32554\tloss: 0.36770\tlr:[2.7801656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:32:02,789 [ hooks.py: 207]:\tstep: 32700\tsteps/sec: 1.38156\tloss: 0.14380\tlr:[2.7794867e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:32:14,478 [ hooks.py: 207]:\tstep: 32710\tsteps/sec: 0.83798\tloss: 0.41177\tlr:[2.7788077e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:32:21,439 [ hooks.py: 207]:\tstep: 32720\tsteps/sec: 1.33438\tloss: 0.19273\tlr:[2.7781287e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:32:30,676 [ hooks.py: 207]:\tstep: 32730\tsteps/sec: 1.15621\tloss: 0.56456\tlr:[2.7774495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:32:38,277 [ hooks.py: 207]:\tstep: 32740\tsteps/sec: 1.27087\tloss: 0.60327\tlr:[2.7767705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:32:47,106 [ hooks.py: 207]:\tstep: 32750\tsteps/sec: 1.12247\tloss: 0.44527\tlr:[2.7760916e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:32:57,410 [ hooks.py: 207]:\tstep: 32760\tsteps/sec: 0.99201\tloss: 0.68579\tlr:[2.7754124e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:33:07,317 [ hooks.py: 207]:\tstep: 32770\tsteps/sec: 0.97289\tloss: 0.40919\tlr:[2.7747332e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:33:17,707 [ hooks.py: 207]:\tstep: 32780\tsteps/sec: 0.99145\tloss: 0.33151\tlr:[2.7740543e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:33:23,984 [ hooks.py: 207]:\tstep: 32790\tsteps/sec: 1.59467\tloss: 0.33089\tlr:[2.7733748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:33:32,380 [ hooks.py: 207]:\tstep: 32800\tsteps/sec: 1.18713\tloss: 0.72727\tlr:[2.772696e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:33:40,437 [ hooks.py: 207]:\tstep: 32810\tsteps/sec: 1.27705\tloss: 0.20545\tlr:[2.772017e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:33:47,628 [ hooks.py: 207]:\tstep: 32820\tsteps/sec: 1.34576\tloss: 0.71746\tlr:[2.7713382e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:33:56,624 [ hooks.py: 207]:\tstep: 32830\tsteps/sec: 1.11139\tloss: 0.28232\tlr:[2.770659e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:34:06,973 [ hooks.py: 207]:\tstep: 32840\tsteps/sec: 0.96463\tloss: 0.54803\tlr:[2.76998e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:34:16,230 [ hooks.py: 207]:\tstep: 32850\tsteps/sec: 1.08272\tloss: 0.68174\tlr:[2.769301e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:34:25,771 [ hooks.py: 207]:\tstep: 32860\tsteps/sec: 1.07235\tloss: 0.35747\tlr:[2.768622e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 04:34:34,657 [ hooks.py: 207]:\tstep: 32870\tsteps/sec: 1.12546\tloss: 0.51909\tlr:[2.767943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:34:41,531 [ hooks.py: 207]:\tstep: 32880\tsteps/sec: 1.42478\tloss: 0.51069\tlr:[2.7672639e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:34:50,024 [ hooks.py: 207]:\tstep: 32890\tsteps/sec: 1.19458\tloss: 0.77445\tlr:[2.766585e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:34:58,795 [ hooks.py: 207]:\tstep: 32900\tsteps/sec: 1.11307\tloss: 0.26786\tlr:[2.7659058e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:35:07,727 [ hooks.py: 207]:\tstep: 32910\tsteps/sec: 1.07459\tloss: 0.30913\tlr:[2.7652266e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:35:16,110 [ hooks.py: 207]:\tstep: 32920\tsteps/sec: 1.26854\tloss: 0.56221\tlr:[2.7645474e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:35:22,827 [ hooks.py: 207]:\tstep: 32930\tsteps/sec: 1.49099\tloss: 0.56701\tlr:[2.7638687e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:35:29,791 [ hooks.py: 207]:\tstep: 32940\tsteps/sec: 1.39187\tloss: 0.67190\tlr:[2.7631895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:35:38,177 [ hooks.py: 207]:\tstep: 32950\tsteps/sec: 1.21506\tloss: 0.99567\tlr:[2.7625105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:35:45,977 [ hooks.py: 207]:\tstep: 32960\tsteps/sec: 1.29202\tloss: 0.36194\tlr:[2.7618315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:35:53,937 [ hooks.py: 207]:\tstep: 32970\tsteps/sec: 1.20152\tloss: 0.30420\tlr:[2.7611526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:05,755 [ hooks.py: 207]:\tstep: 32980\tsteps/sec: 0.84638\tloss: 0.40372\tlr:[2.7604734e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:15,447 [ hooks.py: 207]:\tstep: 32990\tsteps/sec: 1.09244\tloss: 0.55465\tlr:[2.7597942e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:22,573 [ hooks.py: 207]:\tstep: 33000\tsteps/sec: 1.29501\tloss: 0.24824\tlr:[2.7591153e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:22,577 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:22,577 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:22,578 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:36:22,578 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:22,578 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:36:22,578 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:44:17,691 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,693 [ trainer.py: 58]:\t{'acc': 0.7815261044176707, 'loss': 0.5694027551664755}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,693 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,694 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:44:17,694 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:44:17,694 [ trainer.py: 70]:\tacc\t0.7815261044176707\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:44:17,694 [ trainer.py: 70]:\tloss\t0.5694027551664755\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:44:17,694 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,694 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7815261044176707, 'loss': 0.5694027551664755}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,694 [ exporter.py: 64]:\t[Best Exporter]: skip step 33000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,695 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7815261044176707, 'loss': 0.5694027551664755}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,695 [ exporter.py: 104]:\t[Best Exporter]: skip step 33000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:17,696 [monitored_executor.py: 122]:\tsaving step 33000 to ./output/model_33000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:21,057 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_30000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:30,076 [ hooks.py: 207]:\tstep: 33010\tsteps/sec: 0.02054\tloss: 0.75038\tlr:[2.7584365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:36,621 [ hooks.py: 207]:\tstep: 33020\tsteps/sec: 1.47274\tloss: 0.63739\tlr:[2.7577573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:46,064 [ hooks.py: 207]:\tstep: 33030\tsteps/sec: 1.08846\tloss: 0.45066\tlr:[2.757078e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:44:54,912 [ hooks.py: 207]:\tstep: 33040\tsteps/sec: 1.13016\tloss: 0.29145\tlr:[2.7563992e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:45:02,805 [ hooks.py: 207]:\tstep: 33050\tsteps/sec: 1.29645\tloss: 0.63056\tlr:[2.75572e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:45:10,393 [ hooks.py: 207]:\tstep: 33060\tsteps/sec: 1.29588\tloss: 0.52998\tlr:[2.755041e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:45:17,264 [ hooks.py: 207]:\tstep: 33070\tsteps/sec: 1.39304\tloss: 0.69464\tlr:[2.7543618e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:45:23,513 [ hooks.py: 207]:\tstep: 33080\tsteps/sec: 1.51260\tloss: 0.64860\tlr:[2.753683e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:45:32,169 [ hooks.py: 207]:\tstep: 33090\tsteps/sec: 1.20110\tloss: 0.20500\tlr:[2.7530035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:45:41,515 [ hooks.py: 207]:\tstep: 33100\tsteps/sec: 1.02904\tloss: 0.36309\tlr:[2.7523243e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:45:50,834 [ hooks.py: 207]:\tstep: 33110\tsteps/sec: 1.15253\tloss: 0.52670\tlr:[2.7516457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:46:00,508 [ hooks.py: 207]:\tstep: 33120\tsteps/sec: 0.99612\tloss: 0.57512\tlr:[2.7509665e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:46:10,072 [ hooks.py: 207]:\tstep: 33130\tsteps/sec: 1.05434\tloss: 0.69744\tlr:[2.7502872e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:46:18,887 [ hooks.py: 207]:\tstep: 33140\tsteps/sec: 1.13781\tloss: 0.79769\tlr:[2.749608e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:46:27,056 [ hooks.py: 207]:\tstep: 33150\tsteps/sec: 1.18729\tloss: 0.27819\tlr:[2.7489292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:46:34,371 [ hooks.py: 207]:\tstep: 33160\tsteps/sec: 1.43379\tloss: 0.61026\tlr:[2.7482503e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:46:43,536 [ hooks.py: 207]:\tstep: 33170\tsteps/sec: 1.09135\tloss: 0.43400\tlr:[2.7475711e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:46:50,805 [ hooks.py: 207]:\tstep: 33180\tsteps/sec: 1.40217\tloss: 0.26072\tlr:[2.746892e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:47:00,829 [ hooks.py: 207]:\tstep: 33190\tsteps/sec: 0.97138\tloss: 0.34757\tlr:[2.746213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:47:09,924 [ hooks.py: 207]:\tstep: 33200\tsteps/sec: 1.11710\tloss: 0.38694\tlr:[2.745534e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:47:18,436 [ hooks.py: 207]:\tstep: 33210\tsteps/sec: 1.17722\tloss: 0.46527\tlr:[2.7448548e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:47:27,153 [ hooks.py: 207]:\tstep: 33220\tsteps/sec: 1.13491\tloss: 0.63566\tlr:[2.7441758e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:47:35,699 [ hooks.py: 207]:\tstep: 33230\tsteps/sec: 1.18233\tloss: 0.51478\tlr:[2.7434971e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:47:44,464 [ hooks.py: 207]:\tstep: 33240\tsteps/sec: 1.15028\tloss: 0.60280\tlr:[2.7428176e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:47:53,793 [ hooks.py: 207]:\tstep: 33250\tsteps/sec: 1.05024\tloss: 0.23563\tlr:[2.7421383e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:48:02,683 [ hooks.py: 207]:\tstep: 33260\tsteps/sec: 1.17996\tloss: 0.37253\tlr:[2.7414595e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:48:10,412 [ hooks.py: 207]:\tstep: 33270\tsteps/sec: 1.22118\tloss: 0.75003\tlr:[2.7407807e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:48:19,956 [ hooks.py: 207]:\tstep: 33280\tsteps/sec: 1.07525\tloss: 0.71814\tlr:[2.7401014e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:48:27,590 [ hooks.py: 207]:\tstep: 33290\tsteps/sec: 1.27435\tloss: 0.22756\tlr:[2.7394222e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:48:37,072 [ hooks.py: 207]:\tstep: 33300\tsteps/sec: 1.04450\tloss: 0.45686\tlr:[2.7387434e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:48:47,526 [ hooks.py: 207]:\tstep: 33310\tsteps/sec: 0.96554\tloss: 0.81285\tlr:[2.7380645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:48:57,621 [ hooks.py: 207]:\tstep: 33320\tsteps/sec: 0.99392\tloss: 0.88166\tlr:[2.7373853e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 04:49:07,598 [ hooks.py: 207]:\tstep: 33330\tsteps/sec: 1.00716\tloss: 0.24464\tlr:[2.7367061e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:49:17,588 [ hooks.py: 207]:\tstep: 33340\tsteps/sec: 0.97147\tloss: 0.12885\tlr:[2.736027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:49:27,557 [ hooks.py: 207]:\tstep: 33350\tsteps/sec: 1.02687\tloss: 0.27589\tlr:[2.7353482e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:49:37,936 [ hooks.py: 207]:\tstep: 33360\tsteps/sec: 0.97188\tloss: 0.52817\tlr:[2.7346694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:49:45,599 [ hooks.py: 207]:\tstep: 33370\tsteps/sec: 1.27091\tloss: 0.88577\tlr:[2.7339898e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:49:55,090 [ hooks.py: 207]:\tstep: 33380\tsteps/sec: 1.05236\tloss: 0.30530\tlr:[2.733311e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:50:05,716 [ hooks.py: 207]:\tstep: 33390\tsteps/sec: 0.97973\tloss: 0.73615\tlr:[2.7326318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:50:13,860 [ hooks.py: 207]:\tstep: 33400\tsteps/sec: 1.14304\tloss: 0.25822\tlr:[2.7319526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:50:25,191 [ hooks.py: 207]:\tstep: 33410\tsteps/sec: 0.93186\tloss: 0.60023\tlr:[2.7312737e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:50:33,076 [ hooks.py: 207]:\tstep: 33420\tsteps/sec: 1.22020\tloss: 0.54043\tlr:[2.7305949e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:50:41,250 [ hooks.py: 207]:\tstep: 33430\tsteps/sec: 1.25720\tloss: 0.45304\tlr:[2.7299158e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:50:50,697 [ hooks.py: 207]:\tstep: 33440\tsteps/sec: 1.07156\tloss: 0.15842\tlr:[2.7292368e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:50:58,471 [ hooks.py: 207]:\tstep: 33450\tsteps/sec: 1.24550\tloss: 0.15642\tlr:[2.7285578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:51:08,430 [ hooks.py: 207]:\tstep: 33460\tsteps/sec: 0.97050\tloss: 0.26605\tlr:[2.727879e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:51:18,591 [ hooks.py: 207]:\tstep: 33470\tsteps/sec: 1.02457\tloss: 0.54225\tlr:[2.7271997e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:51:26,971 [ hooks.py: 207]:\tstep: 33480\tsteps/sec: 1.18580\tloss: 0.67704\tlr:[2.7265205e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:51:35,443 [ hooks.py: 207]:\tstep: 33490\tsteps/sec: 1.21012\tloss: 0.68098\tlr:[2.7258417e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:51:46,555 [ hooks.py: 207]:\tstep: 33500\tsteps/sec: 0.88863\tloss: 0.25452\tlr:[2.7251624e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:51:53,541 [ hooks.py: 207]:\tstep: 33510\tsteps/sec: 1.42516\tloss: 0.39987\tlr:[2.7244836e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:02,754 [ hooks.py: 207]:\tstep: 33520\tsteps/sec: 1.06461\tloss: 0.77239\tlr:[2.723804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:11,273 [ hooks.py: 207]:\tstep: 33530\tsteps/sec: 1.22775\tloss: 0.53155\tlr:[2.7231252e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:17,998 [ hooks.py: 207]:\tstep: 33540\tsteps/sec: 1.38309\tloss: 0.60652\tlr:[2.7224463e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:26,236 [ hooks.py: 207]:\tstep: 33550\tsteps/sec: 1.29720\tloss: 0.23813\tlr:[2.7217671e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:33,571 [ hooks.py: 207]:\tstep: 33560\tsteps/sec: 1.32902\tloss: 0.45263\tlr:[2.7210881e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:42,157 [ hooks.py: 207]:\tstep: 33570\tsteps/sec: 1.14468\tloss: 0.42503\tlr:[2.7204092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:51,374 [ hooks.py: 207]:\tstep: 33580\tsteps/sec: 1.10964\tloss: 0.83941\tlr:[2.71973e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:52:59,566 [ hooks.py: 207]:\tstep: 33590\tsteps/sec: 1.23750\tloss: 0.24921\tlr:[2.7190508e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:53:06,224 [ hooks.py: 207]:\tstep: 33600\tsteps/sec: 1.38991\tloss: 1.21046\tlr:[2.718372e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:53:15,666 [ hooks.py: 207]:\tstep: 33610\tsteps/sec: 1.06748\tloss: 0.56980\tlr:[2.7176928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:53:23,133 [ hooks.py: 207]:\tstep: 33620\tsteps/sec: 1.37198\tloss: 0.25818\tlr:[2.717014e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:53:33,478 [ hooks.py: 207]:\tstep: 33630\tsteps/sec: 0.96741\tloss: 0.19645\tlr:[2.7163347e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:53:41,575 [ hooks.py: 207]:\tstep: 33640\tsteps/sec: 1.24857\tloss: 0.71856\tlr:[2.7156559e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:53:51,535 [ hooks.py: 207]:\tstep: 33650\tsteps/sec: 1.01401\tloss: 0.49954\tlr:[2.7149768e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:53:58,378 [ hooks.py: 207]:\tstep: 33660\tsteps/sec: 1.43155\tloss: 0.36481\tlr:[2.7142976e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:54:05,522 [ hooks.py: 207]:\tstep: 33670\tsteps/sec: 1.44602\tloss: 0.66589\tlr:[2.7136186e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:54:13,550 [ hooks.py: 207]:\tstep: 33680\tsteps/sec: 1.21500\tloss: 0.98061\tlr:[2.7129396e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:54:20,224 [ hooks.py: 207]:\tstep: 33690\tsteps/sec: 1.45815\tloss: 0.62567\tlr:[2.7122604e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:54:29,249 [ hooks.py: 207]:\tstep: 33700\tsteps/sec: 1.08562\tloss: 0.42670\tlr:[2.7115811e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:54:37,176 [ hooks.py: 207]:\tstep: 33710\tsteps/sec: 1.26012\tloss: 0.30950\tlr:[2.710902e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:54:46,741 [ hooks.py: 207]:\tstep: 33720\tsteps/sec: 1.08380\tloss: 0.73668\tlr:[2.7102235e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:54:54,006 [ hooks.py: 207]:\tstep: 33730\tsteps/sec: 1.37690\tloss: 0.44766\tlr:[2.7095442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:02,125 [ hooks.py: 207]:\tstep: 33740\tsteps/sec: 1.29255\tloss: 0.28410\tlr:[2.708865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:10,043 [ hooks.py: 207]:\tstep: 33750\tsteps/sec: 1.23410\tloss: 0.69777\tlr:[2.7081858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:18,216 [ hooks.py: 207]:\tstep: 33760\tsteps/sec: 1.19698\tloss: 0.36423\tlr:[2.7075072e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:26,140 [ hooks.py: 207]:\tstep: 33770\tsteps/sec: 1.22029\tloss: 0.46061\tlr:[2.7068281e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:34,036 [ hooks.py: 207]:\tstep: 33780\tsteps/sec: 1.30123\tloss: 0.58303\tlr:[2.706149e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:43,619 [ hooks.py: 207]:\tstep: 33790\tsteps/sec: 1.05248\tloss: 0.63357\tlr:[2.7054697e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:51,272 [ hooks.py: 207]:\tstep: 33800\tsteps/sec: 1.35271\tloss: 0.42260\tlr:[2.704791e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:55:57,964 [ hooks.py: 207]:\tstep: 33810\tsteps/sec: 1.43031\tloss: 0.64772\tlr:[2.7041118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:56:07,376 [ hooks.py: 207]:\tstep: 33820\tsteps/sec: 1.06101\tloss: 0.75866\tlr:[2.7034326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:56:17,380 [ hooks.py: 207]:\tstep: 33830\tsteps/sec: 1.00500\tloss: 0.34283\tlr:[2.7027534e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:56:26,179 [ hooks.py: 207]:\tstep: 33840\tsteps/sec: 1.16136\tloss: 0.44140\tlr:[2.7020742e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:56:34,754 [ hooks.py: 207]:\tstep: 33850\tsteps/sec: 1.15146\tloss: 0.75099\tlr:[2.7013952e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:56:42,855 [ hooks.py: 207]:\tstep: 33860\tsteps/sec: 1.17100\tloss: 0.58346\tlr:[2.7007161e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:56:51,584 [ hooks.py: 207]:\tstep: 33870\tsteps/sec: 1.14008\tloss: 0.80550\tlr:[2.7000377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:56:59,533 [ hooks.py: 207]:\tstep: 33880\tsteps/sec: 1.28974\tloss: 0.37848\tlr:[2.6993584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:57:08,081 [ hooks.py: 207]:\tstep: 33890\tsteps/sec: 1.17880\tloss: 0.57683\tlr:[2.6986792e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:57:16,074 [ hooks.py: 207]:\tstep: 33900\tsteps/sec: 1.30204\tloss: 0.54372\tlr:[2.6980004e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:57:23,610 [ hooks.py: 207]:\tstep: 33910\tsteps/sec: 1.23496\tloss: 0.27048\tlr:[2.6973214e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:57:32,828 [ hooks.py: 207]:\tstep: 33920\tsteps/sec: 1.12230\tloss: 0.99522\tlr:[2.6966422e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:57:42,879 [ hooks.py: 207]:\tstep: 33930\tsteps/sec: 1.00156\tloss: 0.20690\tlr:[2.695963e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:57:51,444 [ hooks.py: 207]:\tstep: 33940\tsteps/sec: 1.19429\tloss: 0.27539\tlr:[2.695284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:00,027 [ hooks.py: 207]:\tstep: 33950\tsteps/sec: 1.12950\tloss: 0.51234\tlr:[2.6946049e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:10,368 [ hooks.py: 207]:\tstep: 33960\tsteps/sec: 0.99201\tloss: 0.14010\tlr:[2.6939257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:18,763 [ hooks.py: 207]:\tstep: 33970\tsteps/sec: 1.16284\tloss: 0.58362\tlr:[2.6932465e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 04:58:28,237 [ hooks.py: 207]:\tstep: 33980\tsteps/sec: 1.05780\tloss: 0.51309\tlr:[2.6925676e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:36,013 [ hooks.py: 207]:\tstep: 33990\tsteps/sec: 1.31869\tloss: 0.27666\tlr:[2.6918884e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:42,917 [ hooks.py: 207]:\tstep: 34000\tsteps/sec: 1.41493\tloss: 0.35401\tlr:[2.6912092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:42,918 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:42,918 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:42,918 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 04:58:42,918 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:42,918 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 04:58:42,918 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:06:45,540 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,541 [ trainer.py: 58]:\t{'acc': 0.7859437751004016, 'loss': 0.5567878529620476}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,542 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,543 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:06:45,543 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:06:45,543 [ trainer.py: 70]:\tacc\t0.7859437751004016\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:06:45,543 [ trainer.py: 70]:\tloss\t0.5567878529620476\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:06:45,543 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,543 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7859437751004016, 'loss': 0.5567878529620476}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,543 [ exporter.py: 64]:\t[Best Exporter]: skip step 34000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,544 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7859437751004016, 'loss': 0.5567878529620476}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,544 [ exporter.py: 104]:\t[Best Exporter]: skip step 34000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:45,545 [monitored_executor.py: 122]:\tsaving step 34000 to ./output/model_34000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:48,998 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_31000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:06:58,019 [ hooks.py: 207]:\tstep: 34010\tsteps/sec: 0.02019\tloss: 0.57612\tlr:[2.6905303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:07:06,416 [ hooks.py: 207]:\tstep: 34020\tsteps/sec: 1.23376\tloss: 0.49173\tlr:[2.6898515e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:07:14,370 [ hooks.py: 207]:\tstep: 34030\tsteps/sec: 1.21554\tloss: 0.28831\tlr:[2.6891725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:07:23,232 [ hooks.py: 207]:\tstep: 34040\tsteps/sec: 1.13306\tloss: 1.17180\tlr:[2.6884934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:07:29,902 [ hooks.py: 207]:\tstep: 34050\tsteps/sec: 1.45752\tloss: 0.78005\tlr:[2.6878144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:07:40,498 [ hooks.py: 207]:\tstep: 34060\tsteps/sec: 0.98813\tloss: 1.11861\tlr:[2.6871356e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:07:50,497 [ hooks.py: 207]:\tstep: 34070\tsteps/sec: 0.99079\tloss: 0.42489\tlr:[2.6864564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:07:59,258 [ hooks.py: 207]:\tstep: 34080\tsteps/sec: 1.12502\tloss: 0.42458\tlr:[2.6857771e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:08:07,978 [ hooks.py: 207]:\tstep: 34090\tsteps/sec: 1.12791\tloss: 0.37631\tlr:[2.6850983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:08:18,738 [ hooks.py: 207]:\tstep: 34100\tsteps/sec: 0.93884\tloss: 0.34038\tlr:[2.6844195e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:08:28,268 [ hooks.py: 207]:\tstep: 34110\tsteps/sec: 1.04477\tloss: 0.53526\tlr:[2.6837399e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:08:35,639 [ hooks.py: 207]:\tstep: 34120\tsteps/sec: 1.40467\tloss: 0.49999\tlr:[2.683061e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:08:42,600 [ hooks.py: 207]:\tstep: 34130\tsteps/sec: 1.38801\tloss: 0.75254\tlr:[2.6823822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:08:50,786 [ hooks.py: 207]:\tstep: 34140\tsteps/sec: 1.25548\tloss: 0.45552\tlr:[2.681703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:08:59,206 [ hooks.py: 207]:\tstep: 34150\tsteps/sec: 1.16592\tloss: 0.70727\tlr:[2.6810238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:09:08,559 [ hooks.py: 207]:\tstep: 34160\tsteps/sec: 1.06787\tloss: 0.53697\tlr:[2.6803447e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:09:18,628 [ hooks.py: 207]:\tstep: 34170\tsteps/sec: 0.99904\tloss: 0.32904\tlr:[2.6796659e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:09:28,129 [ hooks.py: 207]:\tstep: 34180\tsteps/sec: 1.03947\tloss: 0.19614\tlr:[2.6789867e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:09:38,464 [ hooks.py: 207]:\tstep: 34190\tsteps/sec: 0.96746\tloss: 0.57325\tlr:[2.6783075e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:09:47,306 [ hooks.py: 207]:\tstep: 34200\tsteps/sec: 1.13934\tloss: 0.53161\tlr:[2.6776286e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:09:53,909 [ hooks.py: 207]:\tstep: 34210\tsteps/sec: 1.51112\tloss: 0.38573\tlr:[2.6769498e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:03,016 [ hooks.py: 207]:\tstep: 34220\tsteps/sec: 1.11659\tloss: 0.36514\tlr:[2.6762706e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:11,626 [ hooks.py: 207]:\tstep: 34230\tsteps/sec: 1.16153\tloss: 0.65895\tlr:[2.675591e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:19,711 [ hooks.py: 207]:\tstep: 34240\tsteps/sec: 1.21556\tloss: 0.98307\tlr:[2.6749125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:28,591 [ hooks.py: 207]:\tstep: 34250\tsteps/sec: 1.12682\tloss: 0.92769\tlr:[2.6742335e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:35,349 [ hooks.py: 207]:\tstep: 34260\tsteps/sec: 1.51475\tloss: 0.62697\tlr:[2.673554e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:41,999 [ hooks.py: 207]:\tstep: 34270\tsteps/sec: 1.49039\tloss: 0.28485\tlr:[2.6728749e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:50,382 [ hooks.py: 207]:\tstep: 34280\tsteps/sec: 1.19914\tloss: 0.19329\tlr:[2.6721962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:10:58,005 [ hooks.py: 207]:\tstep: 34290\tsteps/sec: 1.23467\tloss: 0.49550\tlr:[2.6715168e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:11:06,603 [ hooks.py: 207]:\tstep: 34300\tsteps/sec: 1.23917\tloss: 0.39334\tlr:[2.6708376e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:11:15,305 [ hooks.py: 207]:\tstep: 34310\tsteps/sec: 1.13629\tloss: 0.32899\tlr:[2.6701588e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:11:27,107 [ hooks.py: 207]:\tstep: 34320\tsteps/sec: 0.83055\tloss: 0.88711\tlr:[2.6694797e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:11:35,696 [ hooks.py: 207]:\tstep: 34330\tsteps/sec: 1.16307\tloss: 0.36140\tlr:[2.6688005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:11:43,909 [ hooks.py: 207]:\tstep: 34340\tsteps/sec: 1.18743\tloss: 0.23627\tlr:[2.6681213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:11:51,275 [ hooks.py: 207]:\tstep: 34350\tsteps/sec: 1.40537\tloss: 0.11287\tlr:[2.6674425e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:12:00,038 [ hooks.py: 207]:\tstep: 34360\tsteps/sec: 1.13577\tloss: 0.29765\tlr:[2.666764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:12:08,417 [ hooks.py: 207]:\tstep: 34370\tsteps/sec: 1.21278\tloss: 0.45459\tlr:[2.6660848e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:12:16,126 [ hooks.py: 207]:\tstep: 34380\tsteps/sec: 1.34928\tloss: 0.51967\tlr:[2.6654052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:12:24,298 [ hooks.py: 207]:\tstep: 34390\tsteps/sec: 1.12678\tloss: 0.49287\tlr:[2.6647265e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:12:33,969 [ hooks.py: 207]:\tstep: 34400\tsteps/sec: 1.07734\tloss: 0.22893\tlr:[2.6640477e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:12:44,922 [ hooks.py: 207]:\tstep: 34410\tsteps/sec: 0.91956\tloss: 0.65188\tlr:[2.6633683e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:12:53,457 [ hooks.py: 207]:\tstep: 34420\tsteps/sec: 1.16653\tloss: 0.34628\tlr:[2.662689e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:13:02,908 [ hooks.py: 207]:\tstep: 34430\tsteps/sec: 1.11401\tloss: 0.37938\tlr:[2.66201e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 05:13:11,512 [ hooks.py: 207]:\tstep: 34440\tsteps/sec: 1.12330\tloss: 0.29295\tlr:[2.6613312e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:13:19,309 [ hooks.py: 207]:\tstep: 34450\tsteps/sec: 1.25672\tloss: 0.24464\tlr:[2.660652e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:13:27,245 [ hooks.py: 207]:\tstep: 34460\tsteps/sec: 1.23872\tloss: 0.52183\tlr:[2.6599731e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:13:36,923 [ hooks.py: 207]:\tstep: 34470\tsteps/sec: 1.05127\tloss: 0.39120\tlr:[2.6592943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:13:46,536 [ hooks.py: 207]:\tstep: 34480\tsteps/sec: 0.99533\tloss: 0.64604\tlr:[2.658615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:13:57,861 [ hooks.py: 207]:\tstep: 34490\tsteps/sec: 0.90455\tloss: 0.35544\tlr:[2.6579359e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:14:05,993 [ hooks.py: 207]:\tstep: 34500\tsteps/sec: 1.27076\tloss: 0.61831\tlr:[2.657257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:14:13,729 [ hooks.py: 207]:\tstep: 34510\tsteps/sec: 1.20894\tloss: 0.43465\tlr:[2.656578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:14:23,821 [ hooks.py: 207]:\tstep: 34520\tsteps/sec: 1.03217\tloss: 0.74402\tlr:[2.6558986e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:14:31,722 [ hooks.py: 207]:\tstep: 34530\tsteps/sec: 1.22897\tloss: 0.57905\tlr:[2.6552196e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:14:39,017 [ hooks.py: 207]:\tstep: 34540\tsteps/sec: 1.40673\tloss: 0.25760\tlr:[2.6545405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:14:48,339 [ hooks.py: 207]:\tstep: 34550\tsteps/sec: 1.07137\tloss: 0.54123\tlr:[2.6538619e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:14:57,609 [ hooks.py: 207]:\tstep: 34560\tsteps/sec: 1.07322\tloss: 0.75639\tlr:[2.6531823e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:15:05,837 [ hooks.py: 207]:\tstep: 34570\tsteps/sec: 1.25914\tloss: 0.43524\tlr:[2.6525035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:15:14,258 [ hooks.py: 207]:\tstep: 34580\tsteps/sec: 1.16036\tloss: 0.39376\tlr:[2.6518246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:15:23,567 [ hooks.py: 207]:\tstep: 34590\tsteps/sec: 1.07565\tloss: 0.51709\tlr:[2.6511454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:15:31,603 [ hooks.py: 207]:\tstep: 34600\tsteps/sec: 1.21065\tloss: 0.53527\tlr:[2.6504662e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:15:40,123 [ hooks.py: 207]:\tstep: 34610\tsteps/sec: 1.21983\tloss: 0.31764\tlr:[2.6497873e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:15:49,544 [ hooks.py: 207]:\tstep: 34620\tsteps/sec: 1.00805\tloss: 0.54638\tlr:[2.6491083e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:15:57,954 [ hooks.py: 207]:\tstep: 34630\tsteps/sec: 1.24489\tloss: 0.59460\tlr:[2.6484293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:16:05,038 [ hooks.py: 207]:\tstep: 34640\tsteps/sec: 1.43938\tloss: 0.53977\tlr:[2.64775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:16:12,306 [ hooks.py: 207]:\tstep: 34650\tsteps/sec: 1.33972\tloss: 0.34880\tlr:[2.647071e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:16:21,560 [ hooks.py: 207]:\tstep: 34660\tsteps/sec: 1.11617\tloss: 0.45240\tlr:[2.6463922e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:16:30,186 [ hooks.py: 207]:\tstep: 34670\tsteps/sec: 1.13938\tloss: 0.43698\tlr:[2.645713e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:16:37,930 [ hooks.py: 207]:\tstep: 34680\tsteps/sec: 1.21067\tloss: 0.24897\tlr:[2.6450338e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:16:49,089 [ hooks.py: 207]:\tstep: 34690\tsteps/sec: 0.93467\tloss: 1.05702\tlr:[2.644355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:16:59,038 [ hooks.py: 207]:\tstep: 34700\tsteps/sec: 1.01775\tloss: 0.84734\tlr:[2.6436757e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:17:07,743 [ hooks.py: 207]:\tstep: 34710\tsteps/sec: 1.11952\tloss: 0.43468\tlr:[2.6429965e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:17:14,544 [ hooks.py: 207]:\tstep: 34720\tsteps/sec: 1.48302\tloss: 0.71607\tlr:[2.6423173e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:17:24,784 [ hooks.py: 207]:\tstep: 34730\tsteps/sec: 0.97392\tloss: 0.62886\tlr:[2.6416388e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:17:32,271 [ hooks.py: 207]:\tstep: 34740\tsteps/sec: 1.34285\tloss: 0.65412\tlr:[2.6409596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:17:41,862 [ hooks.py: 207]:\tstep: 34750\tsteps/sec: 1.05683\tloss: 0.49191\tlr:[2.6402804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:17:50,318 [ hooks.py: 207]:\tstep: 34760\tsteps/sec: 1.18916\tloss: 0.37723\tlr:[2.6396012e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:17:59,262 [ hooks.py: 207]:\tstep: 34770\tsteps/sec: 1.11324\tloss: 0.59410\tlr:[2.6389225e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:18:07,140 [ hooks.py: 207]:\tstep: 34780\tsteps/sec: 1.26034\tloss: 0.81360\tlr:[2.6382433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:18:17,100 [ hooks.py: 207]:\tstep: 34790\tsteps/sec: 0.98558\tloss: 0.58959\tlr:[2.6375641e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:18:26,552 [ hooks.py: 207]:\tstep: 34800\tsteps/sec: 1.05286\tloss: 0.31986\tlr:[2.6368853e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:18:34,811 [ hooks.py: 207]:\tstep: 34810\tsteps/sec: 1.21676\tloss: 0.45265\tlr:[2.6362064e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:18:45,027 [ hooks.py: 207]:\tstep: 34820\tsteps/sec: 0.99683\tloss: 0.34692\tlr:[2.6355272e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:18:53,960 [ hooks.py: 207]:\tstep: 34830\tsteps/sec: 1.09940\tloss: 0.49254\tlr:[2.634848e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:19:03,152 [ hooks.py: 207]:\tstep: 34840\tsteps/sec: 1.05811\tloss: 0.31277\tlr:[2.6341691e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:19:11,340 [ hooks.py: 207]:\tstep: 34850\tsteps/sec: 1.30766\tloss: 0.33874\tlr:[2.63349e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:19:17,425 [ hooks.py: 207]:\tstep: 34860\tsteps/sec: 1.49714\tloss: 0.43082\tlr:[2.6328104e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:19:27,206 [ hooks.py: 207]:\tstep: 34870\tsteps/sec: 1.08161\tloss: 0.70510\tlr:[2.6321315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:19:34,153 [ hooks.py: 207]:\tstep: 34880\tsteps/sec: 1.41081\tloss: 0.41384\tlr:[2.6314527e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:19:43,234 [ hooks.py: 207]:\tstep: 34890\tsteps/sec: 1.08933\tloss: 0.31362\tlr:[2.6307735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:19:53,695 [ hooks.py: 207]:\tstep: 34900\tsteps/sec: 0.97295\tloss: 0.99754\tlr:[2.6300942e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:20:01,203 [ hooks.py: 207]:\tstep: 34910\tsteps/sec: 1.37962\tloss: 0.49969\tlr:[2.6294154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:20:08,626 [ hooks.py: 207]:\tstep: 34920\tsteps/sec: 1.28961\tloss: 0.29404\tlr:[2.6287364e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:20:15,600 [ hooks.py: 207]:\tstep: 34930\tsteps/sec: 1.39299\tloss: 0.43045\tlr:[2.6280575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:20:25,489 [ hooks.py: 207]:\tstep: 34940\tsteps/sec: 1.04565\tloss: 0.51624\tlr:[2.6273781e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:20:34,114 [ hooks.py: 207]:\tstep: 34950\tsteps/sec: 1.14377\tloss: 0.43882\tlr:[2.6266991e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:20:43,490 [ hooks.py: 207]:\tstep: 34960\tsteps/sec: 1.06989\tloss: 0.36652\tlr:[2.6260206e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:20:53,747 [ hooks.py: 207]:\tstep: 34970\tsteps/sec: 0.97062\tloss: 0.42230\tlr:[2.625341e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:05,048 [ hooks.py: 207]:\tstep: 34980\tsteps/sec: 0.89451\tloss: 0.38733\tlr:[2.6246618e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:12,501 [ hooks.py: 207]:\tstep: 34990\tsteps/sec: 1.31024\tloss: 0.37703\tlr:[2.623983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:20,362 [ hooks.py: 207]:\tstep: 35000\tsteps/sec: 1.25944\tloss: 0.33005\tlr:[2.6233038e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:20,366 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:20,367 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:20,367 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:21:20,367 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:20,367 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:21:20,367 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:29:22,032 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,034 [ trainer.py: 58]:\t{'acc': 0.7763052208835342, 'loss': 0.5816866824260125}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,034 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,034 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:29:22,035 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:29:22,035 [ trainer.py: 70]:\tacc\t0.7763052208835342\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:29:22,035 [ trainer.py: 70]:\tloss\t0.5816866824260125\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 05:29:22,035 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,036 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7763052208835342, 'loss': 0.5816866824260125}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,036 [ exporter.py: 64]:\t[Best Exporter]: skip step 35000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,036 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7763052208835342, 'loss': 0.5816866824260125}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,036 [ exporter.py: 104]:\t[Best Exporter]: skip step 35000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:22,037 [monitored_executor.py: 122]:\tsaving step 35000 to ./output/model_35000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 05:29:25,235 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_32000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:34,174 [ hooks.py: 207]:\tstep: 35010\tsteps/sec: 0.02026\tloss: 1.02059\tlr:[2.6226246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:42,140 [ hooks.py: 207]:\tstep: 35020\tsteps/sec: 1.20028\tloss: 0.62546\tlr:[2.6219457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:51,207 [ hooks.py: 207]:\tstep: 35030\tsteps/sec: 1.18080\tloss: 0.72147\tlr:[2.6212667e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:29:57,690 [ hooks.py: 207]:\tstep: 35040\tsteps/sec: 1.45618\tloss: 0.46113\tlr:[2.6205877e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:30:07,738 [ hooks.py: 207]:\tstep: 35050\tsteps/sec: 0.96632\tloss: 0.38177\tlr:[2.6199084e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:30:18,941 [ hooks.py: 207]:\tstep: 35060\tsteps/sec: 0.92712\tloss: 0.31964\tlr:[2.6192298e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:30:27,116 [ hooks.py: 207]:\tstep: 35070\tsteps/sec: 1.23138\tloss: 0.86773\tlr:[2.6185506e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:30:36,049 [ hooks.py: 207]:\tstep: 35080\tsteps/sec: 1.13134\tloss: 0.30308\tlr:[2.6178717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:30:43,819 [ hooks.py: 207]:\tstep: 35090\tsteps/sec: 1.26050\tloss: 0.39562\tlr:[2.6171925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:30:54,841 [ hooks.py: 207]:\tstep: 35100\tsteps/sec: 0.90720\tloss: 0.68935\tlr:[2.6165133e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:31:02,036 [ hooks.py: 207]:\tstep: 35110\tsteps/sec: 1.51299\tloss: 0.34085\tlr:[2.6158345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:31:11,473 [ hooks.py: 207]:\tstep: 35120\tsteps/sec: 0.98214\tloss: 0.44545\tlr:[2.6151552e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:31:19,758 [ hooks.py: 207]:\tstep: 35130\tsteps/sec: 1.22600\tloss: 0.51525\tlr:[2.6144764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:31:28,766 [ hooks.py: 207]:\tstep: 35140\tsteps/sec: 1.07172\tloss: 0.49256\tlr:[2.6137974e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:31:36,383 [ hooks.py: 207]:\tstep: 35150\tsteps/sec: 1.40691\tloss: 0.31534\tlr:[2.6131182e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:31:47,063 [ hooks.py: 207]:\tstep: 35160\tsteps/sec: 0.93860\tloss: 0.32893\tlr:[2.612439e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:31:55,898 [ hooks.py: 207]:\tstep: 35170\tsteps/sec: 1.11448\tloss: 0.60153\tlr:[2.6117601e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:32:04,266 [ hooks.py: 207]:\tstep: 35180\tsteps/sec: 1.17243\tloss: 0.38999\tlr:[2.6110813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:32:13,376 [ hooks.py: 207]:\tstep: 35190\tsteps/sec: 1.06662\tloss: 0.43489\tlr:[2.610402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:32:22,979 [ hooks.py: 207]:\tstep: 35200\tsteps/sec: 1.07903\tloss: 0.30976\tlr:[2.6097228e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:32:29,948 [ hooks.py: 207]:\tstep: 35210\tsteps/sec: 1.43288\tloss: 0.25624\tlr:[2.609044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:32:39,208 [ hooks.py: 207]:\tstep: 35220\tsteps/sec: 1.08495\tloss: 0.30719\tlr:[2.6083651e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:32:47,858 [ hooks.py: 207]:\tstep: 35230\tsteps/sec: 1.14899\tloss: 0.52312\tlr:[2.607686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:32:56,812 [ hooks.py: 207]:\tstep: 35240\tsteps/sec: 1.15079\tloss: 0.42320\tlr:[2.6070067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:33:04,708 [ hooks.py: 207]:\tstep: 35250\tsteps/sec: 1.17899\tloss: 0.41199\tlr:[2.6063277e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:33:14,077 [ hooks.py: 207]:\tstep: 35260\tsteps/sec: 1.11340\tloss: 0.28080\tlr:[2.6056488e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:33:22,341 [ hooks.py: 207]:\tstep: 35270\tsteps/sec: 1.22327\tloss: 0.36617\tlr:[2.6049696e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:33:31,828 [ hooks.py: 207]:\tstep: 35280\tsteps/sec: 1.07020\tloss: 0.87556\tlr:[2.6042904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:33:38,194 [ hooks.py: 207]:\tstep: 35290\tsteps/sec: 1.48995\tloss: 0.43873\tlr:[2.6036116e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:33:46,372 [ hooks.py: 207]:\tstep: 35300\tsteps/sec: 1.22766\tloss: 0.38026\tlr:[2.6029324e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:33:54,622 [ hooks.py: 207]:\tstep: 35310\tsteps/sec: 1.22278\tloss: 0.40014\tlr:[2.6022532e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:34:00,975 [ hooks.py: 207]:\tstep: 35320\tsteps/sec: 1.62007\tloss: 0.53648\tlr:[2.6015743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:34:11,233 [ hooks.py: 207]:\tstep: 35330\tsteps/sec: 0.97781\tloss: 0.27590\tlr:[2.6008955e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:34:19,108 [ hooks.py: 207]:\tstep: 35340\tsteps/sec: 1.24558\tloss: 0.54869\tlr:[2.6002162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:34:27,410 [ hooks.py: 207]:\tstep: 35350\tsteps/sec: 1.19678\tloss: 0.56584\tlr:[2.599537e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:34:35,376 [ hooks.py: 207]:\tstep: 35360\tsteps/sec: 1.25572\tloss: 0.52435\tlr:[2.5988582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:34:43,458 [ hooks.py: 207]:\tstep: 35370\tsteps/sec: 1.18296\tloss: 0.57379\tlr:[2.5981792e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:34:51,924 [ hooks.py: 207]:\tstep: 35380\tsteps/sec: 1.22680\tloss: 0.38168\tlr:[2.5975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:35:01,622 [ hooks.py: 207]:\tstep: 35390\tsteps/sec: 1.09375\tloss: 0.48285\tlr:[2.5968206e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:35:10,671 [ hooks.py: 207]:\tstep: 35400\tsteps/sec: 1.02929\tloss: 0.43011\tlr:[2.5961419e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:35:19,334 [ hooks.py: 207]:\tstep: 35410\tsteps/sec: 1.18438\tloss: 0.70940\tlr:[2.595463e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 05:35:27,602 [ hooks.py: 207]:\tstep: 35420\tsteps/sec: 1.19690\tloss: 0.62186\tlr:[2.5947835e-05]\u001b[0m\n", + "IOPub message rate exceeded.\n", + "The notebook server will temporarily stop sending output\n", + "to the client in order to avoid crashing it.\n", + "To change this limit, set the config variable\n", + "`--NotebookApp.iopub_msg_rate_limit`.\n", + "\n", + "Current values:\n", + "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n", + "NotebookApp.rate_limit_window=3.0 (secs)\n", + "\n", + "\u001b[37m[DEBUG] 2019-09-19 06:14:44,247 [ hooks.py: 207]:\tstep: 37040\tsteps/sec: 1.28865\tloss: 0.45994\tlr:[2.4847763e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:14:51,450 [ hooks.py: 207]:\tstep: 37050\tsteps/sec: 1.42377\tloss: 0.29395\tlr:[2.4840974e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:15:03,613 [ hooks.py: 207]:\tstep: 37060\tsteps/sec: 0.79070\tloss: 0.28660\tlr:[2.483418e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:15:15,005 [ hooks.py: 207]:\tstep: 37070\tsteps/sec: 0.91352\tloss: 0.76353\tlr:[2.4827392e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:15:22,075 [ hooks.py: 207]:\tstep: 37080\tsteps/sec: 1.44788\tloss: 0.83951\tlr:[2.4820602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:15:29,799 [ hooks.py: 207]:\tstep: 37090\tsteps/sec: 1.25617\tloss: 0.88195\tlr:[2.4813813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:15:38,788 [ hooks.py: 207]:\tstep: 37100\tsteps/sec: 1.10096\tloss: 0.42317\tlr:[2.480702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:15:48,186 [ hooks.py: 207]:\tstep: 37110\tsteps/sec: 1.09130\tloss: 0.90199\tlr:[2.480023e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:15:56,843 [ hooks.py: 207]:\tstep: 37120\tsteps/sec: 1.11934\tloss: 0.49444\tlr:[2.4793439e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:16:05,540 [ hooks.py: 207]:\tstep: 37130\tsteps/sec: 1.10488\tloss: 0.43914\tlr:[2.4786648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:16:15,068 [ hooks.py: 207]:\tstep: 37140\tsteps/sec: 1.10975\tloss: 0.13463\tlr:[2.4779858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:16:23,937 [ hooks.py: 207]:\tstep: 37150\tsteps/sec: 1.11660\tloss: 0.49945\tlr:[2.4773066e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:16:32,396 [ hooks.py: 207]:\tstep: 37160\tsteps/sec: 1.21967\tloss: 0.33824\tlr:[2.4766277e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:16:41,508 [ hooks.py: 207]:\tstep: 37170\tsteps/sec: 1.09389\tloss: 0.21608\tlr:[2.4759485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:16:48,805 [ hooks.py: 207]:\tstep: 37180\tsteps/sec: 1.34543\tloss: 0.36580\tlr:[2.4752699e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:16:55,377 [ hooks.py: 207]:\tstep: 37190\tsteps/sec: 1.51194\tloss: 0.48545\tlr:[2.4745905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:17:04,156 [ hooks.py: 207]:\tstep: 37200\tsteps/sec: 1.13785\tloss: 0.39924\tlr:[2.4739118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:17:13,032 [ hooks.py: 207]:\tstep: 37210\tsteps/sec: 1.14606\tloss: 0.30570\tlr:[2.4732326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:17:22,192 [ hooks.py: 207]:\tstep: 37220\tsteps/sec: 1.10489\tloss: 0.32462\tlr:[2.4725536e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:17:30,415 [ hooks.py: 207]:\tstep: 37230\tsteps/sec: 1.13213\tloss: 0.35669\tlr:[2.4718744e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 06:17:42,309 [ hooks.py: 207]:\tstep: 37240\tsteps/sec: 0.85766\tloss: 0.98611\tlr:[2.4711953e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:17:51,809 [ hooks.py: 207]:\tstep: 37250\tsteps/sec: 1.08777\tloss: 0.17700\tlr:[2.4705161e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:18:01,052 [ hooks.py: 207]:\tstep: 37260\tsteps/sec: 1.10172\tloss: 0.37778\tlr:[2.4698373e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:18:10,093 [ hooks.py: 207]:\tstep: 37270\tsteps/sec: 1.07166\tloss: 0.70400\tlr:[2.4691582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:18:19,301 [ hooks.py: 207]:\tstep: 37280\tsteps/sec: 1.04313\tloss: 0.60341\tlr:[2.4684789e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:18:29,857 [ hooks.py: 207]:\tstep: 37290\tsteps/sec: 0.98164\tloss: 0.48223\tlr:[2.4678e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:18:37,743 [ hooks.py: 207]:\tstep: 37300\tsteps/sec: 1.27569\tloss: 1.07025\tlr:[2.4671208e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:18:44,533 [ hooks.py: 207]:\tstep: 37310\tsteps/sec: 1.38440\tloss: 0.38846\tlr:[2.466442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:18:53,102 [ hooks.py: 207]:\tstep: 37320\tsteps/sec: 1.24796\tloss: 0.53494\tlr:[2.4657627e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:19:00,332 [ hooks.py: 207]:\tstep: 37330\tsteps/sec: 1.36866\tloss: 0.17646\tlr:[2.465084e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:19:11,347 [ hooks.py: 207]:\tstep: 37340\tsteps/sec: 0.89136\tloss: 0.39807\tlr:[2.4644047e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:19:19,007 [ hooks.py: 207]:\tstep: 37350\tsteps/sec: 1.30216\tloss: 0.33800\tlr:[2.4637258e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:19:29,338 [ hooks.py: 207]:\tstep: 37360\tsteps/sec: 0.97124\tloss: 0.59229\tlr:[2.4630466e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:19:38,219 [ hooks.py: 207]:\tstep: 37370\tsteps/sec: 1.15202\tloss: 0.29886\tlr:[2.4623676e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:19:46,940 [ hooks.py: 207]:\tstep: 37380\tsteps/sec: 1.15020\tloss: 0.42882\tlr:[2.4616884e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:19:55,500 [ hooks.py: 207]:\tstep: 37390\tsteps/sec: 1.14619\tloss: 0.67878\tlr:[2.4610095e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:20:04,605 [ hooks.py: 207]:\tstep: 37400\tsteps/sec: 1.08570\tloss: 0.52433\tlr:[2.4603303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:20:12,513 [ hooks.py: 207]:\tstep: 37410\tsteps/sec: 1.31514\tloss: 0.27805\tlr:[2.4596513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:20:22,053 [ hooks.py: 207]:\tstep: 37420\tsteps/sec: 1.05148\tloss: 0.50950\tlr:[2.4589723e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:20:29,714 [ hooks.py: 207]:\tstep: 37430\tsteps/sec: 1.25420\tloss: 0.49147\tlr:[2.458293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:20:36,710 [ hooks.py: 207]:\tstep: 37440\tsteps/sec: 1.48587\tloss: 0.50461\tlr:[2.4576142e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:20:43,555 [ hooks.py: 207]:\tstep: 37450\tsteps/sec: 1.35176\tloss: 0.32906\tlr:[2.456935e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:20:54,656 [ hooks.py: 207]:\tstep: 37460\tsteps/sec: 0.92525\tloss: 0.34575\tlr:[2.4562562e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:03,281 [ hooks.py: 207]:\tstep: 37470\tsteps/sec: 1.23111\tloss: 0.54544\tlr:[2.455577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:11,834 [ hooks.py: 207]:\tstep: 37480\tsteps/sec: 1.13746\tloss: 0.90525\tlr:[2.454898e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:18,797 [ hooks.py: 207]:\tstep: 37490\tsteps/sec: 1.40409\tloss: 0.79667\tlr:[2.4542189e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:25,653 [ hooks.py: 207]:\tstep: 37500\tsteps/sec: 1.45184\tloss: 0.31801\tlr:[2.4535399e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:34,554 [ hooks.py: 207]:\tstep: 37510\tsteps/sec: 1.12492\tloss: 0.53888\tlr:[2.4528606e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:41,707 [ hooks.py: 207]:\tstep: 37520\tsteps/sec: 1.43703\tloss: 0.19137\tlr:[2.452182e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:50,614 [ hooks.py: 207]:\tstep: 37530\tsteps/sec: 1.09611\tloss: 0.53057\tlr:[2.4515026e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:21:59,990 [ hooks.py: 207]:\tstep: 37540\tsteps/sec: 1.07846\tloss: 0.53466\tlr:[2.4508237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:22:09,137 [ hooks.py: 207]:\tstep: 37550\tsteps/sec: 1.08072\tloss: 0.69469\tlr:[2.4501445e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:22:17,432 [ hooks.py: 207]:\tstep: 37560\tsteps/sec: 1.23955\tloss: 0.40415\tlr:[2.4494657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:22:25,836 [ hooks.py: 207]:\tstep: 37570\tsteps/sec: 1.11357\tloss: 0.19510\tlr:[2.4487865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:22:34,428 [ hooks.py: 207]:\tstep: 37580\tsteps/sec: 1.20202\tloss: 0.56148\tlr:[2.4481074e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:22:45,048 [ hooks.py: 207]:\tstep: 37590\tsteps/sec: 0.95647\tloss: 0.80654\tlr:[2.4474284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:22:53,617 [ hooks.py: 207]:\tstep: 37600\tsteps/sec: 1.15283\tloss: 0.23624\tlr:[2.4467492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:23:05,148 [ hooks.py: 207]:\tstep: 37610\tsteps/sec: 0.86482\tloss: 0.67911\tlr:[2.4460704e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:23:14,453 [ hooks.py: 207]:\tstep: 37620\tsteps/sec: 1.07701\tloss: 0.20282\tlr:[2.4453911e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:23:20,741 [ hooks.py: 207]:\tstep: 37630\tsteps/sec: 1.59758\tloss: 0.75794\tlr:[2.4447123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:23:31,052 [ hooks.py: 207]:\tstep: 37640\tsteps/sec: 0.97373\tloss: 1.00878\tlr:[2.444033e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:23:40,797 [ hooks.py: 207]:\tstep: 37650\tsteps/sec: 0.99795\tloss: 0.41532\tlr:[2.4433542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:23:50,066 [ hooks.py: 207]:\tstep: 37660\tsteps/sec: 1.10973\tloss: 0.30590\tlr:[2.4426748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:23:59,102 [ hooks.py: 207]:\tstep: 37670\tsteps/sec: 1.11084\tloss: 0.38564\tlr:[2.441996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:24:09,660 [ hooks.py: 207]:\tstep: 37680\tsteps/sec: 0.94640\tloss: 0.26465\tlr:[2.4413168e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:24:17,783 [ hooks.py: 207]:\tstep: 37690\tsteps/sec: 1.25827\tloss: 0.32243\tlr:[2.4406378e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:24:26,709 [ hooks.py: 207]:\tstep: 37700\tsteps/sec: 1.07554\tloss: 0.53201\tlr:[2.4399587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:24:35,451 [ hooks.py: 207]:\tstep: 37710\tsteps/sec: 1.12850\tloss: 0.70927\tlr:[2.4392797e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:24:45,716 [ hooks.py: 207]:\tstep: 37720\tsteps/sec: 0.97093\tloss: 0.34676\tlr:[2.4386007e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:24:55,658 [ hooks.py: 207]:\tstep: 37730\tsteps/sec: 1.04074\tloss: 0.46664\tlr:[2.4379215e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:25:05,639 [ hooks.py: 207]:\tstep: 37740\tsteps/sec: 1.00265\tloss: 0.16774\tlr:[2.4372424e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:25:14,088 [ hooks.py: 207]:\tstep: 37750\tsteps/sec: 1.17337\tloss: 0.93663\tlr:[2.4365632e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:25:20,999 [ hooks.py: 207]:\tstep: 37760\tsteps/sec: 1.45953\tloss: 0.90848\tlr:[2.4358844e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:25:28,733 [ hooks.py: 207]:\tstep: 37770\tsteps/sec: 1.22334\tloss: 0.17596\tlr:[2.4352052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:25:36,876 [ hooks.py: 207]:\tstep: 37780\tsteps/sec: 1.32798\tloss: 0.60317\tlr:[2.4345261e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:25:45,672 [ hooks.py: 207]:\tstep: 37790\tsteps/sec: 1.07148\tloss: 0.38273\tlr:[2.4338471e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:25:55,027 [ hooks.py: 207]:\tstep: 37800\tsteps/sec: 1.10027\tloss: 0.28200\tlr:[2.433168e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:26:03,638 [ hooks.py: 207]:\tstep: 37810\tsteps/sec: 1.11465\tloss: 0.46102\tlr:[2.432489e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:26:14,413 [ hooks.py: 207]:\tstep: 37820\tsteps/sec: 0.96665\tloss: 0.28588\tlr:[2.4318102e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:26:21,954 [ hooks.py: 207]:\tstep: 37830\tsteps/sec: 1.36252\tloss: 0.38603\tlr:[2.431131e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:26:28,734 [ hooks.py: 207]:\tstep: 37840\tsteps/sec: 1.36933\tloss: 0.62571\tlr:[2.430452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:26:37,707 [ hooks.py: 207]:\tstep: 37850\tsteps/sec: 1.19819\tloss: 0.61621\tlr:[2.429773e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:26:44,394 [ hooks.py: 207]:\tstep: 37860\tsteps/sec: 1.36491\tloss: 0.44591\tlr:[2.429094e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:26:53,485 [ hooks.py: 207]:\tstep: 37870\tsteps/sec: 1.14159\tloss: 0.55866\tlr:[2.4284147e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:27:00,649 [ hooks.py: 207]:\tstep: 37880\tsteps/sec: 1.43624\tloss: 0.25640\tlr:[2.4277355e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 06:27:07,328 [ hooks.py: 207]:\tstep: 37890\tsteps/sec: 1.44786\tloss: 0.22488\tlr:[2.4270566e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:27:16,266 [ hooks.py: 207]:\tstep: 37900\tsteps/sec: 1.12430\tloss: 0.50528\tlr:[2.4263774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:27:23,160 [ hooks.py: 207]:\tstep: 37910\tsteps/sec: 1.49378\tloss: 0.58086\tlr:[2.4256988e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:27:29,843 [ hooks.py: 207]:\tstep: 37920\tsteps/sec: 1.44356\tloss: 0.46515\tlr:[2.4250196e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:27:39,384 [ hooks.py: 207]:\tstep: 37930\tsteps/sec: 1.05213\tloss: 0.46898\tlr:[2.4243405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:27:46,808 [ hooks.py: 207]:\tstep: 37940\tsteps/sec: 1.36786\tloss: 0.47305\tlr:[2.4236613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:27:54,221 [ hooks.py: 207]:\tstep: 37950\tsteps/sec: 1.32798\tloss: 0.23740\tlr:[2.4229825e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:02,190 [ hooks.py: 207]:\tstep: 37960\tsteps/sec: 1.35352\tloss: 0.20117\tlr:[2.4223033e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:08,794 [ hooks.py: 207]:\tstep: 37970\tsteps/sec: 1.41340\tloss: 0.68261\tlr:[2.4216242e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:15,500 [ hooks.py: 207]:\tstep: 37980\tsteps/sec: 1.40918\tloss: 0.26626\tlr:[2.420945e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:24,501 [ hooks.py: 207]:\tstep: 37990\tsteps/sec: 1.15214\tloss: 0.42966\tlr:[2.4202662e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:32,888 [ hooks.py: 207]:\tstep: 38000\tsteps/sec: 1.22125\tloss: 0.28915\tlr:[2.4195871e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:32,889 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:32,889 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:32,889 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 06:28:32,889 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:32,889 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:28:32,890 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 06:36:36,429 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,430 [ trainer.py: 58]:\t{'acc': 0.7763052208835342, 'loss': 0.6086467042183265}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,431 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,431 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 06:36:36,432 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 06:36:36,432 [ trainer.py: 70]:\tacc\t0.7763052208835342\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 06:36:36,432 [ trainer.py: 70]:\tloss\t0.6086467042183265\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 06:36:36,432 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,432 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7763052208835342, 'loss': 0.6086467042183265}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,432 [ exporter.py: 64]:\t[Best Exporter]: skip step 38000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,432 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7763052208835342, 'loss': 0.6086467042183265}} \n", + "old: {'eval': {'acc': 0.7895582329317269, 'loss': 0.531344682455827}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,433 [ exporter.py: 104]:\t[Best Exporter]: skip step 38000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:36,433 [monitored_executor.py: 122]:\tsaving step 38000 to ./output/model_38000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:39,560 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_35000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:48,153 [ hooks.py: 207]:\tstep: 38010\tsteps/sec: 0.02018\tloss: 0.75509\tlr:[2.4189081e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:36:56,015 [ hooks.py: 207]:\tstep: 38020\tsteps/sec: 1.28746\tloss: 1.03574\tlr:[2.418229e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:37:03,619 [ hooks.py: 207]:\tstep: 38030\tsteps/sec: 1.43347\tloss: 0.86879\tlr:[2.4175497e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:37:12,012 [ hooks.py: 207]:\tstep: 38040\tsteps/sec: 1.12337\tloss: 0.49537\tlr:[2.4168708e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:37:19,100 [ hooks.py: 207]:\tstep: 38050\tsteps/sec: 1.38149\tloss: 0.16871\tlr:[2.4161918e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:37:25,969 [ hooks.py: 207]:\tstep: 38060\tsteps/sec: 1.46482\tloss: 0.45617\tlr:[2.415513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:37:34,265 [ hooks.py: 207]:\tstep: 38070\tsteps/sec: 1.19287\tloss: 0.83183\tlr:[2.4148338e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:37:45,034 [ hooks.py: 207]:\tstep: 38080\tsteps/sec: 0.92965\tloss: 0.35160\tlr:[2.4141547e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:37:54,100 [ hooks.py: 207]:\tstep: 38090\tsteps/sec: 1.10036\tloss: 0.57864\tlr:[2.4134755e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:38:04,095 [ hooks.py: 207]:\tstep: 38100\tsteps/sec: 0.98942\tloss: 0.51646\tlr:[2.4127967e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:38:14,935 [ hooks.py: 207]:\tstep: 38110\tsteps/sec: 0.92805\tloss: 0.66327\tlr:[2.4121173e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:38:24,745 [ hooks.py: 207]:\tstep: 38120\tsteps/sec: 0.99424\tloss: 0.60252\tlr:[2.4114386e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:38:33,106 [ hooks.py: 207]:\tstep: 38130\tsteps/sec: 1.28908\tloss: 0.35652\tlr:[2.4107594e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:38:39,726 [ hooks.py: 207]:\tstep: 38140\tsteps/sec: 1.38896\tloss: 0.79967\tlr:[2.4100806e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:38:48,752 [ hooks.py: 207]:\tstep: 38150\tsteps/sec: 1.15854\tloss: 0.25828\tlr:[2.4094013e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:38:57,569 [ hooks.py: 207]:\tstep: 38160\tsteps/sec: 1.14543\tloss: 0.53342\tlr:[2.4087223e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:39:06,386 [ hooks.py: 207]:\tstep: 38170\tsteps/sec: 1.12102\tloss: 0.54545\tlr:[2.4080431e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:39:13,915 [ hooks.py: 207]:\tstep: 38180\tsteps/sec: 1.24836\tloss: 0.08168\tlr:[2.4073639e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:39:24,254 [ hooks.py: 207]:\tstep: 38190\tsteps/sec: 0.96005\tloss: 0.50305\tlr:[2.406685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:39:35,067 [ hooks.py: 207]:\tstep: 38200\tsteps/sec: 0.96373\tloss: 0.35516\tlr:[2.4060058e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:39:42,497 [ hooks.py: 207]:\tstep: 38210\tsteps/sec: 1.38695\tloss: 0.27938\tlr:[2.405327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:39:48,832 [ hooks.py: 207]:\tstep: 38220\tsteps/sec: 1.59059\tloss: 0.49592\tlr:[2.4046478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:39:56,052 [ hooks.py: 207]:\tstep: 38230\tsteps/sec: 1.34814\tloss: 0.33054\tlr:[2.403969e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:40:05,207 [ hooks.py: 207]:\tstep: 38240\tsteps/sec: 1.08414\tloss: 0.31548\tlr:[2.4032897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:40:13,264 [ hooks.py: 207]:\tstep: 38250\tsteps/sec: 1.28164\tloss: 0.44325\tlr:[2.4026109e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:40:23,662 [ hooks.py: 207]:\tstep: 38260\tsteps/sec: 0.95604\tloss: 0.47870\tlr:[2.4019315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:40:30,544 [ hooks.py: 207]:\tstep: 38270\tsteps/sec: 1.43122\tloss: 0.47708\tlr:[2.4012526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:40:40,587 [ hooks.py: 207]:\tstep: 38280\tsteps/sec: 0.96223\tloss: 0.34302\tlr:[2.4005736e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:40:50,653 [ hooks.py: 207]:\tstep: 38290\tsteps/sec: 1.03690\tloss: 0.71935\tlr:[2.3998946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:40:57,790 [ hooks.py: 207]:\tstep: 38300\tsteps/sec: 1.41799\tloss: 0.38211\tlr:[2.3992156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:41:04,782 [ hooks.py: 207]:\tstep: 38310\tsteps/sec: 1.30982\tloss: 0.33540\tlr:[2.3985362e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:41:13,678 [ hooks.py: 207]:\tstep: 38320\tsteps/sec: 1.20591\tloss: 0.37560\tlr:[2.3978573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:41:21,805 [ hooks.py: 207]:\tstep: 38330\tsteps/sec: 1.16939\tloss: 0.32872\tlr:[2.3971783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:41:30,800 [ hooks.py: 207]:\tstep: 38340\tsteps/sec: 1.11736\tloss: 0.76472\tlr:[2.396499e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 06:41:40,644 [ hooks.py: 207]:\tstep: 38350\tsteps/sec: 1.01433\tloss: 0.70467\tlr:[2.39582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:41:50,280 [ hooks.py: 207]:\tstep: 38360\tsteps/sec: 1.03486\tloss: 0.13933\tlr:[2.3951412e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:41:59,845 [ hooks.py: 207]:\tstep: 38370\tsteps/sec: 1.08890\tloss: 0.26177\tlr:[2.3944618e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:42:10,244 [ hooks.py: 207]:\tstep: 38380\tsteps/sec: 0.95196\tloss: 0.46623\tlr:[2.3937831e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:42:20,455 [ hooks.py: 207]:\tstep: 38390\tsteps/sec: 0.95523\tloss: 0.60905\tlr:[2.3931038e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:42:28,674 [ hooks.py: 207]:\tstep: 38400\tsteps/sec: 1.28750\tloss: 0.69087\tlr:[2.3924249e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:42:35,386 [ hooks.py: 207]:\tstep: 38410\tsteps/sec: 1.43482\tloss: 0.51406\tlr:[2.3917457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:42:43,984 [ hooks.py: 207]:\tstep: 38420\tsteps/sec: 1.18274\tloss: 0.44468\tlr:[2.3910668e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:42:51,054 [ hooks.py: 207]:\tstep: 38430\tsteps/sec: 1.40563\tloss: 0.22833\tlr:[2.3903876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:42:59,582 [ hooks.py: 207]:\tstep: 38440\tsteps/sec: 1.17069\tloss: 0.29312\tlr:[2.3897088e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:43:08,993 [ hooks.py: 207]:\tstep: 38450\tsteps/sec: 1.04168\tloss: 0.38170\tlr:[2.3890296e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 06:43:17,554 [ hooks.py: 207]:\tstep: 38460\tsteps/sec: 1.21459\tloss: 0.18179\tlr:[2.3883504e-05]\u001b[0m\n", + "IOPub message rate exceeded.\n", + "The notebook server will temporarily stop sending output\n", + "to the client in order to avoid crashing it.\n", + "To change this limit, set the config variable\n", + "`--NotebookApp.iopub_msg_rate_limit`.\n", + "\n", + "Current values:\n", + "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n", + "NotebookApp.rate_limit_window=3.0 (secs)\n", + "\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:04,189 [ hooks.py: 207]:\tstep: 39980\tsteps/sec: 1.15165\tloss: 0.62128\tlr:[2.2851338e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:13,434 [ hooks.py: 207]:\tstep: 39990\tsteps/sec: 1.09708\tloss: 0.34658\tlr:[2.2844551e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:21,002 [ hooks.py: 207]:\tstep: 40000\tsteps/sec: 1.28491\tloss: 0.53066\tlr:[2.2837758e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:21,003 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:21,003 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:21,004 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:13:21,004 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:21,004 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:13:21,004 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:21:23,653 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,654 [ trainer.py: 58]:\t{'acc': 0.7899598393574297, 'loss': 0.5478735304413698}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,654 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,654 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:21:23,655 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:21:23,655 [ trainer.py: 70]:\tacc\t0.7899598393574297\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:21:23,655 [ trainer.py: 70]:\tloss\t0.5478735304413698\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:21:23,656 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,656 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7899598393574297, 'loss': 0.5478735304413698}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,656 [ exporter.py: 64]:\t[Best Exporter]: skip step 40000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,656 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7899598393574297, 'loss': 0.5478735304413698}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,656 [ exporter.py: 104]:\t[Best Exporter]: skip step 40000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:23,657 [monitored_executor.py: 122]:\tsaving step 40000 to ./output/model_40000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:26,852 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_37000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:33,928 [ hooks.py: 207]:\tstep: 40010\tsteps/sec: 0.02029\tloss: 0.59512\tlr:[2.283097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:41,814 [ hooks.py: 207]:\tstep: 40020\tsteps/sec: 1.24680\tloss: 0.80191\tlr:[2.2824177e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:49,650 [ hooks.py: 207]:\tstep: 40030\tsteps/sec: 1.27531\tloss: 0.29250\tlr:[2.2817387e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:21:59,942 [ hooks.py: 207]:\tstep: 40040\tsteps/sec: 0.99032\tloss: 0.19200\tlr:[2.2810596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:22:07,108 [ hooks.py: 207]:\tstep: 40050\tsteps/sec: 1.34179\tloss: 0.62874\tlr:[2.2803808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:22:15,777 [ hooks.py: 207]:\tstep: 40060\tsteps/sec: 1.14730\tloss: 0.17010\tlr:[2.2797016e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:22:24,831 [ hooks.py: 207]:\tstep: 40070\tsteps/sec: 1.13314\tloss: 0.42341\tlr:[2.2790226e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:22:34,254 [ hooks.py: 207]:\tstep: 40080\tsteps/sec: 1.07249\tloss: 0.34191\tlr:[2.2783435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:22:44,200 [ hooks.py: 207]:\tstep: 40090\tsteps/sec: 0.96958\tloss: 0.24380\tlr:[2.2776643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:22:54,547 [ hooks.py: 207]:\tstep: 40100\tsteps/sec: 0.99593\tloss: 0.08975\tlr:[2.2769855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:23:04,446 [ hooks.py: 207]:\tstep: 40110\tsteps/sec: 0.98176\tloss: 0.71390\tlr:[2.276306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:23:15,147 [ hooks.py: 207]:\tstep: 40120\tsteps/sec: 0.94994\tloss: 0.51729\tlr:[2.2756272e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:23:21,825 [ hooks.py: 207]:\tstep: 40130\tsteps/sec: 1.49224\tloss: 0.41726\tlr:[2.2749482e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:23:31,191 [ hooks.py: 207]:\tstep: 40140\tsteps/sec: 1.06003\tloss: 0.74197\tlr:[2.2742692e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:23:40,592 [ hooks.py: 207]:\tstep: 40150\tsteps/sec: 1.07093\tloss: 0.70096\tlr:[2.2735901e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:23:47,914 [ hooks.py: 207]:\tstep: 40160\tsteps/sec: 1.35399\tloss: 0.23761\tlr:[2.272911e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:23:56,148 [ hooks.py: 207]:\tstep: 40170\tsteps/sec: 1.21785\tloss: 0.60286\tlr:[2.2722319e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:24:03,963 [ hooks.py: 207]:\tstep: 40180\tsteps/sec: 1.36873\tloss: 0.37825\tlr:[2.2715529e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:24:13,772 [ hooks.py: 207]:\tstep: 40190\tsteps/sec: 0.98613\tloss: 0.27086\tlr:[2.2708737e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:24:21,226 [ hooks.py: 207]:\tstep: 40200\tsteps/sec: 1.26623\tloss: 0.55667\tlr:[2.2701948e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:24:32,042 [ hooks.py: 207]:\tstep: 40210\tsteps/sec: 0.95359\tloss: 0.60843\tlr:[2.2695156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:24:42,635 [ hooks.py: 207]:\tstep: 40220\tsteps/sec: 0.93774\tloss: 0.50515\tlr:[2.2688368e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:24:51,635 [ hooks.py: 207]:\tstep: 40230\tsteps/sec: 1.11104\tloss: 0.55364\tlr:[2.2681577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:24:59,815 [ hooks.py: 207]:\tstep: 40240\tsteps/sec: 1.19898\tloss: 0.30002\tlr:[2.2674782e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:25:08,018 [ hooks.py: 207]:\tstep: 40250\tsteps/sec: 1.20360\tloss: 0.44702\tlr:[2.2667995e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:25:16,281 [ hooks.py: 207]:\tstep: 40260\tsteps/sec: 1.23875\tloss: 0.52830\tlr:[2.2661205e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:25:29,476 [ hooks.py: 207]:\tstep: 40270\tsteps/sec: 0.77773\tloss: 0.48917\tlr:[2.2654414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:25:36,769 [ hooks.py: 207]:\tstep: 40280\tsteps/sec: 1.27035\tloss: 0.06593\tlr:[2.2647622e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:25:45,237 [ hooks.py: 207]:\tstep: 40290\tsteps/sec: 1.18000\tloss: 0.36762\tlr:[2.2640834e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 07:25:54,588 [ hooks.py: 207]:\tstep: 40300\tsteps/sec: 1.11775\tloss: 0.40701\tlr:[2.263404e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:26:03,263 [ hooks.py: 207]:\tstep: 40310\tsteps/sec: 1.17143\tloss: 0.55392\tlr:[2.2627253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:26:09,035 [ hooks.py: 207]:\tstep: 40320\tsteps/sec: 1.63239\tloss: 0.42302\tlr:[2.2620461e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:26:18,130 [ hooks.py: 207]:\tstep: 40330\tsteps/sec: 1.12445\tloss: 0.57011\tlr:[2.2613673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:26:25,704 [ hooks.py: 207]:\tstep: 40340\tsteps/sec: 1.30892\tloss: 0.55068\tlr:[2.260688e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:26:35,526 [ hooks.py: 207]:\tstep: 40350\tsteps/sec: 1.01614\tloss: 0.37968\tlr:[2.2600092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:26:44,340 [ hooks.py: 207]:\tstep: 40360\tsteps/sec: 1.16419\tloss: 0.80540\tlr:[2.25933e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:26:52,832 [ hooks.py: 207]:\tstep: 40370\tsteps/sec: 1.14316\tloss: 0.37294\tlr:[2.258651e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:27:00,258 [ hooks.py: 207]:\tstep: 40380\tsteps/sec: 1.38568\tloss: 0.45117\tlr:[2.2579718e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:27:09,146 [ hooks.py: 207]:\tstep: 40390\tsteps/sec: 1.13083\tloss: 0.78715\tlr:[2.2572925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:27:17,299 [ hooks.py: 207]:\tstep: 40400\tsteps/sec: 1.14854\tloss: 0.77364\tlr:[2.2566137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:27:27,698 [ hooks.py: 207]:\tstep: 40410\tsteps/sec: 0.98056\tloss: 0.14078\tlr:[2.2559345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:27:37,300 [ hooks.py: 207]:\tstep: 40420\tsteps/sec: 1.06173\tloss: 0.27686\tlr:[2.2552555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:27:45,353 [ hooks.py: 207]:\tstep: 40430\tsteps/sec: 1.24045\tloss: 0.39333\tlr:[2.2545764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:27:53,553 [ hooks.py: 207]:\tstep: 40440\tsteps/sec: 1.23816\tloss: 0.33779\tlr:[2.2538974e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:28:01,784 [ hooks.py: 207]:\tstep: 40450\tsteps/sec: 1.20606\tloss: 0.33128\tlr:[2.2532184e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:28:11,723 [ hooks.py: 207]:\tstep: 40460\tsteps/sec: 1.00501\tloss: 1.04985\tlr:[2.2525395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:28:19,244 [ hooks.py: 207]:\tstep: 40470\tsteps/sec: 1.36158\tloss: 0.20505\tlr:[2.2518601e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:28:26,907 [ hooks.py: 207]:\tstep: 40480\tsteps/sec: 1.21844\tloss: 0.52825\tlr:[2.2511813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:28:36,737 [ hooks.py: 207]:\tstep: 40490\tsteps/sec: 1.01498\tloss: 0.20252\tlr:[2.250502e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:28:46,374 [ hooks.py: 207]:\tstep: 40500\tsteps/sec: 1.06283\tloss: 0.66526\tlr:[2.249823e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:28:55,364 [ hooks.py: 207]:\tstep: 40510\tsteps/sec: 1.12461\tloss: 0.39820\tlr:[2.249144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:29:03,890 [ hooks.py: 207]:\tstep: 40520\tsteps/sec: 1.23314\tloss: 0.42420\tlr:[2.2484648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:29:11,507 [ hooks.py: 207]:\tstep: 40530\tsteps/sec: 1.20408\tloss: 0.42442\tlr:[2.2477858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:29:19,712 [ hooks.py: 207]:\tstep: 40540\tsteps/sec: 1.28161\tloss: 0.30306\tlr:[2.2471067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:29:27,792 [ hooks.py: 207]:\tstep: 40550\tsteps/sec: 1.20402\tloss: 0.53379\tlr:[2.2464277e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:29:36,907 [ hooks.py: 207]:\tstep: 40560\tsteps/sec: 1.07069\tloss: 0.17159\tlr:[2.2457485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:29:45,682 [ hooks.py: 207]:\tstep: 40570\tsteps/sec: 1.18818\tloss: 0.28237\tlr:[2.2450698e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:29:53,055 [ hooks.py: 207]:\tstep: 40580\tsteps/sec: 1.35247\tloss: 0.31139\tlr:[2.2443905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:30:02,002 [ hooks.py: 207]:\tstep: 40590\tsteps/sec: 1.15843\tloss: 0.29105\tlr:[2.2437118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:30:13,106 [ hooks.py: 207]:\tstep: 40600\tsteps/sec: 0.87462\tloss: 0.59008\tlr:[2.2430326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:30:21,864 [ hooks.py: 207]:\tstep: 40610\tsteps/sec: 1.15017\tloss: 0.10615\tlr:[2.2423535e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:30:29,394 [ hooks.py: 207]:\tstep: 40620\tsteps/sec: 1.36681\tloss: 0.59138\tlr:[2.2416743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:30:38,391 [ hooks.py: 207]:\tstep: 40630\tsteps/sec: 1.09211\tloss: 0.43688\tlr:[2.2409955e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:30:45,324 [ hooks.py: 207]:\tstep: 40640\tsteps/sec: 1.41998\tloss: 0.35742\tlr:[2.2403163e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:30:53,822 [ hooks.py: 207]:\tstep: 40650\tsteps/sec: 1.19761\tloss: 0.41906\tlr:[2.2396374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:31:01,583 [ hooks.py: 207]:\tstep: 40660\tsteps/sec: 1.31541\tloss: 0.68443\tlr:[2.2389584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:31:11,268 [ hooks.py: 207]:\tstep: 40670\tsteps/sec: 0.96779\tloss: 0.45347\tlr:[2.2382792e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:31:20,699 [ hooks.py: 207]:\tstep: 40680\tsteps/sec: 1.09674\tloss: 0.17835\tlr:[2.2376002e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:31:29,299 [ hooks.py: 207]:\tstep: 40690\tsteps/sec: 1.19532\tloss: 0.28541\tlr:[2.236921e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:31:36,128 [ hooks.py: 207]:\tstep: 40700\tsteps/sec: 1.41153\tloss: 0.19444\tlr:[2.2362421e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:31:43,769 [ hooks.py: 207]:\tstep: 40710\tsteps/sec: 1.28651\tloss: 0.51472\tlr:[2.235563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:31:53,030 [ hooks.py: 207]:\tstep: 40720\tsteps/sec: 1.09308\tloss: 0.37395\tlr:[2.234884e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:32:03,853 [ hooks.py: 207]:\tstep: 40730\tsteps/sec: 0.97275\tloss: 0.13774\tlr:[2.2342048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:32:13,739 [ hooks.py: 207]:\tstep: 40740\tsteps/sec: 0.97933\tloss: 0.41818\tlr:[2.233526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:32:21,147 [ hooks.py: 207]:\tstep: 40750\tsteps/sec: 1.26965\tloss: 0.15492\tlr:[2.2328468e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:32:29,928 [ hooks.py: 207]:\tstep: 40760\tsteps/sec: 1.20356\tloss: 0.26822\tlr:[2.2321678e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:32:38,831 [ hooks.py: 207]:\tstep: 40770\tsteps/sec: 1.07960\tloss: 0.61101\tlr:[2.2314885e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:32:48,354 [ hooks.py: 207]:\tstep: 40780\tsteps/sec: 1.07827\tloss: 0.18843\tlr:[2.2308097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:32:55,285 [ hooks.py: 207]:\tstep: 40790\tsteps/sec: 1.42335\tloss: 0.31550\tlr:[2.2301305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:33:05,380 [ hooks.py: 207]:\tstep: 40800\tsteps/sec: 1.00794\tloss: 0.32888\tlr:[2.2294516e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:33:13,828 [ hooks.py: 207]:\tstep: 40810\tsteps/sec: 1.18874\tloss: 0.80688\tlr:[2.2287722e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:33:21,845 [ hooks.py: 207]:\tstep: 40820\tsteps/sec: 1.22352\tloss: 0.58315\tlr:[2.2280932e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:33:31,697 [ hooks.py: 207]:\tstep: 40830\tsteps/sec: 1.02224\tloss: 0.52660\tlr:[2.2274144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:33:38,930 [ hooks.py: 207]:\tstep: 40840\tsteps/sec: 1.40730\tloss: 0.46870\tlr:[2.2267352e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:33:46,433 [ hooks.py: 207]:\tstep: 40850\tsteps/sec: 1.27158\tloss: 0.40251\tlr:[2.2260561e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:33:55,817 [ hooks.py: 207]:\tstep: 40860\tsteps/sec: 1.08796\tloss: 0.37202\tlr:[2.225377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:34:04,567 [ hooks.py: 207]:\tstep: 40870\tsteps/sec: 1.15154\tloss: 0.37764\tlr:[2.224698e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:34:13,890 [ hooks.py: 207]:\tstep: 40880\tsteps/sec: 1.08323\tloss: 0.43795\tlr:[2.2240189e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:34:23,570 [ hooks.py: 207]:\tstep: 40890\tsteps/sec: 1.02060\tloss: 0.48611\tlr:[2.22334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:34:32,142 [ hooks.py: 207]:\tstep: 40900\tsteps/sec: 1.16964\tloss: 0.30401\tlr:[2.2226608e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:34:40,553 [ hooks.py: 207]:\tstep: 40910\tsteps/sec: 1.17690\tloss: 0.21114\tlr:[2.221982e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:34:49,030 [ hooks.py: 207]:\tstep: 40920\tsteps/sec: 1.20152\tloss: 0.63078\tlr:[2.2213027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:34:55,695 [ hooks.py: 207]:\tstep: 40930\tsteps/sec: 1.46593\tloss: 0.67732\tlr:[2.2206237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:02,897 [ hooks.py: 207]:\tstep: 40940\tsteps/sec: 1.51791\tloss: 0.42541\tlr:[2.2199445e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 07:35:09,444 [ hooks.py: 207]:\tstep: 40950\tsteps/sec: 1.41901\tloss: 0.73979\tlr:[2.2192658e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:20,581 [ hooks.py: 207]:\tstep: 40960\tsteps/sec: 0.89397\tloss: 0.18251\tlr:[2.2185865e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:29,089 [ hooks.py: 207]:\tstep: 40970\tsteps/sec: 1.18593\tloss: 0.36412\tlr:[2.2179072e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:36,817 [ hooks.py: 207]:\tstep: 40980\tsteps/sec: 1.25680\tloss: 0.39615\tlr:[2.2172284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:46,533 [ hooks.py: 207]:\tstep: 40990\tsteps/sec: 1.03075\tloss: 0.52766\tlr:[2.2165492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:53,997 [ hooks.py: 207]:\tstep: 41000\tsteps/sec: 1.38356\tloss: 0.29875\tlr:[2.2158703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:53,998 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:53,998 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:53,998 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:35:53,998 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:53,998 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:35:53,998 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:43:56,180 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,181 [ trainer.py: 58]:\t{'acc': 0.7742971887550201, 'loss': 0.596247603209355}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,182 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,183 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:43:56,183 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:43:56,183 [ trainer.py: 70]:\tacc\t0.7742971887550201\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:43:56,183 [ trainer.py: 70]:\tloss\t0.596247603209355\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 07:43:56,183 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,183 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7742971887550201, 'loss': 0.596247603209355}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,183 [ exporter.py: 64]:\t[Best Exporter]: skip step 41000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,183 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7742971887550201, 'loss': 0.596247603209355}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,184 [ exporter.py: 104]:\t[Best Exporter]: skip step 41000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:56,185 [monitored_executor.py: 122]:\tsaving step 41000 to ./output/model_41000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:43:59,458 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_38000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:44:08,578 [ hooks.py: 207]:\tstep: 41010\tsteps/sec: 0.02022\tloss: 0.58824\tlr:[2.215191e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:44:17,085 [ hooks.py: 207]:\tstep: 41020\tsteps/sec: 1.14098\tloss: 0.46548\tlr:[2.2145121e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:44:26,880 [ hooks.py: 207]:\tstep: 41030\tsteps/sec: 1.03254\tloss: 0.50922\tlr:[2.2138329e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:44:33,992 [ hooks.py: 207]:\tstep: 41040\tsteps/sec: 1.43343\tloss: 0.21027\tlr:[2.213154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:44:42,377 [ hooks.py: 207]:\tstep: 41050\tsteps/sec: 1.16961\tloss: 0.51347\tlr:[2.2124748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:44:51,974 [ hooks.py: 207]:\tstep: 41060\tsteps/sec: 1.03588\tloss: 0.27617\tlr:[2.211796e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:45:00,643 [ hooks.py: 207]:\tstep: 41070\tsteps/sec: 1.20393\tloss: 0.41880\tlr:[2.2111168e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:45:09,286 [ hooks.py: 207]:\tstep: 41080\tsteps/sec: 1.14476\tloss: 0.43289\tlr:[2.210438e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:45:16,724 [ hooks.py: 207]:\tstep: 41090\tsteps/sec: 1.30380\tloss: 0.24160\tlr:[2.2097587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:45:26,812 [ hooks.py: 207]:\tstep: 41100\tsteps/sec: 0.99908\tloss: 0.44722\tlr:[2.2090799e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:45:33,989 [ hooks.py: 207]:\tstep: 41110\tsteps/sec: 1.41268\tloss: 0.37726\tlr:[2.2084008e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:45:43,428 [ hooks.py: 207]:\tstep: 41120\tsteps/sec: 1.04530\tloss: 0.45240\tlr:[2.2077214e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:45:50,738 [ hooks.py: 207]:\tstep: 41130\tsteps/sec: 1.39147\tloss: 0.36525\tlr:[2.2070426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:46:00,696 [ hooks.py: 207]:\tstep: 41140\tsteps/sec: 1.00689\tloss: 0.44451\tlr:[2.2063634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:46:11,484 [ hooks.py: 207]:\tstep: 41150\tsteps/sec: 0.92042\tloss: 0.14924\tlr:[2.2056845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:46:19,766 [ hooks.py: 207]:\tstep: 41160\tsteps/sec: 1.22075\tloss: 0.29342\tlr:[2.2050053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:46:28,027 [ hooks.py: 207]:\tstep: 41170\tsteps/sec: 1.17404\tloss: 0.41165\tlr:[2.2043265e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:46:36,391 [ hooks.py: 207]:\tstep: 41180\tsteps/sec: 1.20031\tloss: 0.76743\tlr:[2.203647e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:46:46,270 [ hooks.py: 207]:\tstep: 41190\tsteps/sec: 1.01951\tloss: 0.48209\tlr:[2.2029682e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:46:54,860 [ hooks.py: 207]:\tstep: 41200\tsteps/sec: 1.18421\tloss: 0.59163\tlr:[2.202289e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:47:04,614 [ hooks.py: 207]:\tstep: 41210\tsteps/sec: 1.02756\tloss: 0.69813\tlr:[2.2016102e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:47:12,975 [ hooks.py: 207]:\tstep: 41220\tsteps/sec: 1.17433\tloss: 0.22005\tlr:[2.200931e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:47:21,795 [ hooks.py: 207]:\tstep: 41230\tsteps/sec: 1.13359\tloss: 0.37264\tlr:[2.2002521e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:47:28,575 [ hooks.py: 207]:\tstep: 41240\tsteps/sec: 1.48065\tloss: 0.58455\tlr:[2.199573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:47:37,333 [ hooks.py: 207]:\tstep: 41250\tsteps/sec: 1.12416\tloss: 0.43051\tlr:[2.1988939e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:47:45,417 [ hooks.py: 207]:\tstep: 41260\tsteps/sec: 1.19618\tloss: 0.16702\tlr:[2.1982149e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:47:54,014 [ hooks.py: 207]:\tstep: 41270\tsteps/sec: 1.24020\tloss: 0.38899\tlr:[2.1975357e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:48:02,771 [ hooks.py: 207]:\tstep: 41280\tsteps/sec: 1.17753\tloss: 0.41008\tlr:[2.1968566e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:48:10,778 [ hooks.py: 207]:\tstep: 41290\tsteps/sec: 1.19504\tloss: 0.51117\tlr:[2.1961776e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:48:19,591 [ hooks.py: 207]:\tstep: 41300\tsteps/sec: 1.15847\tloss: 0.80178\tlr:[2.1954986e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:48:27,038 [ hooks.py: 207]:\tstep: 41310\tsteps/sec: 1.28951\tloss: 0.62717\tlr:[2.1948192e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:48:35,898 [ hooks.py: 207]:\tstep: 41320\tsteps/sec: 1.12675\tloss: 0.80029\tlr:[2.1941403e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:48:46,419 [ hooks.py: 207]:\tstep: 41330\tsteps/sec: 0.92648\tloss: 0.41932\tlr:[2.1934615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:48:56,068 [ hooks.py: 207]:\tstep: 41340\tsteps/sec: 1.06743\tloss: 0.32517\tlr:[2.1927824e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:49:06,783 [ hooks.py: 207]:\tstep: 41350\tsteps/sec: 0.93921\tloss: 0.51737\tlr:[2.1921032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:49:15,011 [ hooks.py: 207]:\tstep: 41360\tsteps/sec: 1.22571\tloss: 0.35584\tlr:[2.1914244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:49:23,408 [ hooks.py: 207]:\tstep: 41370\tsteps/sec: 1.17735\tloss: 0.45253\tlr:[2.1907452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:49:32,294 [ hooks.py: 207]:\tstep: 41380\tsteps/sec: 1.12891\tloss: 0.51496\tlr:[2.1900662e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:49:38,973 [ hooks.py: 207]:\tstep: 41390\tsteps/sec: 1.51243\tloss: 0.49740\tlr:[2.189387e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 07:49:47,431 [ hooks.py: 207]:\tstep: 41400\tsteps/sec: 1.16940\tloss: 0.64497\tlr:[2.1887081e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "IOPub message rate exceeded.\n", + "The notebook server will temporarily stop sending output\n", + "to the client in order to avoid crashing it.\n", + "To change this limit, set the config variable\n", + "`--NotebookApp.iopub_msg_rate_limit`.\n", + "\n", + "Current values:\n", + "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n", + "NotebookApp.rate_limit_window=3.0 (secs)\n", + "\n", + "\u001b[37m[DEBUG] 2019-09-19 08:29:53,888 [ hooks.py: 207]:\tstep: 43050\tsteps/sec: 0.88949\tloss: 0.18220\tlr:[2.0766634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:30:04,896 [ hooks.py: 207]:\tstep: 43060\tsteps/sec: 0.93032\tloss: 0.34885\tlr:[2.0759848e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:30:13,127 [ hooks.py: 207]:\tstep: 43070\tsteps/sec: 1.17766\tloss: 0.26274\tlr:[2.0753056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:30:20,704 [ hooks.py: 207]:\tstep: 43080\tsteps/sec: 1.37526\tloss: 0.16291\tlr:[2.0746267e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:30:27,724 [ hooks.py: 207]:\tstep: 43090\tsteps/sec: 1.37301\tloss: 0.57161\tlr:[2.0739475e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:30:36,229 [ hooks.py: 207]:\tstep: 43100\tsteps/sec: 1.22527\tloss: 0.51577\tlr:[2.0732685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:30:44,422 [ hooks.py: 207]:\tstep: 43110\tsteps/sec: 1.18133\tloss: 0.56439\tlr:[2.0725895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:30:53,497 [ hooks.py: 207]:\tstep: 43120\tsteps/sec: 1.09782\tloss: 0.68214\tlr:[2.0719106e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:31:01,208 [ hooks.py: 207]:\tstep: 43130\tsteps/sec: 1.43210\tloss: 0.45387\tlr:[2.0712312e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:31:11,148 [ hooks.py: 207]:\tstep: 43140\tsteps/sec: 0.96985\tloss: 0.28377\tlr:[2.0705525e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:31:19,559 [ hooks.py: 207]:\tstep: 43150\tsteps/sec: 1.15291\tloss: 0.33157\tlr:[2.0698735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:31:28,390 [ hooks.py: 207]:\tstep: 43160\tsteps/sec: 1.11669\tloss: 0.32563\tlr:[2.0691943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:31:38,621 [ hooks.py: 207]:\tstep: 43170\tsteps/sec: 0.98547\tloss: 0.84459\tlr:[2.0685153e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:31:47,849 [ hooks.py: 207]:\tstep: 43180\tsteps/sec: 1.07382\tloss: 0.62717\tlr:[2.067836e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:31:56,196 [ hooks.py: 207]:\tstep: 43190\tsteps/sec: 1.24350\tloss: 0.66597\tlr:[2.067157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:32:03,529 [ hooks.py: 207]:\tstep: 43200\tsteps/sec: 1.35684\tloss: 0.34807\tlr:[2.066478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:32:12,308 [ hooks.py: 207]:\tstep: 43210\tsteps/sec: 1.07072\tloss: 0.84460\tlr:[2.0657988e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:32:22,511 [ hooks.py: 207]:\tstep: 43220\tsteps/sec: 1.02090\tloss: 0.37098\tlr:[2.0651198e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:32:31,459 [ hooks.py: 207]:\tstep: 43230\tsteps/sec: 1.09650\tloss: 0.61912\tlr:[2.0644407e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:32:41,650 [ hooks.py: 207]:\tstep: 43240\tsteps/sec: 0.96476\tloss: 0.44234\tlr:[2.0637615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:32:50,239 [ hooks.py: 207]:\tstep: 43250\tsteps/sec: 1.23675\tloss: 0.47322\tlr:[2.0630827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:32:58,306 [ hooks.py: 207]:\tstep: 43260\tsteps/sec: 1.15398\tloss: 0.41695\tlr:[2.0624035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:33:08,934 [ hooks.py: 207]:\tstep: 43270\tsteps/sec: 0.94895\tloss: 0.39763\tlr:[2.0617246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:33:16,725 [ hooks.py: 207]:\tstep: 43280\tsteps/sec: 1.29180\tloss: 0.31227\tlr:[2.0610454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:33:25,532 [ hooks.py: 207]:\tstep: 43290\tsteps/sec: 1.20235\tloss: 0.49489\tlr:[2.0603666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:33:32,394 [ hooks.py: 207]:\tstep: 43300\tsteps/sec: 1.41519\tloss: 0.79614\tlr:[2.0596874e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:33:40,312 [ hooks.py: 207]:\tstep: 43310\tsteps/sec: 1.28786\tloss: 0.23236\tlr:[2.0590085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:33:47,176 [ hooks.py: 207]:\tstep: 43320\tsteps/sec: 1.41699\tloss: 0.36641\tlr:[2.0583293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:33:57,481 [ hooks.py: 207]:\tstep: 43330\tsteps/sec: 0.96330\tloss: 0.36242\tlr:[2.0576499e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:34:05,634 [ hooks.py: 207]:\tstep: 43340\tsteps/sec: 1.27236\tloss: 0.35916\tlr:[2.056971e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:34:13,297 [ hooks.py: 207]:\tstep: 43350\tsteps/sec: 1.26268\tloss: 0.72792\tlr:[2.056292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:34:20,200 [ hooks.py: 207]:\tstep: 43360\tsteps/sec: 1.44520\tloss: 0.70874\tlr:[2.055613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:34:28,917 [ hooks.py: 207]:\tstep: 43370\tsteps/sec: 1.15423\tloss: 0.61020\tlr:[2.0549338e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:34:37,319 [ hooks.py: 207]:\tstep: 43380\tsteps/sec: 1.19785\tloss: 0.66327\tlr:[2.0542551e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:34:48,075 [ hooks.py: 207]:\tstep: 43390\tsteps/sec: 0.92112\tloss: 0.36885\tlr:[2.053576e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:34:55,515 [ hooks.py: 207]:\tstep: 43400\tsteps/sec: 1.38493\tloss: 0.43700\tlr:[2.0528969e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:35:04,638 [ hooks.py: 207]:\tstep: 43410\tsteps/sec: 1.07795\tloss: 0.29776\tlr:[2.0522179e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:35:11,460 [ hooks.py: 207]:\tstep: 43420\tsteps/sec: 1.52869\tloss: 0.39793\tlr:[2.0515388e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:35:21,287 [ hooks.py: 207]:\tstep: 43430\tsteps/sec: 1.01722\tloss: 0.67414\tlr:[2.0508596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:35:30,132 [ hooks.py: 207]:\tstep: 43440\tsteps/sec: 1.07625\tloss: 0.42078\tlr:[2.0501808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:35:38,690 [ hooks.py: 207]:\tstep: 43450\tsteps/sec: 1.14589\tloss: 0.31017\tlr:[2.0495016e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:35:46,923 [ hooks.py: 207]:\tstep: 43460\tsteps/sec: 1.28247\tloss: 0.12290\tlr:[2.0488225e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:35:56,214 [ hooks.py: 207]:\tstep: 43470\tsteps/sec: 1.08552\tloss: 1.12098\tlr:[2.0481435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:36:05,599 [ hooks.py: 207]:\tstep: 43480\tsteps/sec: 1.07773\tloss: 0.71082\tlr:[2.0474641e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:36:15,276 [ hooks.py: 207]:\tstep: 43490\tsteps/sec: 1.02184\tloss: 0.54425\tlr:[2.0467854e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:36:24,228 [ hooks.py: 207]:\tstep: 43500\tsteps/sec: 1.08781\tloss: 0.48543\tlr:[2.046106e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:36:32,708 [ hooks.py: 207]:\tstep: 43510\tsteps/sec: 1.18173\tloss: 0.29631\tlr:[2.0454272e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:36:41,987 [ hooks.py: 207]:\tstep: 43520\tsteps/sec: 1.09490\tloss: 0.37401\tlr:[2.044748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:36:49,027 [ hooks.py: 207]:\tstep: 43530\tsteps/sec: 1.31799\tloss: 0.63915\tlr:[2.0440692e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:36:57,810 [ hooks.py: 207]:\tstep: 43540\tsteps/sec: 1.15519\tloss: 0.29770\tlr:[2.04339e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:37:07,552 [ hooks.py: 207]:\tstep: 43550\tsteps/sec: 1.06561\tloss: 0.59679\tlr:[2.0427111e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:37:15,439 [ hooks.py: 207]:\tstep: 43560\tsteps/sec: 1.30513\tloss: 0.53689\tlr:[2.0420319e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:37:22,713 [ hooks.py: 207]:\tstep: 43570\tsteps/sec: 1.31749\tloss: 0.39704\tlr:[2.0413529e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:37:29,313 [ hooks.py: 207]:\tstep: 43580\tsteps/sec: 1.50811\tloss: 0.28172\tlr:[2.0406738e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:37:37,763 [ hooks.py: 207]:\tstep: 43590\tsteps/sec: 1.14856\tloss: 0.38701\tlr:[2.0399946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:37:48,187 [ hooks.py: 207]:\tstep: 43600\tsteps/sec: 0.99461\tloss: 0.97326\tlr:[2.0393156e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:37:57,453 [ hooks.py: 207]:\tstep: 43610\tsteps/sec: 1.03228\tloss: 0.48340\tlr:[2.0386366e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:38:05,833 [ hooks.py: 207]:\tstep: 43620\tsteps/sec: 1.29569\tloss: 0.25632\tlr:[2.0379575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:38:15,147 [ hooks.py: 207]:\tstep: 43630\tsteps/sec: 1.02725\tloss: 0.54656\tlr:[2.0372783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:38:23,899 [ hooks.py: 207]:\tstep: 43640\tsteps/sec: 1.16487\tloss: 0.74178\tlr:[2.0365993e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:38:31,362 [ hooks.py: 207]:\tstep: 43650\tsteps/sec: 1.31688\tloss: 0.47684\tlr:[2.0359203e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:38:41,276 [ hooks.py: 207]:\tstep: 43660\tsteps/sec: 1.05554\tloss: 0.36712\tlr:[2.0352412e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:38:49,559 [ hooks.py: 207]:\tstep: 43670\tsteps/sec: 1.15435\tloss: 0.57028\tlr:[2.0345622e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 08:38:57,185 [ hooks.py: 207]:\tstep: 43680\tsteps/sec: 1.33485\tloss: 0.31375\tlr:[2.0338834e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:05,377 [ hooks.py: 207]:\tstep: 43690\tsteps/sec: 1.19990\tloss: 0.46720\tlr:[2.0332041e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:13,611 [ hooks.py: 207]:\tstep: 43700\tsteps/sec: 1.23092\tloss: 0.31758\tlr:[2.0325251e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:20,282 [ hooks.py: 207]:\tstep: 43710\tsteps/sec: 1.46456\tloss: 0.48585\tlr:[2.031846e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:28,140 [ hooks.py: 207]:\tstep: 43720\tsteps/sec: 1.27176\tloss: 0.34758\tlr:[2.031167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:36,062 [ hooks.py: 207]:\tstep: 43730\tsteps/sec: 1.26804\tloss: 0.60565\tlr:[2.030488e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:43,857 [ hooks.py: 207]:\tstep: 43740\tsteps/sec: 1.30287\tloss: 0.34631\tlr:[2.0298092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:52,165 [ hooks.py: 207]:\tstep: 43750\tsteps/sec: 1.21419\tloss: 0.38199\tlr:[2.02913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:39:59,023 [ hooks.py: 207]:\tstep: 43760\tsteps/sec: 1.42641\tloss: 0.12777\tlr:[2.0284506e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:40:07,010 [ hooks.py: 207]:\tstep: 43770\tsteps/sec: 1.27211\tloss: 0.31712\tlr:[2.0277717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:40:15,420 [ hooks.py: 207]:\tstep: 43780\tsteps/sec: 1.20997\tloss: 0.11687\tlr:[2.0270927e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:40:23,394 [ hooks.py: 207]:\tstep: 43790\tsteps/sec: 1.22813\tloss: 0.59710\tlr:[2.0264139e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:40:31,220 [ hooks.py: 207]:\tstep: 43800\tsteps/sec: 1.26519\tloss: 0.70923\tlr:[2.0257346e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:40:41,195 [ hooks.py: 207]:\tstep: 43810\tsteps/sec: 1.00146\tloss: 0.52306\tlr:[2.0250556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:40:50,781 [ hooks.py: 207]:\tstep: 43820\tsteps/sec: 1.08232\tloss: 0.55002\tlr:[2.0243764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:40:58,133 [ hooks.py: 207]:\tstep: 43830\tsteps/sec: 1.30191\tloss: 0.30522\tlr:[2.0236976e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:41:05,712 [ hooks.py: 207]:\tstep: 43840\tsteps/sec: 1.35993\tloss: 0.58578\tlr:[2.0230182e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:41:12,760 [ hooks.py: 207]:\tstep: 43850\tsteps/sec: 1.37734\tloss: 0.17383\tlr:[2.0223393e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:41:21,323 [ hooks.py: 207]:\tstep: 43860\tsteps/sec: 1.20138\tloss: 0.29914\tlr:[2.0216603e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:41:30,393 [ hooks.py: 207]:\tstep: 43870\tsteps/sec: 1.10441\tloss: 0.68311\tlr:[2.0209813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:41:38,642 [ hooks.py: 207]:\tstep: 43880\tsteps/sec: 1.16435\tloss: 0.12717\tlr:[2.020302e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:41:48,296 [ hooks.py: 207]:\tstep: 43890\tsteps/sec: 1.04102\tloss: 0.45920\tlr:[2.0196234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:41:56,030 [ hooks.py: 207]:\tstep: 43900\tsteps/sec: 1.33200\tloss: 0.28816\tlr:[2.018944e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:42:05,757 [ hooks.py: 207]:\tstep: 43910\tsteps/sec: 1.03354\tloss: 0.79325\tlr:[2.0182648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:42:12,603 [ hooks.py: 207]:\tstep: 43920\tsteps/sec: 1.38942\tloss: 0.77944\tlr:[2.0175861e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:42:22,225 [ hooks.py: 207]:\tstep: 43930\tsteps/sec: 1.06151\tloss: 0.41022\tlr:[2.016907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:42:30,519 [ hooks.py: 207]:\tstep: 43940\tsteps/sec: 1.22334\tloss: 0.48092\tlr:[2.0162279e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:42:40,962 [ hooks.py: 207]:\tstep: 43950\tsteps/sec: 0.96681\tloss: 0.45226\tlr:[2.0155487e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:42:48,696 [ hooks.py: 207]:\tstep: 43960\tsteps/sec: 1.25120\tloss: 0.76411\tlr:[2.0148696e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:42:55,746 [ hooks.py: 207]:\tstep: 43970\tsteps/sec: 1.45649\tloss: 0.18442\tlr:[2.0141906e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:03,090 [ hooks.py: 207]:\tstep: 43980\tsteps/sec: 1.39897\tloss: 0.59976\tlr:[2.0135116e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:10,642 [ hooks.py: 207]:\tstep: 43990\tsteps/sec: 1.28280\tloss: 0.54836\tlr:[2.0128324e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:18,717 [ hooks.py: 207]:\tstep: 44000\tsteps/sec: 1.15440\tloss: 0.25020\tlr:[2.0121535e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:18,718 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:18,718 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:18,718 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 08:43:18,718 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:18,718 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:43:18,719 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 08:51:22,269 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,270 [ trainer.py: 58]:\t{'acc': 0.7819277108433735, 'loss': 0.5945944198622153}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,271 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,272 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 08:51:22,272 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 08:51:22,272 [ trainer.py: 70]:\tacc\t0.7819277108433735\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 08:51:22,272 [ trainer.py: 70]:\tloss\t0.5945944198622153\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 08:51:22,272 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,272 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7819277108433735, 'loss': 0.5945944198622153}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,272 [ exporter.py: 64]:\t[Best Exporter]: skip step 44000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,273 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7819277108433735, 'loss': 0.5945944198622153}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,273 [ exporter.py: 104]:\t[Best Exporter]: skip step 44000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:22,274 [monitored_executor.py: 122]:\tsaving step 44000 to ./output/model_44000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:25,470 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_41000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:33,241 [ hooks.py: 207]:\tstep: 44010\tsteps/sec: 0.02024\tloss: 0.53003\tlr:[2.0114743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:43,537 [ hooks.py: 207]:\tstep: 44020\tsteps/sec: 0.97391\tloss: 0.35818\tlr:[2.0107955e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:51:53,408 [ hooks.py: 207]:\tstep: 44030\tsteps/sec: 1.00080\tloss: 0.69287\tlr:[2.0101164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:52:03,170 [ hooks.py: 207]:\tstep: 44040\tsteps/sec: 1.09555\tloss: 0.76589\tlr:[2.0094374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:52:09,461 [ hooks.py: 207]:\tstep: 44050\tsteps/sec: 1.39016\tloss: 0.84515\tlr:[2.0087582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:52:18,856 [ hooks.py: 207]:\tstep: 44060\tsteps/sec: 1.10843\tloss: 0.39045\tlr:[2.0080792e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:52:27,372 [ hooks.py: 207]:\tstep: 44070\tsteps/sec: 1.16466\tloss: 0.40807\tlr:[2.0074001e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:52:36,961 [ hooks.py: 207]:\tstep: 44080\tsteps/sec: 1.06555\tloss: 0.52170\tlr:[2.006721e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:52:46,954 [ hooks.py: 207]:\tstep: 44090\tsteps/sec: 0.97469\tloss: 0.40015\tlr:[2.006042e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:52:55,503 [ hooks.py: 207]:\tstep: 44100\tsteps/sec: 1.16155\tloss: 0.29567\tlr:[2.0053629e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:53:04,977 [ hooks.py: 207]:\tstep: 44110\tsteps/sec: 1.07682\tloss: 0.26186\tlr:[2.0046842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:53:13,923 [ hooks.py: 207]:\tstep: 44120\tsteps/sec: 1.12191\tloss: 0.09809\tlr:[2.004005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:53:22,242 [ hooks.py: 207]:\tstep: 44130\tsteps/sec: 1.21096\tloss: 0.53633\tlr:[2.003326e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 08:53:30,276 [ hooks.py: 207]:\tstep: 44140\tsteps/sec: 1.25935\tloss: 0.46256\tlr:[2.0026468e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:53:40,501 [ hooks.py: 207]:\tstep: 44150\tsteps/sec: 0.97493\tloss: 0.38938\tlr:[2.001968e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:53:49,328 [ hooks.py: 207]:\tstep: 44160\tsteps/sec: 1.06341\tloss: 0.22975\tlr:[2.0012887e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:53:58,793 [ hooks.py: 207]:\tstep: 44170\tsteps/sec: 1.09979\tloss: 0.46090\tlr:[2.0006097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:54:06,397 [ hooks.py: 207]:\tstep: 44180\tsteps/sec: 1.31222\tloss: 0.41565\tlr:[1.9999305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:54:15,751 [ hooks.py: 207]:\tstep: 44190\tsteps/sec: 1.09186\tloss: 0.34399\tlr:[1.9992516e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:54:26,295 [ hooks.py: 207]:\tstep: 44200\tsteps/sec: 0.92888\tloss: 0.33728\tlr:[1.9985724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:54:35,030 [ hooks.py: 207]:\tstep: 44210\tsteps/sec: 1.15225\tloss: 0.37759\tlr:[1.9978932e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:54:43,118 [ hooks.py: 207]:\tstep: 44220\tsteps/sec: 1.23917\tloss: 0.70447\tlr:[1.9972143e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:54:51,576 [ hooks.py: 207]:\tstep: 44230\tsteps/sec: 1.18120\tloss: 0.31135\tlr:[1.9965351e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:00,810 [ hooks.py: 207]:\tstep: 44240\tsteps/sec: 1.12510\tloss: 0.31728\tlr:[1.9958563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:06,783 [ hooks.py: 207]:\tstep: 44250\tsteps/sec: 1.65545\tloss: 0.38047\tlr:[1.995177e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:16,266 [ hooks.py: 207]:\tstep: 44260\tsteps/sec: 1.04847\tloss: 0.48343\tlr:[1.9944982e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:24,473 [ hooks.py: 207]:\tstep: 44270\tsteps/sec: 1.17774\tloss: 0.45145\tlr:[1.9938188e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:32,926 [ hooks.py: 207]:\tstep: 44280\tsteps/sec: 1.17981\tloss: 0.26361\tlr:[1.99314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:41,992 [ hooks.py: 207]:\tstep: 44290\tsteps/sec: 1.14974\tloss: 0.40656\tlr:[1.992461e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:48,969 [ hooks.py: 207]:\tstep: 44300\tsteps/sec: 1.37315\tloss: 0.71637\tlr:[1.9917818e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:55:57,940 [ hooks.py: 207]:\tstep: 44310\tsteps/sec: 1.10076\tloss: 0.23394\tlr:[1.9911027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:56:05,844 [ hooks.py: 207]:\tstep: 44320\tsteps/sec: 1.33128\tloss: 0.87708\tlr:[1.9904237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:56:14,888 [ hooks.py: 207]:\tstep: 44330\tsteps/sec: 1.07556\tloss: 0.69390\tlr:[1.9897445e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:56:23,111 [ hooks.py: 207]:\tstep: 44340\tsteps/sec: 1.20885\tloss: 0.33705\tlr:[1.9890656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:56:31,943 [ hooks.py: 207]:\tstep: 44350\tsteps/sec: 1.12257\tloss: 0.51944\tlr:[1.9883864e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:56:39,541 [ hooks.py: 207]:\tstep: 44360\tsteps/sec: 1.33709\tloss: 0.37097\tlr:[1.9877072e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:56:46,736 [ hooks.py: 207]:\tstep: 44370\tsteps/sec: 1.46387\tloss: 0.44931\tlr:[1.9870284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:56:54,274 [ hooks.py: 207]:\tstep: 44380\tsteps/sec: 1.27101\tloss: 0.72502\tlr:[1.9863492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:57:03,569 [ hooks.py: 207]:\tstep: 44390\tsteps/sec: 1.08506\tloss: 0.40024\tlr:[1.9856705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:57:11,530 [ hooks.py: 207]:\tstep: 44400\tsteps/sec: 1.30343\tloss: 0.62184\tlr:[1.9849911e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:57:22,294 [ hooks.py: 207]:\tstep: 44410\tsteps/sec: 0.91920\tloss: 0.60818\tlr:[1.9843123e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:57:30,897 [ hooks.py: 207]:\tstep: 44420\tsteps/sec: 1.10975\tloss: 0.57050\tlr:[1.983633e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:57:39,573 [ hooks.py: 207]:\tstep: 44430\tsteps/sec: 1.12397\tloss: 0.31306\tlr:[1.982954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:57:51,053 [ hooks.py: 207]:\tstep: 44440\tsteps/sec: 0.89161\tloss: 0.32635\tlr:[1.982275e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:58:00,387 [ hooks.py: 207]:\tstep: 44450\tsteps/sec: 1.11476\tloss: 0.67473\tlr:[1.981596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 08:58:09,317 [ hooks.py: 207]:\tstep: 44460\tsteps/sec: 1.09137\tloss: 0.44976\tlr:[1.980917e-05]\u001b[0m\n", + "IOPub message rate exceeded.\n", + "The notebook server will temporarily stop sending output\n", + "to the client in order to avoid crashing it.\n", + "To change this limit, set the config variable\n", + "`--NotebookApp.iopub_msg_rate_limit`.\n", + "\n", + "Current values:\n", + "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n", + "NotebookApp.rate_limit_window=3.0 (secs)\n", + "\n", + "\u001b[37m[DEBUG] 2019-09-19 09:36:30,859 [ hooks.py: 207]:\tstep: 46020\tsteps/sec: 1.30306\tloss: 0.58037\tlr:[1.874984e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:36:39,352 [ hooks.py: 207]:\tstep: 46030\tsteps/sec: 1.20339\tloss: 0.38571\tlr:[1.874305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:36:47,816 [ hooks.py: 207]:\tstep: 46040\tsteps/sec: 1.22792\tloss: 0.57116\tlr:[1.873626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:36:56,964 [ hooks.py: 207]:\tstep: 46050\tsteps/sec: 1.10393\tloss: 0.26001\tlr:[1.8729468e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:37:06,765 [ hooks.py: 207]:\tstep: 46060\tsteps/sec: 0.99752\tloss: 0.38964\tlr:[1.8722681e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:37:15,105 [ hooks.py: 207]:\tstep: 46070\tsteps/sec: 1.18670\tloss: 0.80924\tlr:[1.871589e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:37:22,578 [ hooks.py: 207]:\tstep: 46080\tsteps/sec: 1.36295\tloss: 0.38007\tlr:[1.87091e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:37:30,803 [ hooks.py: 207]:\tstep: 46090\tsteps/sec: 1.19908\tloss: 0.28955\tlr:[1.8702309e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:37:38,564 [ hooks.py: 207]:\tstep: 46100\tsteps/sec: 1.22647\tloss: 0.13643\tlr:[1.8695519e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:37:48,707 [ hooks.py: 207]:\tstep: 46110\tsteps/sec: 1.01700\tloss: 0.56892\tlr:[1.8688726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:00,069 [ hooks.py: 207]:\tstep: 46120\tsteps/sec: 0.86351\tloss: 0.19472\tlr:[1.8681934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:08,640 [ hooks.py: 207]:\tstep: 46130\tsteps/sec: 1.15903\tloss: 0.06948\tlr:[1.8675146e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:16,306 [ hooks.py: 207]:\tstep: 46140\tsteps/sec: 1.38510\tloss: 0.54459\tlr:[1.8668354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:24,497 [ hooks.py: 207]:\tstep: 46150\tsteps/sec: 1.19916\tloss: 0.53376\tlr:[1.8661563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:34,438 [ hooks.py: 207]:\tstep: 46160\tsteps/sec: 1.00609\tloss: 0.45551\tlr:[1.8654771e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:41,750 [ hooks.py: 207]:\tstep: 46170\tsteps/sec: 1.41500\tloss: 0.43646\tlr:[1.8647983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:48,803 [ hooks.py: 207]:\tstep: 46180\tsteps/sec: 1.37534\tloss: 0.52712\tlr:[1.864119e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:38:57,954 [ hooks.py: 207]:\tstep: 46190\tsteps/sec: 1.11792\tloss: 0.26621\tlr:[1.86344e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:39:06,161 [ hooks.py: 207]:\tstep: 46200\tsteps/sec: 1.18951\tloss: 0.25033\tlr:[1.862761e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:39:15,190 [ hooks.py: 207]:\tstep: 46210\tsteps/sec: 1.05710\tloss: 0.31370\tlr:[1.8620822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:39:24,859 [ hooks.py: 207]:\tstep: 46220\tsteps/sec: 1.04631\tloss: 0.93153\tlr:[1.861403e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:39:32,664 [ hooks.py: 207]:\tstep: 46230\tsteps/sec: 1.36467\tloss: 0.37703\tlr:[1.8607241e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:39:41,889 [ hooks.py: 207]:\tstep: 46240\tsteps/sec: 1.09817\tloss: 0.81874\tlr:[1.8600449e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:39:49,959 [ hooks.py: 207]:\tstep: 46250\tsteps/sec: 1.18843\tloss: 0.48312\tlr:[1.8593659e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:39:57,183 [ hooks.py: 207]:\tstep: 46260\tsteps/sec: 1.45690\tloss: 0.42260\tlr:[1.8586868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:40:04,592 [ hooks.py: 207]:\tstep: 46270\tsteps/sec: 1.24781\tloss: 0.42217\tlr:[1.8580076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:40:12,912 [ hooks.py: 207]:\tstep: 46280\tsteps/sec: 1.27865\tloss: 0.10518\tlr:[1.8573286e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:40:21,918 [ hooks.py: 207]:\tstep: 46290\tsteps/sec: 1.11691\tloss: 0.22620\tlr:[1.8566494e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:40:29,528 [ hooks.py: 207]:\tstep: 46300\tsteps/sec: 1.21837\tloss: 0.53937\tlr:[1.8559705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:40:39,949 [ hooks.py: 207]:\tstep: 46310\tsteps/sec: 0.99090\tloss: 0.60156\tlr:[1.8552913e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 09:40:48,826 [ hooks.py: 207]:\tstep: 46320\tsteps/sec: 1.07502\tloss: 0.94310\tlr:[1.8546125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:40:56,806 [ hooks.py: 207]:\tstep: 46330\tsteps/sec: 1.35105\tloss: 0.63789\tlr:[1.8539333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:41:06,200 [ hooks.py: 207]:\tstep: 46340\tsteps/sec: 1.04116\tloss: 0.34402\tlr:[1.8532544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:41:15,386 [ hooks.py: 207]:\tstep: 46350\tsteps/sec: 1.09278\tloss: 0.34306\tlr:[1.8525752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:41:24,023 [ hooks.py: 207]:\tstep: 46360\tsteps/sec: 1.15260\tloss: 0.29288\tlr:[1.8518964e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:41:33,838 [ hooks.py: 207]:\tstep: 46370\tsteps/sec: 1.02741\tloss: 0.37546\tlr:[1.8512172e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:41:42,168 [ hooks.py: 207]:\tstep: 46380\tsteps/sec: 1.19191\tloss: 0.91345\tlr:[1.8505383e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:41:52,192 [ hooks.py: 207]:\tstep: 46390\tsteps/sec: 1.04129\tloss: 0.42507\tlr:[1.849859e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:41:59,418 [ hooks.py: 207]:\tstep: 46400\tsteps/sec: 1.26975\tloss: 0.79659\tlr:[1.8491803e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:42:09,608 [ hooks.py: 207]:\tstep: 46410\tsteps/sec: 1.00357\tloss: 0.65642\tlr:[1.8485009e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:42:18,389 [ hooks.py: 207]:\tstep: 46420\tsteps/sec: 1.11343\tloss: 0.31551\tlr:[1.8478217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:42:27,309 [ hooks.py: 207]:\tstep: 46430\tsteps/sec: 1.14243\tloss: 0.31943\tlr:[1.8471428e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:42:38,344 [ hooks.py: 207]:\tstep: 46440\tsteps/sec: 0.92730\tloss: 0.54672\tlr:[1.8464636e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:42:46,677 [ hooks.py: 207]:\tstep: 46450\tsteps/sec: 1.20214\tloss: 0.42254\tlr:[1.845785e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:42:53,784 [ hooks.py: 207]:\tstep: 46460\tsteps/sec: 1.36535\tloss: 0.19579\tlr:[1.8451057e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:43:01,799 [ hooks.py: 207]:\tstep: 46470\tsteps/sec: 1.32495\tloss: 0.49429\tlr:[1.8444269e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:43:11,034 [ hooks.py: 207]:\tstep: 46480\tsteps/sec: 1.04428\tloss: 0.86134\tlr:[1.8437477e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:43:17,882 [ hooks.py: 207]:\tstep: 46490\tsteps/sec: 1.42944\tloss: 0.34138\tlr:[1.8430686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:43:25,153 [ hooks.py: 207]:\tstep: 46500\tsteps/sec: 1.32401\tloss: 0.27822\tlr:[1.8423894e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:43:34,889 [ hooks.py: 207]:\tstep: 46510\tsteps/sec: 1.06458\tloss: 0.61927\tlr:[1.8417104e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:43:44,315 [ hooks.py: 207]:\tstep: 46520\tsteps/sec: 1.02445\tloss: 0.27155\tlr:[1.8410314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:43:54,655 [ hooks.py: 207]:\tstep: 46530\tsteps/sec: 0.99516\tloss: 0.43907\tlr:[1.8403525e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:44:02,397 [ hooks.py: 207]:\tstep: 46540\tsteps/sec: 1.42628\tloss: 0.33886\tlr:[1.8396733e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:44:10,934 [ hooks.py: 207]:\tstep: 46550\tsteps/sec: 1.09520\tloss: 0.33565\tlr:[1.8389945e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:44:20,288 [ hooks.py: 207]:\tstep: 46560\tsteps/sec: 1.06430\tloss: 0.53916\tlr:[1.838315e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:44:29,774 [ hooks.py: 207]:\tstep: 46570\tsteps/sec: 1.04764\tloss: 0.25028\tlr:[1.837636e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:44:37,132 [ hooks.py: 207]:\tstep: 46580\tsteps/sec: 1.34810\tloss: 0.83476\tlr:[1.836957e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:44:46,435 [ hooks.py: 207]:\tstep: 46590\tsteps/sec: 1.10058\tloss: 0.60389\tlr:[1.836278e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:44:53,617 [ hooks.py: 207]:\tstep: 46600\tsteps/sec: 1.37886\tloss: 0.21004\tlr:[1.835599e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:45:01,087 [ hooks.py: 207]:\tstep: 46610\tsteps/sec: 1.31130\tloss: 0.54675\tlr:[1.83492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:45:11,547 [ hooks.py: 207]:\tstep: 46620\tsteps/sec: 0.97582\tloss: 0.23266\tlr:[1.8342407e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:45:19,823 [ hooks.py: 207]:\tstep: 46630\tsteps/sec: 1.17429\tloss: 0.25261\tlr:[1.8335617e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:45:28,212 [ hooks.py: 207]:\tstep: 46640\tsteps/sec: 1.24531\tloss: 0.39967\tlr:[1.8328827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:45:37,231 [ hooks.py: 207]:\tstep: 46650\tsteps/sec: 1.11374\tloss: 0.34529\tlr:[1.8322036e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:45:45,539 [ hooks.py: 207]:\tstep: 46660\tsteps/sec: 1.16351\tloss: 0.26231\tlr:[1.8315248e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:45:56,479 [ hooks.py: 207]:\tstep: 46670\tsteps/sec: 0.89312\tloss: 0.49514\tlr:[1.8308454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:46:05,002 [ hooks.py: 207]:\tstep: 46680\tsteps/sec: 1.21802\tloss: 0.40464\tlr:[1.8301667e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:46:12,361 [ hooks.py: 207]:\tstep: 46690\tsteps/sec: 1.39057\tloss: 0.47334\tlr:[1.8294875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:46:21,960 [ hooks.py: 207]:\tstep: 46700\tsteps/sec: 1.03257\tloss: 0.71093\tlr:[1.8288087e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:46:29,430 [ hooks.py: 207]:\tstep: 46710\tsteps/sec: 1.30008\tloss: 0.59886\tlr:[1.8281293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:46:37,704 [ hooks.py: 207]:\tstep: 46720\tsteps/sec: 1.20914\tloss: 0.22118\tlr:[1.82745e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:46:48,702 [ hooks.py: 207]:\tstep: 46730\tsteps/sec: 0.92538\tloss: 0.13703\tlr:[1.8267712e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:46:58,308 [ hooks.py: 207]:\tstep: 46740\tsteps/sec: 1.05500\tloss: 0.31766\tlr:[1.826092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:47:05,527 [ hooks.py: 207]:\tstep: 46750\tsteps/sec: 1.35051\tloss: 0.47392\tlr:[1.825413e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:47:13,482 [ hooks.py: 207]:\tstep: 46760\tsteps/sec: 1.28253\tloss: 0.45428\tlr:[1.824734e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:47:23,337 [ hooks.py: 207]:\tstep: 46770\tsteps/sec: 0.97646\tloss: 0.24979\tlr:[1.824055e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:47:33,067 [ hooks.py: 207]:\tstep: 46780\tsteps/sec: 1.07329\tloss: 0.25782\tlr:[1.8233757e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:47:41,427 [ hooks.py: 207]:\tstep: 46790\tsteps/sec: 1.17197\tloss: 0.44984\tlr:[1.8226969e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:47:48,300 [ hooks.py: 207]:\tstep: 46800\tsteps/sec: 1.39942\tloss: 0.27028\tlr:[1.8220177e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:47:59,047 [ hooks.py: 207]:\tstep: 46810\tsteps/sec: 0.95859\tloss: 0.20031\tlr:[1.821339e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:48:05,994 [ hooks.py: 207]:\tstep: 46820\tsteps/sec: 1.38708\tloss: 0.49348\tlr:[1.8206596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:48:14,396 [ hooks.py: 207]:\tstep: 46830\tsteps/sec: 1.20877\tloss: 0.75082\tlr:[1.8199808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:48:23,951 [ hooks.py: 207]:\tstep: 46840\tsteps/sec: 1.01464\tloss: 0.47216\tlr:[1.8193015e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:48:33,040 [ hooks.py: 207]:\tstep: 46850\tsteps/sec: 1.15170\tloss: 0.66946\tlr:[1.8186225e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:48:41,715 [ hooks.py: 207]:\tstep: 46860\tsteps/sec: 1.12075\tloss: 0.22510\tlr:[1.8179435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:48:50,251 [ hooks.py: 207]:\tstep: 46870\tsteps/sec: 1.15552\tloss: 0.32497\tlr:[1.8172643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:48:58,616 [ hooks.py: 207]:\tstep: 46880\tsteps/sec: 1.27629\tloss: 0.46863\tlr:[1.8165852e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:49:05,679 [ hooks.py: 207]:\tstep: 46890\tsteps/sec: 1.37161\tloss: 0.35328\tlr:[1.8159062e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:49:13,865 [ hooks.py: 207]:\tstep: 46900\tsteps/sec: 1.21974\tloss: 0.14069\tlr:[1.8152272e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:49:21,700 [ hooks.py: 207]:\tstep: 46910\tsteps/sec: 1.30140\tloss: 0.80753\tlr:[1.814548e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:49:31,532 [ hooks.py: 207]:\tstep: 46920\tsteps/sec: 0.98268\tloss: 0.18646\tlr:[1.8138693e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:49:41,166 [ hooks.py: 207]:\tstep: 46930\tsteps/sec: 1.05739\tloss: 0.43811\tlr:[1.81319e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:49:51,493 [ hooks.py: 207]:\tstep: 46940\tsteps/sec: 0.97751\tloss: 0.56070\tlr:[1.8125113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:00,618 [ hooks.py: 207]:\tstep: 46950\tsteps/sec: 1.09162\tloss: 0.39891\tlr:[1.811832e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:08,181 [ hooks.py: 207]:\tstep: 46960\tsteps/sec: 1.34719\tloss: 0.60344\tlr:[1.811153e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 09:50:18,373 [ hooks.py: 207]:\tstep: 46970\tsteps/sec: 0.93918\tloss: 0.26965\tlr:[1.8104738e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:27,786 [ hooks.py: 207]:\tstep: 46980\tsteps/sec: 1.12374\tloss: 0.27166\tlr:[1.8097948e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:34,236 [ hooks.py: 207]:\tstep: 46990\tsteps/sec: 1.41837\tloss: 0.34891\tlr:[1.8091157e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:43,576 [ hooks.py: 207]:\tstep: 47000\tsteps/sec: 1.12039\tloss: 0.22786\tlr:[1.8084369e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:43,577 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:43,577 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:43,577 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 09:50:43,577 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:43,578 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:50:43,578 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 09:58:47,037 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,038 [ trainer.py: 58]:\t{'acc': 0.7674698795180723, 'loss': 0.6217247162682887}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,039 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,040 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 09:58:47,040 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 09:58:47,040 [ trainer.py: 70]:\tacc\t0.7674698795180723\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 09:58:47,040 [ trainer.py: 70]:\tloss\t0.6217247162682887\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 09:58:47,041 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,041 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7674698795180723, 'loss': 0.6217247162682887}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,041 [ exporter.py: 64]:\t[Best Exporter]: skip step 47000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,041 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7674698795180723, 'loss': 0.6217247162682887}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,041 [ exporter.py: 104]:\t[Best Exporter]: skip step 47000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:47,042 [monitored_executor.py: 122]:\tsaving step 47000 to ./output/model_47000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:50,272 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_44000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:58:58,865 [ hooks.py: 207]:\tstep: 47010\tsteps/sec: 0.02020\tloss: 0.74304\tlr:[1.8077577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:59:08,814 [ hooks.py: 207]:\tstep: 47020\tsteps/sec: 1.00006\tloss: 0.62848\tlr:[1.8070783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:59:16,956 [ hooks.py: 207]:\tstep: 47030\tsteps/sec: 1.22916\tloss: 0.71216\tlr:[1.8063996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:59:24,752 [ hooks.py: 207]:\tstep: 47040\tsteps/sec: 1.24826\tloss: 0.32294\tlr:[1.8057202e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:59:32,906 [ hooks.py: 207]:\tstep: 47050\tsteps/sec: 1.27769\tloss: 0.41060\tlr:[1.8050414e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:59:40,624 [ hooks.py: 207]:\tstep: 47060\tsteps/sec: 1.20787\tloss: 0.33574\tlr:[1.8043624e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:59:48,953 [ hooks.py: 207]:\tstep: 47070\tsteps/sec: 1.26229\tloss: 0.35177\tlr:[1.8036833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 09:59:56,448 [ hooks.py: 207]:\tstep: 47080\tsteps/sec: 1.24574\tloss: 0.27777\tlr:[1.8030041e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:00:06,561 [ hooks.py: 207]:\tstep: 47090\tsteps/sec: 1.02535\tloss: 0.65348\tlr:[1.8023253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:00:15,595 [ hooks.py: 207]:\tstep: 47100\tsteps/sec: 1.11319\tloss: 0.21102\tlr:[1.801646e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:00:22,592 [ hooks.py: 207]:\tstep: 47110\tsteps/sec: 1.42453\tloss: 0.90391\tlr:[1.8009672e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:00:32,682 [ hooks.py: 207]:\tstep: 47120\tsteps/sec: 1.01895\tloss: 0.48261\tlr:[1.800288e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:00:40,944 [ hooks.py: 207]:\tstep: 47130\tsteps/sec: 1.13850\tloss: 0.28921\tlr:[1.799609e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:00:49,955 [ hooks.py: 207]:\tstep: 47140\tsteps/sec: 1.10291\tloss: 0.41500\tlr:[1.79893e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:00:59,582 [ hooks.py: 207]:\tstep: 47150\tsteps/sec: 1.04161\tloss: 0.90525\tlr:[1.7982507e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:01:09,554 [ hooks.py: 207]:\tstep: 47160\tsteps/sec: 1.02651\tloss: 0.66665\tlr:[1.7975719e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:01:16,079 [ hooks.py: 207]:\tstep: 47170\tsteps/sec: 1.45350\tloss: 0.29955\tlr:[1.7968925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:01:24,112 [ hooks.py: 207]:\tstep: 47180\tsteps/sec: 1.32495\tloss: 0.52198\tlr:[1.7962138e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:01:30,850 [ hooks.py: 207]:\tstep: 47190\tsteps/sec: 1.40247\tloss: 0.34714\tlr:[1.7955346e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:01:40,197 [ hooks.py: 207]:\tstep: 47200\tsteps/sec: 1.07103\tloss: 0.49231\tlr:[1.7948558e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:01:49,179 [ hooks.py: 207]:\tstep: 47210\tsteps/sec: 1.15780\tloss: 0.26476\tlr:[1.7941766e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:01:58,454 [ hooks.py: 207]:\tstep: 47220\tsteps/sec: 1.11354\tloss: 0.35657\tlr:[1.7934975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:02:05,763 [ hooks.py: 207]:\tstep: 47230\tsteps/sec: 1.30830\tloss: 0.38882\tlr:[1.7928183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:02:13,868 [ hooks.py: 207]:\tstep: 47240\tsteps/sec: 1.24070\tloss: 0.50055\tlr:[1.7921395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:02:23,384 [ hooks.py: 207]:\tstep: 47250\tsteps/sec: 1.08422\tloss: 0.33388\tlr:[1.7914603e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:02:31,572 [ hooks.py: 207]:\tstep: 47260\tsteps/sec: 1.18836\tloss: 0.46056\tlr:[1.7907814e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:02:38,955 [ hooks.py: 207]:\tstep: 47270\tsteps/sec: 1.36619\tloss: 0.54897\tlr:[1.7901022e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:02:46,019 [ hooks.py: 207]:\tstep: 47280\tsteps/sec: 1.39056\tloss: 0.56509\tlr:[1.7894234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:02:54,736 [ hooks.py: 207]:\tstep: 47290\tsteps/sec: 1.13682\tloss: 0.59621\tlr:[1.7887442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:03:05,320 [ hooks.py: 207]:\tstep: 47300\tsteps/sec: 0.95290\tloss: 0.50909\tlr:[1.7880648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:03:13,875 [ hooks.py: 207]:\tstep: 47310\tsteps/sec: 1.18483\tloss: 0.48036\tlr:[1.787386e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:03:21,377 [ hooks.py: 207]:\tstep: 47320\tsteps/sec: 1.23798\tloss: 0.70484\tlr:[1.7867069e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:03:32,737 [ hooks.py: 207]:\tstep: 47330\tsteps/sec: 0.90573\tloss: 0.43336\tlr:[1.7860279e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:03:40,477 [ hooks.py: 207]:\tstep: 47340\tsteps/sec: 1.30398\tloss: 0.43773\tlr:[1.7853486e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:03:50,860 [ hooks.py: 207]:\tstep: 47350\tsteps/sec: 0.96769\tloss: 0.55479\tlr:[1.7846696e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:03:58,381 [ hooks.py: 207]:\tstep: 47360\tsteps/sec: 1.36773\tloss: 0.32219\tlr:[1.7839906e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:04:06,650 [ hooks.py: 207]:\tstep: 47370\tsteps/sec: 1.18990\tloss: 0.49031\tlr:[1.7833116e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:04:13,226 [ hooks.py: 207]:\tstep: 47380\tsteps/sec: 1.54224\tloss: 0.86436\tlr:[1.7826324e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:04:20,447 [ hooks.py: 207]:\tstep: 47390\tsteps/sec: 1.34914\tloss: 0.63334\tlr:[1.7819535e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:04:30,888 [ hooks.py: 207]:\tstep: 47400\tsteps/sec: 1.02957\tloss: 0.61395\tlr:[1.7812745e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:04:39,410 [ hooks.py: 207]:\tstep: 47410\tsteps/sec: 1.08308\tloss: 0.39524\tlr:[1.7805954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:04:47,456 [ hooks.py: 207]:\tstep: 47420\tsteps/sec: 1.27480\tloss: 0.56640\tlr:[1.7799162e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 10:04:54,470 [ hooks.py: 207]:\tstep: 47430\tsteps/sec: 1.37895\tloss: 0.51404\tlr:[1.7792374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:05:05,155 [ hooks.py: 207]:\tstep: 47440\tsteps/sec: 0.93383\tloss: 0.44998\tlr:[1.7785584e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:05:14,160 [ hooks.py: 207]:\tstep: 47450\tsteps/sec: 1.12215\tloss: 0.44438\tlr:[1.777879e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:05:23,183 [ hooks.py: 207]:\tstep: 47460\tsteps/sec: 1.09379\tloss: 0.32545\tlr:[1.7772001e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:05:32,146 [ hooks.py: 207]:\tstep: 47470\tsteps/sec: 1.13782\tloss: 0.67146\tlr:[1.7765211e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:05:40,519 [ hooks.py: 207]:\tstep: 47480\tsteps/sec: 1.18943\tloss: 0.42202\tlr:[1.775842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:05:48,717 [ hooks.py: 207]:\tstep: 47490\tsteps/sec: 1.18910\tloss: 0.53400\tlr:[1.7751629e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:05:56,809 [ hooks.py: 207]:\tstep: 47500\tsteps/sec: 1.26662\tloss: 0.15906\tlr:[1.774484e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:06:05,039 [ hooks.py: 207]:\tstep: 47510\tsteps/sec: 1.19338\tloss: 0.33807\tlr:[1.7738048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:06:13,208 [ hooks.py: 207]:\tstep: 47520\tsteps/sec: 1.27290\tloss: 0.29612\tlr:[1.773126e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:06:21,957 [ hooks.py: 207]:\tstep: 47530\tsteps/sec: 1.13698\tloss: 0.59158\tlr:[1.7724466e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:06:30,816 [ hooks.py: 207]:\tstep: 47540\tsteps/sec: 1.11242\tloss: 0.52271\tlr:[1.7717679e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:06:41,084 [ hooks.py: 207]:\tstep: 47550\tsteps/sec: 0.94932\tloss: 0.48627\tlr:[1.7710885e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:06:50,596 [ hooks.py: 207]:\tstep: 47560\tsteps/sec: 1.07894\tloss: 0.23501\tlr:[1.7704097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:06:59,038 [ hooks.py: 207]:\tstep: 47570\tsteps/sec: 1.23705\tloss: 0.30490\tlr:[1.7697304e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:07:06,242 [ hooks.py: 207]:\tstep: 47580\tsteps/sec: 1.25249\tloss: 0.25947\tlr:[1.7690516e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:07:14,922 [ hooks.py: 207]:\tstep: 47590\tsteps/sec: 1.16452\tloss: 0.22083\tlr:[1.7683724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:07:25,053 [ hooks.py: 207]:\tstep: 47600\tsteps/sec: 1.02098\tloss: 0.90155\tlr:[1.767693e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:07:34,483 [ hooks.py: 207]:\tstep: 47610\tsteps/sec: 1.01662\tloss: 0.54110\tlr:[1.7670141e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:07:43,698 [ hooks.py: 207]:\tstep: 47620\tsteps/sec: 1.16492\tloss: 0.74106\tlr:[1.766335e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:07:52,776 [ hooks.py: 207]:\tstep: 47630\tsteps/sec: 1.10154\tloss: 0.61227\tlr:[1.765656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:08:00,379 [ hooks.py: 207]:\tstep: 47640\tsteps/sec: 1.35292\tloss: 0.42981\tlr:[1.7649769e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:08:08,685 [ hooks.py: 207]:\tstep: 47650\tsteps/sec: 1.17556\tloss: 0.51040\tlr:[1.764298e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:08:15,822 [ hooks.py: 207]:\tstep: 47660\tsteps/sec: 1.32189\tloss: 0.60900\tlr:[1.7636188e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:08:27,695 [ hooks.py: 207]:\tstep: 47670\tsteps/sec: 0.91958\tloss: 0.69840\tlr:[1.76294e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:08:37,287 [ hooks.py: 207]:\tstep: 47680\tsteps/sec: 0.97230\tloss: 0.38203\tlr:[1.7622608e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:08:43,687 [ hooks.py: 207]:\tstep: 47690\tsteps/sec: 1.47251\tloss: 0.47295\tlr:[1.761582e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:08:53,844 [ hooks.py: 207]:\tstep: 47700\tsteps/sec: 1.03553\tloss: 0.46673\tlr:[1.7609027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:09:03,661 [ hooks.py: 207]:\tstep: 47710\tsteps/sec: 1.02111\tloss: 0.62360\tlr:[1.7602237e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:09:10,933 [ hooks.py: 207]:\tstep: 47720\tsteps/sec: 1.32291\tloss: 0.45446\tlr:[1.7595446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:09:18,928 [ hooks.py: 207]:\tstep: 47730\tsteps/sec: 1.20807\tloss: 0.25796\tlr:[1.7588658e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:09:26,308 [ hooks.py: 207]:\tstep: 47740\tsteps/sec: 1.34665\tloss: 0.59141\tlr:[1.7581864e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:09:35,636 [ hooks.py: 207]:\tstep: 47750\tsteps/sec: 1.11176\tloss: 0.66069\tlr:[1.7575074e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:09:44,472 [ hooks.py: 207]:\tstep: 47760\tsteps/sec: 1.12173\tloss: 0.29256\tlr:[1.7568285e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:09:52,655 [ hooks.py: 207]:\tstep: 47770\tsteps/sec: 1.22641\tloss: 0.29396\tlr:[1.7561493e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:10:02,741 [ hooks.py: 207]:\tstep: 47780\tsteps/sec: 1.01926\tloss: 0.30751\tlr:[1.7554705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:10:14,021 [ hooks.py: 207]:\tstep: 47790\tsteps/sec: 0.93450\tloss: 0.21350\tlr:[1.7547913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:10:22,205 [ hooks.py: 207]:\tstep: 47800\tsteps/sec: 1.12234\tloss: 0.59803\tlr:[1.7541124e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:10:30,346 [ hooks.py: 207]:\tstep: 47810\tsteps/sec: 1.21995\tloss: 0.60989\tlr:[1.7534332e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:10:39,515 [ hooks.py: 207]:\tstep: 47820\tsteps/sec: 1.08399\tloss: 0.59833\tlr:[1.7527542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:10:46,162 [ hooks.py: 207]:\tstep: 47830\tsteps/sec: 1.48913\tloss: 0.58001\tlr:[1.7520751e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:10:54,652 [ hooks.py: 207]:\tstep: 47840\tsteps/sec: 1.19894\tloss: 0.58433\tlr:[1.751396e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:11:03,235 [ hooks.py: 207]:\tstep: 47850\tsteps/sec: 1.20644\tloss: 0.71072\tlr:[1.750717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:11:13,347 [ hooks.py: 207]:\tstep: 47860\tsteps/sec: 0.99556\tloss: 0.65280\tlr:[1.7500379e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:11:21,018 [ hooks.py: 207]:\tstep: 47870\tsteps/sec: 1.25559\tloss: 0.68861\tlr:[1.7493587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:11:27,873 [ hooks.py: 207]:\tstep: 47880\tsteps/sec: 1.48314\tloss: 0.57745\tlr:[1.7486796e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:11:37,401 [ hooks.py: 207]:\tstep: 47890\tsteps/sec: 1.04512\tloss: 0.85445\tlr:[1.7480006e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:11:46,534 [ hooks.py: 207]:\tstep: 47900\tsteps/sec: 1.06996\tloss: 0.33820\tlr:[1.7473214e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:11:56,052 [ hooks.py: 207]:\tstep: 47910\tsteps/sec: 1.05491\tloss: 0.60273\tlr:[1.7466424e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:12:02,896 [ hooks.py: 207]:\tstep: 47920\tsteps/sec: 1.51074\tloss: 0.43619\tlr:[1.7459632e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:12:10,585 [ hooks.py: 207]:\tstep: 47930\tsteps/sec: 1.26380\tloss: 0.62263\tlr:[1.7452845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:12:18,589 [ hooks.py: 207]:\tstep: 47940\tsteps/sec: 1.38579\tloss: 0.30119\tlr:[1.7446053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:12:26,035 [ hooks.py: 207]:\tstep: 47950\tsteps/sec: 1.19867\tloss: 0.27320\tlr:[1.7439264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:12:36,068 [ hooks.py: 207]:\tstep: 47960\tsteps/sec: 1.00598\tloss: 0.29503\tlr:[1.743247e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:12:43,281 [ hooks.py: 207]:\tstep: 47970\tsteps/sec: 1.44257\tloss: 0.40088\tlr:[1.7425682e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:12:50,402 [ hooks.py: 207]:\tstep: 47980\tsteps/sec: 1.34252\tloss: 0.11674\tlr:[1.741889e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:13:00,620 [ hooks.py: 207]:\tstep: 47990\tsteps/sec: 1.23261\tloss: 0.56938\tlr:[1.7412101e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:13:09,516 [ hooks.py: 207]:\tstep: 48000\tsteps/sec: 0.91520\tloss: 0.74720\tlr:[1.740531e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:13:09,516 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:13:09,517 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:13:09,517 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:13:09,517 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:13:09,517 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:13:09,517 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:21:08,737 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,739 [ trainer.py: 58]:\t{'acc': 0.7803212851405622, 'loss': 0.5845942366390656}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,739 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,740 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:21:08,740 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:21:08,740 [ trainer.py: 70]:\tacc\t0.7803212851405622\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:21:08,740 [ trainer.py: 70]:\tloss\t0.5845942366390656\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:21:08,740 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,740 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7803212851405622, 'loss': 0.5845942366390656}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,741 [ exporter.py: 64]:\t[Best Exporter]: skip step 48000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,741 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7803212851405622, 'loss': 0.5845942366390656}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,741 [ exporter.py: 104]:\t[Best Exporter]: skip step 48000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:08,742 [monitored_executor.py: 122]:\tsaving step 48000 to ./output/model_48000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 10:21:12,522 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_45000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:21,673 [ hooks.py: 207]:\tstep: 48010\tsteps/sec: 0.02032\tloss: 0.36003\tlr:[1.739852e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:30,328 [ hooks.py: 207]:\tstep: 48020\tsteps/sec: 1.14979\tloss: 0.71991\tlr:[1.7391729e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:39,426 [ hooks.py: 207]:\tstep: 48030\tsteps/sec: 1.11433\tloss: 0.64022\tlr:[1.738494e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:48,396 [ hooks.py: 207]:\tstep: 48040\tsteps/sec: 1.12410\tloss: 0.30861\tlr:[1.7378148e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:21:56,225 [ hooks.py: 207]:\tstep: 48050\tsteps/sec: 1.24085\tloss: 0.51936\tlr:[1.7371356e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:22:04,677 [ hooks.py: 207]:\tstep: 48060\tsteps/sec: 1.17353\tloss: 0.56922\tlr:[1.736457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:22:14,315 [ hooks.py: 207]:\tstep: 48070\tsteps/sec: 1.00619\tloss: 0.51021\tlr:[1.7357776e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:22:22,284 [ hooks.py: 207]:\tstep: 48080\tsteps/sec: 1.28905\tloss: 0.91258\tlr:[1.7350987e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:22:30,247 [ hooks.py: 207]:\tstep: 48090\tsteps/sec: 1.26875\tloss: 0.52582\tlr:[1.7344195e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:22:39,572 [ hooks.py: 207]:\tstep: 48100\tsteps/sec: 1.08079\tloss: 0.15541\tlr:[1.7337405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:22:49,423 [ hooks.py: 207]:\tstep: 48110\tsteps/sec: 1.03021\tloss: 0.50759\tlr:[1.7330613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:22:57,799 [ hooks.py: 207]:\tstep: 48120\tsteps/sec: 1.18790\tloss: 0.37497\tlr:[1.7323824e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:23:06,510 [ hooks.py: 207]:\tstep: 48130\tsteps/sec: 1.14113\tloss: 0.64909\tlr:[1.7317034e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:23:15,574 [ hooks.py: 207]:\tstep: 48140\tsteps/sec: 1.09635\tloss: 0.25876\tlr:[1.7310243e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:23:24,270 [ hooks.py: 207]:\tstep: 48150\tsteps/sec: 1.18279\tloss: 0.67942\tlr:[1.7303453e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:23:33,377 [ hooks.py: 207]:\tstep: 48160\tsteps/sec: 1.06587\tloss: 0.48545\tlr:[1.7296663e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:23:45,256 [ hooks.py: 207]:\tstep: 48170\tsteps/sec: 0.84195\tloss: 0.57353\tlr:[1.728987e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:23:53,249 [ hooks.py: 207]:\tstep: 48180\tsteps/sec: 1.30729\tloss: 0.67961\tlr:[1.7283079e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:24:03,026 [ hooks.py: 207]:\tstep: 48190\tsteps/sec: 1.00907\tloss: 0.21662\tlr:[1.727629e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:24:09,815 [ hooks.py: 207]:\tstep: 48200\tsteps/sec: 1.42585\tloss: 0.46978\tlr:[1.7269498e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:24:18,631 [ hooks.py: 207]:\tstep: 48210\tsteps/sec: 1.17500\tloss: 0.51384\tlr:[1.7262708e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:24:27,616 [ hooks.py: 207]:\tstep: 48220\tsteps/sec: 1.10057\tloss: 0.45266\tlr:[1.7255918e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:24:37,913 [ hooks.py: 207]:\tstep: 48230\tsteps/sec: 0.97453\tloss: 0.42798\tlr:[1.7249127e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:24:48,029 [ hooks.py: 207]:\tstep: 48240\tsteps/sec: 0.99636\tloss: 0.38948\tlr:[1.7242335e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:24:56,599 [ hooks.py: 207]:\tstep: 48250\tsteps/sec: 1.13839\tloss: 0.45404\tlr:[1.7235545e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:25:06,308 [ hooks.py: 207]:\tstep: 48260\tsteps/sec: 1.02186\tloss: 0.18347\tlr:[1.7228753e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:25:16,223 [ hooks.py: 207]:\tstep: 48270\tsteps/sec: 1.01242\tloss: 0.70607\tlr:[1.7221966e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:25:23,282 [ hooks.py: 207]:\tstep: 48280\tsteps/sec: 1.46902\tloss: 0.35509\tlr:[1.7215174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:25:34,209 [ hooks.py: 207]:\tstep: 48290\tsteps/sec: 0.90816\tloss: 0.16649\tlr:[1.7208387e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:25:44,227 [ hooks.py: 207]:\tstep: 48300\tsteps/sec: 0.99759\tloss: 0.73638\tlr:[1.7201593e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:25:51,012 [ hooks.py: 207]:\tstep: 48310\tsteps/sec: 1.43641\tloss: 0.36383\tlr:[1.7194805e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:26:00,858 [ hooks.py: 207]:\tstep: 48320\tsteps/sec: 1.06224\tloss: 0.50720\tlr:[1.7188013e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:26:09,971 [ hooks.py: 207]:\tstep: 48330\tsteps/sec: 1.04450\tloss: 0.42979\tlr:[1.718122e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:26:20,101 [ hooks.py: 207]:\tstep: 48340\tsteps/sec: 0.95804\tloss: 0.42888\tlr:[1.717443e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:26:29,000 [ hooks.py: 207]:\tstep: 48350\tsteps/sec: 1.22482\tloss: 0.59659\tlr:[1.716764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:26:35,247 [ hooks.py: 207]:\tstep: 48360\tsteps/sec: 1.48400\tloss: 0.21515\tlr:[1.716085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:26:44,755 [ hooks.py: 207]:\tstep: 48370\tsteps/sec: 1.06791\tloss: 0.43799\tlr:[1.7154058e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:26:51,818 [ hooks.py: 207]:\tstep: 48380\tsteps/sec: 1.41412\tloss: 0.45021\tlr:[1.7147271e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:27:01,024 [ hooks.py: 207]:\tstep: 48390\tsteps/sec: 1.08983\tloss: 0.34134\tlr:[1.7140477e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:27:08,896 [ hooks.py: 207]:\tstep: 48400\tsteps/sec: 1.29508\tloss: 0.75302\tlr:[1.713369e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:27:21,289 [ hooks.py: 207]:\tstep: 48410\tsteps/sec: 0.77160\tloss: 0.43177\tlr:[1.7126897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:27:31,165 [ hooks.py: 207]:\tstep: 48420\tsteps/sec: 1.04239\tloss: 0.18303\tlr:[1.7120108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:27:40,019 [ hooks.py: 207]:\tstep: 48430\tsteps/sec: 1.13250\tloss: 0.51625\tlr:[1.7113318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:27:47,296 [ hooks.py: 207]:\tstep: 48440\tsteps/sec: 1.38218\tloss: 0.70819\tlr:[1.7106528e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:27:55,185 [ hooks.py: 207]:\tstep: 48450\tsteps/sec: 1.26251\tloss: 0.50063\tlr:[1.7099735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:28:03,187 [ hooks.py: 207]:\tstep: 48460\tsteps/sec: 1.28892\tloss: 0.17637\tlr:[1.7092947e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:28:12,535 [ hooks.py: 207]:\tstep: 48470\tsteps/sec: 1.05118\tloss: 0.09912\tlr:[1.7086153e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:28:21,767 [ hooks.py: 207]:\tstep: 48480\tsteps/sec: 1.03765\tloss: 0.21742\tlr:[1.7079363e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:28:31,254 [ hooks.py: 207]:\tstep: 48490\tsteps/sec: 1.09682\tloss: 0.29433\tlr:[1.7072574e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:28:39,186 [ hooks.py: 207]:\tstep: 48500\tsteps/sec: 1.31087\tloss: 0.31475\tlr:[1.706578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:28:46,226 [ hooks.py: 207]:\tstep: 48510\tsteps/sec: 1.34893\tloss: 0.39517\tlr:[1.7058992e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:28:55,546 [ hooks.py: 207]:\tstep: 48520\tsteps/sec: 1.06713\tloss: 0.42684\tlr:[1.7052202e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:29:04,940 [ hooks.py: 207]:\tstep: 48530\tsteps/sec: 1.08193\tloss: 0.49722\tlr:[1.7045411e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:29:14,187 [ hooks.py: 207]:\tstep: 48540\tsteps/sec: 1.11007\tloss: 0.58547\tlr:[1.703862e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:29:22,298 [ hooks.py: 207]:\tstep: 48550\tsteps/sec: 1.20233\tloss: 0.41718\tlr:[1.703183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:29:37,480 [ hooks.py: 207]:\tstep: 48560\tsteps/sec: 1.54249\tloss: 0.23598\tlr:[1.7025039e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:29:46,050 [ hooks.py: 207]:\tstep: 48570\tsteps/sec: 0.57483\tloss: 0.61280\tlr:[1.7018248e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:29:54,144 [ hooks.py: 207]:\tstep: 48580\tsteps/sec: 1.26355\tloss: 0.30000\tlr:[1.7011458e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:30:02,073 [ hooks.py: 207]:\tstep: 48590\tsteps/sec: 1.22387\tloss: 0.28447\tlr:[1.7004668e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:30:09,483 [ hooks.py: 207]:\tstep: 48600\tsteps/sec: 1.39383\tloss: 0.49334\tlr:[1.6997878e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:30:17,721 [ hooks.py: 207]:\tstep: 48610\tsteps/sec: 1.20920\tloss: 0.82548\tlr:[1.6991089e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:30:28,871 [ hooks.py: 207]:\tstep: 48620\tsteps/sec: 0.92387\tloss: 0.41691\tlr:[1.6984297e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:30:38,395 [ hooks.py: 207]:\tstep: 48630\tsteps/sec: 1.03288\tloss: 0.60778\tlr:[1.6977503e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:30:46,577 [ hooks.py: 207]:\tstep: 48640\tsteps/sec: 1.19329\tloss: 0.43937\tlr:[1.6970716e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 10:30:54,871 [ hooks.py: 207]:\tstep: 48650\tsteps/sec: 1.21060\tloss: 0.25743\tlr:[1.6963922e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:31:03,528 [ hooks.py: 207]:\tstep: 48660\tsteps/sec: 1.19585\tloss: 0.23949\tlr:[1.6957134e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:31:11,638 [ hooks.py: 207]:\tstep: 48670\tsteps/sec: 1.18106\tloss: 0.36427\tlr:[1.6950344e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:31:19,713 [ hooks.py: 207]:\tstep: 48680\tsteps/sec: 1.17842\tloss: 0.17897\tlr:[1.6943553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:31:27,564 [ hooks.py: 207]:\tstep: 48690\tsteps/sec: 1.34786\tloss: 0.77200\tlr:[1.6936761e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:31:36,240 [ hooks.py: 207]:\tstep: 48700\tsteps/sec: 1.15146\tloss: 0.24731\tlr:[1.6929971e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:31:44,299 [ hooks.py: 207]:\tstep: 48710\tsteps/sec: 1.27519\tloss: 0.22381\tlr:[1.6923179e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:31:52,806 [ hooks.py: 207]:\tstep: 48720\tsteps/sec: 1.11909\tloss: 0.14254\tlr:[1.6916392e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:32:01,473 [ hooks.py: 207]:\tstep: 48730\tsteps/sec: 1.23121\tloss: 0.76520\tlr:[1.69096e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:32:11,595 [ hooks.py: 207]:\tstep: 48740\tsteps/sec: 0.91066\tloss: 0.35915\tlr:[1.6902812e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:32:21,759 [ hooks.py: 207]:\tstep: 48750\tsteps/sec: 1.03405\tloss: 0.14392\tlr:[1.689602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:32:29,940 [ hooks.py: 207]:\tstep: 48760\tsteps/sec: 1.16575\tloss: 0.37771\tlr:[1.688923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:32:40,806 [ hooks.py: 207]:\tstep: 48770\tsteps/sec: 0.95193\tloss: 0.20913\tlr:[1.6882437e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:32:47,700 [ hooks.py: 207]:\tstep: 48780\tsteps/sec: 1.48427\tloss: 0.25924\tlr:[1.6875645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:32:56,279 [ hooks.py: 207]:\tstep: 48790\tsteps/sec: 1.15113\tloss: 0.41849\tlr:[1.6868858e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:33:06,504 [ hooks.py: 207]:\tstep: 48800\tsteps/sec: 0.96872\tloss: 0.35326\tlr:[1.6862065e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:33:13,936 [ hooks.py: 207]:\tstep: 48810\tsteps/sec: 1.39087\tloss: 0.17457\tlr:[1.6855276e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:33:24,683 [ hooks.py: 207]:\tstep: 48820\tsteps/sec: 0.92287\tloss: 0.65567\tlr:[1.6848484e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:33:34,643 [ hooks.py: 207]:\tstep: 48830\tsteps/sec: 1.01274\tloss: 0.41316\tlr:[1.6841694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:33:41,841 [ hooks.py: 207]:\tstep: 48840\tsteps/sec: 1.35464\tloss: 0.33530\tlr:[1.6834903e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:33:50,344 [ hooks.py: 207]:\tstep: 48850\tsteps/sec: 1.17919\tloss: 0.60937\tlr:[1.6828113e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:00,317 [ hooks.py: 207]:\tstep: 48860\tsteps/sec: 1.04597\tloss: 0.46446\tlr:[1.6821323e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:09,623 [ hooks.py: 207]:\tstep: 48870\tsteps/sec: 1.05453\tloss: 0.16485\tlr:[1.6814533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:17,616 [ hooks.py: 207]:\tstep: 48880\tsteps/sec: 1.22443\tloss: 0.37560\tlr:[1.680774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:26,396 [ hooks.py: 207]:\tstep: 48890\tsteps/sec: 1.13489\tloss: 0.46372\tlr:[1.6800954e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:35,017 [ hooks.py: 207]:\tstep: 48900\tsteps/sec: 1.16204\tloss: 0.37130\tlr:[1.6794162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:41,706 [ hooks.py: 207]:\tstep: 48910\tsteps/sec: 1.49295\tloss: 0.30220\tlr:[1.6787371e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:50,183 [ hooks.py: 207]:\tstep: 48920\tsteps/sec: 1.18019\tloss: 0.74063\tlr:[1.6780581e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:34:56,683 [ hooks.py: 207]:\tstep: 48930\tsteps/sec: 1.45131\tloss: 0.62313\tlr:[1.6773789e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:05,123 [ hooks.py: 207]:\tstep: 48940\tsteps/sec: 1.23425\tloss: 0.42075\tlr:[1.6766999e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:15,049 [ hooks.py: 207]:\tstep: 48950\tsteps/sec: 1.00746\tloss: 0.21715\tlr:[1.6760207e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:23,525 [ hooks.py: 207]:\tstep: 48960\tsteps/sec: 1.22373\tloss: 0.46984\tlr:[1.6753418e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:30,361 [ hooks.py: 207]:\tstep: 48970\tsteps/sec: 1.40558\tloss: 0.39061\tlr:[1.6746626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:37,059 [ hooks.py: 207]:\tstep: 48980\tsteps/sec: 1.40265\tloss: 0.48111\tlr:[1.6739838e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:45,983 [ hooks.py: 207]:\tstep: 48990\tsteps/sec: 1.16615\tloss: 0.61999\tlr:[1.6733044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:54,367 [ hooks.py: 207]:\tstep: 49000\tsteps/sec: 1.23556\tloss: 0.84743\tlr:[1.6726255e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:54,367 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:54,367 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:54,367 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:35:54,367 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:54,368 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:35:54,368 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:43:54,530 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,531 [ trainer.py: 58]:\t{'acc': 0.7734939759036145, 'loss': 0.6058546351507689}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,531 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,531 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:43:54,531 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:43:54,531 [ trainer.py: 70]:\tacc\t0.7734939759036145\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:43:54,531 [ trainer.py: 70]:\tloss\t0.6058546351507689\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:43:54,532 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,533 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7734939759036145, 'loss': 0.6058546351507689}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,533 [ exporter.py: 64]:\t[Best Exporter]: skip step 49000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,533 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7734939759036145, 'loss': 0.6058546351507689}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,534 [ exporter.py: 104]:\t[Best Exporter]: skip step 49000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:54,534 [monitored_executor.py: 122]:\tsaving step 49000 to ./output/model_49000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:43:57,686 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_46000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:44:06,101 [ hooks.py: 207]:\tstep: 49010\tsteps/sec: 0.02031\tloss: 0.52791\tlr:[1.6719465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:44:16,473 [ hooks.py: 207]:\tstep: 49020\tsteps/sec: 0.99025\tloss: 0.25213\tlr:[1.6712676e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:44:25,758 [ hooks.py: 207]:\tstep: 49030\tsteps/sec: 1.03709\tloss: 0.22487\tlr:[1.6705882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:44:33,737 [ hooks.py: 207]:\tstep: 49040\tsteps/sec: 1.36478\tloss: 0.52580\tlr:[1.6699094e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:44:43,229 [ hooks.py: 207]:\tstep: 49050\tsteps/sec: 1.04678\tloss: 0.30163\tlr:[1.6692302e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:44:49,955 [ hooks.py: 207]:\tstep: 49060\tsteps/sec: 1.46910\tloss: 0.47701\tlr:[1.6685513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:44:57,161 [ hooks.py: 207]:\tstep: 49070\tsteps/sec: 1.37547\tloss: 0.45206\tlr:[1.667872e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:45:04,846 [ hooks.py: 207]:\tstep: 49080\tsteps/sec: 1.32999\tloss: 0.57822\tlr:[1.667193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:45:13,211 [ hooks.py: 207]:\tstep: 49090\tsteps/sec: 1.18480\tloss: 0.40826\tlr:[1.6665139e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:45:22,617 [ hooks.py: 207]:\tstep: 49100\tsteps/sec: 1.04561\tloss: 0.75303\tlr:[1.6658347e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 10:45:31,364 [ hooks.py: 207]:\tstep: 49110\tsteps/sec: 1.13778\tloss: 0.35365\tlr:[1.6651558e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:45:39,624 [ hooks.py: 207]:\tstep: 49120\tsteps/sec: 1.23887\tloss: 0.61784\tlr:[1.6644766e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:45:48,898 [ hooks.py: 207]:\tstep: 49130\tsteps/sec: 1.10717\tloss: 0.77643\tlr:[1.6637978e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:45:57,384 [ hooks.py: 207]:\tstep: 49140\tsteps/sec: 1.13420\tloss: 0.25511\tlr:[1.6631184e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:46:06,417 [ hooks.py: 207]:\tstep: 49150\tsteps/sec: 1.10685\tloss: 0.88184\tlr:[1.6624397e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:46:13,805 [ hooks.py: 207]:\tstep: 49160\tsteps/sec: 1.38835\tloss: 0.23639\tlr:[1.6617605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:46:24,692 [ hooks.py: 207]:\tstep: 49170\tsteps/sec: 0.91506\tloss: 0.18004\tlr:[1.6610817e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:46:33,248 [ hooks.py: 207]:\tstep: 49180\tsteps/sec: 1.17064\tloss: 0.48720\tlr:[1.6604024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:46:41,479 [ hooks.py: 207]:\tstep: 49190\tsteps/sec: 1.17957\tloss: 0.31322\tlr:[1.6597234e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:46:52,879 [ hooks.py: 207]:\tstep: 49200\tsteps/sec: 0.88048\tloss: 0.26811\tlr:[1.6590442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:01,681 [ hooks.py: 207]:\tstep: 49210\tsteps/sec: 1.16419\tloss: 0.59318\tlr:[1.6583654e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:08,448 [ hooks.py: 207]:\tstep: 49220\tsteps/sec: 1.41460\tloss: 0.20025\tlr:[1.6576862e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:19,368 [ hooks.py: 207]:\tstep: 49230\tsteps/sec: 0.95317\tloss: 0.63778\tlr:[1.657007e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:26,936 [ hooks.py: 207]:\tstep: 49240\tsteps/sec: 1.26876\tloss: 0.32811\tlr:[1.6563281e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:34,379 [ hooks.py: 207]:\tstep: 49250\tsteps/sec: 1.40107\tloss: 0.51577\tlr:[1.6556489e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:41,335 [ hooks.py: 207]:\tstep: 49260\tsteps/sec: 1.37931\tloss: 0.32803\tlr:[1.6549702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:49,545 [ hooks.py: 207]:\tstep: 49270\tsteps/sec: 1.25805\tloss: 0.55588\tlr:[1.654291e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:47:57,358 [ hooks.py: 207]:\tstep: 49280\tsteps/sec: 1.23911\tloss: 0.32169\tlr:[1.653612e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:48:04,592 [ hooks.py: 207]:\tstep: 49290\tsteps/sec: 1.42058\tloss: 0.42205\tlr:[1.652933e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:48:13,551 [ hooks.py: 207]:\tstep: 49300\tsteps/sec: 1.07666\tloss: 0.62057\tlr:[1.652254e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:48:22,118 [ hooks.py: 207]:\tstep: 49310\tsteps/sec: 1.16612\tloss: 0.53068\tlr:[1.6515747e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:48:31,046 [ hooks.py: 207]:\tstep: 49320\tsteps/sec: 1.12669\tloss: 0.29655\tlr:[1.6508959e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:48:39,994 [ hooks.py: 207]:\tstep: 49330\tsteps/sec: 1.13987\tloss: 0.48549\tlr:[1.6502167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:48:48,636 [ hooks.py: 207]:\tstep: 49340\tsteps/sec: 1.17375\tloss: 1.10032\tlr:[1.6495378e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:48:57,023 [ hooks.py: 207]:\tstep: 49350\tsteps/sec: 1.15385\tloss: 0.35164\tlr:[1.6488586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:49:04,562 [ hooks.py: 207]:\tstep: 49360\tsteps/sec: 1.32597\tloss: 0.13787\tlr:[1.6481794e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:49:14,262 [ hooks.py: 207]:\tstep: 49370\tsteps/sec: 1.06938\tloss: 0.73155\tlr:[1.6475005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:49:21,230 [ hooks.py: 207]:\tstep: 49380\tsteps/sec: 1.37279\tloss: 0.28912\tlr:[1.6468213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:49:27,520 [ hooks.py: 207]:\tstep: 49390\tsteps/sec: 1.49466\tloss: 0.41841\tlr:[1.6461425e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:49:36,667 [ hooks.py: 207]:\tstep: 49400\tsteps/sec: 1.14473\tloss: 0.44397\tlr:[1.6454633e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:49:44,556 [ hooks.py: 207]:\tstep: 49410\tsteps/sec: 1.25005\tloss: 0.59152\tlr:[1.6447842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:49:53,265 [ hooks.py: 207]:\tstep: 49420\tsteps/sec: 1.18553\tloss: 0.37052\tlr:[1.6441052e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:00,509 [ hooks.py: 207]:\tstep: 49430\tsteps/sec: 1.43959\tloss: 0.42524\tlr:[1.643426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:09,709 [ hooks.py: 207]:\tstep: 49440\tsteps/sec: 1.01997\tloss: 0.38732\tlr:[1.642747e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:19,105 [ hooks.py: 207]:\tstep: 49450\tsteps/sec: 1.12585\tloss: 0.20084\tlr:[1.642068e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:26,480 [ hooks.py: 207]:\tstep: 49460\tsteps/sec: 1.27915\tloss: 0.48053\tlr:[1.641389e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:35,444 [ hooks.py: 207]:\tstep: 49470\tsteps/sec: 1.11417\tloss: 0.23182\tlr:[1.64071e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:42,059 [ hooks.py: 207]:\tstep: 49480\tsteps/sec: 1.50072\tloss: 0.28731\tlr:[1.6400309e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:50,255 [ hooks.py: 207]:\tstep: 49490\tsteps/sec: 1.25478\tloss: 0.72759\tlr:[1.6393518e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:50:58,616 [ hooks.py: 207]:\tstep: 49500\tsteps/sec: 1.21778\tloss: 0.75950\tlr:[1.6386726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:51:07,787 [ hooks.py: 207]:\tstep: 49510\tsteps/sec: 1.01957\tloss: 0.37535\tlr:[1.6379936e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:51:18,240 [ hooks.py: 207]:\tstep: 49520\tsteps/sec: 1.01106\tloss: 0.21323\tlr:[1.6373147e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:51:26,231 [ hooks.py: 207]:\tstep: 49530\tsteps/sec: 1.20710\tloss: 0.19354\tlr:[1.6366354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:51:37,284 [ hooks.py: 207]:\tstep: 49540\tsteps/sec: 0.91695\tloss: 0.58058\tlr:[1.6359565e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:51:45,911 [ hooks.py: 207]:\tstep: 49550\tsteps/sec: 1.09707\tloss: 0.66294\tlr:[1.6352773e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:51:55,522 [ hooks.py: 207]:\tstep: 49560\tsteps/sec: 1.09254\tloss: 0.23864\tlr:[1.6345983e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:52:03,208 [ hooks.py: 207]:\tstep: 49570\tsteps/sec: 1.33447\tloss: 0.56427\tlr:[1.6339192e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:52:10,301 [ hooks.py: 207]:\tstep: 49580\tsteps/sec: 1.39526\tloss: 0.62446\tlr:[1.6332402e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:52:17,832 [ hooks.py: 207]:\tstep: 49590\tsteps/sec: 1.30596\tloss: 0.34900\tlr:[1.632561e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:52:26,881 [ hooks.py: 207]:\tstep: 49600\tsteps/sec: 1.06644\tloss: 0.26257\tlr:[1.6318822e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:52:34,984 [ hooks.py: 207]:\tstep: 49610\tsteps/sec: 1.29564\tloss: 0.28270\tlr:[1.631203e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:52:45,187 [ hooks.py: 207]:\tstep: 49620\tsteps/sec: 0.96992\tloss: 0.31978\tlr:[1.6305241e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:52:54,786 [ hooks.py: 207]:\tstep: 49630\tsteps/sec: 1.06955\tloss: 0.53091\tlr:[1.6298449e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:53:05,325 [ hooks.py: 207]:\tstep: 49640\tsteps/sec: 0.93782\tloss: 0.11915\tlr:[1.629166e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:53:13,953 [ hooks.py: 207]:\tstep: 49650\tsteps/sec: 1.17687\tloss: 0.68275\tlr:[1.6284866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:53:22,885 [ hooks.py: 207]:\tstep: 49660\tsteps/sec: 1.08171\tloss: 0.75912\tlr:[1.6278076e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:53:32,040 [ hooks.py: 207]:\tstep: 49670\tsteps/sec: 1.11107\tloss: 0.63127\tlr:[1.6271286e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:53:39,379 [ hooks.py: 207]:\tstep: 49680\tsteps/sec: 1.40961\tloss: 0.30962\tlr:[1.6264496e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:53:46,828 [ hooks.py: 207]:\tstep: 49690\tsteps/sec: 1.21867\tloss: 0.25086\tlr:[1.6257705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:53:56,175 [ hooks.py: 207]:\tstep: 49700\tsteps/sec: 1.12389\tloss: 0.36763\tlr:[1.6250913e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:54:03,866 [ hooks.py: 207]:\tstep: 49710\tsteps/sec: 1.37880\tloss: 0.87363\tlr:[1.6244125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:54:11,010 [ hooks.py: 207]:\tstep: 49720\tsteps/sec: 1.31550\tloss: 0.33108\tlr:[1.6237333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:54:20,043 [ hooks.py: 207]:\tstep: 49730\tsteps/sec: 1.13520\tloss: 1.02602\tlr:[1.6230544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:54:27,521 [ hooks.py: 207]:\tstep: 49740\tsteps/sec: 1.29874\tloss: 0.57247\tlr:[1.6223752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:54:35,457 [ hooks.py: 207]:\tstep: 49750\tsteps/sec: 1.26663\tloss: 0.29535\tlr:[1.6216965e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 10:54:41,996 [ hooks.py: 207]:\tstep: 49760\tsteps/sec: 1.52917\tloss: 0.26462\tlr:[1.6210171e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:54:50,374 [ hooks.py: 207]:\tstep: 49770\tsteps/sec: 1.21636\tloss: 0.22677\tlr:[1.6203383e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:54:57,073 [ hooks.py: 207]:\tstep: 49780\tsteps/sec: 1.43960\tloss: 0.48804\tlr:[1.619659e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:55:05,128 [ hooks.py: 207]:\tstep: 49790\tsteps/sec: 1.28441\tloss: 0.55918\tlr:[1.6189802e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:55:11,834 [ hooks.py: 207]:\tstep: 49800\tsteps/sec: 1.45665\tloss: 0.36845\tlr:[1.618301e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:55:20,056 [ hooks.py: 207]:\tstep: 49810\tsteps/sec: 1.23296\tloss: 0.64938\tlr:[1.6176218e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:55:28,682 [ hooks.py: 207]:\tstep: 49820\tsteps/sec: 1.16533\tloss: 0.26750\tlr:[1.616943e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:55:36,742 [ hooks.py: 207]:\tstep: 49830\tsteps/sec: 1.21589\tloss: 0.20849\tlr:[1.6162638e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:55:46,915 [ hooks.py: 207]:\tstep: 49840\tsteps/sec: 0.97686\tloss: 0.35498\tlr:[1.615585e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:55:55,047 [ hooks.py: 207]:\tstep: 49850\tsteps/sec: 1.24629\tloss: 0.41639\tlr:[1.6149057e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:02,136 [ hooks.py: 207]:\tstep: 49860\tsteps/sec: 1.38486\tloss: 0.42574\tlr:[1.6142269e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:09,306 [ hooks.py: 207]:\tstep: 49870\tsteps/sec: 1.45041\tloss: 0.74187\tlr:[1.6135476e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:15,863 [ hooks.py: 207]:\tstep: 49880\tsteps/sec: 1.46721\tloss: 0.23978\tlr:[1.6128688e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:23,778 [ hooks.py: 207]:\tstep: 49890\tsteps/sec: 1.30317\tloss: 0.55087\tlr:[1.6121896e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:32,921 [ hooks.py: 207]:\tstep: 49900\tsteps/sec: 1.05736\tloss: 0.49506\tlr:[1.6115106e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:41,581 [ hooks.py: 207]:\tstep: 49910\tsteps/sec: 1.17877\tloss: 0.47505\tlr:[1.6108314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:48,691 [ hooks.py: 207]:\tstep: 49920\tsteps/sec: 1.45527\tloss: 0.49925\tlr:[1.6101525e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:56:56,011 [ hooks.py: 207]:\tstep: 49930\tsteps/sec: 1.23951\tloss: 0.24347\tlr:[1.6094733e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:06,460 [ hooks.py: 207]:\tstep: 49940\tsteps/sec: 0.96846\tloss: 0.54068\tlr:[1.6087944e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:15,538 [ hooks.py: 207]:\tstep: 49950\tsteps/sec: 1.09310\tloss: 0.45011\tlr:[1.6081152e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:24,331 [ hooks.py: 207]:\tstep: 49960\tsteps/sec: 1.27500\tloss: 0.19514\tlr:[1.6074358e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:34,305 [ hooks.py: 207]:\tstep: 49970\tsteps/sec: 0.98426\tloss: 0.16961\tlr:[1.6067572e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:40,391 [ hooks.py: 207]:\tstep: 49980\tsteps/sec: 1.45415\tloss: 0.10746\tlr:[1.6060778e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:48,414 [ hooks.py: 207]:\tstep: 49990\tsteps/sec: 1.35803\tloss: 0.52491\tlr:[1.605399e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:56,112 [ hooks.py: 207]:\tstep: 50000\tsteps/sec: 1.24402\tloss: 0.65404\tlr:[1.6047197e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:56,112 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:56,113 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:56,113 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 10:57:56,113 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:56,113 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 10:57:56,113 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:05:59,492 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,493 [ trainer.py: 58]:\t{'acc': 0.7799196787148595, 'loss': 0.5849914938593522}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,494 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,494 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:05:59,495 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:05:59,495 [ trainer.py: 70]:\tacc\t0.7799196787148595\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:05:59,495 [ trainer.py: 70]:\tloss\t0.5849914938593522\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:05:59,495 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,495 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7799196787148595, 'loss': 0.5849914938593522}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,495 [ exporter.py: 64]:\t[Best Exporter]: skip step 50000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,495 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7799196787148595, 'loss': 0.5849914938593522}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,495 [ exporter.py: 104]:\t[Best Exporter]: skip step 50000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:05:59,496 [monitored_executor.py: 122]:\tsaving step 50000 to ./output/model_50000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:06:02,905 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_47000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:06:09,977 [ hooks.py: 207]:\tstep: 50010\tsteps/sec: 0.02025\tloss: 0.14070\tlr:[1.6040409e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:06:20,226 [ hooks.py: 207]:\tstep: 50020\tsteps/sec: 1.02181\tloss: 0.08300\tlr:[1.6033619e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:06:29,649 [ hooks.py: 207]:\tstep: 50030\tsteps/sec: 1.03562\tloss: 0.27222\tlr:[1.6026828e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:06:37,491 [ hooks.py: 207]:\tstep: 50040\tsteps/sec: 1.27208\tloss: 0.34640\tlr:[1.6020036e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:06:45,577 [ hooks.py: 207]:\tstep: 50050\tsteps/sec: 1.24210\tloss: 0.21613\tlr:[1.6013246e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:06:52,128 [ hooks.py: 207]:\tstep: 50060\tsteps/sec: 1.52725\tloss: 0.93327\tlr:[1.6006454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:02,012 [ hooks.py: 207]:\tstep: 50070\tsteps/sec: 0.99875\tloss: 0.50686\tlr:[1.5999667e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:09,387 [ hooks.py: 207]:\tstep: 50080\tsteps/sec: 1.44323\tloss: 0.49672\tlr:[1.5992873e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:16,285 [ hooks.py: 207]:\tstep: 50090\tsteps/sec: 1.33416\tloss: 0.47388\tlr:[1.5986087e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:25,077 [ hooks.py: 207]:\tstep: 50100\tsteps/sec: 1.16055\tloss: 0.21147\tlr:[1.5979293e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:33,375 [ hooks.py: 207]:\tstep: 50110\tsteps/sec: 1.21889\tloss: 0.22762\tlr:[1.5972502e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:41,286 [ hooks.py: 207]:\tstep: 50120\tsteps/sec: 1.19821\tloss: 0.69608\tlr:[1.5965714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:49,030 [ hooks.py: 207]:\tstep: 50130\tsteps/sec: 1.34668\tloss: 0.45031\tlr:[1.595892e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:07:58,221 [ hooks.py: 207]:\tstep: 50140\tsteps/sec: 1.09400\tloss: 0.24038\tlr:[1.5952131e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:08:06,629 [ hooks.py: 207]:\tstep: 50150\tsteps/sec: 1.18876\tloss: 0.73494\tlr:[1.594534e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:08:14,896 [ hooks.py: 207]:\tstep: 50160\tsteps/sec: 1.24937\tloss: 0.75931\tlr:[1.5938549e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:08:22,431 [ hooks.py: 207]:\tstep: 50170\tsteps/sec: 1.27156\tloss: 0.56543\tlr:[1.5931759e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:08:30,625 [ hooks.py: 207]:\tstep: 50180\tsteps/sec: 1.25360\tloss: 0.79878\tlr:[1.5924968e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:08:38,966 [ hooks.py: 207]:\tstep: 50190\tsteps/sec: 1.22356\tloss: 0.52267\tlr:[1.5918176e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:08:45,625 [ hooks.py: 207]:\tstep: 50200\tsteps/sec: 1.43421\tloss: 0.28057\tlr:[1.5911388e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:08:52,588 [ hooks.py: 207]:\tstep: 50210\tsteps/sec: 1.44820\tloss: 0.50984\tlr:[1.5904596e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 11:09:01,733 [ hooks.py: 207]:\tstep: 50220\tsteps/sec: 1.08325\tloss: 0.33773\tlr:[1.5897807e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:09:08,574 [ hooks.py: 207]:\tstep: 50230\tsteps/sec: 1.50276\tloss: 0.62412\tlr:[1.5891015e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:09:16,752 [ hooks.py: 207]:\tstep: 50240\tsteps/sec: 1.20032\tloss: 0.49778\tlr:[1.5884225e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:09:22,934 [ hooks.py: 207]:\tstep: 50250\tsteps/sec: 1.51920\tloss: 0.31601\tlr:[1.5877435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:09:33,010 [ hooks.py: 207]:\tstep: 50260\tsteps/sec: 1.01770\tloss: 0.48771\tlr:[1.5870643e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:09:42,273 [ hooks.py: 207]:\tstep: 50270\tsteps/sec: 1.09268\tloss: 0.54262\tlr:[1.5863852e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:09:53,821 [ hooks.py: 207]:\tstep: 50280\tsteps/sec: 0.88317\tloss: 0.49703\tlr:[1.5857062e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:01,953 [ hooks.py: 207]:\tstep: 50290\tsteps/sec: 1.15943\tloss: 0.61067\tlr:[1.5850273e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:10,527 [ hooks.py: 207]:\tstep: 50300\tsteps/sec: 1.25178\tloss: 0.33133\tlr:[1.5843481e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:17,144 [ hooks.py: 207]:\tstep: 50310\tsteps/sec: 1.38410\tloss: 0.05028\tlr:[1.5836693e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:26,804 [ hooks.py: 207]:\tstep: 50320\tsteps/sec: 1.06911\tloss: 0.37138\tlr:[1.58299e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:35,381 [ hooks.py: 207]:\tstep: 50330\tsteps/sec: 1.19951\tloss: 0.47471\tlr:[1.5823112e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:42,616 [ hooks.py: 207]:\tstep: 50340\tsteps/sec: 1.27142\tloss: 0.12975\tlr:[1.5816318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:51,567 [ hooks.py: 207]:\tstep: 50350\tsteps/sec: 1.17515\tloss: 0.36108\tlr:[1.5809532e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:10:57,735 [ hooks.py: 207]:\tstep: 50360\tsteps/sec: 1.54264\tloss: 0.42931\tlr:[1.580274e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:11:05,414 [ hooks.py: 207]:\tstep: 50370\tsteps/sec: 1.39941\tloss: 0.39516\tlr:[1.579595e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:11:13,139 [ hooks.py: 207]:\tstep: 50380\tsteps/sec: 1.24414\tloss: 0.68047\tlr:[1.5789159e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:11:20,996 [ hooks.py: 207]:\tstep: 50390\tsteps/sec: 1.27730\tloss: 0.36837\tlr:[1.5782367e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:11:31,354 [ hooks.py: 207]:\tstep: 50400\tsteps/sec: 0.97675\tloss: 0.26306\tlr:[1.5775577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:11:37,158 [ hooks.py: 207]:\tstep: 50410\tsteps/sec: 1.60226\tloss: 0.36734\tlr:[1.5768785e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:11:46,163 [ hooks.py: 207]:\tstep: 50420\tsteps/sec: 1.10392\tloss: 0.35947\tlr:[1.5761996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:11:54,080 [ hooks.py: 207]:\tstep: 50430\tsteps/sec: 1.37855\tloss: 0.45462\tlr:[1.5755204e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:12:01,422 [ hooks.py: 207]:\tstep: 50440\tsteps/sec: 1.39519\tloss: 0.62933\tlr:[1.5748416e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:12:10,609 [ hooks.py: 207]:\tstep: 50450\tsteps/sec: 1.06235\tloss: 0.50804\tlr:[1.5741622e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:12:19,881 [ hooks.py: 207]:\tstep: 50460\tsteps/sec: 1.04737\tloss: 0.20970\tlr:[1.5734833e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:12:29,783 [ hooks.py: 207]:\tstep: 50470\tsteps/sec: 1.05475\tloss: 0.14376\tlr:[1.5728041e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:12:37,105 [ hooks.py: 207]:\tstep: 50480\tsteps/sec: 1.25862\tloss: 0.29335\tlr:[1.5721253e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:12:46,441 [ hooks.py: 207]:\tstep: 50490\tsteps/sec: 1.09801\tloss: 0.25608\tlr:[1.571446e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:12:53,598 [ hooks.py: 207]:\tstep: 50500\tsteps/sec: 1.36976\tloss: 0.57721\tlr:[1.5707672e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:13:03,379 [ hooks.py: 207]:\tstep: 50510\tsteps/sec: 1.04559\tloss: 0.70122\tlr:[1.570088e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:13:13,074 [ hooks.py: 207]:\tstep: 50520\tsteps/sec: 1.02296\tloss: 0.40778\tlr:[1.569409e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:13:21,719 [ hooks.py: 207]:\tstep: 50530\tsteps/sec: 1.12928\tloss: 0.69457\tlr:[1.5687297e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:13:29,623 [ hooks.py: 207]:\tstep: 50540\tsteps/sec: 1.35709\tloss: 0.88051\tlr:[1.5680507e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:13:39,000 [ hooks.py: 207]:\tstep: 50550\tsteps/sec: 1.05535\tloss: 0.49741\tlr:[1.5673717e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:13:47,725 [ hooks.py: 207]:\tstep: 50560\tsteps/sec: 1.12869\tloss: 0.44173\tlr:[1.5666925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:13:54,841 [ hooks.py: 207]:\tstep: 50570\tsteps/sec: 1.38957\tloss: 0.34759\tlr:[1.5660136e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:14:05,938 [ hooks.py: 207]:\tstep: 50580\tsteps/sec: 0.89812\tloss: 0.37761\tlr:[1.5653344e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:14:14,243 [ hooks.py: 207]:\tstep: 50590\tsteps/sec: 1.25131\tloss: 0.53897\tlr:[1.5646556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:14:22,532 [ hooks.py: 207]:\tstep: 50600\tsteps/sec: 1.17400\tloss: 0.36337\tlr:[1.5639764e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:14:31,347 [ hooks.py: 207]:\tstep: 50610\tsteps/sec: 1.13048\tloss: 0.17329\tlr:[1.5632975e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:14:40,021 [ hooks.py: 207]:\tstep: 50620\tsteps/sec: 1.19168\tloss: 0.36587\tlr:[1.5626183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:14:49,054 [ hooks.py: 207]:\tstep: 50630\tsteps/sec: 1.07256\tloss: 0.16119\tlr:[1.5619395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:14:58,708 [ hooks.py: 207]:\tstep: 50640\tsteps/sec: 1.06147\tloss: 0.50888\tlr:[1.5612603e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:15:06,489 [ hooks.py: 207]:\tstep: 50650\tsteps/sec: 1.24082\tloss: 0.15574\tlr:[1.5605812e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:15:13,703 [ hooks.py: 207]:\tstep: 50660\tsteps/sec: 1.42942\tloss: 0.40777\tlr:[1.559902e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:15:21,791 [ hooks.py: 207]:\tstep: 50670\tsteps/sec: 1.20975\tloss: 0.23965\tlr:[1.5592232e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:15:31,370 [ hooks.py: 207]:\tstep: 50680\tsteps/sec: 1.05452\tloss: 0.15274\tlr:[1.558544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:15:40,124 [ hooks.py: 207]:\tstep: 50690\tsteps/sec: 1.17456\tloss: 0.49040\tlr:[1.5578647e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:15:47,337 [ hooks.py: 207]:\tstep: 50700\tsteps/sec: 1.34193\tloss: 0.40410\tlr:[1.5571859e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:15:55,777 [ hooks.py: 207]:\tstep: 50710\tsteps/sec: 1.19159\tloss: 0.86582\tlr:[1.5565067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:16:02,946 [ hooks.py: 207]:\tstep: 50720\tsteps/sec: 1.38582\tloss: 0.45927\tlr:[1.555828e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:16:11,296 [ hooks.py: 207]:\tstep: 50730\tsteps/sec: 1.18049\tloss: 0.34382\tlr:[1.5551486e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:16:19,712 [ hooks.py: 207]:\tstep: 50740\tsteps/sec: 1.23294\tloss: 0.56571\tlr:[1.5544698e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:16:27,809 [ hooks.py: 207]:\tstep: 50750\tsteps/sec: 1.21435\tloss: 0.51623\tlr:[1.5537906e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:16:35,921 [ hooks.py: 207]:\tstep: 50760\tsteps/sec: 1.17590\tloss: 0.43111\tlr:[1.5531115e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:16:45,808 [ hooks.py: 207]:\tstep: 50770\tsteps/sec: 1.05200\tloss: 0.43295\tlr:[1.5524325e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:16:52,554 [ hooks.py: 207]:\tstep: 50780\tsteps/sec: 1.46390\tloss: 0.71197\tlr:[1.5517537e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:01,493 [ hooks.py: 207]:\tstep: 50790\tsteps/sec: 1.14498\tloss: 0.33346\tlr:[1.5510743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:09,969 [ hooks.py: 207]:\tstep: 50800\tsteps/sec: 1.17643\tloss: 0.71727\tlr:[1.5503956e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:17,474 [ hooks.py: 207]:\tstep: 50810\tsteps/sec: 1.28978\tloss: 0.41247\tlr:[1.5497162e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:26,961 [ hooks.py: 207]:\tstep: 50820\tsteps/sec: 1.05367\tloss: 0.39705\tlr:[1.5490374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:33,440 [ hooks.py: 207]:\tstep: 50830\tsteps/sec: 1.53281\tloss: 0.12863\tlr:[1.5483583e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:42,332 [ hooks.py: 207]:\tstep: 50840\tsteps/sec: 1.14082\tloss: 0.57493\tlr:[1.5476791e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:50,734 [ hooks.py: 207]:\tstep: 50850\tsteps/sec: 1.15153\tloss: 0.40515\tlr:[1.5470001e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:17:59,115 [ hooks.py: 207]:\tstep: 50860\tsteps/sec: 1.26272\tloss: 0.08455\tlr:[1.546321e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 11:18:07,137 [ hooks.py: 207]:\tstep: 50870\tsteps/sec: 1.21198\tloss: 0.71547\tlr:[1.545642e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:18:17,007 [ hooks.py: 207]:\tstep: 50880\tsteps/sec: 1.01116\tloss: 0.12551\tlr:[1.5449628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:18:24,117 [ hooks.py: 207]:\tstep: 50890\tsteps/sec: 1.39328\tloss: 0.36564\tlr:[1.544284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:18:34,608 [ hooks.py: 207]:\tstep: 50900\tsteps/sec: 0.98797\tloss: 0.14386\tlr:[1.5436048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:18:44,153 [ hooks.py: 207]:\tstep: 50910\tsteps/sec: 1.04158\tloss: 0.57177\tlr:[1.5429257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:18:52,783 [ hooks.py: 207]:\tstep: 50920\tsteps/sec: 1.14244\tloss: 0.44444\tlr:[1.5422465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:19:00,348 [ hooks.py: 207]:\tstep: 50930\tsteps/sec: 1.41596\tloss: 0.17953\tlr:[1.5415677e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:19:08,763 [ hooks.py: 207]:\tstep: 50940\tsteps/sec: 1.13245\tloss: 0.40041\tlr:[1.5408887e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:19:18,439 [ hooks.py: 207]:\tstep: 50950\tsteps/sec: 1.00202\tloss: 0.47210\tlr:[1.5402098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:19:27,481 [ hooks.py: 207]:\tstep: 50960\tsteps/sec: 1.12765\tloss: 0.35904\tlr:[1.5395306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:19:34,306 [ hooks.py: 207]:\tstep: 50970\tsteps/sec: 1.44048\tloss: 0.81146\tlr:[1.5388518e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:19:43,191 [ hooks.py: 207]:\tstep: 50980\tsteps/sec: 1.14045\tloss: 0.43705\tlr:[1.5381724e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:19:51,558 [ hooks.py: 207]:\tstep: 50990\tsteps/sec: 1.18247\tloss: 0.23197\tlr:[1.5374932e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:20:00,787 [ hooks.py: 207]:\tstep: 51000\tsteps/sec: 1.18055\tloss: 0.42550\tlr:[1.5368143e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:20:00,788 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:20:00,788 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:20:00,788 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:20:00,788 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:20:00,788 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:20:00,789 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:28:08,419 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,420 [ trainer.py: 58]:\t{'acc': 0.7811244979919679, 'loss': 0.5841030561102506}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,421 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,421 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:28:08,422 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:28:08,422 [ trainer.py: 70]:\tacc\t0.7811244979919679\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:28:08,422 [ trainer.py: 70]:\tloss\t0.5841030561102506\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:28:08,422 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,422 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7811244979919679, 'loss': 0.5841030561102506}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,422 [ exporter.py: 64]:\t[Best Exporter]: skip step 51000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,422 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7811244979919679, 'loss': 0.5841030561102506}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,422 [ exporter.py: 104]:\t[Best Exporter]: skip step 51000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:08,423 [monitored_executor.py: 122]:\tsaving step 51000 to ./output/model_51000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:11,615 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_48000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:19,719 [ hooks.py: 207]:\tstep: 51010\tsteps/sec: 0.02002\tloss: 0.09905\tlr:[1.5361351e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:27,615 [ hooks.py: 207]:\tstep: 51020\tsteps/sec: 1.22324\tloss: 0.53872\tlr:[1.535456e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:38,921 [ hooks.py: 207]:\tstep: 51030\tsteps/sec: 0.91162\tloss: 0.32257\tlr:[1.534777e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:47,389 [ hooks.py: 207]:\tstep: 51040\tsteps/sec: 1.15929\tloss: 0.42235\tlr:[1.534098e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:28:57,007 [ hooks.py: 207]:\tstep: 51050\tsteps/sec: 1.03761\tloss: 0.98934\tlr:[1.5334188e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:29:05,935 [ hooks.py: 207]:\tstep: 51060\tsteps/sec: 1.07852\tloss: 0.39504\tlr:[1.5327401e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:29:15,265 [ hooks.py: 207]:\tstep: 51070\tsteps/sec: 1.11108\tloss: 0.53503\tlr:[1.532061e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:29:24,749 [ hooks.py: 207]:\tstep: 51080\tsteps/sec: 1.05972\tloss: 0.26341\tlr:[1.531382e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:29:33,042 [ hooks.py: 207]:\tstep: 51090\tsteps/sec: 1.23117\tloss: 0.37783\tlr:[1.5307027e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:29:41,709 [ hooks.py: 207]:\tstep: 51100\tsteps/sec: 1.20102\tloss: 0.52048\tlr:[1.5300238e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:29:51,243 [ hooks.py: 207]:\tstep: 51110\tsteps/sec: 0.99814\tloss: 0.36496\tlr:[1.5293448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:29:58,639 [ hooks.py: 207]:\tstep: 51120\tsteps/sec: 1.37126\tloss: 0.81554\tlr:[1.5286656e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:30:06,361 [ hooks.py: 207]:\tstep: 51130\tsteps/sec: 1.20455\tloss: 0.20939\tlr:[1.5279866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:30:15,864 [ hooks.py: 207]:\tstep: 51140\tsteps/sec: 1.09069\tloss: 0.22027\tlr:[1.5273074e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:30:25,401 [ hooks.py: 207]:\tstep: 51150\tsteps/sec: 1.05859\tloss: 0.20531\tlr:[1.5266283e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:30:34,510 [ hooks.py: 207]:\tstep: 51160\tsteps/sec: 1.06867\tloss: 0.31430\tlr:[1.5259491e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:30:42,411 [ hooks.py: 207]:\tstep: 51170\tsteps/sec: 1.27084\tloss: 0.28162\tlr:[1.5252703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:30:50,898 [ hooks.py: 207]:\tstep: 51180\tsteps/sec: 1.21632\tloss: 0.22289\tlr:[1.52459115e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:30:58,861 [ hooks.py: 207]:\tstep: 51190\tsteps/sec: 1.27287\tloss: 0.30333\tlr:[1.5239122e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:31:08,963 [ hooks.py: 207]:\tstep: 51200\tsteps/sec: 1.00058\tloss: 0.85460\tlr:[1.523233e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:31:16,641 [ hooks.py: 207]:\tstep: 51210\tsteps/sec: 1.26195\tloss: 0.29471\tlr:[1.5225542e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:31:24,831 [ hooks.py: 207]:\tstep: 51220\tsteps/sec: 1.23118\tloss: 0.20649\tlr:[1.521875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:31:34,340 [ hooks.py: 207]:\tstep: 51230\tsteps/sec: 1.06392\tloss: 0.78680\tlr:[1.5211962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:31:43,109 [ hooks.py: 207]:\tstep: 51240\tsteps/sec: 1.13112\tloss: 0.42302\tlr:[1.520517e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:31:51,438 [ hooks.py: 207]:\tstep: 51250\tsteps/sec: 1.18242\tloss: 0.64163\tlr:[1.51983795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:32:00,231 [ hooks.py: 207]:\tstep: 51260\tsteps/sec: 1.10464\tloss: 0.14251\tlr:[1.5191589e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:32:08,943 [ hooks.py: 207]:\tstep: 51270\tsteps/sec: 1.22007\tloss: 0.28733\tlr:[1.51848e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:32:18,308 [ hooks.py: 207]:\tstep: 51280\tsteps/sec: 1.06428\tloss: 0.32207\tlr:[1.5178008e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:32:24,835 [ hooks.py: 207]:\tstep: 51290\tsteps/sec: 1.49990\tloss: 0.19220\tlr:[1.5171216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:32:33,359 [ hooks.py: 207]:\tstep: 51300\tsteps/sec: 1.18471\tloss: 0.57760\tlr:[1.5164427e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:32:42,161 [ hooks.py: 207]:\tstep: 51310\tsteps/sec: 1.16726\tloss: 0.27356\tlr:[1.5157635e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:32:49,209 [ hooks.py: 207]:\tstep: 51320\tsteps/sec: 1.38715\tloss: 0.30122\tlr:[1.5150847e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 11:32:57,043 [ hooks.py: 207]:\tstep: 51330\tsteps/sec: 1.22523\tloss: 0.19971\tlr:[1.5144053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:33:07,001 [ hooks.py: 207]:\tstep: 51340\tsteps/sec: 1.00629\tloss: 0.43921\tlr:[1.5137264e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:33:17,988 [ hooks.py: 207]:\tstep: 51350\tsteps/sec: 0.92693\tloss: 0.47318\tlr:[1.5130472e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:33:26,012 [ hooks.py: 207]:\tstep: 51360\tsteps/sec: 1.21690\tloss: 0.37327\tlr:[1.5123683e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:33:34,184 [ hooks.py: 207]:\tstep: 51370\tsteps/sec: 1.25800\tloss: 0.45884\tlr:[1.5116891e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:33:43,449 [ hooks.py: 207]:\tstep: 51380\tsteps/sec: 1.07506\tloss: 0.28513\tlr:[1.5110102e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:33:52,435 [ hooks.py: 207]:\tstep: 51390\tsteps/sec: 1.06340\tloss: 0.47050\tlr:[1.510331e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:34:03,030 [ hooks.py: 207]:\tstep: 51400\tsteps/sec: 0.97519\tloss: 0.22343\tlr:[1.509652e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:34:13,353 [ hooks.py: 207]:\tstep: 51410\tsteps/sec: 0.98111\tloss: 0.38458\tlr:[1.5089728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:34:21,406 [ hooks.py: 207]:\tstep: 51420\tsteps/sec: 1.22723\tloss: 0.20249\tlr:[1.508294e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:34:30,222 [ hooks.py: 207]:\tstep: 51430\tsteps/sec: 1.13892\tloss: 0.53264\tlr:[1.5076148e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:34:37,955 [ hooks.py: 207]:\tstep: 51440\tsteps/sec: 1.30004\tloss: 0.40990\tlr:[1.5069356e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:34:45,607 [ hooks.py: 207]:\tstep: 51450\tsteps/sec: 1.26687\tloss: 0.43510\tlr:[1.5062566e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:34:53,726 [ hooks.py: 207]:\tstep: 51460\tsteps/sec: 1.26162\tloss: 0.27119\tlr:[1.5055775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:35:03,254 [ hooks.py: 207]:\tstep: 51470\tsteps/sec: 1.05041\tloss: 0.10308\tlr:[1.5048986e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:35:11,620 [ hooks.py: 207]:\tstep: 51480\tsteps/sec: 1.13729\tloss: 0.84369\tlr:[1.5042195e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:35:20,351 [ hooks.py: 207]:\tstep: 51490\tsteps/sec: 1.18583\tloss: 0.50329\tlr:[1.5035405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:35:28,450 [ hooks.py: 207]:\tstep: 51500\tsteps/sec: 1.26455\tloss: 0.39631\tlr:[1.5028613e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:35:36,120 [ hooks.py: 207]:\tstep: 51510\tsteps/sec: 1.25919\tloss: 0.62902\tlr:[1.5021825e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:35:45,503 [ hooks.py: 207]:\tstep: 51520\tsteps/sec: 1.07007\tloss: 0.22377\tlr:[1.5015033e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:35:55,075 [ hooks.py: 207]:\tstep: 51530\tsteps/sec: 1.01634\tloss: 0.13667\tlr:[1.5008243e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:36:04,462 [ hooks.py: 207]:\tstep: 51540\tsteps/sec: 1.13342\tloss: 0.26238\tlr:[1.5001451e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:36:11,251 [ hooks.py: 207]:\tstep: 51550\tsteps/sec: 1.36977\tloss: 0.43201\tlr:[1.49946645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:36:19,761 [ hooks.py: 207]:\tstep: 51560\tsteps/sec: 1.17954\tloss: 0.53771\tlr:[1.4987872e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:36:29,060 [ hooks.py: 207]:\tstep: 51570\tsteps/sec: 1.12048\tloss: 0.90421\tlr:[1.498108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:36:36,801 [ hooks.py: 207]:\tstep: 51580\tsteps/sec: 1.27858\tloss: 0.44267\tlr:[1.4974291e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:36:44,336 [ hooks.py: 207]:\tstep: 51590\tsteps/sec: 1.36280\tloss: 0.42227\tlr:[1.49675e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:36:51,777 [ hooks.py: 207]:\tstep: 51600\tsteps/sec: 1.30796\tloss: 0.46595\tlr:[1.496071e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:37:00,228 [ hooks.py: 207]:\tstep: 51610\tsteps/sec: 1.18089\tloss: 0.35073\tlr:[1.4953918e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:37:09,495 [ hooks.py: 207]:\tstep: 51620\tsteps/sec: 1.09116\tloss: 0.58005\tlr:[1.494713e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:37:20,446 [ hooks.py: 207]:\tstep: 51630\tsteps/sec: 0.90269\tloss: 0.24224\tlr:[1.4940336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:37:29,688 [ hooks.py: 207]:\tstep: 51640\tsteps/sec: 1.08387\tloss: 0.41405\tlr:[1.4933548e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:37:40,617 [ hooks.py: 207]:\tstep: 51650\tsteps/sec: 0.91103\tloss: 0.38457\tlr:[1.4926755e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:37:50,988 [ hooks.py: 207]:\tstep: 51660\tsteps/sec: 0.96914\tloss: 0.16524\tlr:[1.4919968e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:00,653 [ hooks.py: 207]:\tstep: 51670\tsteps/sec: 1.03711\tloss: 0.19090\tlr:[1.4913174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:07,062 [ hooks.py: 207]:\tstep: 51680\tsteps/sec: 1.46578\tloss: 0.15121\tlr:[1.4906386e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:15,112 [ hooks.py: 207]:\tstep: 51690\tsteps/sec: 1.25397\tloss: 0.53484\tlr:[1.4899593e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:23,837 [ hooks.py: 207]:\tstep: 51700\tsteps/sec: 1.15972\tloss: 0.32319\tlr:[1.4892804e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:32,913 [ hooks.py: 207]:\tstep: 51710\tsteps/sec: 1.14612\tloss: 0.35688\tlr:[1.4886014e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:39,714 [ hooks.py: 207]:\tstep: 51720\tsteps/sec: 1.51193\tloss: 0.31584\tlr:[1.4879221e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:46,551 [ hooks.py: 207]:\tstep: 51730\tsteps/sec: 1.40137\tloss: 0.67226\tlr:[1.4872433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:38:54,789 [ hooks.py: 207]:\tstep: 51740\tsteps/sec: 1.21465\tloss: 1.04917\tlr:[1.4865641e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:39:02,174 [ hooks.py: 207]:\tstep: 51750\tsteps/sec: 1.33284\tloss: 0.31372\tlr:[1.4858851e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:39:10,481 [ hooks.py: 207]:\tstep: 51760\tsteps/sec: 1.22080\tloss: 0.58707\tlr:[1.48520585e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:39:20,172 [ hooks.py: 207]:\tstep: 51770\tsteps/sec: 1.03131\tloss: 0.60980\tlr:[1.4845271e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:39:29,366 [ hooks.py: 207]:\tstep: 51780\tsteps/sec: 1.10896\tloss: 0.25247\tlr:[1.4838479e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:39:36,461 [ hooks.py: 207]:\tstep: 51790\tsteps/sec: 1.37241\tloss: 0.38006\tlr:[1.48316885e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:39:44,103 [ hooks.py: 207]:\tstep: 51800\tsteps/sec: 1.34476\tloss: 0.40053\tlr:[1.4824898e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:39:52,443 [ hooks.py: 207]:\tstep: 51810\tsteps/sec: 1.16114\tloss: 0.28346\tlr:[1.4818109e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:40:00,583 [ hooks.py: 207]:\tstep: 51820\tsteps/sec: 1.24701\tloss: 0.19314\tlr:[1.4811318e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:40:08,165 [ hooks.py: 207]:\tstep: 51830\tsteps/sec: 1.34125\tloss: 0.72608\tlr:[1.4804528e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:40:15,830 [ hooks.py: 207]:\tstep: 51840\tsteps/sec: 1.26268\tloss: 0.29424\tlr:[1.4797736e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:40:23,694 [ hooks.py: 207]:\tstep: 51850\tsteps/sec: 1.26630\tloss: 0.28823\tlr:[1.4790948e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:40:33,458 [ hooks.py: 207]:\tstep: 51860\tsteps/sec: 1.04785\tloss: 0.56990\tlr:[1.4784154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:40:44,815 [ hooks.py: 207]:\tstep: 51870\tsteps/sec: 0.88067\tloss: 0.33183\tlr:[1.4777362e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:40:54,622 [ hooks.py: 207]:\tstep: 51880\tsteps/sec: 1.01529\tloss: 0.50727\tlr:[1.4770573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:41:04,493 [ hooks.py: 207]:\tstep: 51890\tsteps/sec: 1.02984\tloss: 0.19671\tlr:[1.4763783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:41:13,318 [ hooks.py: 207]:\tstep: 51900\tsteps/sec: 1.12083\tloss: 0.39424\tlr:[1.4756992e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:41:20,658 [ hooks.py: 207]:\tstep: 51910\tsteps/sec: 1.27897\tloss: 0.42415\tlr:[1.47502005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:41:30,606 [ hooks.py: 207]:\tstep: 51920\tsteps/sec: 1.04331\tloss: 0.06887\tlr:[1.4743411e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:41:38,675 [ hooks.py: 207]:\tstep: 51930\tsteps/sec: 1.21311\tloss: 0.17048\tlr:[1.4736619e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:41:49,119 [ hooks.py: 207]:\tstep: 51940\tsteps/sec: 0.99029\tloss: 0.38188\tlr:[1.4729831e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:41:57,408 [ hooks.py: 207]:\tstep: 51950\tsteps/sec: 1.18020\tloss: 0.47014\tlr:[1.47230385e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:04,127 [ hooks.py: 207]:\tstep: 51960\tsteps/sec: 1.51954\tloss: 0.72427\tlr:[1.4716249e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:11,034 [ hooks.py: 207]:\tstep: 51970\tsteps/sec: 1.35215\tloss: 0.57179\tlr:[1.4709459e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 11:42:19,772 [ hooks.py: 207]:\tstep: 51980\tsteps/sec: 1.22053\tloss: 0.42384\tlr:[1.47026685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:30,019 [ hooks.py: 207]:\tstep: 51990\tsteps/sec: 0.96341\tloss: 0.56964\tlr:[1.4695876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:38,308 [ hooks.py: 207]:\tstep: 52000\tsteps/sec: 1.22980\tloss: 0.83626\tlr:[1.4689088e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:38,308 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:38,308 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:38,308 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:42:38,309 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:38,309 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:42:38,309 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:50:44,996 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:44,998 [ trainer.py: 58]:\t{'acc': 0.778714859437751, 'loss': 0.6016215128967395}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:44,998 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:44,999 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:50:44,999 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:50:44,999 [ trainer.py: 70]:\tacc\t0.778714859437751\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:50:44,999 [ trainer.py: 70]:\tloss\t0.6016215128967395\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 11:50:44,999 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:45,000 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.778714859437751, 'loss': 0.6016215128967395}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:45,000 [ exporter.py: 64]:\t[Best Exporter]: skip step 52000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:45,000 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.778714859437751, 'loss': 0.6016215128967395}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:45,000 [ exporter.py: 104]:\t[Best Exporter]: skip step 52000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:45,001 [monitored_executor.py: 122]:\tsaving step 52000 to ./output/model_52000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:48,392 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_49000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:50:55,351 [ hooks.py: 207]:\tstep: 52010\tsteps/sec: 0.02011\tloss: 0.76061\tlr:[1.4682296e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:51:02,789 [ hooks.py: 207]:\tstep: 52020\tsteps/sec: 1.31458\tloss: 0.86669\tlr:[1.4675504e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:51:13,253 [ hooks.py: 207]:\tstep: 52030\tsteps/sec: 0.97089\tloss: 0.52870\tlr:[1.4668714e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:51:21,120 [ hooks.py: 207]:\tstep: 52040\tsteps/sec: 1.21346\tloss: 0.31825\tlr:[1.4661922e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:51:30,521 [ hooks.py: 207]:\tstep: 52050\tsteps/sec: 1.09094\tloss: 0.39621\tlr:[1.4655134e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:51:38,901 [ hooks.py: 207]:\tstep: 52060\tsteps/sec: 1.23475\tloss: 0.34727\tlr:[1.46483435e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:51:46,933 [ hooks.py: 207]:\tstep: 52070\tsteps/sec: 1.20773\tloss: 0.37843\tlr:[1.4641554e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:51:56,419 [ hooks.py: 207]:\tstep: 52080\tsteps/sec: 1.01935\tloss: 0.22326\tlr:[1.4634761e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:52:03,546 [ hooks.py: 207]:\tstep: 52090\tsteps/sec: 1.51980\tloss: 0.56950\tlr:[1.4627972e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:52:11,670 [ hooks.py: 207]:\tstep: 52100\tsteps/sec: 1.19956\tloss: 0.21688\tlr:[1.462118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:52:19,864 [ hooks.py: 207]:\tstep: 52110\tsteps/sec: 1.25370\tloss: 0.21776\tlr:[1.4614392e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:52:29,335 [ hooks.py: 207]:\tstep: 52120\tsteps/sec: 1.05224\tloss: 0.77365\tlr:[1.4607599e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:52:37,030 [ hooks.py: 207]:\tstep: 52130\tsteps/sec: 1.26580\tloss: 0.62719\tlr:[1.460081e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:52:45,696 [ hooks.py: 207]:\tstep: 52140\tsteps/sec: 1.11268\tloss: 0.74927\tlr:[1.45940185e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:52:55,367 [ hooks.py: 207]:\tstep: 52150\tsteps/sec: 1.04336\tloss: 0.23975\tlr:[1.4587229e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:53:04,344 [ hooks.py: 207]:\tstep: 52160\tsteps/sec: 1.18844\tloss: 0.50388\tlr:[1.4580439e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:53:11,600 [ hooks.py: 207]:\tstep: 52170\tsteps/sec: 1.33256\tloss: 0.49997\tlr:[1.4573647e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:53:20,582 [ hooks.py: 207]:\tstep: 52180\tsteps/sec: 1.12391\tloss: 0.52066\tlr:[1.4566857e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:53:29,071 [ hooks.py: 207]:\tstep: 52190\tsteps/sec: 1.22547\tloss: 0.56141\tlr:[1.4560064e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:53:36,610 [ hooks.py: 207]:\tstep: 52200\tsteps/sec: 1.22537\tloss: 0.16556\tlr:[1.4553275e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:53:46,271 [ hooks.py: 207]:\tstep: 52210\tsteps/sec: 1.05002\tloss: 0.22060\tlr:[1.4546484e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:53:54,810 [ hooks.py: 207]:\tstep: 52220\tsteps/sec: 1.18428\tloss: 0.32397\tlr:[1.4539694e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:54:03,979 [ hooks.py: 207]:\tstep: 52230\tsteps/sec: 1.06737\tloss: 0.52982\tlr:[1.4532902e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:54:13,310 [ hooks.py: 207]:\tstep: 52240\tsteps/sec: 1.10249\tloss: 0.85792\tlr:[1.4526114e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:54:21,576 [ hooks.py: 207]:\tstep: 52250\tsteps/sec: 1.15350\tloss: 0.27796\tlr:[1.4519322e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:54:33,309 [ hooks.py: 207]:\tstep: 52260\tsteps/sec: 0.85737\tloss: 0.28912\tlr:[1.4512532e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:54:43,532 [ hooks.py: 207]:\tstep: 52270\tsteps/sec: 1.00066\tloss: 0.51188\tlr:[1.450574e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:54:52,142 [ hooks.py: 207]:\tstep: 52280\tsteps/sec: 1.15901\tloss: 0.39205\tlr:[1.4498952e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:55:02,849 [ hooks.py: 207]:\tstep: 52290\tsteps/sec: 0.93293\tloss: 0.48117\tlr:[1.449216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:55:12,440 [ hooks.py: 207]:\tstep: 52300\tsteps/sec: 1.04128\tloss: 0.43055\tlr:[1.4485371e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:55:22,158 [ hooks.py: 207]:\tstep: 52310\tsteps/sec: 1.02762\tloss: 0.31137\tlr:[1.4478579e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:55:29,364 [ hooks.py: 207]:\tstep: 52320\tsteps/sec: 1.42532\tloss: 0.27179\tlr:[1.4471789e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:55:36,476 [ hooks.py: 207]:\tstep: 52330\tsteps/sec: 1.30738\tloss: 0.38619\tlr:[1.4464999e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:55:46,880 [ hooks.py: 207]:\tstep: 52340\tsteps/sec: 1.09859\tloss: 0.40832\tlr:[1.4458206e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:55:54,582 [ hooks.py: 207]:\tstep: 52350\tsteps/sec: 1.18192\tloss: 0.53060\tlr:[1.4451417e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:56:03,728 [ hooks.py: 207]:\tstep: 52360\tsteps/sec: 1.08976\tloss: 0.37333\tlr:[1.4444627e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:56:13,651 [ hooks.py: 207]:\tstep: 52370\tsteps/sec: 1.00401\tloss: 0.38534\tlr:[1.4437837e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:56:20,472 [ hooks.py: 207]:\tstep: 52380\tsteps/sec: 1.45717\tloss: 0.24783\tlr:[1.4431044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:56:29,663 [ hooks.py: 207]:\tstep: 52390\tsteps/sec: 1.10703\tloss: 0.04180\tlr:[1.4424257e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:56:36,465 [ hooks.py: 207]:\tstep: 52400\tsteps/sec: 1.40995\tloss: 0.37159\tlr:[1.4417465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:56:46,600 [ hooks.py: 207]:\tstep: 52410\tsteps/sec: 0.99678\tloss: 0.39637\tlr:[1.4410675e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:56:54,519 [ hooks.py: 207]:\tstep: 52420\tsteps/sec: 1.23244\tloss: 0.28652\tlr:[1.4403882e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:57:01,615 [ hooks.py: 207]:\tstep: 52430\tsteps/sec: 1.37677\tloss: 0.19783\tlr:[1.4397095e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 11:57:11,905 [ hooks.py: 207]:\tstep: 52440\tsteps/sec: 1.01091\tloss: 0.35647\tlr:[1.4390303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:57:19,037 [ hooks.py: 207]:\tstep: 52450\tsteps/sec: 1.36214\tloss: 0.30212\tlr:[1.4383514e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:57:29,810 [ hooks.py: 207]:\tstep: 52460\tsteps/sec: 0.95930\tloss: 0.32247\tlr:[1.4376722e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:57:37,515 [ hooks.py: 207]:\tstep: 52470\tsteps/sec: 1.24775\tloss: 0.88453\tlr:[1.4369928e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:57:45,889 [ hooks.py: 207]:\tstep: 52480\tsteps/sec: 1.21607\tloss: 0.54633\tlr:[1.436314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:57:53,127 [ hooks.py: 207]:\tstep: 52490\tsteps/sec: 1.31779\tloss: 0.38122\tlr:[1.4356349e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:58:03,093 [ hooks.py: 207]:\tstep: 52500\tsteps/sec: 1.03456\tloss: 0.32300\tlr:[1.4349559e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:58:12,437 [ hooks.py: 207]:\tstep: 52510\tsteps/sec: 1.06993\tloss: 0.30057\tlr:[1.4342767e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:58:20,067 [ hooks.py: 207]:\tstep: 52520\tsteps/sec: 1.35249\tloss: 0.20413\tlr:[1.43359775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:58:27,208 [ hooks.py: 207]:\tstep: 52530\tsteps/sec: 1.33818\tloss: 0.58610\tlr:[1.4329185e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:58:37,382 [ hooks.py: 207]:\tstep: 52540\tsteps/sec: 0.98996\tloss: 0.26738\tlr:[1.4322397e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:58:44,815 [ hooks.py: 207]:\tstep: 52550\tsteps/sec: 1.41942\tloss: 0.37783\tlr:[1.4315605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:58:51,824 [ hooks.py: 207]:\tstep: 52560\tsteps/sec: 1.36331\tloss: 0.11697\tlr:[1.43088155e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:59:02,027 [ hooks.py: 207]:\tstep: 52570\tsteps/sec: 0.97596\tloss: 0.61330\tlr:[1.4302024e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:59:11,981 [ hooks.py: 207]:\tstep: 52580\tsteps/sec: 1.00028\tloss: 0.68584\tlr:[1.4295235e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:59:21,923 [ hooks.py: 207]:\tstep: 52590\tsteps/sec: 1.01723\tloss: 0.47417\tlr:[1.4288443e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:59:30,492 [ hooks.py: 207]:\tstep: 52600\tsteps/sec: 1.12704\tloss: 0.33107\tlr:[1.4281651e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:59:40,087 [ hooks.py: 207]:\tstep: 52610\tsteps/sec: 1.15557\tloss: 0.38864\tlr:[1.4274862e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:59:48,596 [ hooks.py: 207]:\tstep: 52620\tsteps/sec: 1.10503\tloss: 0.39997\tlr:[1.426807e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 11:59:56,218 [ hooks.py: 207]:\tstep: 52630\tsteps/sec: 1.27192\tloss: 0.30161\tlr:[1.4261281e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:00:05,679 [ hooks.py: 207]:\tstep: 52640\tsteps/sec: 1.06081\tloss: 0.35929\tlr:[1.42544895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:00:14,970 [ hooks.py: 207]:\tstep: 52650\tsteps/sec: 1.11248\tloss: 0.06335\tlr:[1.42477e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:00:22,684 [ hooks.py: 207]:\tstep: 52660\tsteps/sec: 1.24559\tloss: 0.67408\tlr:[1.4240908e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:00:30,860 [ hooks.py: 207]:\tstep: 52670\tsteps/sec: 1.23465\tloss: 0.43451\tlr:[1.423412e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:00:41,803 [ hooks.py: 207]:\tstep: 52680\tsteps/sec: 0.95394\tloss: 0.70762\tlr:[1.42273275e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:00:48,931 [ hooks.py: 207]:\tstep: 52690\tsteps/sec: 1.33979\tloss: 0.24172\tlr:[1.4220538e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:00:59,212 [ hooks.py: 207]:\tstep: 52700\tsteps/sec: 0.98047\tloss: 0.25595\tlr:[1.4213748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:01:07,480 [ hooks.py: 207]:\tstep: 52710\tsteps/sec: 1.16965\tloss: 0.35785\tlr:[1.4206959e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:01:15,784 [ hooks.py: 207]:\tstep: 52720\tsteps/sec: 1.22506\tloss: 0.09453\tlr:[1.4200167e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:01:24,075 [ hooks.py: 207]:\tstep: 52730\tsteps/sec: 1.23131\tloss: 0.11012\tlr:[1.4193377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:01:32,288 [ hooks.py: 207]:\tstep: 52740\tsteps/sec: 1.18043\tloss: 0.36560\tlr:[1.4186586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:01:39,457 [ hooks.py: 207]:\tstep: 52750\tsteps/sec: 1.44707\tloss: 0.36605\tlr:[1.4179793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:01:48,472 [ hooks.py: 207]:\tstep: 52760\tsteps/sec: 1.09346\tloss: 0.39867\tlr:[1.4173005e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:01:59,429 [ hooks.py: 207]:\tstep: 52770\tsteps/sec: 0.92617\tloss: 0.36509\tlr:[1.4166213e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:02:06,006 [ hooks.py: 207]:\tstep: 52780\tsteps/sec: 1.44229\tloss: 0.49134\tlr:[1.4159423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:02:15,049 [ hooks.py: 207]:\tstep: 52790\tsteps/sec: 1.09131\tloss: 0.18819\tlr:[1.4152631e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:02:23,624 [ hooks.py: 207]:\tstep: 52800\tsteps/sec: 1.20182\tloss: 0.41322\tlr:[1.4145842e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:02:32,317 [ hooks.py: 207]:\tstep: 52810\tsteps/sec: 1.14324\tloss: 1.02637\tlr:[1.413905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:02:41,157 [ hooks.py: 207]:\tstep: 52820\tsteps/sec: 1.19796\tloss: 0.24306\tlr:[1.4132261e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:02:48,709 [ hooks.py: 207]:\tstep: 52830\tsteps/sec: 1.27981\tloss: 0.27693\tlr:[1.4125469e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:02:56,041 [ hooks.py: 207]:\tstep: 52840\tsteps/sec: 1.33779\tloss: 0.56551\tlr:[1.411868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:03:04,125 [ hooks.py: 207]:\tstep: 52850\tsteps/sec: 1.25711\tloss: 0.57496\tlr:[1.4111888e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:03:11,487 [ hooks.py: 207]:\tstep: 52860\tsteps/sec: 1.32300\tloss: 0.31566\tlr:[1.4105099e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:03:20,236 [ hooks.py: 207]:\tstep: 52870\tsteps/sec: 1.15031\tloss: 0.50236\tlr:[1.40983075e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:03:30,217 [ hooks.py: 207]:\tstep: 52880\tsteps/sec: 1.00299\tloss: 0.31795\tlr:[1.409152e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:03:39,212 [ hooks.py: 207]:\tstep: 52890\tsteps/sec: 1.13061\tloss: 0.48511\tlr:[1.4084728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:03:47,153 [ hooks.py: 207]:\tstep: 52900\tsteps/sec: 1.21301\tloss: 0.66130\tlr:[1.4077934e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:03:55,150 [ hooks.py: 207]:\tstep: 52910\tsteps/sec: 1.27756\tloss: 0.37804\tlr:[1.4071147e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:04:02,320 [ hooks.py: 207]:\tstep: 52920\tsteps/sec: 1.28228\tloss: 0.56823\tlr:[1.4064355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:04:10,996 [ hooks.py: 207]:\tstep: 52930\tsteps/sec: 1.18200\tloss: 0.17634\tlr:[1.4057564e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:04:21,328 [ hooks.py: 207]:\tstep: 52940\tsteps/sec: 1.00009\tloss: 0.45029\tlr:[1.4050774e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:04:29,944 [ hooks.py: 207]:\tstep: 52950\tsteps/sec: 1.17650\tloss: 0.44992\tlr:[1.4043985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:04:41,758 [ hooks.py: 207]:\tstep: 52960\tsteps/sec: 0.87434\tloss: 0.24859\tlr:[1.4037193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:04:51,161 [ hooks.py: 207]:\tstep: 52970\tsteps/sec: 1.01153\tloss: 0.37458\tlr:[1.4030404e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:04:59,764 [ hooks.py: 207]:\tstep: 52980\tsteps/sec: 1.14301\tloss: 0.44028\tlr:[1.4023611e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:05:07,129 [ hooks.py: 207]:\tstep: 52990\tsteps/sec: 1.30485\tloss: 0.57589\tlr:[1.4016823e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:05:15,518 [ hooks.py: 207]:\tstep: 53000\tsteps/sec: 1.27678\tloss: 0.24716\tlr:[1.401003e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:05:15,519 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:05:15,520 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:05:15,520 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:05:15,520 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:05:15,520 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:05:15,520 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:13:16,836 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,837 [ trainer.py: 58]:\t{'acc': 0.7879518072289157, 'loss': 0.5892255254901755}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,838 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,839 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:13:16,839 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:13:16,839 [ trainer.py: 70]:\tacc\t0.7879518072289157\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:13:16,839 [ trainer.py: 70]:\tloss\t0.5892255254901755\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:13:16,839 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,839 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7879518072289157, 'loss': 0.5892255254901755}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,839 [ exporter.py: 64]:\t[Best Exporter]: skip step 53000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,839 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7879518072289157, 'loss': 0.5892255254901755}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,839 [ exporter.py: 104]:\t[Best Exporter]: skip step 53000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:16,841 [monitored_executor.py: 122]:\tsaving step 53000 to ./output/model_53000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 12:13:19,951 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_50000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:27,459 [ hooks.py: 207]:\tstep: 53010\tsteps/sec: 0.02032\tloss: 0.82579\tlr:[1.40032425e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:35,063 [ hooks.py: 207]:\tstep: 53020\tsteps/sec: 1.33780\tloss: 0.56264\tlr:[1.3996449e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:46,591 [ hooks.py: 207]:\tstep: 53030\tsteps/sec: 0.85406\tloss: 0.44572\tlr:[1.3989661e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:13:56,164 [ hooks.py: 207]:\tstep: 53040\tsteps/sec: 1.05382\tloss: 0.62524\tlr:[1.3982868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:14:06,791 [ hooks.py: 207]:\tstep: 53050\tsteps/sec: 0.92900\tloss: 0.11478\tlr:[1.3976078e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:14:16,763 [ hooks.py: 207]:\tstep: 53060\tsteps/sec: 1.00326\tloss: 0.40520\tlr:[1.3969287e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:14:26,548 [ hooks.py: 207]:\tstep: 53070\tsteps/sec: 0.99771\tloss: 0.56367\tlr:[1.3962495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:14:35,437 [ hooks.py: 207]:\tstep: 53080\tsteps/sec: 1.20127\tloss: 0.21596\tlr:[1.3955706e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:14:42,601 [ hooks.py: 207]:\tstep: 53090\tsteps/sec: 1.39101\tloss: 0.19393\tlr:[1.3948914e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:14:49,941 [ hooks.py: 207]:\tstep: 53100\tsteps/sec: 1.37001\tloss: 0.45306\tlr:[1.3942125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:14:59,319 [ hooks.py: 207]:\tstep: 53110\tsteps/sec: 1.05218\tloss: 0.43912\tlr:[1.3935333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:15:07,840 [ hooks.py: 207]:\tstep: 53120\tsteps/sec: 1.16149\tloss: 0.51164\tlr:[1.3928544e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:15:14,931 [ hooks.py: 207]:\tstep: 53130\tsteps/sec: 1.44612\tloss: 0.83043\tlr:[1.3921752e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:15:22,433 [ hooks.py: 207]:\tstep: 53140\tsteps/sec: 1.23580\tloss: 0.57474\tlr:[1.3914963e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:15:31,744 [ hooks.py: 207]:\tstep: 53150\tsteps/sec: 1.08252\tloss: 0.20479\tlr:[1.3908171e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:15:42,501 [ hooks.py: 207]:\tstep: 53160\tsteps/sec: 0.99118\tloss: 0.42095\tlr:[1.3901383e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:15:50,187 [ hooks.py: 207]:\tstep: 53170\tsteps/sec: 1.28209\tloss: 0.13869\tlr:[1.3894591e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:16:00,907 [ hooks.py: 207]:\tstep: 53180\tsteps/sec: 0.92788\tloss: 0.45939\tlr:[1.3887801e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:16:09,051 [ hooks.py: 207]:\tstep: 53190\tsteps/sec: 1.22045\tloss: 0.46995\tlr:[1.3881009e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:16:17,808 [ hooks.py: 207]:\tstep: 53200\tsteps/sec: 1.12036\tloss: 0.41193\tlr:[1.3874217e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:16:25,994 [ hooks.py: 207]:\tstep: 53210\tsteps/sec: 1.23776\tloss: 0.16426\tlr:[1.386743e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:16:34,238 [ hooks.py: 207]:\tstep: 53220\tsteps/sec: 1.22269\tloss: 0.78191\tlr:[1.38606365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:16:43,779 [ hooks.py: 207]:\tstep: 53230\tsteps/sec: 1.01640\tloss: 0.16442\tlr:[1.3853848e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:16:54,235 [ hooks.py: 207]:\tstep: 53240\tsteps/sec: 0.99107\tloss: 0.22909\tlr:[1.3847056e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:00,987 [ hooks.py: 207]:\tstep: 53250\tsteps/sec: 1.44674\tloss: 0.41411\tlr:[1.3840267e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:09,394 [ hooks.py: 207]:\tstep: 53260\tsteps/sec: 1.21264\tloss: 0.34162\tlr:[1.3833476e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:16,919 [ hooks.py: 207]:\tstep: 53270\tsteps/sec: 1.23387\tloss: 0.13689\tlr:[1.3826686e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:24,819 [ hooks.py: 207]:\tstep: 53280\tsteps/sec: 1.37858\tloss: 0.44942\tlr:[1.3819896e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:32,161 [ hooks.py: 207]:\tstep: 53290\tsteps/sec: 1.31273\tloss: 0.42456\tlr:[1.38131045e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:40,777 [ hooks.py: 207]:\tstep: 53300\tsteps/sec: 1.25387\tloss: 0.41044\tlr:[1.3806314e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:47,394 [ hooks.py: 207]:\tstep: 53310\tsteps/sec: 1.31564\tloss: 0.42076\tlr:[1.3799526e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:17:55,390 [ hooks.py: 207]:\tstep: 53320\tsteps/sec: 1.35271\tloss: 0.58966\tlr:[1.3792734e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:18:03,814 [ hooks.py: 207]:\tstep: 53330\tsteps/sec: 1.17127\tloss: 0.64805\tlr:[1.3785944e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:18:12,011 [ hooks.py: 207]:\tstep: 53340\tsteps/sec: 1.20079\tloss: 0.45447\tlr:[1.3779151e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:18:22,182 [ hooks.py: 207]:\tstep: 53350\tsteps/sec: 0.97567\tloss: 0.35230\tlr:[1.3772361e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:18:30,999 [ hooks.py: 207]:\tstep: 53360\tsteps/sec: 1.09642\tloss: 0.19069\tlr:[1.3765572e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:18:40,062 [ hooks.py: 207]:\tstep: 53370\tsteps/sec: 1.19540\tloss: 0.17574\tlr:[1.37587795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:18:49,363 [ hooks.py: 207]:\tstep: 53380\tsteps/sec: 1.02536\tloss: 0.72635\tlr:[1.3751991e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:18:57,252 [ hooks.py: 207]:\tstep: 53390\tsteps/sec: 1.29385\tloss: 0.50524\tlr:[1.3745199e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:19:07,156 [ hooks.py: 207]:\tstep: 53400\tsteps/sec: 0.97343\tloss: 0.45979\tlr:[1.3738409e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:19:17,655 [ hooks.py: 207]:\tstep: 53410\tsteps/sec: 0.97766\tloss: 0.09906\tlr:[1.37316165e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:19:27,199 [ hooks.py: 207]:\tstep: 53420\tsteps/sec: 1.02240\tloss: 0.75221\tlr:[1.3724827e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:19:37,102 [ hooks.py: 207]:\tstep: 53430\tsteps/sec: 1.04276\tloss: 0.53847\tlr:[1.3718036e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:19:45,120 [ hooks.py: 207]:\tstep: 53440\tsteps/sec: 1.30000\tloss: 0.19661\tlr:[1.37112465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:19:51,791 [ hooks.py: 207]:\tstep: 53450\tsteps/sec: 1.37515\tloss: 0.82625\tlr:[1.3704454e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:20:01,094 [ hooks.py: 207]:\tstep: 53460\tsteps/sec: 1.11457\tloss: 0.31725\tlr:[1.3697666e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:20:09,391 [ hooks.py: 207]:\tstep: 53470\tsteps/sec: 1.21914\tloss: 0.62866\tlr:[1.3690874e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:20:17,063 [ hooks.py: 207]:\tstep: 53480\tsteps/sec: 1.28020\tloss: 0.50004\tlr:[1.36840845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:20:26,105 [ hooks.py: 207]:\tstep: 53490\tsteps/sec: 1.09693\tloss: 0.27411\tlr:[1.3677292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:20:34,814 [ hooks.py: 207]:\tstep: 53500\tsteps/sec: 1.16968\tloss: 0.99261\tlr:[1.3670501e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:20:46,662 [ hooks.py: 207]:\tstep: 53510\tsteps/sec: 0.80882\tloss: 0.54653\tlr:[1.3663712e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:20:56,121 [ hooks.py: 207]:\tstep: 53520\tsteps/sec: 1.10827\tloss: 0.53944\tlr:[1.365692e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:21:04,033 [ hooks.py: 207]:\tstep: 53530\tsteps/sec: 1.28229\tloss: 0.35941\tlr:[1.3650132e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:21:14,233 [ hooks.py: 207]:\tstep: 53540\tsteps/sec: 0.98460\tloss: 0.30881\tlr:[1.364334e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:21:21,253 [ hooks.py: 207]:\tstep: 53550\tsteps/sec: 1.40902\tloss: 0.70062\tlr:[1.3636552e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:21:29,830 [ hooks.py: 207]:\tstep: 53560\tsteps/sec: 1.17301\tloss: 0.40110\tlr:[1.3629758e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:21:39,487 [ hooks.py: 207]:\tstep: 53570\tsteps/sec: 1.05450\tloss: 0.53029\tlr:[1.3622971e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:21:48,211 [ hooks.py: 207]:\tstep: 53580\tsteps/sec: 1.09521\tloss: 0.19397\tlr:[1.3616179e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:21:59,217 [ hooks.py: 207]:\tstep: 53590\tsteps/sec: 0.92891\tloss: 0.18383\tlr:[1.36093895e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:22:07,178 [ hooks.py: 207]:\tstep: 53600\tsteps/sec: 1.23732\tloss: 0.34353\tlr:[1.3602597e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:22:14,226 [ hooks.py: 207]:\tstep: 53610\tsteps/sec: 1.43847\tloss: 0.54791\tlr:[1.3595809e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:22:21,300 [ hooks.py: 207]:\tstep: 53620\tsteps/sec: 1.39974\tloss: 0.58528\tlr:[1.3589017e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:22:29,418 [ hooks.py: 207]:\tstep: 53630\tsteps/sec: 1.20484\tloss: 0.06987\tlr:[1.3582227e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:22:37,333 [ hooks.py: 207]:\tstep: 53640\tsteps/sec: 1.20642\tloss: 0.10872\tlr:[1.3575436e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 12:22:48,241 [ hooks.py: 207]:\tstep: 53650\tsteps/sec: 0.96370\tloss: 0.22435\tlr:[1.3568644e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:22:55,403 [ hooks.py: 207]:\tstep: 53660\tsteps/sec: 1.34761\tloss: 0.30739\tlr:[1.3561855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:23:05,827 [ hooks.py: 207]:\tstep: 53670\tsteps/sec: 0.98559\tloss: 0.62173\tlr:[1.3555063e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:23:13,832 [ hooks.py: 207]:\tstep: 53680\tsteps/sec: 1.24484\tloss: 0.18586\tlr:[1.3548274e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:23:22,759 [ hooks.py: 207]:\tstep: 53690\tsteps/sec: 1.06732\tloss: 0.64982\tlr:[1.3541482e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:23:32,207 [ hooks.py: 207]:\tstep: 53700\tsteps/sec: 1.10865\tloss: 0.49679\tlr:[1.3534692e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:23:39,196 [ hooks.py: 207]:\tstep: 53710\tsteps/sec: 1.45082\tloss: 0.17421\tlr:[1.35279015e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:23:48,415 [ hooks.py: 207]:\tstep: 53720\tsteps/sec: 1.06032\tloss: 0.86754\tlr:[1.3521112e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:23:55,953 [ hooks.py: 207]:\tstep: 53730\tsteps/sec: 1.29389\tloss: 0.39702\tlr:[1.351432e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:24:05,008 [ hooks.py: 207]:\tstep: 53740\tsteps/sec: 1.17351\tloss: 0.29532\tlr:[1.3507532e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:24:12,519 [ hooks.py: 207]:\tstep: 53750\tsteps/sec: 1.28712\tloss: 0.49742\tlr:[1.3500739e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:24:20,711 [ hooks.py: 207]:\tstep: 53760\tsteps/sec: 1.18882\tloss: 0.40622\tlr:[1.349395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:24:30,139 [ hooks.py: 207]:\tstep: 53770\tsteps/sec: 1.10291\tloss: 0.62415\tlr:[1.3487158e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:24:37,432 [ hooks.py: 207]:\tstep: 53780\tsteps/sec: 1.33162\tloss: 0.47926\tlr:[1.3480367e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:24:45,847 [ hooks.py: 207]:\tstep: 53790\tsteps/sec: 1.20817\tloss: 0.29879\tlr:[1.3473575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:24:52,739 [ hooks.py: 207]:\tstep: 53800\tsteps/sec: 1.36599\tloss: 0.31015\tlr:[1.3466784e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:25:01,082 [ hooks.py: 207]:\tstep: 53810\tsteps/sec: 1.27787\tloss: 0.39019\tlr:[1.3459995e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:25:09,180 [ hooks.py: 207]:\tstep: 53820\tsteps/sec: 1.18091\tloss: 0.18615\tlr:[1.3453203e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:25:17,548 [ hooks.py: 207]:\tstep: 53830\tsteps/sec: 1.20389\tloss: 0.65963\tlr:[1.3446415e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:25:27,221 [ hooks.py: 207]:\tstep: 53840\tsteps/sec: 1.05091\tloss: 0.32446\tlr:[1.3439624e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:25:38,837 [ hooks.py: 207]:\tstep: 53850\tsteps/sec: 0.84367\tloss: 0.47758\tlr:[1.3432835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:25:49,368 [ hooks.py: 207]:\tstep: 53860\tsteps/sec: 0.98643\tloss: 0.26672\tlr:[1.3426043e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:25:56,333 [ hooks.py: 207]:\tstep: 53870\tsteps/sec: 1.33478\tloss: 0.55604\tlr:[1.3419254e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:26:06,414 [ hooks.py: 207]:\tstep: 53880\tsteps/sec: 1.02775\tloss: 0.25319\tlr:[1.3412462e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:26:13,852 [ hooks.py: 207]:\tstep: 53890\tsteps/sec: 1.31054\tloss: 0.32750\tlr:[1.3405672e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:26:24,620 [ hooks.py: 207]:\tstep: 53900\tsteps/sec: 0.97202\tloss: 0.36971\tlr:[1.3398881e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:26:32,911 [ hooks.py: 207]:\tstep: 53910\tsteps/sec: 1.15297\tloss: 0.42486\tlr:[1.3392092e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:26:42,723 [ hooks.py: 207]:\tstep: 53920\tsteps/sec: 1.04369\tloss: 0.18359\tlr:[1.3385298e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:26:51,088 [ hooks.py: 207]:\tstep: 53930\tsteps/sec: 1.19200\tloss: 0.14351\tlr:[1.3378508e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:00,141 [ hooks.py: 207]:\tstep: 53940\tsteps/sec: 1.08914\tloss: 0.44201\tlr:[1.3371719e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:08,126 [ hooks.py: 207]:\tstep: 53950\tsteps/sec: 1.25267\tloss: 0.29272\tlr:[1.33649255e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:17,662 [ hooks.py: 207]:\tstep: 53960\tsteps/sec: 1.05442\tloss: 0.35066\tlr:[1.3358137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:24,279 [ hooks.py: 207]:\tstep: 53970\tsteps/sec: 1.46743\tloss: 0.68513\tlr:[1.3351345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:32,228 [ hooks.py: 207]:\tstep: 53980\tsteps/sec: 1.29273\tloss: 0.30165\tlr:[1.3344557e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:40,468 [ hooks.py: 207]:\tstep: 53990\tsteps/sec: 1.23891\tloss: 0.30037\tlr:[1.3337765e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:48,433 [ hooks.py: 207]:\tstep: 54000\tsteps/sec: 1.17663\tloss: 0.46466\tlr:[1.3330976e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:48,434 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:48,434 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:48,434 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:27:48,434 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:48,434 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:27:48,434 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:35:48,990 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,991 [ trainer.py: 58]:\t{'acc': 0.7779116465863454, 'loss': 0.6124191965716772}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,992 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,992 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:35:48,992 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:35:48,993 [ trainer.py: 70]:\tacc\t0.7779116465863454\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:35:48,993 [ trainer.py: 70]:\tloss\t0.6124191965716772\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:35:48,993 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,993 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.6124191965716772}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,993 [ exporter.py: 64]:\t[Best Exporter]: skip step 54000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,993 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.6124191965716772}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,993 [ exporter.py: 104]:\t[Best Exporter]: skip step 54000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:48,994 [monitored_executor.py: 122]:\tsaving step 54000 to ./output/model_54000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:35:52,189 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_51000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:01,607 [ hooks.py: 207]:\tstep: 54010\tsteps/sec: 0.02029\tloss: 0.56055\tlr:[1.3324183e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:10,449 [ hooks.py: 207]:\tstep: 54020\tsteps/sec: 1.15522\tloss: 0.36105\tlr:[1.33173935e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:17,819 [ hooks.py: 207]:\tstep: 54030\tsteps/sec: 1.32502\tloss: 0.38507\tlr:[1.3310603e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:25,716 [ hooks.py: 207]:\tstep: 54040\tsteps/sec: 1.29486\tloss: 0.47561\tlr:[1.3303815e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:33,291 [ hooks.py: 207]:\tstep: 54050\tsteps/sec: 1.27757\tloss: 0.32375\tlr:[1.3297023e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:42,492 [ hooks.py: 207]:\tstep: 54060\tsteps/sec: 1.14947\tloss: 0.18923\tlr:[1.3290233e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:50,632 [ hooks.py: 207]:\tstep: 54070\tsteps/sec: 1.19886\tloss: 0.05738\tlr:[1.3283442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:36:58,382 [ hooks.py: 207]:\tstep: 54080\tsteps/sec: 1.24182\tloss: 0.21657\tlr:[1.3276648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:37:08,192 [ hooks.py: 207]:\tstep: 54090\tsteps/sec: 1.02392\tloss: 0.58364\tlr:[1.3269861e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:37:16,407 [ hooks.py: 207]:\tstep: 54100\tsteps/sec: 1.17776\tloss: 0.14015\tlr:[1.32630685e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 12:37:24,870 [ hooks.py: 207]:\tstep: 54110\tsteps/sec: 1.25321\tloss: 0.23139\tlr:[1.325628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:37:32,917 [ hooks.py: 207]:\tstep: 54120\tsteps/sec: 1.21133\tloss: 0.37564\tlr:[1.3249488e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:37:39,604 [ hooks.py: 207]:\tstep: 54130\tsteps/sec: 1.53919\tloss: 0.57870\tlr:[1.32426985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:37:49,188 [ hooks.py: 207]:\tstep: 54140\tsteps/sec: 1.03404\tloss: 0.45536\tlr:[1.3235907e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:37:56,786 [ hooks.py: 207]:\tstep: 54150\tsteps/sec: 1.28907\tloss: 0.25886\tlr:[1.3229118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:38:06,233 [ hooks.py: 207]:\tstep: 54160\tsteps/sec: 1.06737\tloss: 0.49030\tlr:[1.3222326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:38:13,942 [ hooks.py: 207]:\tstep: 54170\tsteps/sec: 1.27906\tloss: 0.37051\tlr:[1.3215537e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:38:23,610 [ hooks.py: 207]:\tstep: 54180\tsteps/sec: 1.02520\tloss: 0.43129\tlr:[1.3208745e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:38:34,677 [ hooks.py: 207]:\tstep: 54190\tsteps/sec: 0.93268\tloss: 0.35864\tlr:[1.3201956e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:38:45,299 [ hooks.py: 207]:\tstep: 54200\tsteps/sec: 0.95308\tloss: 0.55625\tlr:[1.3195164e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:38:54,590 [ hooks.py: 207]:\tstep: 54210\tsteps/sec: 1.05841\tloss: 0.32223\tlr:[1.3188375e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:02,955 [ hooks.py: 207]:\tstep: 54220\tsteps/sec: 1.17226\tloss: 0.60838\tlr:[1.3181583e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:10,070 [ hooks.py: 207]:\tstep: 54230\tsteps/sec: 1.46408\tloss: 0.24050\tlr:[1.3174791e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:19,869 [ hooks.py: 207]:\tstep: 54240\tsteps/sec: 0.97982\tloss: 0.26876\tlr:[1.3168003e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:27,841 [ hooks.py: 207]:\tstep: 54250\tsteps/sec: 1.28084\tloss: 0.28574\tlr:[1.3161209e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:35,893 [ hooks.py: 207]:\tstep: 54260\tsteps/sec: 1.27302\tloss: 0.43880\tlr:[1.3154421e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:43,300 [ hooks.py: 207]:\tstep: 54270\tsteps/sec: 1.32430\tloss: 0.49601\tlr:[1.3147628e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:49,886 [ hooks.py: 207]:\tstep: 54280\tsteps/sec: 1.56310\tloss: 0.07531\tlr:[1.3140841e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:39:57,404 [ hooks.py: 207]:\tstep: 54290\tsteps/sec: 1.23011\tloss: 0.20045\tlr:[1.3134047e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:40:07,199 [ hooks.py: 207]:\tstep: 54300\tsteps/sec: 1.06233\tloss: 0.93656\tlr:[1.312726e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:40:15,438 [ hooks.py: 207]:\tstep: 54310\tsteps/sec: 1.19194\tloss: 0.16246\tlr:[1.3120466e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:40:23,220 [ hooks.py: 207]:\tstep: 54320\tsteps/sec: 1.29809\tloss: 0.37360\tlr:[1.3113678e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:40:31,197 [ hooks.py: 207]:\tstep: 54330\tsteps/sec: 1.24115\tloss: 0.66968\tlr:[1.31068855e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:40:41,053 [ hooks.py: 207]:\tstep: 54340\tsteps/sec: 1.09626\tloss: 0.28051\tlr:[1.3100096e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:40:49,607 [ hooks.py: 207]:\tstep: 54350\tsteps/sec: 1.10999\tloss: 0.04703\tlr:[1.3093306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:40:57,059 [ hooks.py: 207]:\tstep: 54360\tsteps/sec: 1.25191\tloss: 0.27663\tlr:[1.30865155e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:41:08,341 [ hooks.py: 207]:\tstep: 54370\tsteps/sec: 0.91683\tloss: 0.64092\tlr:[1.3079723e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:41:18,973 [ hooks.py: 207]:\tstep: 54380\tsteps/sec: 0.94140\tloss: 0.21968\tlr:[1.3072931e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:41:27,553 [ hooks.py: 207]:\tstep: 54390\tsteps/sec: 1.15802\tloss: 0.46787\tlr:[1.3066144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:41:35,354 [ hooks.py: 207]:\tstep: 54400\tsteps/sec: 1.32536\tloss: 0.23838\tlr:[1.3059352e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:41:47,448 [ hooks.py: 207]:\tstep: 54410\tsteps/sec: 0.79476\tloss: 0.44257\tlr:[1.3052563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:41:55,322 [ hooks.py: 207]:\tstep: 54420\tsteps/sec: 1.36368\tloss: 0.57354\tlr:[1.3045771e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:04,117 [ hooks.py: 207]:\tstep: 54430\tsteps/sec: 1.09969\tloss: 0.29762\tlr:[1.3038982e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:11,290 [ hooks.py: 207]:\tstep: 54440\tsteps/sec: 1.43297\tloss: 0.33025\tlr:[1.30321905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:18,465 [ hooks.py: 207]:\tstep: 54450\tsteps/sec: 1.34425\tloss: 0.36219\tlr:[1.3025401e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:25,908 [ hooks.py: 207]:\tstep: 54460\tsteps/sec: 1.32171\tloss: 0.27067\tlr:[1.3018609e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:34,599 [ hooks.py: 207]:\tstep: 54470\tsteps/sec: 1.21026\tloss: 0.88791\tlr:[1.3011821e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:42,152 [ hooks.py: 207]:\tstep: 54480\tsteps/sec: 1.43071\tloss: 0.11951\tlr:[1.3005028e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:48,635 [ hooks.py: 207]:\tstep: 54490\tsteps/sec: 1.36731\tloss: 0.15081\tlr:[1.2998239e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:42:59,623 [ hooks.py: 207]:\tstep: 54500\tsteps/sec: 0.92823\tloss: 0.26920\tlr:[1.2991447e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:43:07,588 [ hooks.py: 207]:\tstep: 54510\tsteps/sec: 1.18429\tloss: 0.26379\tlr:[1.29846585e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:43:15,400 [ hooks.py: 207]:\tstep: 54520\tsteps/sec: 1.31182\tloss: 0.83857\tlr:[1.2977866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:43:25,599 [ hooks.py: 207]:\tstep: 54530\tsteps/sec: 1.03163\tloss: 0.56087\tlr:[1.2971073e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:43:34,508 [ hooks.py: 207]:\tstep: 54540\tsteps/sec: 1.09910\tloss: 0.06880\tlr:[1.2964284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:43:44,890 [ hooks.py: 207]:\tstep: 54550\tsteps/sec: 0.97459\tloss: 0.07289\tlr:[1.2957492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:43:52,477 [ hooks.py: 207]:\tstep: 54560\tsteps/sec: 1.25558\tloss: 0.86618\tlr:[1.2950703e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:43:59,507 [ hooks.py: 207]:\tstep: 54570\tsteps/sec: 1.45781\tloss: 0.32528\tlr:[1.2943911e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:44:08,190 [ hooks.py: 207]:\tstep: 54580\tsteps/sec: 1.13775\tloss: 0.60713\tlr:[1.2937122e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:44:17,894 [ hooks.py: 207]:\tstep: 54590\tsteps/sec: 1.02846\tloss: 0.24936\tlr:[1.2930331e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:44:26,003 [ hooks.py: 207]:\tstep: 54600\tsteps/sec: 1.27267\tloss: 0.23601\tlr:[1.2923541e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:44:34,819 [ hooks.py: 207]:\tstep: 54610\tsteps/sec: 1.13081\tloss: 0.12497\tlr:[1.2916749e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:44:43,402 [ hooks.py: 207]:\tstep: 54620\tsteps/sec: 1.13089\tloss: 0.16090\tlr:[1.2909962e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:44:52,793 [ hooks.py: 207]:\tstep: 54630\tsteps/sec: 1.06171\tloss: 0.21072\tlr:[1.290317e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:02,760 [ hooks.py: 207]:\tstep: 54640\tsteps/sec: 1.01845\tloss: 0.28697\tlr:[1.2896381e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:11,140 [ hooks.py: 207]:\tstep: 54650\tsteps/sec: 1.21854\tloss: 0.26741\tlr:[1.2889587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:18,625 [ hooks.py: 207]:\tstep: 54660\tsteps/sec: 1.24100\tloss: 0.34600\tlr:[1.28828e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:27,142 [ hooks.py: 207]:\tstep: 54670\tsteps/sec: 1.22672\tloss: 0.53799\tlr:[1.2876008e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:34,809 [ hooks.py: 207]:\tstep: 54680\tsteps/sec: 1.33006\tloss: 0.56628\tlr:[1.2869216e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:42,657 [ hooks.py: 207]:\tstep: 54690\tsteps/sec: 1.28890\tloss: 0.56865\tlr:[1.2862427e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:50,826 [ hooks.py: 207]:\tstep: 54700\tsteps/sec: 1.24878\tloss: 0.47746\tlr:[1.2855635e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:45:57,713 [ hooks.py: 207]:\tstep: 54710\tsteps/sec: 1.31300\tloss: 0.35809\tlr:[1.2848846e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:46:06,348 [ hooks.py: 207]:\tstep: 54720\tsteps/sec: 1.16738\tloss: 0.10602\tlr:[1.2842054e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:46:14,194 [ hooks.py: 207]:\tstep: 54730\tsteps/sec: 1.26883\tloss: 0.33997\tlr:[1.2835266e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:46:23,558 [ hooks.py: 207]:\tstep: 54740\tsteps/sec: 1.05191\tloss: 0.28387\tlr:[1.2828474e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:46:32,532 [ hooks.py: 207]:\tstep: 54750\tsteps/sec: 1.16963\tloss: 0.46787\tlr:[1.2821684e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 12:46:40,132 [ hooks.py: 207]:\tstep: 54760\tsteps/sec: 1.31551\tloss: 0.29755\tlr:[1.2814892e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:46:49,389 [ hooks.py: 207]:\tstep: 54770\tsteps/sec: 1.09540\tloss: 0.73232\tlr:[1.2808104e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:46:56,520 [ hooks.py: 207]:\tstep: 54780\tsteps/sec: 1.41369\tloss: 0.23233\tlr:[1.280131e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:47:03,973 [ hooks.py: 207]:\tstep: 54790\tsteps/sec: 1.29947\tloss: 0.30803\tlr:[1.2794521e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:47:11,092 [ hooks.py: 207]:\tstep: 54800\tsteps/sec: 1.47288\tloss: 0.37635\tlr:[1.2787729e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:47:18,793 [ hooks.py: 207]:\tstep: 54810\tsteps/sec: 1.24647\tloss: 0.15514\tlr:[1.2780937e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:47:27,970 [ hooks.py: 207]:\tstep: 54820\tsteps/sec: 1.08418\tloss: 0.25087\tlr:[1.2774149e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:47:36,265 [ hooks.py: 207]:\tstep: 54830\tsteps/sec: 1.23386\tloss: 0.28328\tlr:[1.27673575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:47:45,980 [ hooks.py: 207]:\tstep: 54840\tsteps/sec: 1.02952\tloss: 0.47492\tlr:[1.2760567e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:47:54,980 [ hooks.py: 207]:\tstep: 54850\tsteps/sec: 1.11100\tloss: 0.58911\tlr:[1.2753775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:48:01,751 [ hooks.py: 207]:\tstep: 54860\tsteps/sec: 1.48665\tloss: 0.78766\tlr:[1.2746987e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:48:07,885 [ hooks.py: 207]:\tstep: 54870\tsteps/sec: 1.57312\tloss: 0.06688\tlr:[1.27401945e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:48:16,206 [ hooks.py: 207]:\tstep: 54880\tsteps/sec: 1.23558\tloss: 0.25690\tlr:[1.2733405e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:48:26,262 [ hooks.py: 207]:\tstep: 54890\tsteps/sec: 0.98813\tloss: 0.28083\tlr:[1.2726614e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:48:35,330 [ hooks.py: 207]:\tstep: 54900\tsteps/sec: 1.14258\tloss: 0.68524\tlr:[1.2719825e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:48:43,824 [ hooks.py: 207]:\tstep: 54910\tsteps/sec: 1.07556\tloss: 0.50646\tlr:[1.2713032e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:48:51,352 [ hooks.py: 207]:\tstep: 54920\tsteps/sec: 1.35514\tloss: 0.18122\tlr:[1.2706244e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:01,463 [ hooks.py: 207]:\tstep: 54930\tsteps/sec: 1.04010\tloss: 0.48142\tlr:[1.2699452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:08,346 [ hooks.py: 207]:\tstep: 54940\tsteps/sec: 1.40732\tloss: 0.57958\tlr:[1.26926625e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:15,233 [ hooks.py: 207]:\tstep: 54950\tsteps/sec: 1.49317\tloss: 0.37729\tlr:[1.268587e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:23,783 [ hooks.py: 207]:\tstep: 54960\tsteps/sec: 1.14563\tloss: 0.21945\tlr:[1.2679079e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:30,596 [ hooks.py: 207]:\tstep: 54970\tsteps/sec: 1.51639\tloss: 0.25944\tlr:[1.2672292e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:37,707 [ hooks.py: 207]:\tstep: 54980\tsteps/sec: 1.37154\tloss: 0.42981\tlr:[1.2665498e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:44,379 [ hooks.py: 207]:\tstep: 54990\tsteps/sec: 1.40438\tloss: 0.44504\tlr:[1.265871e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:52,433 [ hooks.py: 207]:\tstep: 55000\tsteps/sec: 1.29821\tloss: 0.30254\tlr:[1.2651919e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:52,434 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:52,434 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:52,434 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:49:52,434 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:52,435 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:49:52,435 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:57:54,853 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,854 [ trainer.py: 58]:\t{'acc': 0.7895582329317269, 'loss': 0.6197588161016122}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,855 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,855 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:57:54,856 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:57:54,856 [ trainer.py: 70]:\tacc\t0.7895582329317269\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:57:54,856 [ trainer.py: 70]:\tloss\t0.6197588161016122\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 12:57:54,856 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,856 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7895582329317269, 'loss': 0.6197588161016122}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,856 [ exporter.py: 64]:\t[Best Exporter]: skip step 55000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,856 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7895582329317269, 'loss': 0.6197588161016122}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,856 [ exporter.py: 104]:\t[Best Exporter]: skip step 55000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:54,857 [monitored_executor.py: 122]:\tsaving step 55000 to ./output/model_55000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:57:58,142 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_52000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:58:08,258 [ hooks.py: 207]:\tstep: 55010\tsteps/sec: 0.02017\tloss: 0.18260\tlr:[1.264513e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:58:17,955 [ hooks.py: 207]:\tstep: 55020\tsteps/sec: 1.02608\tloss: 0.44266\tlr:[1.26383375e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:58:26,103 [ hooks.py: 207]:\tstep: 55030\tsteps/sec: 1.27655\tloss: 0.72709\tlr:[1.2631549e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:58:34,337 [ hooks.py: 207]:\tstep: 55040\tsteps/sec: 1.17140\tloss: 0.49594\tlr:[1.2624757e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:58:43,133 [ hooks.py: 207]:\tstep: 55050\tsteps/sec: 1.10768\tloss: 0.56588\tlr:[1.26179675e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:58:51,537 [ hooks.py: 207]:\tstep: 55060\tsteps/sec: 1.26246\tloss: 0.13798\tlr:[1.2611175e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:59:01,352 [ hooks.py: 207]:\tstep: 55070\tsteps/sec: 1.02311\tloss: 0.37240\tlr:[1.2604387e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:59:09,177 [ hooks.py: 207]:\tstep: 55080\tsteps/sec: 1.23642\tloss: 0.21880\tlr:[1.2597595e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:59:17,984 [ hooks.py: 207]:\tstep: 55090\tsteps/sec: 1.13336\tloss: 0.46253\tlr:[1.2590805e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:59:26,605 [ hooks.py: 207]:\tstep: 55100\tsteps/sec: 1.17948\tloss: 0.28648\tlr:[1.2584014e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:59:37,354 [ hooks.py: 207]:\tstep: 55110\tsteps/sec: 0.92328\tloss: 0.36473\tlr:[1.257722e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:59:45,480 [ hooks.py: 207]:\tstep: 55120\tsteps/sec: 1.24754\tloss: 0.33408\tlr:[1.2570433e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 12:59:54,572 [ hooks.py: 207]:\tstep: 55130\tsteps/sec: 1.08584\tloss: 0.33462\tlr:[1.256364e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:00:01,966 [ hooks.py: 207]:\tstep: 55140\tsteps/sec: 1.36513\tloss: 0.32784\tlr:[1.255685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:00:09,971 [ hooks.py: 207]:\tstep: 55150\tsteps/sec: 1.26063\tloss: 0.56926\tlr:[1.2550058e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:00:19,521 [ hooks.py: 207]:\tstep: 55160\tsteps/sec: 1.03023\tloss: 0.06726\tlr:[1.254327e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:00:27,734 [ hooks.py: 207]:\tstep: 55170\tsteps/sec: 1.19920\tloss: 0.30632\tlr:[1.2536478e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:00:37,961 [ hooks.py: 207]:\tstep: 55180\tsteps/sec: 0.98733\tloss: 0.42535\tlr:[1.2529688e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:00:48,010 [ hooks.py: 207]:\tstep: 55190\tsteps/sec: 0.99662\tloss: 0.37611\tlr:[1.2522897e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:00:56,993 [ hooks.py: 207]:\tstep: 55200\tsteps/sec: 1.11502\tloss: 0.49839\tlr:[1.2516108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:01:04,355 [ hooks.py: 207]:\tstep: 55210\tsteps/sec: 1.34683\tloss: 0.20916\tlr:[1.2509316e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 13:01:10,900 [ hooks.py: 207]:\tstep: 55220\tsteps/sec: 1.60418\tloss: 0.20004\tlr:[1.2502528e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:01:20,215 [ hooks.py: 207]:\tstep: 55230\tsteps/sec: 1.06544\tloss: 0.46491\tlr:[1.2495736e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:01:29,261 [ hooks.py: 207]:\tstep: 55240\tsteps/sec: 1.04837\tloss: 0.25982\tlr:[1.2488946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:01:38,357 [ hooks.py: 207]:\tstep: 55250\tsteps/sec: 1.14369\tloss: 0.55633\tlr:[1.2482155e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:01:46,043 [ hooks.py: 207]:\tstep: 55260\tsteps/sec: 1.35794\tloss: 0.24421\tlr:[1.2475363e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:01:57,199 [ hooks.py: 207]:\tstep: 55270\tsteps/sec: 0.87384\tloss: 0.20110\tlr:[1.2468573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:02:03,285 [ hooks.py: 207]:\tstep: 55280\tsteps/sec: 1.58300\tloss: 0.57532\tlr:[1.2461781e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:02:12,389 [ hooks.py: 207]:\tstep: 55290\tsteps/sec: 1.11053\tloss: 0.12903\tlr:[1.2454993e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:02:20,374 [ hooks.py: 207]:\tstep: 55300\tsteps/sec: 1.30219\tloss: 0.19955\tlr:[1.2448201e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:02:29,385 [ hooks.py: 207]:\tstep: 55310\tsteps/sec: 1.08537\tloss: 0.70476\tlr:[1.2441412e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:02:38,746 [ hooks.py: 207]:\tstep: 55320\tsteps/sec: 1.06677\tloss: 0.56832\tlr:[1.243462e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:02:48,193 [ hooks.py: 207]:\tstep: 55330\tsteps/sec: 1.05540\tloss: 0.24829\tlr:[1.242783e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:02:57,074 [ hooks.py: 207]:\tstep: 55340\tsteps/sec: 1.12940\tloss: 0.56306\tlr:[1.2421038e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:03:06,021 [ hooks.py: 207]:\tstep: 55350\tsteps/sec: 1.15110\tloss: 0.45811\tlr:[1.241425e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:03:12,369 [ hooks.py: 207]:\tstep: 55360\tsteps/sec: 1.52950\tloss: 0.04922\tlr:[1.2407457e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:03:23,352 [ hooks.py: 207]:\tstep: 55370\tsteps/sec: 0.87584\tloss: 0.10854\tlr:[1.240067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:03:32,002 [ hooks.py: 207]:\tstep: 55380\tsteps/sec: 1.22811\tloss: 0.07169\tlr:[1.2393876e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:03:40,220 [ hooks.py: 207]:\tstep: 55390\tsteps/sec: 1.25490\tloss: 0.22745\tlr:[1.2387089e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:03:47,641 [ hooks.py: 207]:\tstep: 55400\tsteps/sec: 1.28152\tloss: 0.19363\tlr:[1.2380296e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:04:00,705 [ hooks.py: 207]:\tstep: 55410\tsteps/sec: 0.77753\tloss: 0.70861\tlr:[1.2373505e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:04:08,908 [ hooks.py: 207]:\tstep: 55420\tsteps/sec: 1.17931\tloss: 0.19347\tlr:[1.2366716e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:04:16,999 [ hooks.py: 207]:\tstep: 55430\tsteps/sec: 1.24044\tloss: 0.57878\tlr:[1.2359923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:04:28,128 [ hooks.py: 207]:\tstep: 55440\tsteps/sec: 0.89263\tloss: 0.10407\tlr:[1.2353135e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:04:37,058 [ hooks.py: 207]:\tstep: 55450\tsteps/sec: 1.08053\tloss: 0.34930\tlr:[1.2346342e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:04:45,776 [ hooks.py: 207]:\tstep: 55460\tsteps/sec: 1.24114\tloss: 0.26533\tlr:[1.2339555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:04:53,471 [ hooks.py: 207]:\tstep: 55470\tsteps/sec: 1.25369\tloss: 0.56812\tlr:[1.2332762e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:05:00,658 [ hooks.py: 207]:\tstep: 55480\tsteps/sec: 1.45031\tloss: 0.20805\tlr:[1.2325972e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:05:11,381 [ hooks.py: 207]:\tstep: 55490\tsteps/sec: 0.90916\tloss: 0.11491\tlr:[1.2319181e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:05:21,116 [ hooks.py: 207]:\tstep: 55500\tsteps/sec: 1.07488\tloss: 0.09476\tlr:[1.2312391e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:05:30,261 [ hooks.py: 207]:\tstep: 55510\tsteps/sec: 1.03449\tloss: 0.33716\tlr:[1.2305599e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:05:38,226 [ hooks.py: 207]:\tstep: 55520\tsteps/sec: 1.22389\tloss: 0.07888\tlr:[1.2298811e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:05:46,409 [ hooks.py: 207]:\tstep: 55530\tsteps/sec: 1.35678\tloss: 0.75156\tlr:[1.2292018e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:05:54,571 [ hooks.py: 207]:\tstep: 55540\tsteps/sec: 1.16223\tloss: 0.53739\tlr:[1.2285229e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:06:01,538 [ hooks.py: 207]:\tstep: 55550\tsteps/sec: 1.48533\tloss: 0.27436\tlr:[1.2278438e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:06:08,018 [ hooks.py: 207]:\tstep: 55560\tsteps/sec: 1.40296\tloss: 0.53465\tlr:[1.22716465e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:06:17,539 [ hooks.py: 207]:\tstep: 55570\tsteps/sec: 1.09347\tloss: 0.09915\tlr:[1.2264856e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:06:27,051 [ hooks.py: 207]:\tstep: 55580\tsteps/sec: 1.05688\tloss: 0.34610\tlr:[1.2258064e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:06:35,197 [ hooks.py: 207]:\tstep: 55590\tsteps/sec: 1.24422\tloss: 0.17023\tlr:[1.2251276e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:06:45,728 [ hooks.py: 207]:\tstep: 55600\tsteps/sec: 0.92242\tloss: 0.63358\tlr:[1.2244485e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:06:57,456 [ hooks.py: 207]:\tstep: 55610\tsteps/sec: 0.85855\tloss: 0.29028\tlr:[1.2237696e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:07:06,553 [ hooks.py: 207]:\tstep: 55620\tsteps/sec: 1.14396\tloss: 0.12136\tlr:[1.2230903e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:07:14,067 [ hooks.py: 207]:\tstep: 55630\tsteps/sec: 1.21850\tloss: 0.14566\tlr:[1.2224115e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:07:22,904 [ hooks.py: 207]:\tstep: 55640\tsteps/sec: 1.17022\tloss: 0.32593\tlr:[1.2217323e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:07:31,508 [ hooks.py: 207]:\tstep: 55650\tsteps/sec: 1.22290\tloss: 0.85649\tlr:[1.2210533e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:07:40,573 [ hooks.py: 207]:\tstep: 55660\tsteps/sec: 1.12875\tloss: 0.31891\tlr:[1.2203741e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:07:49,075 [ hooks.py: 207]:\tstep: 55670\tsteps/sec: 1.10488\tloss: 0.30178\tlr:[1.21969515e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:07:57,661 [ hooks.py: 207]:\tstep: 55680\tsteps/sec: 1.17263\tloss: 0.52930\tlr:[1.2190161e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:08:06,066 [ hooks.py: 207]:\tstep: 55690\tsteps/sec: 1.21938\tloss: 0.23864\tlr:[1.2183372e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:08:14,915 [ hooks.py: 207]:\tstep: 55700\tsteps/sec: 1.11009\tloss: 0.21539\tlr:[1.2176578e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:08:24,605 [ hooks.py: 207]:\tstep: 55710\tsteps/sec: 1.02157\tloss: 0.80960\tlr:[1.2169787e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:08:35,425 [ hooks.py: 207]:\tstep: 55720\tsteps/sec: 0.94194\tloss: 0.32476\tlr:[1.2162999e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:08:42,699 [ hooks.py: 207]:\tstep: 55730\tsteps/sec: 1.32251\tloss: 0.48594\tlr:[1.2156207e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:08:52,209 [ hooks.py: 207]:\tstep: 55740\tsteps/sec: 1.01642\tloss: 0.18955\tlr:[1.2149417e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:09:01,825 [ hooks.py: 207]:\tstep: 55750\tsteps/sec: 1.11234\tloss: 0.40348\tlr:[1.2142626e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:09:11,291 [ hooks.py: 207]:\tstep: 55760\tsteps/sec: 1.05579\tloss: 0.45550\tlr:[1.2135838e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:09:18,792 [ hooks.py: 207]:\tstep: 55770\tsteps/sec: 1.30634\tloss: 0.12524\tlr:[1.2129044e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:09:27,913 [ hooks.py: 207]:\tstep: 55780\tsteps/sec: 1.08593\tloss: 0.27536\tlr:[1.2122256e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:09:34,496 [ hooks.py: 207]:\tstep: 55790\tsteps/sec: 1.52258\tloss: 0.09175\tlr:[1.2115464e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:09:42,251 [ hooks.py: 207]:\tstep: 55800\tsteps/sec: 1.41699\tloss: 0.21751\tlr:[1.2108675e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:09:50,850 [ hooks.py: 207]:\tstep: 55810\tsteps/sec: 1.11723\tloss: 0.12109\tlr:[1.2101883e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:10:00,862 [ hooks.py: 207]:\tstep: 55820\tsteps/sec: 0.99828\tloss: 0.29857\tlr:[1.20950945e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:10:09,511 [ hooks.py: 207]:\tstep: 55830\tsteps/sec: 1.11251\tloss: 0.23290\tlr:[1.2088303e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:10:16,664 [ hooks.py: 207]:\tstep: 55840\tsteps/sec: 1.44889\tloss: 0.27330\tlr:[1.2081512e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:10:23,820 [ hooks.py: 207]:\tstep: 55850\tsteps/sec: 1.29863\tloss: 0.48336\tlr:[1.2074723e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:10:32,821 [ hooks.py: 207]:\tstep: 55860\tsteps/sec: 1.15534\tloss: 0.21193\tlr:[1.206793e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 13:10:38,759 [ hooks.py: 207]:\tstep: 55870\tsteps/sec: 1.68932\tloss: 0.47572\tlr:[1.2061141e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:10:48,204 [ hooks.py: 207]:\tstep: 55880\tsteps/sec: 1.05308\tloss: 0.39769\tlr:[1.2054348e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:10:58,852 [ hooks.py: 207]:\tstep: 55890\tsteps/sec: 0.93754\tloss: 0.48302\tlr:[1.2047559e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:11:06,438 [ hooks.py: 207]:\tstep: 55900\tsteps/sec: 1.38210\tloss: 0.29142\tlr:[1.2040768e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:11:14,161 [ hooks.py: 207]:\tstep: 55910\tsteps/sec: 1.19460\tloss: 0.27627\tlr:[1.2033978e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:11:24,285 [ hooks.py: 207]:\tstep: 55920\tsteps/sec: 1.02715\tloss: 0.46165\tlr:[1.2027187e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:11:32,692 [ hooks.py: 207]:\tstep: 55930\tsteps/sec: 1.17122\tloss: 0.43259\tlr:[1.2020397e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:11:42,580 [ hooks.py: 207]:\tstep: 55940\tsteps/sec: 1.01340\tloss: 0.29252\tlr:[1.2013605e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:11:53,209 [ hooks.py: 207]:\tstep: 55950\tsteps/sec: 0.93893\tloss: 0.44666\tlr:[1.2006817e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:02,004 [ hooks.py: 207]:\tstep: 55960\tsteps/sec: 1.16322\tloss: 0.28549\tlr:[1.2000025e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:11,525 [ hooks.py: 207]:\tstep: 55970\tsteps/sec: 1.05672\tloss: 0.43527\tlr:[1.19932365e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:18,573 [ hooks.py: 207]:\tstep: 55980\tsteps/sec: 1.36306\tloss: 0.18068\tlr:[1.1986444e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:27,296 [ hooks.py: 207]:\tstep: 55990\tsteps/sec: 1.17352\tloss: 0.61940\tlr:[1.1979652e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:36,482 [ hooks.py: 207]:\tstep: 56000\tsteps/sec: 1.09835\tloss: 0.09541\tlr:[1.1972863e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:36,483 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:36,483 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:36,483 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:12:36,483 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:36,483 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:12:36,484 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:20:46,868 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,869 [ trainer.py: 58]:\t{'acc': 0.7855421686746988, 'loss': 0.6036989387029257}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,870 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,870 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:20:46,871 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:20:46,871 [ trainer.py: 70]:\tacc\t0.7855421686746988\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:20:46,871 [ trainer.py: 70]:\tloss\t0.6036989387029257\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:20:46,871 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,871 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6036989387029257}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,871 [ exporter.py: 64]:\t[Best Exporter]: skip step 56000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,871 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6036989387029257}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,871 [ exporter.py: 104]:\t[Best Exporter]: skip step 56000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:46,872 [monitored_executor.py: 122]:\tsaving step 56000 to ./output/model_56000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:50,047 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_53000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:20:58,692 [ hooks.py: 207]:\tstep: 56010\tsteps/sec: 0.01991\tloss: 0.34896\tlr:[1.1966071e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:21:06,036 [ hooks.py: 207]:\tstep: 56020\tsteps/sec: 1.40941\tloss: 0.36055\tlr:[1.1959282e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:21:14,280 [ hooks.py: 207]:\tstep: 56030\tsteps/sec: 1.17914\tloss: 0.42855\tlr:[1.1952491e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:21:22,750 [ hooks.py: 207]:\tstep: 56040\tsteps/sec: 1.12113\tloss: 0.40352\tlr:[1.1945702e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:21:31,191 [ hooks.py: 207]:\tstep: 56050\tsteps/sec: 1.29028\tloss: 0.24038\tlr:[1.1938909e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:21:41,984 [ hooks.py: 207]:\tstep: 56060\tsteps/sec: 0.95797\tloss: 0.69546\tlr:[1.1932119e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:21:48,739 [ hooks.py: 207]:\tstep: 56070\tsteps/sec: 1.32958\tloss: 0.23633\tlr:[1.1925328e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:21:57,397 [ hooks.py: 207]:\tstep: 56080\tsteps/sec: 1.20849\tloss: 0.47885\tlr:[1.1918539e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:22:08,424 [ hooks.py: 207]:\tstep: 56090\tsteps/sec: 0.88530\tloss: 0.19648\tlr:[1.1911748e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:22:17,322 [ hooks.py: 207]:\tstep: 56100\tsteps/sec: 1.14329\tloss: 0.14712\tlr:[1.1904958e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:22:27,940 [ hooks.py: 207]:\tstep: 56110\tsteps/sec: 0.92886\tloss: 0.51161\tlr:[1.1898166e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:22:34,784 [ hooks.py: 207]:\tstep: 56120\tsteps/sec: 1.36758\tloss: 0.38155\tlr:[1.1891377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:22:43,006 [ hooks.py: 207]:\tstep: 56130\tsteps/sec: 1.20609\tloss: 0.49216\tlr:[1.1884586e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:22:52,533 [ hooks.py: 207]:\tstep: 56140\tsteps/sec: 1.12129\tloss: 0.46998\tlr:[1.1877793e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:01,957 [ hooks.py: 207]:\tstep: 56150\tsteps/sec: 1.08975\tloss: 0.63867\tlr:[1.1871004e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:08,761 [ hooks.py: 207]:\tstep: 56160\tsteps/sec: 1.34365\tloss: 0.94132\tlr:[1.1864212e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:19,166 [ hooks.py: 207]:\tstep: 56170\tsteps/sec: 0.98815\tloss: 0.54204\tlr:[1.1857423e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:27,702 [ hooks.py: 207]:\tstep: 56180\tsteps/sec: 1.19153\tloss: 0.43918\tlr:[1.18506305e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:34,914 [ hooks.py: 207]:\tstep: 56190\tsteps/sec: 1.29963\tloss: 0.30666\tlr:[1.1843843e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:43,592 [ hooks.py: 207]:\tstep: 56200\tsteps/sec: 1.18697\tloss: 0.20007\tlr:[1.1837051e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:50,261 [ hooks.py: 207]:\tstep: 56210\tsteps/sec: 1.47738\tloss: 0.24642\tlr:[1.1830261e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:23:59,495 [ hooks.py: 207]:\tstep: 56220\tsteps/sec: 1.11682\tloss: 0.18025\tlr:[1.1823469e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:24:06,294 [ hooks.py: 207]:\tstep: 56230\tsteps/sec: 1.49949\tloss: 0.41268\tlr:[1.1816681e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:24:18,226 [ hooks.py: 207]:\tstep: 56240\tsteps/sec: 0.82283\tloss: 0.17773\tlr:[1.180989e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:24:26,573 [ hooks.py: 207]:\tstep: 56250\tsteps/sec: 1.25519\tloss: 0.40691\tlr:[1.1803099e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:24:33,640 [ hooks.py: 207]:\tstep: 56260\tsteps/sec: 1.34055\tloss: 0.45725\tlr:[1.1796307e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:24:44,187 [ hooks.py: 207]:\tstep: 56270\tsteps/sec: 0.95311\tloss: 0.16612\tlr:[1.178952e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:24:53,647 [ hooks.py: 207]:\tstep: 56280\tsteps/sec: 1.04886\tloss: 0.86100\tlr:[1.1782728e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:25:03,074 [ hooks.py: 207]:\tstep: 56290\tsteps/sec: 1.06246\tloss: 0.10312\tlr:[1.17759355e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:25:11,878 [ hooks.py: 207]:\tstep: 56300\tsteps/sec: 1.18279\tloss: 0.32632\tlr:[1.1769146e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:25:20,243 [ hooks.py: 207]:\tstep: 56310\tsteps/sec: 1.12908\tloss: 0.25285\tlr:[1.1762354e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:25:31,856 [ hooks.py: 207]:\tstep: 56320\tsteps/sec: 0.89297\tloss: 0.55229\tlr:[1.1755566e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 13:25:38,634 [ hooks.py: 207]:\tstep: 56330\tsteps/sec: 1.36426\tloss: 0.57736\tlr:[1.17487725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:25:47,112 [ hooks.py: 207]:\tstep: 56340\tsteps/sec: 1.19801\tloss: 0.31775\tlr:[1.1741985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:25:56,450 [ hooks.py: 207]:\tstep: 56350\tsteps/sec: 1.13485\tloss: 0.06195\tlr:[1.1735193e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:26:07,843 [ hooks.py: 207]:\tstep: 56360\tsteps/sec: 0.85735\tloss: 0.25055\tlr:[1.17284035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:26:16,060 [ hooks.py: 207]:\tstep: 56370\tsteps/sec: 1.20176\tloss: 0.48130\tlr:[1.1721612e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:26:25,885 [ hooks.py: 207]:\tstep: 56380\tsteps/sec: 1.04053\tloss: 0.68303\tlr:[1.1714823e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:26:32,725 [ hooks.py: 207]:\tstep: 56390\tsteps/sec: 1.33861\tloss: 0.68063\tlr:[1.1708031e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:26:42,919 [ hooks.py: 207]:\tstep: 56400\tsteps/sec: 1.07537\tloss: 0.21453\tlr:[1.1701241e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:26:51,192 [ hooks.py: 207]:\tstep: 56410\tsteps/sec: 1.15379\tloss: 0.16491\tlr:[1.1694448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:26:59,296 [ hooks.py: 207]:\tstep: 56420\tsteps/sec: 1.21563\tloss: 0.33019\tlr:[1.1687662e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:27:08,008 [ hooks.py: 207]:\tstep: 56430\tsteps/sec: 1.16444\tloss: 0.37170\tlr:[1.1680868e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:27:18,561 [ hooks.py: 207]:\tstep: 56440\tsteps/sec: 0.93818\tloss: 0.65056\tlr:[1.16740775e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:27:25,254 [ hooks.py: 207]:\tstep: 56450\tsteps/sec: 1.44770\tloss: 0.41423\tlr:[1.1667288e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:27:34,599 [ hooks.py: 207]:\tstep: 56460\tsteps/sec: 1.10888\tloss: 0.66537\tlr:[1.1660495e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:27:44,136 [ hooks.py: 207]:\tstep: 56470\tsteps/sec: 1.03439\tloss: 0.15830\tlr:[1.1653707e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:27:54,152 [ hooks.py: 207]:\tstep: 56480\tsteps/sec: 1.00300\tloss: 0.37070\tlr:[1.1646915e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:28:04,111 [ hooks.py: 207]:\tstep: 56490\tsteps/sec: 1.01610\tloss: 0.32297\tlr:[1.1640125e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:28:12,750 [ hooks.py: 207]:\tstep: 56500\tsteps/sec: 1.14214\tloss: 0.20729\tlr:[1.1633333e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:28:22,016 [ hooks.py: 207]:\tstep: 56510\tsteps/sec: 1.12734\tloss: 0.16433\tlr:[1.1626545e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:28:29,796 [ hooks.py: 207]:\tstep: 56520\tsteps/sec: 1.20888\tloss: 0.29533\tlr:[1.1619753e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:28:37,537 [ hooks.py: 207]:\tstep: 56530\tsteps/sec: 1.25104\tloss: 0.77797\tlr:[1.1612965e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:28:45,624 [ hooks.py: 207]:\tstep: 56540\tsteps/sec: 1.25242\tloss: 0.28750\tlr:[1.1606173e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:28:54,247 [ hooks.py: 207]:\tstep: 56550\tsteps/sec: 1.19335\tloss: 0.38240\tlr:[1.15993835e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:29:03,509 [ hooks.py: 207]:\tstep: 56560\tsteps/sec: 1.07743\tloss: 0.18497\tlr:[1.1592592e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:29:12,185 [ hooks.py: 207]:\tstep: 56570\tsteps/sec: 1.18969\tloss: 0.52830\tlr:[1.1585802e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:29:20,143 [ hooks.py: 207]:\tstep: 56580\tsteps/sec: 1.22376\tloss: 0.14324\tlr:[1.1579011e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:29:28,735 [ hooks.py: 207]:\tstep: 56590\tsteps/sec: 1.15388\tloss: 0.07884\tlr:[1.1572219e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:29:35,994 [ hooks.py: 207]:\tstep: 56600\tsteps/sec: 1.42677\tloss: 0.48475\tlr:[1.1565429e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:29:44,354 [ hooks.py: 207]:\tstep: 56610\tsteps/sec: 1.17627\tloss: 0.28538\tlr:[1.1558638e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:29:52,277 [ hooks.py: 207]:\tstep: 56620\tsteps/sec: 1.31155\tloss: 0.34259\tlr:[1.155185e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:01,063 [ hooks.py: 207]:\tstep: 56630\tsteps/sec: 1.12600\tloss: 0.48834\tlr:[1.15450575e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:08,554 [ hooks.py: 207]:\tstep: 56640\tsteps/sec: 1.27756\tloss: 0.14865\tlr:[1.1538267e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:18,882 [ hooks.py: 207]:\tstep: 56650\tsteps/sec: 0.97936\tloss: 0.54102\tlr:[1.1531475e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:27,867 [ hooks.py: 207]:\tstep: 56660\tsteps/sec: 1.06225\tloss: 0.10589\tlr:[1.1524688e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:35,731 [ hooks.py: 207]:\tstep: 56670\tsteps/sec: 1.36411\tloss: 0.07793\tlr:[1.1517894e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:44,281 [ hooks.py: 207]:\tstep: 56680\tsteps/sec: 1.15950\tloss: 0.30000\tlr:[1.1511106e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:50,703 [ hooks.py: 207]:\tstep: 56690\tsteps/sec: 1.61626\tloss: 0.73779\tlr:[1.1504313e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:30:58,754 [ hooks.py: 207]:\tstep: 56700\tsteps/sec: 1.19535\tloss: 0.65713\tlr:[1.1497524e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:31:07,836 [ hooks.py: 207]:\tstep: 56710\tsteps/sec: 1.10062\tloss: 0.14102\tlr:[1.14907325e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:31:19,305 [ hooks.py: 207]:\tstep: 56720\tsteps/sec: 0.87110\tloss: 0.22262\tlr:[1.1483945e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:31:28,732 [ hooks.py: 207]:\tstep: 56730\tsteps/sec: 1.07068\tloss: 0.81272\tlr:[1.1477153e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:31:35,044 [ hooks.py: 207]:\tstep: 56740\tsteps/sec: 1.49466\tloss: 0.43539\tlr:[1.1470361e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:31:46,250 [ hooks.py: 207]:\tstep: 56750\tsteps/sec: 0.94162\tloss: 0.12196\tlr:[1.1463571e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:31:53,714 [ hooks.py: 207]:\tstep: 56760\tsteps/sec: 1.28214\tloss: 0.30210\tlr:[1.1456779e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:32:02,904 [ hooks.py: 207]:\tstep: 56770\tsteps/sec: 1.10316\tloss: 0.15772\tlr:[1.1449989e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:32:13,595 [ hooks.py: 207]:\tstep: 56780\tsteps/sec: 0.92978\tloss: 0.33639\tlr:[1.1443199e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:32:22,870 [ hooks.py: 207]:\tstep: 56790\tsteps/sec: 1.08136\tloss: 0.16305\tlr:[1.143641e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:32:32,652 [ hooks.py: 207]:\tstep: 56800\tsteps/sec: 1.03696\tloss: 0.40147\tlr:[1.1429617e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:32:39,225 [ hooks.py: 207]:\tstep: 56810\tsteps/sec: 1.42348\tloss: 0.30453\tlr:[1.1422829e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:32:49,583 [ hooks.py: 207]:\tstep: 56820\tsteps/sec: 1.00203\tloss: 0.36844\tlr:[1.14160375e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:32:56,726 [ hooks.py: 207]:\tstep: 56830\tsteps/sec: 1.37717\tloss: 0.13513\tlr:[1.1409247e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:33:04,524 [ hooks.py: 207]:\tstep: 56840\tsteps/sec: 1.29384\tloss: 0.44300\tlr:[1.1402456e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:33:14,016 [ hooks.py: 207]:\tstep: 56850\tsteps/sec: 1.04297\tloss: 0.68189\tlr:[1.1395668e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:33:22,497 [ hooks.py: 207]:\tstep: 56860\tsteps/sec: 1.16130\tloss: 0.56699\tlr:[1.1388875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:33:31,018 [ hooks.py: 207]:\tstep: 56870\tsteps/sec: 1.22444\tloss: 0.09534\tlr:[1.1382085e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:33:38,884 [ hooks.py: 207]:\tstep: 56880\tsteps/sec: 1.24209\tloss: 0.25948\tlr:[1.1375294e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:33:46,635 [ hooks.py: 207]:\tstep: 56890\tsteps/sec: 1.35489\tloss: 0.41754\tlr:[1.1368502e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:33:56,494 [ hooks.py: 207]:\tstep: 56900\tsteps/sec: 0.97117\tloss: 0.60787\tlr:[1.1361712e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:34:05,355 [ hooks.py: 207]:\tstep: 56910\tsteps/sec: 1.14683\tloss: 0.33344\tlr:[1.135492e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:34:13,638 [ hooks.py: 207]:\tstep: 56920\tsteps/sec: 1.19883\tloss: 0.50361\tlr:[1.1348132e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:34:21,830 [ hooks.py: 207]:\tstep: 56930\tsteps/sec: 1.27739\tloss: 0.38123\tlr:[1.1341339e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:34:31,561 [ hooks.py: 207]:\tstep: 56940\tsteps/sec: 1.03213\tloss: 0.30673\tlr:[1.133455e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:34:37,989 [ hooks.py: 207]:\tstep: 56950\tsteps/sec: 1.48569\tloss: 0.26740\tlr:[1.1327759e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:34:46,636 [ hooks.py: 207]:\tstep: 56960\tsteps/sec: 1.19076\tloss: 0.16220\tlr:[1.1320969e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:34:53,879 [ hooks.py: 207]:\tstep: 56970\tsteps/sec: 1.26614\tloss: 0.07877\tlr:[1.1314178e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 13:35:04,593 [ hooks.py: 207]:\tstep: 56980\tsteps/sec: 0.92970\tloss: 0.07730\tlr:[1.1307389e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:35:13,471 [ hooks.py: 207]:\tstep: 56990\tsteps/sec: 1.16438\tloss: 0.03199\tlr:[1.1300596e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:35:23,030 [ hooks.py: 207]:\tstep: 57000\tsteps/sec: 1.06059\tloss: 0.06464\tlr:[1.1293808e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:35:23,031 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:35:23,031 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:35:23,031 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:35:23,031 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:35:23,031 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:35:23,031 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:43:34,687 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:34,688 [ trainer.py: 58]:\t{'acc': 0.793574297188755, 'loss': 0.6033463256958967}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:34,689 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:34,689 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:43:34,690 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:43:34,690 [ trainer.py: 70]:\tacc\t0.793574297188755\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:43:34,690 [ trainer.py: 70]:\tloss\t0.6033463256958967\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:43:34,690 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:34,691 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:34,691 [ exporter.py: 52]:\t[Best Exporter]: export to ./output/best_model\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:34,692 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_39000']\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:34,692 [monitored_executor.py: 122]:\tsaving step 57000 to ./output/best_model/model_57000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:35,647 [monitored_executor.py: 138]:\tNo. of ckpt exceed 1, clean up: ./output/best_model/model_39000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:35,648 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}} \n", + "old: {'eval': {'acc': 0.7919678714859437, 'loss': 0.5662454146509751}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:35,648 [ exporter.py: 77]:\t[Best Exporter]: export to ./output/best\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:35,649 [ exporter.py: 90]:\t[Best Exporter]: save inference model: \"inference\" to ./output/best/inference\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:36,909 [monitored_executor.py: 122]:\tsaving step 57000 to ./output/model_57000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:40,242 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_54000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:48,353 [ hooks.py: 207]:\tstep: 57010\tsteps/sec: 0.01979\tloss: 0.09690\tlr:[1.1287017e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:43:58,270 [ hooks.py: 207]:\tstep: 57020\tsteps/sec: 1.00217\tloss: 0.51780\tlr:[1.1280224e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:44:08,243 [ hooks.py: 207]:\tstep: 57030\tsteps/sec: 1.01002\tloss: 0.30172\tlr:[1.1273434e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:44:19,155 [ hooks.py: 207]:\tstep: 57040\tsteps/sec: 0.89903\tloss: 0.44026\tlr:[1.1266644e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:44:27,431 [ hooks.py: 207]:\tstep: 57050\tsteps/sec: 1.22479\tloss: 0.61750\tlr:[1.1259854e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:44:38,452 [ hooks.py: 207]:\tstep: 57060\tsteps/sec: 0.87805\tloss: 0.39321\tlr:[1.12530615e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:44:48,554 [ hooks.py: 207]:\tstep: 57070\tsteps/sec: 1.03709\tloss: 0.62982\tlr:[1.1246273e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:44:56,360 [ hooks.py: 207]:\tstep: 57080\tsteps/sec: 1.25084\tloss: 0.30863\tlr:[1.1239482e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:45:06,126 [ hooks.py: 207]:\tstep: 57090\tsteps/sec: 1.05024\tloss: 0.49033\tlr:[1.12326925e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:45:16,594 [ hooks.py: 207]:\tstep: 57100\tsteps/sec: 0.96695\tloss: 0.22689\tlr:[1.12259e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:45:25,368 [ hooks.py: 207]:\tstep: 57110\tsteps/sec: 1.11201\tloss: 0.66676\tlr:[1.1219111e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:45:32,667 [ hooks.py: 207]:\tstep: 57120\tsteps/sec: 1.34165\tloss: 0.06504\tlr:[1.121232e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:45:42,125 [ hooks.py: 207]:\tstep: 57130\tsteps/sec: 1.17791\tloss: 0.40743\tlr:[1.120553e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:45:50,935 [ hooks.py: 207]:\tstep: 57140\tsteps/sec: 1.05058\tloss: 0.45531\tlr:[1.1198738e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:45:59,276 [ hooks.py: 207]:\tstep: 57150\tsteps/sec: 1.17126\tloss: 0.56582\tlr:[1.1191951e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:46:07,778 [ hooks.py: 207]:\tstep: 57160\tsteps/sec: 1.22262\tloss: 0.17449\tlr:[1.118516e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:46:14,511 [ hooks.py: 207]:\tstep: 57170\tsteps/sec: 1.36980\tloss: 0.22856\tlr:[1.1178367e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:46:24,087 [ hooks.py: 207]:\tstep: 57180\tsteps/sec: 1.08687\tloss: 0.31703\tlr:[1.1171577e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:46:32,846 [ hooks.py: 207]:\tstep: 57190\tsteps/sec: 1.16438\tloss: 0.25963\tlr:[1.1164786e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:46:40,828 [ hooks.py: 207]:\tstep: 57200\tsteps/sec: 1.28328\tloss: 0.18695\tlr:[1.1157996e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:46:49,731 [ hooks.py: 207]:\tstep: 57210\tsteps/sec: 1.07034\tloss: 0.40310\tlr:[1.11512045e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:46:56,936 [ hooks.py: 207]:\tstep: 57220\tsteps/sec: 1.44908\tloss: 0.51321\tlr:[1.1144416e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:47:05,117 [ hooks.py: 207]:\tstep: 57230\tsteps/sec: 1.19472\tloss: 0.58738\tlr:[1.1137624e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:47:13,382 [ hooks.py: 207]:\tstep: 57240\tsteps/sec: 1.19586\tloss: 0.25353\tlr:[1.11308345e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:47:21,162 [ hooks.py: 207]:\tstep: 57250\tsteps/sec: 1.27193\tloss: 0.23252\tlr:[1.1124043e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:47:30,749 [ hooks.py: 207]:\tstep: 57260\tsteps/sec: 1.07014\tloss: 0.79141\tlr:[1.1117252e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:47:37,929 [ hooks.py: 207]:\tstep: 57270\tsteps/sec: 1.39103\tloss: 0.65600\tlr:[1.111046e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:47:47,195 [ hooks.py: 207]:\tstep: 57280\tsteps/sec: 1.10043\tloss: 0.54555\tlr:[1.1103673e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:47:56,033 [ hooks.py: 207]:\tstep: 57290\tsteps/sec: 1.10477\tloss: 0.38990\tlr:[1.10968795e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:48:04,556 [ hooks.py: 207]:\tstep: 57300\tsteps/sec: 1.11807\tloss: 0.30196\tlr:[1.1090091e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:48:15,531 [ hooks.py: 207]:\tstep: 57310\tsteps/sec: 0.93231\tloss: 0.43212\tlr:[1.1083299e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:48:23,813 [ hooks.py: 207]:\tstep: 57320\tsteps/sec: 1.19753\tloss: 0.14085\tlr:[1.1076508e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:48:32,363 [ hooks.py: 207]:\tstep: 57330\tsteps/sec: 1.23882\tloss: 0.03473\tlr:[1.1069718e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:48:40,022 [ hooks.py: 207]:\tstep: 57340\tsteps/sec: 1.25891\tloss: 0.27214\tlr:[1.1062926e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:48:48,748 [ hooks.py: 207]:\tstep: 57350\tsteps/sec: 1.12965\tloss: 0.34758\tlr:[1.1056137e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:48:57,562 [ hooks.py: 207]:\tstep: 57360\tsteps/sec: 1.17741\tloss: 0.14968\tlr:[1.1049346e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:49:06,358 [ hooks.py: 207]:\tstep: 57370\tsteps/sec: 1.11992\tloss: 0.45500\tlr:[1.1042556e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:49:14,654 [ hooks.py: 207]:\tstep: 57380\tsteps/sec: 1.18761\tloss: 0.26876\tlr:[1.1035765e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:49:22,993 [ hooks.py: 207]:\tstep: 57390\tsteps/sec: 1.18003\tloss: 0.09902\tlr:[1.1028975e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 13:49:34,081 [ hooks.py: 207]:\tstep: 57400\tsteps/sec: 0.91076\tloss: 0.33519\tlr:[1.1022184e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:49:43,045 [ hooks.py: 207]:\tstep: 57410\tsteps/sec: 1.10212\tloss: 0.42895\tlr:[1.1015395e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:49:52,766 [ hooks.py: 207]:\tstep: 57420\tsteps/sec: 1.07841\tloss: 0.69676\tlr:[1.1008602e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:01,994 [ hooks.py: 207]:\tstep: 57430\tsteps/sec: 1.08838\tloss: 0.13605\tlr:[1.1001813e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:09,697 [ hooks.py: 207]:\tstep: 57440\tsteps/sec: 1.23616\tloss: 0.41969\tlr:[1.0995022e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:16,727 [ hooks.py: 207]:\tstep: 57450\tsteps/sec: 1.48052\tloss: 0.34476\tlr:[1.0988232e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:24,120 [ hooks.py: 207]:\tstep: 57460\tsteps/sec: 1.31462\tloss: 0.25723\tlr:[1.0981442e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:31,461 [ hooks.py: 207]:\tstep: 57470\tsteps/sec: 1.34584\tloss: 0.37427\tlr:[1.0974648e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:40,301 [ hooks.py: 207]:\tstep: 57480\tsteps/sec: 1.20583\tloss: 0.28758\tlr:[1.0967861e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:48,558 [ hooks.py: 207]:\tstep: 57490\tsteps/sec: 1.14950\tloss: 0.21933\tlr:[1.0961067e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:50:56,398 [ hooks.py: 207]:\tstep: 57500\tsteps/sec: 1.24609\tloss: 0.20660\tlr:[1.0954279e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:51:04,315 [ hooks.py: 207]:\tstep: 57510\tsteps/sec: 1.28573\tloss: 0.39484\tlr:[1.0947488e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:51:11,414 [ hooks.py: 207]:\tstep: 57520\tsteps/sec: 1.37912\tloss: 0.25937\tlr:[1.0940699e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:51:19,479 [ hooks.py: 207]:\tstep: 57530\tsteps/sec: 1.22650\tloss: 0.20643\tlr:[1.0933906e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:51:30,155 [ hooks.py: 207]:\tstep: 57540\tsteps/sec: 0.96299\tloss: 0.02677\tlr:[1.0927118e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:51:40,854 [ hooks.py: 207]:\tstep: 57550\tsteps/sec: 0.98600\tloss: 0.09481\tlr:[1.0920326e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:51:48,266 [ hooks.py: 207]:\tstep: 57560\tsteps/sec: 1.19392\tloss: 0.06520\tlr:[1.0913536e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:51:57,382 [ hooks.py: 207]:\tstep: 57570\tsteps/sec: 1.18226\tloss: 0.45781\tlr:[1.0906744e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:52:05,359 [ hooks.py: 207]:\tstep: 57580\tsteps/sec: 1.19577\tloss: 0.32237\tlr:[1.0899957e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:52:13,623 [ hooks.py: 207]:\tstep: 57590\tsteps/sec: 1.22582\tloss: 0.09684\tlr:[1.08931645e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:52:20,832 [ hooks.py: 207]:\tstep: 57600\tsteps/sec: 1.39856\tloss: 0.47834\tlr:[1.0886374e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:52:28,719 [ hooks.py: 207]:\tstep: 57610\tsteps/sec: 1.19326\tloss: 0.12127\tlr:[1.0879583e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:52:38,461 [ hooks.py: 207]:\tstep: 57620\tsteps/sec: 1.07083\tloss: 0.39796\tlr:[1.0872791e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:52:48,765 [ hooks.py: 207]:\tstep: 57630\tsteps/sec: 0.97413\tloss: 0.45396\tlr:[1.0866002e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:52:56,348 [ hooks.py: 207]:\tstep: 57640\tsteps/sec: 1.34587\tloss: 0.33470\tlr:[1.0859209e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:53:06,309 [ hooks.py: 207]:\tstep: 57650\tsteps/sec: 1.00110\tloss: 0.24601\tlr:[1.0852421e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:53:14,904 [ hooks.py: 207]:\tstep: 57660\tsteps/sec: 1.14291\tloss: 0.38151\tlr:[1.084563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:53:25,597 [ hooks.py: 207]:\tstep: 57670\tsteps/sec: 0.91054\tloss: 0.39870\tlr:[1.0838839e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:53:36,644 [ hooks.py: 207]:\tstep: 57680\tsteps/sec: 0.95506\tloss: 0.69400\tlr:[1.0832048e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:53:46,477 [ hooks.py: 207]:\tstep: 57690\tsteps/sec: 0.97452\tloss: 0.33731\tlr:[1.0825259e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:53:54,775 [ hooks.py: 207]:\tstep: 57700\tsteps/sec: 1.22630\tloss: 0.14595\tlr:[1.0818467e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:54:05,092 [ hooks.py: 207]:\tstep: 57710\tsteps/sec: 0.96946\tloss: 0.49286\tlr:[1.0811679e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:54:13,172 [ hooks.py: 207]:\tstep: 57720\tsteps/sec: 1.25433\tloss: 0.50604\tlr:[1.0804886e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:54:21,758 [ hooks.py: 207]:\tstep: 57730\tsteps/sec: 1.18658\tloss: 0.33758\tlr:[1.0798097e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:54:30,343 [ hooks.py: 207]:\tstep: 57740\tsteps/sec: 1.14524\tloss: 0.51687\tlr:[1.0791306e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:54:36,983 [ hooks.py: 207]:\tstep: 57750\tsteps/sec: 1.47066\tloss: 0.08549\tlr:[1.0784517e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:54:46,116 [ hooks.py: 207]:\tstep: 57760\tsteps/sec: 1.11395\tloss: 0.06778\tlr:[1.0777725e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:54:54,294 [ hooks.py: 207]:\tstep: 57770\tsteps/sec: 1.16208\tloss: 0.06556\tlr:[1.0770932e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:55:02,888 [ hooks.py: 207]:\tstep: 57780\tsteps/sec: 1.24261\tloss: 0.30110\tlr:[1.0764144e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:55:10,610 [ hooks.py: 207]:\tstep: 57790\tsteps/sec: 1.21325\tloss: 0.47432\tlr:[1.0757352e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:55:22,220 [ hooks.py: 207]:\tstep: 57800\tsteps/sec: 0.87603\tloss: 0.47143\tlr:[1.0750563e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:55:32,153 [ hooks.py: 207]:\tstep: 57810\tsteps/sec: 1.05550\tloss: 0.67732\tlr:[1.074377e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:55:40,919 [ hooks.py: 207]:\tstep: 57820\tsteps/sec: 1.16597\tloss: 0.24217\tlr:[1.0736981e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:55:47,843 [ hooks.py: 207]:\tstep: 57830\tsteps/sec: 1.35862\tloss: 0.31502\tlr:[1.07301885e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:55:56,668 [ hooks.py: 207]:\tstep: 57840\tsteps/sec: 1.14438\tloss: 0.19589\tlr:[1.0723401e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:56:04,788 [ hooks.py: 207]:\tstep: 57850\tsteps/sec: 1.19853\tloss: 0.49236\tlr:[1.0716609e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:56:12,222 [ hooks.py: 207]:\tstep: 57860\tsteps/sec: 1.43004\tloss: 0.38686\tlr:[1.07098185e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:56:23,005 [ hooks.py: 207]:\tstep: 57870\tsteps/sec: 0.91260\tloss: 0.35384\tlr:[1.0703028e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:56:32,321 [ hooks.py: 207]:\tstep: 57880\tsteps/sec: 1.06267\tloss: 0.37141\tlr:[1.0696239e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:56:41,964 [ hooks.py: 207]:\tstep: 57890\tsteps/sec: 1.12314\tloss: 0.15384\tlr:[1.0689448e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:56:50,241 [ hooks.py: 207]:\tstep: 57900\tsteps/sec: 1.08775\tloss: 0.15638\tlr:[1.0682657e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:56:58,930 [ hooks.py: 207]:\tstep: 57910\tsteps/sec: 1.16257\tloss: 0.28207\tlr:[1.0675866e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:57:05,921 [ hooks.py: 207]:\tstep: 57920\tsteps/sec: 1.42719\tloss: 0.27425\tlr:[1.0669074e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:57:13,738 [ hooks.py: 207]:\tstep: 57930\tsteps/sec: 1.26625\tloss: 0.35547\tlr:[1.0662284e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:57:22,177 [ hooks.py: 207]:\tstep: 57940\tsteps/sec: 1.23924\tloss: 0.15418\tlr:[1.06554935e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:57:31,480 [ hooks.py: 207]:\tstep: 57950\tsteps/sec: 1.06911\tloss: 0.17870\tlr:[1.0648704e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:57:39,268 [ hooks.py: 207]:\tstep: 57960\tsteps/sec: 1.23412\tloss: 0.47845\tlr:[1.0641912e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:57:46,968 [ hooks.py: 207]:\tstep: 57970\tsteps/sec: 1.28558\tloss: 0.20050\tlr:[1.06351235e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:57:55,451 [ hooks.py: 207]:\tstep: 57980\tsteps/sec: 1.19998\tloss: 0.39417\tlr:[1.0628331e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:58:03,712 [ hooks.py: 207]:\tstep: 57990\tsteps/sec: 1.20572\tloss: 0.37581\tlr:[1.0621541e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:58:12,205 [ hooks.py: 207]:\tstep: 58000\tsteps/sec: 1.22471\tloss: 0.54477\tlr:[1.0614751e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:58:12,206 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:58:12,206 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:58:12,206 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 13:58:12,206 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:58:12,206 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 13:58:12,206 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-19 14:06:25,783 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,785 [ trainer.py: 58]:\t{'acc': 0.7871485943775101, 'loss': 0.6148386448144149}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,785 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,786 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:06:25,786 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:06:25,786 [ trainer.py: 70]:\tacc\t0.7871485943775101\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:06:25,786 [ trainer.py: 70]:\tloss\t0.6148386448144149\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:06:25,787 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,787 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7871485943775101, 'loss': 0.6148386448144149}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,787 [ exporter.py: 64]:\t[Best Exporter]: skip step 58000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,787 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7871485943775101, 'loss': 0.6148386448144149}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,787 [ exporter.py: 104]:\t[Best Exporter]: skip step 58000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:25,788 [monitored_executor.py: 122]:\tsaving step 58000 to ./output/model_58000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:28,980 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_55000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:36,969 [ hooks.py: 207]:\tstep: 58010\tsteps/sec: 0.01981\tloss: 0.02397\tlr:[1.0607961e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:47,077 [ hooks.py: 207]:\tstep: 58020\tsteps/sec: 0.99710\tloss: 0.28637\tlr:[1.060117e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:06:56,966 [ hooks.py: 207]:\tstep: 58030\tsteps/sec: 0.99326\tloss: 0.50033\tlr:[1.059438e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:07:07,439 [ hooks.py: 207]:\tstep: 58040\tsteps/sec: 0.97504\tloss: 0.46614\tlr:[1.0587589e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:07:14,637 [ hooks.py: 207]:\tstep: 58050\tsteps/sec: 1.30377\tloss: 0.40581\tlr:[1.05807985e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:07:23,670 [ hooks.py: 207]:\tstep: 58060\tsteps/sec: 1.12084\tloss: 0.43400\tlr:[1.0574007e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:07:33,865 [ hooks.py: 207]:\tstep: 58070\tsteps/sec: 0.98158\tloss: 0.06973\tlr:[1.0567215e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:07:43,265 [ hooks.py: 207]:\tstep: 58080\tsteps/sec: 1.09669\tloss: 0.31633\tlr:[1.0560426e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:07:49,943 [ hooks.py: 207]:\tstep: 58090\tsteps/sec: 1.43998\tloss: 0.43766\tlr:[1.0553634e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:07:57,898 [ hooks.py: 207]:\tstep: 58100\tsteps/sec: 1.31360\tloss: 0.44747\tlr:[1.0546845e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:08:04,894 [ hooks.py: 207]:\tstep: 58110\tsteps/sec: 1.36116\tloss: 0.34499\tlr:[1.0540053e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:08:14,141 [ hooks.py: 207]:\tstep: 58120\tsteps/sec: 1.08116\tloss: 0.54361\tlr:[1.0533266e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:08:24,102 [ hooks.py: 207]:\tstep: 58130\tsteps/sec: 1.00355\tloss: 0.40163\tlr:[1.05264735e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:08:32,602 [ hooks.py: 207]:\tstep: 58140\tsteps/sec: 1.21726\tloss: 0.15303\tlr:[1.0519685e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:08:40,670 [ hooks.py: 207]:\tstep: 58150\tsteps/sec: 1.24970\tloss: 0.47326\tlr:[1.0512894e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:08:49,261 [ hooks.py: 207]:\tstep: 58160\tsteps/sec: 1.11723\tloss: 0.08300\tlr:[1.05061035e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:08:55,752 [ hooks.py: 207]:\tstep: 58170\tsteps/sec: 1.46465\tloss: 0.05796\tlr:[1.04993105e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:09:05,202 [ hooks.py: 207]:\tstep: 58180\tsteps/sec: 1.10406\tloss: 0.46658\tlr:[1.0492521e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:09:12,526 [ hooks.py: 207]:\tstep: 58190\tsteps/sec: 1.42972\tloss: 0.51801\tlr:[1.048573e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:09:19,787 [ hooks.py: 207]:\tstep: 58200\tsteps/sec: 1.30513\tloss: 0.08708\tlr:[1.0478937e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:09:26,894 [ hooks.py: 207]:\tstep: 58210\tsteps/sec: 1.44876\tloss: 0.14148\tlr:[1.047215e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:09:36,008 [ hooks.py: 207]:\tstep: 58220\tsteps/sec: 1.07386\tloss: 0.75120\tlr:[1.0465357e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:09:44,500 [ hooks.py: 207]:\tstep: 58230\tsteps/sec: 1.18029\tloss: 0.19852\tlr:[1.0458568e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:09:52,847 [ hooks.py: 207]:\tstep: 58240\tsteps/sec: 1.18611\tloss: 0.24634\tlr:[1.0451776e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:10:03,690 [ hooks.py: 207]:\tstep: 58250\tsteps/sec: 0.93163\tloss: 0.40634\tlr:[1.0444986e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:10:11,938 [ hooks.py: 207]:\tstep: 58260\tsteps/sec: 1.25319\tloss: 0.32587\tlr:[1.0438194e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:10:19,282 [ hooks.py: 207]:\tstep: 58270\tsteps/sec: 1.31220\tloss: 0.98831\tlr:[1.0431406e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:10:25,862 [ hooks.py: 207]:\tstep: 58280\tsteps/sec: 1.50752\tloss: 0.38287\tlr:[1.0424614e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:10:35,294 [ hooks.py: 207]:\tstep: 58290\tsteps/sec: 1.06670\tloss: 0.22484\tlr:[1.0417824e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:10:44,419 [ hooks.py: 207]:\tstep: 58300\tsteps/sec: 1.08191\tloss: 0.23166\tlr:[1.0411033e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:10:53,150 [ hooks.py: 207]:\tstep: 58310\tsteps/sec: 1.20048\tloss: 0.30681\tlr:[1.0404245e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:11:00,463 [ hooks.py: 207]:\tstep: 58320\tsteps/sec: 1.33139\tloss: 0.29037\tlr:[1.0397452e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:11:10,186 [ hooks.py: 207]:\tstep: 58330\tsteps/sec: 1.02004\tloss: 0.57875\tlr:[1.0390663e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:11:18,777 [ hooks.py: 207]:\tstep: 58340\tsteps/sec: 1.17426\tloss: 0.54712\tlr:[1.0383871e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:11:26,082 [ hooks.py: 207]:\tstep: 58350\tsteps/sec: 1.34903\tloss: 0.23926\tlr:[1.037708e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:11:34,854 [ hooks.py: 207]:\tstep: 58360\tsteps/sec: 1.12615\tloss: 0.29908\tlr:[1.03702905e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:11:43,833 [ hooks.py: 207]:\tstep: 58370\tsteps/sec: 1.13187\tloss: 0.34994\tlr:[1.0363499e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:11:52,892 [ hooks.py: 207]:\tstep: 58380\tsteps/sec: 1.13602\tloss: 0.44555\tlr:[1.0356709e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:12:03,146 [ hooks.py: 207]:\tstep: 58390\tsteps/sec: 0.92985\tloss: 0.15598\tlr:[1.0349919e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:12:13,015 [ hooks.py: 207]:\tstep: 58400\tsteps/sec: 1.07063\tloss: 0.39121\tlr:[1.0343129e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:12:20,945 [ hooks.py: 207]:\tstep: 58410\tsteps/sec: 1.15812\tloss: 0.53134\tlr:[1.0336336e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:12:30,266 [ hooks.py: 207]:\tstep: 58420\tsteps/sec: 1.08739\tloss: 0.18563\tlr:[1.0329549e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:12:41,778 [ hooks.py: 207]:\tstep: 58430\tsteps/sec: 0.94940\tloss: 0.15942\tlr:[1.0322757e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:12:50,286 [ hooks.py: 207]:\tstep: 58440\tsteps/sec: 1.09369\tloss: 0.16058\tlr:[1.0315968e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:12:57,805 [ hooks.py: 207]:\tstep: 58450\tsteps/sec: 1.39091\tloss: 0.78559\tlr:[1.0309174e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:13:05,504 [ hooks.py: 207]:\tstep: 58460\tsteps/sec: 1.23272\tloss: 0.24423\tlr:[1.0302386e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:13:12,783 [ hooks.py: 207]:\tstep: 58470\tsteps/sec: 1.44980\tloss: 0.37780\tlr:[1.0295594e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:13:20,875 [ hooks.py: 207]:\tstep: 58480\tsteps/sec: 1.18165\tloss: 0.18102\tlr:[1.0288805e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:13:29,705 [ hooks.py: 207]:\tstep: 58490\tsteps/sec: 1.13132\tloss: 0.79545\tlr:[1.0282012e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:13:38,225 [ hooks.py: 207]:\tstep: 58500\tsteps/sec: 1.20287\tloss: 0.33272\tlr:[1.0275221e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:13:45,154 [ hooks.py: 207]:\tstep: 58510\tsteps/sec: 1.38977\tloss: 0.15675\tlr:[1.0268432e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:13:53,777 [ hooks.py: 207]:\tstep: 58520\tsteps/sec: 1.18593\tloss: 0.29027\tlr:[1.0261641e-05]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 14:14:00,314 [ hooks.py: 207]:\tstep: 58530\tsteps/sec: 1.42421\tloss: 0.06522\tlr:[1.0254851e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:14:11,094 [ hooks.py: 207]:\tstep: 58540\tsteps/sec: 0.99482\tloss: 0.57352\tlr:[1.024806e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:14:18,518 [ hooks.py: 207]:\tstep: 58550\tsteps/sec: 1.33206\tloss: 0.30781\tlr:[1.02412705e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:14:25,721 [ hooks.py: 207]:\tstep: 58560\tsteps/sec: 1.25627\tloss: 0.19503\tlr:[1.0234479e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:14:36,057 [ hooks.py: 207]:\tstep: 58570\tsteps/sec: 1.00818\tloss: 0.53293\tlr:[1.0227689e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:14:45,321 [ hooks.py: 207]:\tstep: 58580\tsteps/sec: 1.08813\tloss: 0.38980\tlr:[1.0220899e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:14:55,257 [ hooks.py: 207]:\tstep: 58590\tsteps/sec: 0.99921\tloss: 0.15233\tlr:[1.0214108e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:15:04,087 [ hooks.py: 207]:\tstep: 58600\tsteps/sec: 1.11061\tloss: 0.82050\tlr:[1.0207316e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:15:11,780 [ hooks.py: 207]:\tstep: 58610\tsteps/sec: 1.36301\tloss: 0.54575\tlr:[1.0200528e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:15:20,392 [ hooks.py: 207]:\tstep: 58620\tsteps/sec: 1.14761\tloss: 0.47099\tlr:[1.0193734e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:15:28,822 [ hooks.py: 207]:\tstep: 58630\tsteps/sec: 1.19179\tloss: 0.25949\tlr:[1.0186946e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:15:38,599 [ hooks.py: 207]:\tstep: 58640\tsteps/sec: 1.02791\tloss: 0.22864\tlr:[1.0180154e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:15:49,935 [ hooks.py: 207]:\tstep: 58650\tsteps/sec: 0.86769\tloss: 0.18795\tlr:[1.0173363e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:15:58,533 [ hooks.py: 207]:\tstep: 58660\tsteps/sec: 1.21166\tloss: 0.17166\tlr:[1.0166574e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:16:07,786 [ hooks.py: 207]:\tstep: 58670\tsteps/sec: 1.04553\tloss: 0.23485\tlr:[1.0159782e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:16:16,011 [ hooks.py: 207]:\tstep: 58680\tsteps/sec: 1.21926\tloss: 0.55187\tlr:[1.0152994e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:16:24,629 [ hooks.py: 207]:\tstep: 58690\tsteps/sec: 1.16016\tloss: 0.65326\tlr:[1.01462e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:16:34,675 [ hooks.py: 207]:\tstep: 58700\tsteps/sec: 1.02072\tloss: 0.15313\tlr:[1.0139413e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:16:42,571 [ hooks.py: 207]:\tstep: 58710\tsteps/sec: 1.34079\tloss: 0.18235\tlr:[1.0132621e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:16:50,013 [ hooks.py: 207]:\tstep: 58720\tsteps/sec: 1.22030\tloss: 0.43953\tlr:[1.0125831e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:16:58,612 [ hooks.py: 207]:\tstep: 58730\tsteps/sec: 1.20337\tloss: 0.18304\tlr:[1.011904e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:17:05,637 [ hooks.py: 207]:\tstep: 58740\tsteps/sec: 1.37724\tloss: 0.31989\tlr:[1.0112251e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:17:13,669 [ hooks.py: 207]:\tstep: 58750\tsteps/sec: 1.17690\tloss: 0.12060\tlr:[1.0105458e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:17:22,099 [ hooks.py: 207]:\tstep: 58760\tsteps/sec: 1.23679\tloss: 0.27009\tlr:[1.0098669e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:17:30,330 [ hooks.py: 207]:\tstep: 58770\tsteps/sec: 1.23961\tloss: 0.12847\tlr:[1.0091877e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:17:38,740 [ hooks.py: 207]:\tstep: 58780\tsteps/sec: 1.19226\tloss: 0.25380\tlr:[1.00850875e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:17:45,316 [ hooks.py: 207]:\tstep: 58790\tsteps/sec: 1.50527\tloss: 0.28932\tlr:[1.0078297e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:17:53,694 [ hooks.py: 207]:\tstep: 58800\tsteps/sec: 1.21671\tloss: 0.60273\tlr:[1.0071504e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:18:02,101 [ hooks.py: 207]:\tstep: 58810\tsteps/sec: 1.19153\tloss: 0.62015\tlr:[1.0064716e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:18:13,107 [ hooks.py: 207]:\tstep: 58820\tsteps/sec: 0.91751\tloss: 0.41843\tlr:[1.0057923e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:18:21,744 [ hooks.py: 207]:\tstep: 58830\tsteps/sec: 1.10586\tloss: 0.21169\tlr:[1.0051134e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:18:34,181 [ hooks.py: 207]:\tstep: 58840\tsteps/sec: 0.81859\tloss: 0.16824\tlr:[1.0044343e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:18:43,126 [ hooks.py: 207]:\tstep: 58850\tsteps/sec: 1.08700\tloss: 0.04514\tlr:[1.0037555e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:18:50,962 [ hooks.py: 207]:\tstep: 58860\tsteps/sec: 1.25041\tloss: 0.72922\tlr:[1.0030762e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:18:59,980 [ hooks.py: 207]:\tstep: 58870\tsteps/sec: 1.14926\tloss: 0.24101\tlr:[1.0023973e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:19:09,396 [ hooks.py: 207]:\tstep: 58880\tsteps/sec: 1.05444\tloss: 0.56714\tlr:[1.0017182e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:19:18,247 [ hooks.py: 207]:\tstep: 58890\tsteps/sec: 1.17959\tloss: 0.40650\tlr:[1.0010392e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:19:26,267 [ hooks.py: 207]:\tstep: 58900\tsteps/sec: 1.13966\tloss: 0.30806\tlr:[1.00036e-05]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:19:37,113 [ hooks.py: 207]:\tstep: 58910\tsteps/sec: 0.98537\tloss: 0.36998\tlr:[9.996811e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:19:48,620 [ hooks.py: 207]:\tstep: 58920\tsteps/sec: 0.86275\tloss: 0.25144\tlr:[9.99002e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:19:55,836 [ hooks.py: 207]:\tstep: 58930\tsteps/sec: 1.35734\tloss: 0.21482\tlr:[9.9832305e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:03,846 [ hooks.py: 207]:\tstep: 58940\tsteps/sec: 1.23547\tloss: 0.46074\tlr:[9.976438e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:12,123 [ hooks.py: 207]:\tstep: 58950\tsteps/sec: 1.24673\tloss: 0.47748\tlr:[9.969646e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:20,810 [ hooks.py: 207]:\tstep: 58960\tsteps/sec: 1.12322\tloss: 0.75682\tlr:[9.962857e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:30,309 [ hooks.py: 207]:\tstep: 58970\tsteps/sec: 1.07997\tloss: 0.14575\tlr:[9.956065e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:41,260 [ hooks.py: 207]:\tstep: 58980\tsteps/sec: 0.94793\tloss: 0.41287\tlr:[9.949277e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:49,946 [ hooks.py: 207]:\tstep: 58990\tsteps/sec: 1.07716\tloss: 0.11709\tlr:[9.942483e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:58,687 [ hooks.py: 207]:\tstep: 59000\tsteps/sec: 1.15367\tloss: 0.53564\tlr:[9.935696e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:58,692 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:58,692 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:58,692 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:20:58,692 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:58,692 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:20:58,692 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:29:01,399 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,400 [ trainer.py: 58]:\t{'acc': 0.7779116465863454, 'loss': 0.6601010672748089}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,401 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,401 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:29:01,402 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:29:01,402 [ trainer.py: 70]:\tacc\t0.7779116465863454\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:29:01,402 [ trainer.py: 70]:\tloss\t0.6601010672748089\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:29:01,402 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,402 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.6601010672748089}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,402 [ exporter.py: 64]:\t[Best Exporter]: skip step 59000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,402 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7779116465863454, 'loss': 0.6601010672748089}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,402 [ exporter.py: 104]:\t[Best Exporter]: skip step 59000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:01,403 [monitored_executor.py: 122]:\tsaving step 59000 to ./output/model_59000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 14:29:05,564 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_56000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:14,770 [ hooks.py: 207]:\tstep: 59010\tsteps/sec: 0.02015\tloss: 0.13538\tlr:[9.928903e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:24,091 [ hooks.py: 207]:\tstep: 59020\tsteps/sec: 1.08893\tloss: 0.38227\tlr:[9.922113e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:31,797 [ hooks.py: 207]:\tstep: 59030\tsteps/sec: 1.28765\tloss: 0.11340\tlr:[9.915323e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:38,729 [ hooks.py: 207]:\tstep: 59040\tsteps/sec: 1.48917\tloss: 0.28606\tlr:[9.908534e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:46,381 [ hooks.py: 207]:\tstep: 59050\tsteps/sec: 1.25970\tloss: 0.35547\tlr:[9.901742e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:29:54,909 [ hooks.py: 207]:\tstep: 59060\tsteps/sec: 1.17821\tloss: 0.24271\tlr:[9.894953e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:30:03,197 [ hooks.py: 207]:\tstep: 59070\tsteps/sec: 1.25481\tloss: 1.01881\tlr:[9.888161e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:30:13,146 [ hooks.py: 207]:\tstep: 59080\tsteps/sec: 1.00353\tloss: 0.41832\tlr:[9.881372e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:30:20,319 [ hooks.py: 207]:\tstep: 59090\tsteps/sec: 1.34120\tloss: 0.14159\tlr:[9.8745795e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:30:28,486 [ hooks.py: 207]:\tstep: 59100\tsteps/sec: 1.25899\tloss: 0.35472\tlr:[9.867788e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:30:36,273 [ hooks.py: 207]:\tstep: 59110\tsteps/sec: 1.23246\tloss: 0.79543\tlr:[9.860999e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:30:44,868 [ hooks.py: 207]:\tstep: 59120\tsteps/sec: 1.12834\tloss: 1.24911\tlr:[9.854207e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:30:53,440 [ hooks.py: 207]:\tstep: 59130\tsteps/sec: 1.27187\tloss: 0.33020\tlr:[9.847418e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:00,326 [ hooks.py: 207]:\tstep: 59140\tsteps/sec: 1.38258\tloss: 0.42267\tlr:[9.840627e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:06,360 [ hooks.py: 207]:\tstep: 59150\tsteps/sec: 1.65483\tloss: 0.17030\tlr:[9.833838e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:13,913 [ hooks.py: 207]:\tstep: 59160\tsteps/sec: 1.35689\tloss: 0.08875\tlr:[9.827045e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:21,529 [ hooks.py: 207]:\tstep: 59170\tsteps/sec: 1.27389\tloss: 0.54407\tlr:[9.820256e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:32,052 [ hooks.py: 207]:\tstep: 59180\tsteps/sec: 0.96491\tloss: 0.31619\tlr:[9.813463e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:40,793 [ hooks.py: 207]:\tstep: 59190\tsteps/sec: 1.18951\tloss: 0.54694\tlr:[9.806675e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:48,871 [ hooks.py: 207]:\tstep: 59200\tsteps/sec: 1.13786\tloss: 0.40050\tlr:[9.799884e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:31:58,895 [ hooks.py: 207]:\tstep: 59210\tsteps/sec: 1.03617\tloss: 0.47808\tlr:[9.793093e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:32:06,919 [ hooks.py: 207]:\tstep: 59220\tsteps/sec: 1.19892\tloss: 0.33098\tlr:[9.786302e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:32:15,610 [ hooks.py: 207]:\tstep: 59230\tsteps/sec: 1.18277\tloss: 0.16164\tlr:[9.77951e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:32:22,942 [ hooks.py: 207]:\tstep: 59240\tsteps/sec: 1.41355\tloss: 0.08239\tlr:[9.772722e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:32:31,739 [ hooks.py: 207]:\tstep: 59250\tsteps/sec: 1.05905\tloss: 0.23824\tlr:[9.7659295e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:32:42,523 [ hooks.py: 207]:\tstep: 59260\tsteps/sec: 1.01688\tloss: 0.57163\tlr:[9.759139e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:32:51,838 [ hooks.py: 207]:\tstep: 59270\tsteps/sec: 1.01928\tloss: 0.11963\tlr:[9.752349e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:32:58,897 [ hooks.py: 207]:\tstep: 59280\tsteps/sec: 1.33312\tloss: 0.41010\tlr:[9.7455595e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:33:08,910 [ hooks.py: 207]:\tstep: 59290\tsteps/sec: 1.05480\tloss: 0.09734\tlr:[9.738768e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:33:18,371 [ hooks.py: 207]:\tstep: 59300\tsteps/sec: 1.06646\tloss: 0.21778\tlr:[9.73198e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:33:26,879 [ hooks.py: 207]:\tstep: 59310\tsteps/sec: 1.14892\tloss: 0.10358\tlr:[9.725187e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:33:34,025 [ hooks.py: 207]:\tstep: 59320\tsteps/sec: 1.42501\tloss: 0.49695\tlr:[9.718398e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:33:42,924 [ hooks.py: 207]:\tstep: 59330\tsteps/sec: 1.08576\tloss: 0.07962\tlr:[9.711606e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:33:51,794 [ hooks.py: 207]:\tstep: 59340\tsteps/sec: 1.13723\tloss: 0.20421\tlr:[9.704818e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:34:00,113 [ hooks.py: 207]:\tstep: 59350\tsteps/sec: 1.20611\tloss: 0.56832\tlr:[9.698025e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:34:08,079 [ hooks.py: 207]:\tstep: 59360\tsteps/sec: 1.30025\tloss: 0.15504\tlr:[9.691236e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:34:16,805 [ hooks.py: 207]:\tstep: 59370\tsteps/sec: 1.11466\tloss: 0.18408\tlr:[9.684445e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:34:28,054 [ hooks.py: 207]:\tstep: 59380\tsteps/sec: 0.90588\tloss: 0.63266\tlr:[9.677652e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:34:37,665 [ hooks.py: 207]:\tstep: 59390\tsteps/sec: 1.03806\tloss: 0.23219\tlr:[9.670863e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:34:45,489 [ hooks.py: 207]:\tstep: 59400\tsteps/sec: 1.24722\tloss: 0.11974\tlr:[9.6640715e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:34:53,894 [ hooks.py: 207]:\tstep: 59410\tsteps/sec: 1.22593\tloss: 0.60256\tlr:[9.657282e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:35:01,924 [ hooks.py: 207]:\tstep: 59420\tsteps/sec: 1.16388\tloss: 0.55766\tlr:[9.65049e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:35:10,626 [ hooks.py: 207]:\tstep: 59430\tsteps/sec: 1.18583\tloss: 0.46186\tlr:[9.643702e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:35:18,987 [ hooks.py: 207]:\tstep: 59440\tsteps/sec: 1.22168\tloss: 0.26083\tlr:[9.6369085e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:35:29,092 [ hooks.py: 207]:\tstep: 59450\tsteps/sec: 0.98959\tloss: 0.49076\tlr:[9.63012e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:35:37,739 [ hooks.py: 207]:\tstep: 59460\tsteps/sec: 1.17116\tloss: 0.61940\tlr:[9.623329e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:35:44,581 [ hooks.py: 207]:\tstep: 59470\tsteps/sec: 1.39252\tloss: 0.25605\tlr:[9.616539e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:35:53,396 [ hooks.py: 207]:\tstep: 59480\tsteps/sec: 1.18832\tloss: 0.59741\tlr:[9.609747e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:36:02,985 [ hooks.py: 207]:\tstep: 59490\tsteps/sec: 1.04268\tloss: 0.19156\tlr:[9.602958e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:36:09,248 [ hooks.py: 207]:\tstep: 59500\tsteps/sec: 1.44752\tloss: 0.33306\tlr:[9.596167e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:36:18,737 [ hooks.py: 207]:\tstep: 59510\tsteps/sec: 1.11244\tloss: 0.66574\tlr:[9.589378e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:36:27,771 [ hooks.py: 207]:\tstep: 59520\tsteps/sec: 1.07187\tloss: 0.10088\tlr:[9.582584e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:36:36,656 [ hooks.py: 207]:\tstep: 59530\tsteps/sec: 1.12982\tloss: 0.61029\tlr:[9.575793e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:36:47,899 [ hooks.py: 207]:\tstep: 59540\tsteps/sec: 0.89034\tloss: 0.32021\tlr:[9.569004e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:36:56,131 [ hooks.py: 207]:\tstep: 59550\tsteps/sec: 1.22854\tloss: 0.33504\tlr:[9.562213e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:37:02,980 [ hooks.py: 207]:\tstep: 59560\tsteps/sec: 1.50618\tloss: 0.79832\tlr:[9.555423e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:37:11,027 [ hooks.py: 207]:\tstep: 59570\tsteps/sec: 1.20447\tloss: 0.11195\tlr:[9.548633e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:37:20,219 [ hooks.py: 207]:\tstep: 59580\tsteps/sec: 1.08973\tloss: 0.35115\tlr:[9.541843e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:37:28,577 [ hooks.py: 207]:\tstep: 59590\tsteps/sec: 1.22608\tloss: 0.27411\tlr:[9.535051e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:37:36,462 [ hooks.py: 207]:\tstep: 59600\tsteps/sec: 1.22358\tloss: 0.14198\tlr:[9.528262e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:37:45,690 [ hooks.py: 207]:\tstep: 59610\tsteps/sec: 1.08722\tloss: 0.56516\tlr:[9.521469e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:37:55,042 [ hooks.py: 207]:\tstep: 59620\tsteps/sec: 1.03122\tloss: 0.48007\tlr:[9.514681e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:38:02,634 [ hooks.py: 207]:\tstep: 59630\tsteps/sec: 1.36526\tloss: 0.59440\tlr:[9.507889e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:38:11,627 [ hooks.py: 207]:\tstep: 59640\tsteps/sec: 1.10926\tloss: 0.06910\tlr:[9.5011e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:38:20,757 [ hooks.py: 207]:\tstep: 59650\tsteps/sec: 1.07657\tloss: 0.29769\tlr:[9.494309e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 14:38:31,335 [ hooks.py: 207]:\tstep: 59660\tsteps/sec: 0.98501\tloss: 0.31987\tlr:[9.487519e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:38:38,545 [ hooks.py: 207]:\tstep: 59670\tsteps/sec: 1.39650\tloss: 0.20260\tlr:[9.480727e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:38:49,069 [ hooks.py: 207]:\tstep: 59680\tsteps/sec: 0.90655\tloss: 0.28542\tlr:[9.473934e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:38:58,185 [ hooks.py: 207]:\tstep: 59690\tsteps/sec: 1.18760\tloss: 0.42820\tlr:[9.467146e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:39:05,163 [ hooks.py: 207]:\tstep: 59700\tsteps/sec: 1.30980\tloss: 0.12369\tlr:[9.460355e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:39:14,516 [ hooks.py: 207]:\tstep: 59710\tsteps/sec: 1.10710\tloss: 0.29593\tlr:[9.453565e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:39:21,272 [ hooks.py: 207]:\tstep: 59720\tsteps/sec: 1.50214\tloss: 0.06489\tlr:[9.446773e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:39:28,562 [ hooks.py: 207]:\tstep: 59730\tsteps/sec: 1.41238\tloss: 0.45271\tlr:[9.439986e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:39:36,100 [ hooks.py: 207]:\tstep: 59740\tsteps/sec: 1.21014\tloss: 0.29130\tlr:[9.433193e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:39:45,858 [ hooks.py: 207]:\tstep: 59750\tsteps/sec: 1.07075\tloss: 0.29058\tlr:[9.426403e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:39:53,129 [ hooks.py: 207]:\tstep: 59760\tsteps/sec: 1.42930\tloss: 0.51210\tlr:[9.419611e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:40:02,732 [ hooks.py: 207]:\tstep: 59770\tsteps/sec: 1.02622\tloss: 0.33576\tlr:[9.412824e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:40:09,129 [ hooks.py: 207]:\tstep: 59780\tsteps/sec: 1.55999\tloss: 0.10555\tlr:[9.4060315e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:40:17,891 [ hooks.py: 207]:\tstep: 59790\tsteps/sec: 1.15448\tloss: 0.36645\tlr:[9.399242e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:40:25,204 [ hooks.py: 207]:\tstep: 59800\tsteps/sec: 1.31691\tloss: 0.37871\tlr:[9.392451e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:40:33,410 [ hooks.py: 207]:\tstep: 59810\tsteps/sec: 1.25487\tloss: 0.39034\tlr:[9.385661e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:40:41,592 [ hooks.py: 207]:\tstep: 59820\tsteps/sec: 1.26989\tloss: 0.66093\tlr:[9.378869e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:40:51,325 [ hooks.py: 207]:\tstep: 59830\tsteps/sec: 0.98546\tloss: 0.44149\tlr:[9.372077e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:00,814 [ hooks.py: 207]:\tstep: 59840\tsteps/sec: 1.04636\tloss: 0.04723\tlr:[9.365288e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:09,019 [ hooks.py: 207]:\tstep: 59850\tsteps/sec: 1.25848\tloss: 0.69383\tlr:[9.358496e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:18,476 [ hooks.py: 207]:\tstep: 59860\tsteps/sec: 1.06222\tloss: 0.41518\tlr:[9.351707e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:25,247 [ hooks.py: 207]:\tstep: 59870\tsteps/sec: 1.39815\tloss: 0.08172\tlr:[9.344914e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:34,216 [ hooks.py: 207]:\tstep: 59880\tsteps/sec: 1.15963\tloss: 0.47989\tlr:[9.338126e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:41,271 [ hooks.py: 207]:\tstep: 59890\tsteps/sec: 1.48672\tloss: 0.08796\tlr:[9.331334e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:49,561 [ hooks.py: 207]:\tstep: 59900\tsteps/sec: 1.09281\tloss: 0.34643\tlr:[9.324545e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:41:59,030 [ hooks.py: 207]:\tstep: 59910\tsteps/sec: 1.11792\tloss: 0.07064\tlr:[9.317753e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:42:06,121 [ hooks.py: 207]:\tstep: 59920\tsteps/sec: 1.36133\tloss: 0.44254\tlr:[9.310964e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:42:16,583 [ hooks.py: 207]:\tstep: 59930\tsteps/sec: 0.91846\tloss: 0.16970\tlr:[9.304172e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:42:27,402 [ hooks.py: 207]:\tstep: 59940\tsteps/sec: 0.97422\tloss: 0.14394\tlr:[9.297381e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:42:36,497 [ hooks.py: 207]:\tstep: 59950\tsteps/sec: 1.07180\tloss: 0.24615\tlr:[9.29059e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:42:46,131 [ hooks.py: 207]:\tstep: 59960\tsteps/sec: 1.05141\tloss: 1.02077\tlr:[9.283802e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:42:53,506 [ hooks.py: 207]:\tstep: 59970\tsteps/sec: 1.33547\tloss: 0.23717\tlr:[9.277009e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:04,453 [ hooks.py: 207]:\tstep: 59980\tsteps/sec: 0.91878\tloss: 0.44698\tlr:[9.2702185e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:15,096 [ hooks.py: 207]:\tstep: 59990\tsteps/sec: 0.94259\tloss: 0.32151\tlr:[9.26343e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:21,195 [ hooks.py: 207]:\tstep: 60000\tsteps/sec: 1.57148\tloss: 0.22274\tlr:[9.256637e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:21,195 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:21,195 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:21,196 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:43:21,196 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:21,196 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:43:21,196 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:51:22,411 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,412 [ trainer.py: 58]:\t{'acc': 0.7799196787148595, 'loss': 0.6426207561236925}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,412 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,413 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:51:22,413 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:51:22,413 [ trainer.py: 70]:\tacc\t0.7799196787148595\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:51:22,414 [ trainer.py: 70]:\tloss\t0.6426207561236925\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 14:51:22,414 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,414 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7799196787148595, 'loss': 0.6426207561236925}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,414 [ exporter.py: 64]:\t[Best Exporter]: skip step 60000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,414 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7799196787148595, 'loss': 0.6426207561236925}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,414 [ exporter.py: 104]:\t[Best Exporter]: skip step 60000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:22,415 [monitored_executor.py: 122]:\tsaving step 60000 to ./output/model_60000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:25,982 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_57000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:35,508 [ hooks.py: 207]:\tstep: 60010\tsteps/sec: 0.02024\tloss: 0.45992\tlr:[9.2498485e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:44,275 [ hooks.py: 207]:\tstep: 60020\tsteps/sec: 1.18751\tloss: 0.12132\tlr:[9.243056e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:51:54,009 [ hooks.py: 207]:\tstep: 60030\tsteps/sec: 1.02623\tloss: 0.96505\tlr:[9.236268e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:52:01,039 [ hooks.py: 207]:\tstep: 60040\tsteps/sec: 1.31462\tloss: 0.13985\tlr:[9.229477e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:52:10,321 [ hooks.py: 207]:\tstep: 60050\tsteps/sec: 1.10899\tloss: 0.04631\tlr:[9.2226865e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:52:19,826 [ hooks.py: 207]:\tstep: 60060\tsteps/sec: 1.06167\tloss: 0.15633\tlr:[9.215895e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:52:27,635 [ hooks.py: 207]:\tstep: 60070\tsteps/sec: 1.30831\tloss: 0.18945\tlr:[9.209107e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:52:36,289 [ hooks.py: 207]:\tstep: 60080\tsteps/sec: 1.13328\tloss: 0.77659\tlr:[9.202314e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:52:43,892 [ hooks.py: 207]:\tstep: 60090\tsteps/sec: 1.36062\tloss: 0.27872\tlr:[9.195524e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:52:51,075 [ hooks.py: 207]:\tstep: 60100\tsteps/sec: 1.34892\tloss: 0.11094\tlr:[9.188732e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:53:00,675 [ hooks.py: 207]:\tstep: 60110\tsteps/sec: 1.02950\tloss: 0.82968\tlr:[9.181944e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:53:09,399 [ hooks.py: 207]:\tstep: 60120\tsteps/sec: 1.11252\tloss: 0.16011\tlr:[9.175152e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 14:53:17,976 [ hooks.py: 207]:\tstep: 60130\tsteps/sec: 1.23934\tloss: 0.11212\tlr:[9.1683605e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:53:27,939 [ hooks.py: 207]:\tstep: 60140\tsteps/sec: 1.00662\tloss: 0.54724\tlr:[9.161571e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:53:35,803 [ hooks.py: 207]:\tstep: 60150\tsteps/sec: 1.24592\tloss: 0.24474\tlr:[9.154778e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:53:44,766 [ hooks.py: 207]:\tstep: 60160\tsteps/sec: 1.09851\tloss: 0.25444\tlr:[9.14799e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:53:52,849 [ hooks.py: 207]:\tstep: 60170\tsteps/sec: 1.29448\tloss: 0.16299\tlr:[9.1411985e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:54:01,073 [ hooks.py: 207]:\tstep: 60180\tsteps/sec: 1.13917\tloss: 0.05779\tlr:[9.134408e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:54:10,860 [ hooks.py: 207]:\tstep: 60190\tsteps/sec: 1.05492\tloss: 0.26710\tlr:[9.127617e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:54:20,904 [ hooks.py: 207]:\tstep: 60200\tsteps/sec: 0.99998\tloss: 0.57360\tlr:[9.1208285e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:54:29,528 [ hooks.py: 207]:\tstep: 60210\tsteps/sec: 1.17310\tloss: 0.40907\tlr:[9.114036e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:54:37,749 [ hooks.py: 207]:\tstep: 60220\tsteps/sec: 1.17477\tloss: 0.20845\tlr:[9.107248e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:54:51,143 [ hooks.py: 207]:\tstep: 60230\tsteps/sec: 0.75943\tloss: 0.48289\tlr:[9.100455e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:54:58,484 [ hooks.py: 207]:\tstep: 60240\tsteps/sec: 1.41708\tloss: 0.23853\tlr:[9.093666e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:55:07,097 [ hooks.py: 207]:\tstep: 60250\tsteps/sec: 1.10178\tloss: 0.41299\tlr:[9.086875e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:55:15,208 [ hooks.py: 207]:\tstep: 60260\tsteps/sec: 1.25035\tloss: 0.29348\tlr:[9.080086e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:55:21,799 [ hooks.py: 207]:\tstep: 60270\tsteps/sec: 1.43389\tloss: 0.56688\tlr:[9.073294e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:55:30,665 [ hooks.py: 207]:\tstep: 60280\tsteps/sec: 1.17976\tloss: 0.17957\tlr:[9.066503e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:55:38,049 [ hooks.py: 207]:\tstep: 60290\tsteps/sec: 1.33727\tloss: 0.76345\tlr:[9.059713e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:55:48,297 [ hooks.py: 207]:\tstep: 60300\tsteps/sec: 1.02456\tloss: 0.51399\tlr:[9.052921e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:55:56,356 [ hooks.py: 207]:\tstep: 60310\tsteps/sec: 1.12743\tloss: 0.35042\tlr:[9.046131e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:56:04,797 [ hooks.py: 207]:\tstep: 60320\tsteps/sec: 1.24623\tloss: 0.27368\tlr:[9.03934e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:56:12,848 [ hooks.py: 207]:\tstep: 60330\tsteps/sec: 1.22021\tloss: 0.13452\tlr:[9.032551e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:56:20,538 [ hooks.py: 207]:\tstep: 60340\tsteps/sec: 1.31021\tloss: 0.25980\tlr:[9.025759e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:56:30,269 [ hooks.py: 207]:\tstep: 60350\tsteps/sec: 1.03334\tloss: 0.36848\tlr:[9.01897e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:56:37,504 [ hooks.py: 207]:\tstep: 60360\tsteps/sec: 1.40696\tloss: 0.74015\tlr:[9.012178e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:56:46,281 [ hooks.py: 207]:\tstep: 60370\tsteps/sec: 1.13273\tloss: 0.16511\tlr:[9.005389e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:56:53,715 [ hooks.py: 207]:\tstep: 60380\tsteps/sec: 1.41045\tloss: 0.31045\tlr:[8.998598e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:57:02,902 [ hooks.py: 207]:\tstep: 60390\tsteps/sec: 1.02382\tloss: 0.52767\tlr:[8.991809e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:57:12,177 [ hooks.py: 207]:\tstep: 60400\tsteps/sec: 1.08388\tloss: 0.24775\tlr:[8.985016e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:57:21,937 [ hooks.py: 207]:\tstep: 60410\tsteps/sec: 1.03212\tloss: 0.17547\tlr:[8.978225e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:57:30,016 [ hooks.py: 207]:\tstep: 60420\tsteps/sec: 1.25574\tloss: 0.38472\tlr:[8.971435e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:57:38,236 [ hooks.py: 207]:\tstep: 60430\tsteps/sec: 1.23794\tloss: 0.89036\tlr:[8.964644e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:57:49,623 [ hooks.py: 207]:\tstep: 60440\tsteps/sec: 0.86674\tloss: 0.39105\tlr:[8.957854e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:57:58,432 [ hooks.py: 207]:\tstep: 60450\tsteps/sec: 1.17446\tloss: 0.21498\tlr:[8.951062e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:58:05,601 [ hooks.py: 207]:\tstep: 60460\tsteps/sec: 1.32082\tloss: 0.36084\tlr:[8.944274e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:58:14,009 [ hooks.py: 207]:\tstep: 60470\tsteps/sec: 1.25710\tloss: 0.24390\tlr:[8.937481e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:58:23,822 [ hooks.py: 207]:\tstep: 60480\tsteps/sec: 1.01395\tloss: 0.42036\tlr:[8.930692e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:58:32,299 [ hooks.py: 207]:\tstep: 60490\tsteps/sec: 1.12576\tloss: 0.38303\tlr:[8.9239e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:58:40,296 [ hooks.py: 207]:\tstep: 60500\tsteps/sec: 1.28425\tloss: 0.44597\tlr:[8.917112e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:58:48,167 [ hooks.py: 207]:\tstep: 60510\tsteps/sec: 1.28103\tloss: 0.12618\tlr:[8.910319e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:58:57,790 [ hooks.py: 207]:\tstep: 60520\tsteps/sec: 1.03844\tloss: 0.61725\tlr:[8.903531e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:59:06,695 [ hooks.py: 207]:\tstep: 60530\tsteps/sec: 1.04818\tloss: 0.05361\tlr:[8.896738e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:59:16,295 [ hooks.py: 207]:\tstep: 60540\tsteps/sec: 1.08809\tloss: 0.48862\tlr:[8.88995e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:59:24,911 [ hooks.py: 207]:\tstep: 60550\tsteps/sec: 1.16098\tloss: 0.52472\tlr:[8.883158e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:59:35,540 [ hooks.py: 207]:\tstep: 60560\tsteps/sec: 0.91394\tloss: 0.32495\tlr:[8.876366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:59:47,250 [ hooks.py: 207]:\tstep: 60570\tsteps/sec: 0.87252\tloss: 0.19144\tlr:[8.869577e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 14:59:53,878 [ hooks.py: 207]:\tstep: 60580\tsteps/sec: 1.53186\tloss: 0.08451\tlr:[8.862785e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:00:04,610 [ hooks.py: 207]:\tstep: 60590\tsteps/sec: 0.94135\tloss: 0.54942\tlr:[8.855996e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:00:12,026 [ hooks.py: 207]:\tstep: 60600\tsteps/sec: 1.30521\tloss: 0.90450\tlr:[8.849203e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:00:21,600 [ hooks.py: 207]:\tstep: 60610\tsteps/sec: 1.05323\tloss: 0.37492\tlr:[8.842415e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:00:30,426 [ hooks.py: 207]:\tstep: 60620\tsteps/sec: 1.13316\tloss: 0.85025\tlr:[8.835623e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:00:39,145 [ hooks.py: 207]:\tstep: 60630\tsteps/sec: 1.14777\tloss: 0.37817\tlr:[8.828834e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:00:48,950 [ hooks.py: 207]:\tstep: 60640\tsteps/sec: 1.00792\tloss: 0.35425\tlr:[8.822041e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:00:57,491 [ hooks.py: 207]:\tstep: 60650\tsteps/sec: 1.19108\tloss: 0.36475\tlr:[8.815254e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:05,728 [ hooks.py: 207]:\tstep: 60660\tsteps/sec: 1.20970\tloss: 0.17596\tlr:[8.808462e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:13,792 [ hooks.py: 207]:\tstep: 60670\tsteps/sec: 1.30112\tloss: 0.34366\tlr:[8.801672e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:21,097 [ hooks.py: 207]:\tstep: 60680\tsteps/sec: 1.30400\tloss: 0.10636\tlr:[8.79488e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:27,965 [ hooks.py: 207]:\tstep: 60690\tsteps/sec: 1.50739\tloss: 0.35956\tlr:[8.788092e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:34,734 [ hooks.py: 207]:\tstep: 60700\tsteps/sec: 1.44975\tloss: 0.57549\tlr:[8.7813e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:42,318 [ hooks.py: 207]:\tstep: 60710\tsteps/sec: 1.30956\tloss: 0.10450\tlr:[8.7745075e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:50,179 [ hooks.py: 207]:\tstep: 60720\tsteps/sec: 1.27532\tloss: 0.47885\tlr:[8.767719e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:01:59,256 [ hooks.py: 207]:\tstep: 60730\tsteps/sec: 1.12304\tloss: 0.09784\tlr:[8.760927e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:02:09,521 [ hooks.py: 207]:\tstep: 60740\tsteps/sec: 0.92997\tloss: 0.10451\tlr:[8.7541375e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:02:19,008 [ hooks.py: 207]:\tstep: 60750\tsteps/sec: 1.07632\tloss: 0.31533\tlr:[8.747345e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:02:27,890 [ hooks.py: 207]:\tstep: 60760\tsteps/sec: 1.14960\tloss: 0.71977\tlr:[8.740556e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:02:37,893 [ hooks.py: 207]:\tstep: 60770\tsteps/sec: 1.00004\tloss: 0.08221\tlr:[8.733764e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:02:45,570 [ hooks.py: 207]:\tstep: 60780\tsteps/sec: 1.22412\tloss: 0.36997\tlr:[8.726975e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 15:02:55,711 [ hooks.py: 207]:\tstep: 60790\tsteps/sec: 0.98732\tloss: 0.52906\tlr:[8.720183e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:03:04,086 [ hooks.py: 207]:\tstep: 60800\tsteps/sec: 1.27706\tloss: 0.42483\tlr:[8.713393e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:03:12,379 [ hooks.py: 207]:\tstep: 60810\tsteps/sec: 1.17907\tloss: 0.76294\tlr:[8.706602e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:03:20,606 [ hooks.py: 207]:\tstep: 60820\tsteps/sec: 1.16982\tloss: 0.05047\tlr:[8.699814e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:03:29,034 [ hooks.py: 207]:\tstep: 60830\tsteps/sec: 1.29752\tloss: 0.41265\tlr:[8.693022e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:03:38,188 [ hooks.py: 207]:\tstep: 60840\tsteps/sec: 1.06656\tloss: 0.43789\tlr:[8.686234e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:03:48,240 [ hooks.py: 207]:\tstep: 60850\tsteps/sec: 0.99005\tloss: 0.28739\tlr:[8.679441e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:03:56,479 [ hooks.py: 207]:\tstep: 60860\tsteps/sec: 1.19820\tloss: 0.03283\tlr:[8.6726495e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:04:05,437 [ hooks.py: 207]:\tstep: 60870\tsteps/sec: 1.07151\tloss: 0.10524\tlr:[8.665859e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:04:14,803 [ hooks.py: 207]:\tstep: 60880\tsteps/sec: 1.12185\tloss: 0.15209\tlr:[8.659068e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:04:22,967 [ hooks.py: 207]:\tstep: 60890\tsteps/sec: 1.21898\tloss: 0.20575\tlr:[8.6522805e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:04:30,414 [ hooks.py: 207]:\tstep: 60900\tsteps/sec: 1.26348\tloss: 0.25161\tlr:[8.6454875e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:04:40,018 [ hooks.py: 207]:\tstep: 60910\tsteps/sec: 1.10863\tloss: 0.38604\tlr:[8.638698e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:04:50,429 [ hooks.py: 207]:\tstep: 60920\tsteps/sec: 0.94664\tloss: 0.34516\tlr:[8.631907e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:04:59,595 [ hooks.py: 207]:\tstep: 60930\tsteps/sec: 1.05918\tloss: 0.50496\tlr:[8.625116e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:10,192 [ hooks.py: 207]:\tstep: 60940\tsteps/sec: 0.96595\tloss: 0.39799\tlr:[8.618325e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:19,545 [ hooks.py: 207]:\tstep: 60950\tsteps/sec: 1.09075\tloss: 0.67006\tlr:[8.611536e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:29,782 [ hooks.py: 207]:\tstep: 60960\tsteps/sec: 0.97567\tloss: 0.21732\tlr:[8.604745e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:38,076 [ hooks.py: 207]:\tstep: 60970\tsteps/sec: 1.21934\tloss: 0.26277\tlr:[8.5979555e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:45,266 [ hooks.py: 207]:\tstep: 60980\tsteps/sec: 1.33850\tloss: 0.49453\tlr:[8.591163e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:52,701 [ hooks.py: 207]:\tstep: 60990\tsteps/sec: 1.36771\tloss: 0.56895\tlr:[8.584374e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:59,312 [ hooks.py: 207]:\tstep: 61000\tsteps/sec: 1.54028\tloss: 0.62912\tlr:[8.577583e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:59,313 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:59,313 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:59,313 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:05:59,313 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:59,313 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:05:59,313 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:14:00,048 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,050 [ trainer.py: 58]:\t{'acc': 0.7855421686746988, 'loss': 0.6394722750171636}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,050 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,051 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:14:00,051 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:14:00,051 [ trainer.py: 70]:\tacc\t0.7855421686746988\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:14:00,051 [ trainer.py: 70]:\tloss\t0.6394722750171636\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:14:00,051 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,051 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6394722750171636}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,052 [ exporter.py: 64]:\t[Best Exporter]: skip step 61000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,052 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6394722750171636}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,052 [ exporter.py: 104]:\t[Best Exporter]: skip step 61000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:00,053 [monitored_executor.py: 122]:\tsaving step 61000 to ./output/model_61000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:03,502 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_58000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:11,410 [ hooks.py: 207]:\tstep: 61010\tsteps/sec: 0.02031\tloss: 0.19525\tlr:[8.570792e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:21,200 [ hooks.py: 207]:\tstep: 61020\tsteps/sec: 1.00860\tloss: 0.31003\tlr:[8.564002e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:31,471 [ hooks.py: 207]:\tstep: 61030\tsteps/sec: 0.98260\tloss: 0.62665\tlr:[8.557211e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:41,476 [ hooks.py: 207]:\tstep: 61040\tsteps/sec: 1.04549\tloss: 0.38456\tlr:[8.550421e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:14:51,105 [ hooks.py: 207]:\tstep: 61050\tsteps/sec: 0.98432\tloss: 0.23951\tlr:[8.543629e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:15:00,864 [ hooks.py: 207]:\tstep: 61060\tsteps/sec: 1.03143\tloss: 0.19884\tlr:[8.53684e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:15:09,768 [ hooks.py: 207]:\tstep: 61070\tsteps/sec: 1.14339\tloss: 0.21479\tlr:[8.530048e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:15:17,151 [ hooks.py: 207]:\tstep: 61080\tsteps/sec: 1.31486\tloss: 0.15209\tlr:[8.523259e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:15:25,759 [ hooks.py: 207]:\tstep: 61090\tsteps/sec: 1.16189\tloss: 0.14816\tlr:[8.5164675e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:15:37,244 [ hooks.py: 207]:\tstep: 61100\tsteps/sec: 0.87292\tloss: 0.04472\tlr:[8.509677e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:15:46,041 [ hooks.py: 207]:\tstep: 61110\tsteps/sec: 1.15180\tloss: 0.41019\tlr:[8.502886e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:15:52,974 [ hooks.py: 207]:\tstep: 61120\tsteps/sec: 1.38138\tloss: 0.23918\tlr:[8.496098e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:01,247 [ hooks.py: 207]:\tstep: 61130\tsteps/sec: 1.20868\tloss: 0.78136\tlr:[8.4893045e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:10,379 [ hooks.py: 207]:\tstep: 61140\tsteps/sec: 1.12957\tloss: 0.27875\tlr:[8.482516e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:16,797 [ hooks.py: 207]:\tstep: 61150\tsteps/sec: 1.55416\tloss: 0.56844\tlr:[8.475724e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:26,584 [ hooks.py: 207]:\tstep: 61160\tsteps/sec: 1.01745\tloss: 0.15631\tlr:[8.468932e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:33,952 [ hooks.py: 207]:\tstep: 61170\tsteps/sec: 1.42879\tloss: 0.38790\tlr:[8.462144e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:42,824 [ hooks.py: 207]:\tstep: 61180\tsteps/sec: 1.14162\tloss: 0.50070\tlr:[8.455351e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:50,186 [ hooks.py: 207]:\tstep: 61190\tsteps/sec: 1.30775\tloss: 0.63960\tlr:[8.448563e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:16:57,092 [ hooks.py: 207]:\tstep: 61200\tsteps/sec: 1.41147\tloss: 0.25933\tlr:[8.441772e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:17:05,742 [ hooks.py: 207]:\tstep: 61210\tsteps/sec: 1.15205\tloss: 0.25138\tlr:[8.434981e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:17:14,615 [ hooks.py: 207]:\tstep: 61220\tsteps/sec: 1.16321\tloss: 0.48833\tlr:[8.42819e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:17:23,483 [ hooks.py: 207]:\tstep: 61230\tsteps/sec: 1.12804\tloss: 0.23595\tlr:[8.421402e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:17:30,410 [ hooks.py: 207]:\tstep: 61240\tsteps/sec: 1.40223\tloss: 0.73389\tlr:[8.414608e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:17:37,005 [ hooks.py: 207]:\tstep: 61250\tsteps/sec: 1.50730\tloss: 0.17271\tlr:[8.407819e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 15:17:43,803 [ hooks.py: 207]:\tstep: 61260\tsteps/sec: 1.53695\tloss: 0.18771\tlr:[8.401027e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:17:51,507 [ hooks.py: 207]:\tstep: 61270\tsteps/sec: 1.25051\tloss: 0.24641\tlr:[8.39424e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:17:58,555 [ hooks.py: 207]:\tstep: 61280\tsteps/sec: 1.46333\tloss: 0.23633\tlr:[8.387447e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:18:05,226 [ hooks.py: 207]:\tstep: 61290\tsteps/sec: 1.44674\tloss: 0.34918\tlr:[8.380659e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:18:15,004 [ hooks.py: 207]:\tstep: 61300\tsteps/sec: 1.04627\tloss: 0.27413\tlr:[8.373867e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:18:22,843 [ hooks.py: 207]:\tstep: 61310\tsteps/sec: 1.25024\tloss: 0.47219\tlr:[8.367074e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:18:29,757 [ hooks.py: 207]:\tstep: 61320\tsteps/sec: 1.47299\tloss: 0.39550\tlr:[8.360285e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:18:37,612 [ hooks.py: 207]:\tstep: 61330\tsteps/sec: 1.24195\tloss: 0.11817\tlr:[8.353493e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:18:46,319 [ hooks.py: 207]:\tstep: 61340\tsteps/sec: 1.14188\tloss: 0.44037\tlr:[8.346704e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:18:54,985 [ hooks.py: 207]:\tstep: 61350\tsteps/sec: 1.20109\tloss: 0.38042\tlr:[8.339912e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:19:02,796 [ hooks.py: 207]:\tstep: 61360\tsteps/sec: 1.23833\tloss: 0.62252\tlr:[8.333122e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:19:09,896 [ hooks.py: 207]:\tstep: 61370\tsteps/sec: 1.44163\tloss: 0.32896\tlr:[8.32633e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:19:20,375 [ hooks.py: 207]:\tstep: 61380\tsteps/sec: 0.93939\tloss: 0.70933\tlr:[8.319542e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:19:29,467 [ hooks.py: 207]:\tstep: 61390\tsteps/sec: 1.14592\tloss: 0.45162\tlr:[8.31275e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:19:36,522 [ hooks.py: 207]:\tstep: 61400\tsteps/sec: 1.35686\tloss: 0.28766\tlr:[8.30596e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:19:45,894 [ hooks.py: 207]:\tstep: 61410\tsteps/sec: 1.05227\tloss: 0.85227\tlr:[8.299169e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:19:54,031 [ hooks.py: 207]:\tstep: 61420\tsteps/sec: 1.23517\tloss: 0.43132\tlr:[8.29238e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:20:03,181 [ hooks.py: 207]:\tstep: 61430\tsteps/sec: 1.10713\tloss: 0.27298\tlr:[8.285589e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:20:13,943 [ hooks.py: 207]:\tstep: 61440\tsteps/sec: 0.94455\tloss: 0.44950\tlr:[8.278799e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:20:21,275 [ hooks.py: 207]:\tstep: 61450\tsteps/sec: 1.30595\tloss: 0.31586\tlr:[8.272008e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:20:30,652 [ hooks.py: 207]:\tstep: 61460\tsteps/sec: 1.03634\tloss: 0.35796\tlr:[8.265216e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:20:42,231 [ hooks.py: 207]:\tstep: 61470\tsteps/sec: 0.91657\tloss: 0.13658\tlr:[8.2584265e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:20:51,055 [ hooks.py: 207]:\tstep: 61480\tsteps/sec: 1.09668\tloss: 0.49831\tlr:[8.251634e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:20:59,306 [ hooks.py: 207]:\tstep: 61490\tsteps/sec: 1.25558\tloss: 0.29863\tlr:[8.244847e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:21:06,601 [ hooks.py: 207]:\tstep: 61500\tsteps/sec: 1.31533\tloss: 0.29293\tlr:[8.238055e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:21:15,865 [ hooks.py: 207]:\tstep: 61510\tsteps/sec: 1.07513\tloss: 0.36603\tlr:[8.2312645e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:21:24,644 [ hooks.py: 207]:\tstep: 61520\tsteps/sec: 1.17532\tloss: 0.38680\tlr:[8.224473e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:21:32,808 [ hooks.py: 207]:\tstep: 61530\tsteps/sec: 1.18684\tloss: 0.08109\tlr:[8.217684e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:21:42,160 [ hooks.py: 207]:\tstep: 61540\tsteps/sec: 1.12108\tloss: 0.33955\tlr:[8.210893e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:21:50,272 [ hooks.py: 207]:\tstep: 61550\tsteps/sec: 1.20709\tloss: 0.16975\tlr:[8.204102e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:21:57,203 [ hooks.py: 207]:\tstep: 61560\tsteps/sec: 1.37999\tloss: 0.97466\tlr:[8.197311e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:22:05,545 [ hooks.py: 207]:\tstep: 61570\tsteps/sec: 1.21921\tloss: 0.37534\tlr:[8.190522e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:22:13,991 [ hooks.py: 207]:\tstep: 61580\tsteps/sec: 1.20709\tloss: 0.69915\tlr:[8.18373e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:22:21,920 [ hooks.py: 207]:\tstep: 61590\tsteps/sec: 1.22830\tloss: 0.22511\tlr:[8.1769385e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:22:31,592 [ hooks.py: 207]:\tstep: 61600\tsteps/sec: 1.04082\tloss: 0.42083\tlr:[8.170149e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:22:39,407 [ hooks.py: 207]:\tstep: 61610\tsteps/sec: 1.31900\tloss: 0.08739\tlr:[8.163358e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:22:48,465 [ hooks.py: 207]:\tstep: 61620\tsteps/sec: 1.09069\tloss: 0.12905\tlr:[8.156569e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:22:56,795 [ hooks.py: 207]:\tstep: 61630\tsteps/sec: 1.17563\tloss: 0.15440\tlr:[8.149776e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:23:04,304 [ hooks.py: 207]:\tstep: 61640\tsteps/sec: 1.38839\tloss: 0.67161\tlr:[8.142987e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:23:12,278 [ hooks.py: 207]:\tstep: 61650\tsteps/sec: 1.20378\tloss: 0.40387\tlr:[8.136195e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:23:20,257 [ hooks.py: 207]:\tstep: 61660\tsteps/sec: 1.30344\tloss: 0.20544\tlr:[8.1294065e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:23:28,657 [ hooks.py: 207]:\tstep: 61670\tsteps/sec: 1.18452\tloss: 0.15086\tlr:[8.1226135e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:23:36,650 [ hooks.py: 207]:\tstep: 61680\tsteps/sec: 1.21490\tloss: 0.30118\tlr:[8.115826e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:23:44,368 [ hooks.py: 207]:\tstep: 61690\tsteps/sec: 1.34106\tloss: 0.27679\tlr:[8.109035e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:23:52,339 [ hooks.py: 207]:\tstep: 61700\tsteps/sec: 1.22023\tloss: 0.12272\tlr:[8.102244e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:24:00,631 [ hooks.py: 207]:\tstep: 61710\tsteps/sec: 1.16138\tloss: 0.05202\tlr:[8.095452e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:24:10,041 [ hooks.py: 207]:\tstep: 61720\tsteps/sec: 1.11955\tloss: 0.31717\tlr:[8.088664e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:24:18,554 [ hooks.py: 207]:\tstep: 61730\tsteps/sec: 1.16752\tloss: 0.31763\tlr:[8.081872e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:24:26,153 [ hooks.py: 207]:\tstep: 61740\tsteps/sec: 1.23241\tloss: 0.49813\tlr:[8.075081e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:24:35,669 [ hooks.py: 207]:\tstep: 61750\tsteps/sec: 1.08967\tloss: 0.57752\tlr:[8.068291e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:24:46,503 [ hooks.py: 207]:\tstep: 61760\tsteps/sec: 0.92156\tloss: 0.44519\tlr:[8.061499e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:24:56,949 [ hooks.py: 207]:\tstep: 61770\tsteps/sec: 0.95543\tloss: 0.40031\tlr:[8.05471e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:25:06,959 [ hooks.py: 207]:\tstep: 61780\tsteps/sec: 0.96811\tloss: 0.55619\tlr:[8.047918e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:25:15,749 [ hooks.py: 207]:\tstep: 61790\tsteps/sec: 1.13060\tloss: 0.08462\tlr:[8.04113e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:25:25,091 [ hooks.py: 207]:\tstep: 61800\tsteps/sec: 1.14789\tloss: 0.24486\tlr:[8.034336e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:25:34,573 [ hooks.py: 207]:\tstep: 61810\tsteps/sec: 1.02722\tloss: 0.15742\tlr:[8.027549e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:25:44,439 [ hooks.py: 207]:\tstep: 61820\tsteps/sec: 1.05387\tloss: 0.43314\tlr:[8.020756e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:25:51,340 [ hooks.py: 207]:\tstep: 61830\tsteps/sec: 1.38586\tloss: 0.14770\tlr:[8.013967e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:25:59,212 [ hooks.py: 207]:\tstep: 61840\tsteps/sec: 1.32894\tloss: 0.29672\tlr:[8.007174e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:26:07,547 [ hooks.py: 207]:\tstep: 61850\tsteps/sec: 1.15672\tloss: 0.31752\tlr:[8.0003865e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:26:16,132 [ hooks.py: 207]:\tstep: 61860\tsteps/sec: 1.11010\tloss: 0.26906\tlr:[7.993595e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:26:24,431 [ hooks.py: 207]:\tstep: 61870\tsteps/sec: 1.30172\tloss: 0.06471\tlr:[7.986805e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:26:31,699 [ hooks.py: 207]:\tstep: 61880\tsteps/sec: 1.33442\tloss: 0.35068\tlr:[7.980014e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:26:40,905 [ hooks.py: 207]:\tstep: 61890\tsteps/sec: 1.12259\tloss: 0.23516\tlr:[7.973222e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:26:49,840 [ hooks.py: 207]:\tstep: 61900\tsteps/sec: 1.09474\tloss: 0.22470\tlr:[7.966431e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:27:00,171 [ hooks.py: 207]:\tstep: 61910\tsteps/sec: 0.93290\tloss: 0.48997\tlr:[7.95964e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 15:27:10,144 [ hooks.py: 207]:\tstep: 61920\tsteps/sec: 1.06365\tloss: 0.27265\tlr:[7.952852e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:27:18,097 [ hooks.py: 207]:\tstep: 61930\tsteps/sec: 1.20548\tloss: 0.72618\tlr:[7.94606e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:27:28,450 [ hooks.py: 207]:\tstep: 61940\tsteps/sec: 0.97436\tloss: 0.83249\tlr:[7.93927e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:27:36,305 [ hooks.py: 207]:\tstep: 61950\tsteps/sec: 1.25792\tloss: 0.50538\tlr:[7.932479e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:27:43,810 [ hooks.py: 207]:\tstep: 61960\tsteps/sec: 1.38379\tloss: 0.15190\tlr:[7.925691e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:27:52,691 [ hooks.py: 207]:\tstep: 61970\tsteps/sec: 1.05707\tloss: 0.22888\tlr:[7.918898e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:02,028 [ hooks.py: 207]:\tstep: 61980\tsteps/sec: 1.10575\tloss: 0.46219\tlr:[7.912109e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:11,717 [ hooks.py: 207]:\tstep: 61990\tsteps/sec: 1.03778\tloss: 0.34318\tlr:[7.905318e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:18,448 [ hooks.py: 207]:\tstep: 62000\tsteps/sec: 1.52039\tloss: 0.52057\tlr:[7.898529e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:18,449 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:18,449 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:18,449 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:28:18,449 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:18,449 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:28:18,449 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:36:20,667 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,668 [ trainer.py: 58]:\t{'acc': 0.791566265060241, 'loss': 0.621344296548229}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,669 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,669 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:36:20,669 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:36:20,669 [ trainer.py: 70]:\tacc\t0.791566265060241\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:36:20,669 [ trainer.py: 70]:\tloss\t0.621344296548229\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:36:20,669 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,669 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.791566265060241, 'loss': 0.621344296548229}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,669 [ exporter.py: 64]:\t[Best Exporter]: skip step 62000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,670 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.791566265060241, 'loss': 0.621344296548229}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,670 [ exporter.py: 104]:\t[Best Exporter]: skip step 62000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:20,671 [monitored_executor.py: 122]:\tsaving step 62000 to ./output/model_62000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:23,728 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_59000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:31,567 [ hooks.py: 207]:\tstep: 62010\tsteps/sec: 0.02027\tloss: 0.41330\tlr:[7.891736e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:41,529 [ hooks.py: 207]:\tstep: 62020\tsteps/sec: 1.03739\tloss: 0.05907\tlr:[7.884946e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:36:51,516 [ hooks.py: 207]:\tstep: 62030\tsteps/sec: 0.97043\tloss: 0.40679\tlr:[7.878155e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:37:01,990 [ hooks.py: 207]:\tstep: 62040\tsteps/sec: 0.96718\tloss: 0.26660\tlr:[7.871363e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:37:09,236 [ hooks.py: 207]:\tstep: 62050\tsteps/sec: 1.33763\tloss: 0.25671\tlr:[7.8645735e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:37:16,615 [ hooks.py: 207]:\tstep: 62060\tsteps/sec: 1.32463\tloss: 0.50202\tlr:[7.857781e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:37:26,546 [ hooks.py: 207]:\tstep: 62070\tsteps/sec: 1.00264\tloss: 0.58043\tlr:[7.850993e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:37:36,636 [ hooks.py: 207]:\tstep: 62080\tsteps/sec: 1.02342\tloss: 1.24279\tlr:[7.844202e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:37:45,394 [ hooks.py: 207]:\tstep: 62090\tsteps/sec: 1.17950\tloss: 0.08069\tlr:[7.837411e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:37:54,575 [ hooks.py: 207]:\tstep: 62100\tsteps/sec: 1.09550\tloss: 0.31552\tlr:[7.830619e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:38:02,211 [ hooks.py: 207]:\tstep: 62110\tsteps/sec: 1.21258\tloss: 0.46067\tlr:[7.823832e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:38:11,531 [ hooks.py: 207]:\tstep: 62120\tsteps/sec: 1.10367\tloss: 0.47331\tlr:[7.817041e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:38:19,672 [ hooks.py: 207]:\tstep: 62130\tsteps/sec: 1.27597\tloss: 0.37309\tlr:[7.81025e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:38:27,000 [ hooks.py: 207]:\tstep: 62140\tsteps/sec: 1.32057\tloss: 0.27223\tlr:[7.803458e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:38:33,490 [ hooks.py: 207]:\tstep: 62150\tsteps/sec: 1.47260\tloss: 0.39312\tlr:[7.79667e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:38:44,411 [ hooks.py: 207]:\tstep: 62160\tsteps/sec: 0.96296\tloss: 0.37981\tlr:[7.789877e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:38:56,200 [ hooks.py: 207]:\tstep: 62170\tsteps/sec: 0.82297\tloss: 0.16399\tlr:[7.783088e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:39:04,349 [ hooks.py: 207]:\tstep: 62180\tsteps/sec: 1.28727\tloss: 0.14233\tlr:[7.776297e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:39:12,336 [ hooks.py: 207]:\tstep: 62190\tsteps/sec: 1.20465\tloss: 0.73031\tlr:[7.769505e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:39:22,342 [ hooks.py: 207]:\tstep: 62200\tsteps/sec: 0.99960\tloss: 0.54664\tlr:[7.762715e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:39:29,891 [ hooks.py: 207]:\tstep: 62210\tsteps/sec: 1.36269\tloss: 0.41263\tlr:[7.755924e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:39:39,776 [ hooks.py: 207]:\tstep: 62220\tsteps/sec: 1.02346\tloss: 0.19705\tlr:[7.749136e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:39:47,014 [ hooks.py: 207]:\tstep: 62230\tsteps/sec: 1.32731\tloss: 0.49101\tlr:[7.742344e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:39:53,795 [ hooks.py: 207]:\tstep: 62240\tsteps/sec: 1.50393\tloss: 0.40961\tlr:[7.7355535e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:40:00,538 [ hooks.py: 207]:\tstep: 62250\tsteps/sec: 1.49088\tloss: 0.23850\tlr:[7.728762e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:40:08,006 [ hooks.py: 207]:\tstep: 62260\tsteps/sec: 1.24789\tloss: 0.69918\tlr:[7.721972e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:40:17,769 [ hooks.py: 207]:\tstep: 62270\tsteps/sec: 1.05267\tloss: 0.15365\tlr:[7.715181e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:40:27,332 [ hooks.py: 207]:\tstep: 62280\tsteps/sec: 1.04604\tloss: 0.12936\tlr:[7.708392e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:40:38,305 [ hooks.py: 207]:\tstep: 62290\tsteps/sec: 0.91295\tloss: 0.45558\tlr:[7.7016e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:40:47,710 [ hooks.py: 207]:\tstep: 62300\tsteps/sec: 1.07646\tloss: 0.35269\tlr:[7.694811e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:40:55,044 [ hooks.py: 207]:\tstep: 62310\tsteps/sec: 1.39915\tloss: 0.06630\tlr:[7.68802e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:41:04,739 [ hooks.py: 207]:\tstep: 62320\tsteps/sec: 1.02934\tloss: 0.17980\tlr:[7.68123e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:41:14,650 [ hooks.py: 207]:\tstep: 62330\tsteps/sec: 1.01117\tloss: 0.49144\tlr:[7.674438e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:41:23,918 [ hooks.py: 207]:\tstep: 62340\tsteps/sec: 1.08491\tloss: 0.27333\tlr:[7.667646e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:41:30,795 [ hooks.py: 207]:\tstep: 62350\tsteps/sec: 1.36762\tloss: 0.49962\tlr:[7.660857e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:41:40,432 [ hooks.py: 207]:\tstep: 62360\tsteps/sec: 1.07983\tloss: 0.73812\tlr:[7.6540655e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:41:48,107 [ hooks.py: 207]:\tstep: 62370\tsteps/sec: 1.25923\tloss: 0.69288\tlr:[7.647276e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:41:58,923 [ hooks.py: 207]:\tstep: 62380\tsteps/sec: 0.94521\tloss: 0.21727\tlr:[7.640485e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 15:42:07,354 [ hooks.py: 207]:\tstep: 62390\tsteps/sec: 1.15689\tloss: 0.48227\tlr:[7.6336955e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:42:16,072 [ hooks.py: 207]:\tstep: 62400\tsteps/sec: 1.16617\tloss: 0.29268\tlr:[7.626903e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:42:24,680 [ hooks.py: 207]:\tstep: 62410\tsteps/sec: 1.14360\tloss: 0.44032\tlr:[7.6201145e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:42:34,118 [ hooks.py: 207]:\tstep: 62420\tsteps/sec: 1.09079\tloss: 0.13390\tlr:[7.613323e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:42:42,470 [ hooks.py: 207]:\tstep: 62430\tsteps/sec: 1.19968\tloss: 0.47472\tlr:[7.606534e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:42:51,105 [ hooks.py: 207]:\tstep: 62440\tsteps/sec: 1.12531\tloss: 0.20277\tlr:[7.599741e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:42:58,474 [ hooks.py: 207]:\tstep: 62450\tsteps/sec: 1.36655\tloss: 0.34203\tlr:[7.592952e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:43:05,749 [ hooks.py: 207]:\tstep: 62460\tsteps/sec: 1.39835\tloss: 0.22708\tlr:[7.586161e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:43:13,725 [ hooks.py: 207]:\tstep: 62470\tsteps/sec: 1.19420\tloss: 0.14670\tlr:[7.579373e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:43:23,137 [ hooks.py: 207]:\tstep: 62480\tsteps/sec: 1.09561\tloss: 0.24076\tlr:[7.5725807e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:43:30,947 [ hooks.py: 207]:\tstep: 62490\tsteps/sec: 1.22149\tloss: 0.58882\tlr:[7.565788e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:43:40,210 [ hooks.py: 207]:\tstep: 62500\tsteps/sec: 1.17080\tloss: 0.30293\tlr:[7.5589983e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:43:47,589 [ hooks.py: 207]:\tstep: 62510\tsteps/sec: 1.27962\tloss: 0.04882\tlr:[7.552207e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:43:56,334 [ hooks.py: 207]:\tstep: 62520\tsteps/sec: 1.13271\tloss: 0.46535\tlr:[7.545418e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:03,076 [ hooks.py: 207]:\tstep: 62530\tsteps/sec: 1.51490\tloss: 0.29613\tlr:[7.5386247e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:10,468 [ hooks.py: 207]:\tstep: 62540\tsteps/sec: 1.38734\tloss: 0.10370\tlr:[7.531837e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:17,513 [ hooks.py: 207]:\tstep: 62550\tsteps/sec: 1.37983\tloss: 0.41043\tlr:[7.525045e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:25,966 [ hooks.py: 207]:\tstep: 62560\tsteps/sec: 1.17349\tloss: 0.66108\tlr:[7.518256e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:33,623 [ hooks.py: 207]:\tstep: 62570\tsteps/sec: 1.27983\tloss: 0.16716\tlr:[7.5114644e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:42,455 [ hooks.py: 207]:\tstep: 62580\tsteps/sec: 1.20155\tloss: 0.56809\tlr:[7.504676e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:50,189 [ hooks.py: 207]:\tstep: 62590\tsteps/sec: 1.28484\tloss: 0.19075\tlr:[7.4978834e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:44:57,654 [ hooks.py: 207]:\tstep: 62600\tsteps/sec: 1.29064\tloss: 0.50245\tlr:[7.4910936e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:45:05,571 [ hooks.py: 207]:\tstep: 62610\tsteps/sec: 1.29056\tloss: 0.43461\tlr:[7.4843033e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:45:15,133 [ hooks.py: 207]:\tstep: 62620\tsteps/sec: 1.06458\tloss: 0.61165\tlr:[7.477511e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:45:25,693 [ hooks.py: 207]:\tstep: 62630\tsteps/sec: 0.93452\tloss: 0.34057\tlr:[7.4707214e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:45:32,207 [ hooks.py: 207]:\tstep: 62640\tsteps/sec: 1.42591\tloss: 0.52856\tlr:[7.463929e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:45:41,480 [ hooks.py: 207]:\tstep: 62650\tsteps/sec: 1.19290\tloss: 0.49067\tlr:[7.457141e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:45:50,307 [ hooks.py: 207]:\tstep: 62660\tsteps/sec: 1.10108\tloss: 0.28550\tlr:[7.4503487e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:45:59,077 [ hooks.py: 207]:\tstep: 62670\tsteps/sec: 1.12795\tloss: 0.38806\tlr:[7.4435607e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:46:06,840 [ hooks.py: 207]:\tstep: 62680\tsteps/sec: 1.25371\tloss: 0.38413\tlr:[7.4367676e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:46:13,501 [ hooks.py: 207]:\tstep: 62690\tsteps/sec: 1.48913\tloss: 0.52214\tlr:[7.4299787e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:46:22,481 [ hooks.py: 207]:\tstep: 62700\tsteps/sec: 1.11953\tloss: 0.91259\tlr:[7.423187e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:46:29,862 [ hooks.py: 207]:\tstep: 62710\tsteps/sec: 1.40986\tloss: 0.17874\tlr:[7.4163986e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:46:37,168 [ hooks.py: 207]:\tstep: 62720\tsteps/sec: 1.26852\tloss: 0.81750\tlr:[7.4096065e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:46:45,817 [ hooks.py: 207]:\tstep: 62730\tsteps/sec: 1.21787\tloss: 0.31889\tlr:[7.4028176e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:46:52,397 [ hooks.py: 207]:\tstep: 62740\tsteps/sec: 1.42051\tloss: 0.28069\tlr:[7.396024e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:47:02,586 [ hooks.py: 207]:\tstep: 62750\tsteps/sec: 1.00587\tloss: 0.32492\tlr:[7.389235e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:47:11,437 [ hooks.py: 207]:\tstep: 62760\tsteps/sec: 1.10697\tloss: 0.38648\tlr:[7.382444e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:47:22,059 [ hooks.py: 207]:\tstep: 62770\tsteps/sec: 0.96760\tloss: 0.23384\tlr:[7.3756523e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:47:30,192 [ hooks.py: 207]:\tstep: 62780\tsteps/sec: 1.27872\tloss: 0.40232\tlr:[7.3688634e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:47:37,004 [ hooks.py: 207]:\tstep: 62790\tsteps/sec: 1.37361\tloss: 0.68957\tlr:[7.3620713e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:47:46,644 [ hooks.py: 207]:\tstep: 62800\tsteps/sec: 1.01131\tloss: 0.29125\tlr:[7.3552824e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:47:55,101 [ hooks.py: 207]:\tstep: 62810\tsteps/sec: 1.28232\tloss: 0.09979\tlr:[7.3484903e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:48:01,873 [ hooks.py: 207]:\tstep: 62820\tsteps/sec: 1.34745\tloss: 0.13868\tlr:[7.3417013e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:48:10,075 [ hooks.py: 207]:\tstep: 62830\tsteps/sec: 1.31366\tloss: 0.52343\tlr:[7.334909e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:48:17,882 [ hooks.py: 207]:\tstep: 62840\tsteps/sec: 1.23668\tloss: 0.72245\tlr:[7.32812e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:48:28,255 [ hooks.py: 207]:\tstep: 62850\tsteps/sec: 0.94096\tloss: 0.50078\tlr:[7.3213278e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:48:37,488 [ hooks.py: 207]:\tstep: 62860\tsteps/sec: 1.11493\tloss: 0.22956\tlr:[7.31454e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:48:45,452 [ hooks.py: 207]:\tstep: 62870\tsteps/sec: 1.27627\tloss: 0.16074\tlr:[7.3077467e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:48:52,407 [ hooks.py: 207]:\tstep: 62880\tsteps/sec: 1.42331\tloss: 0.19838\tlr:[7.3009587e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:49:00,852 [ hooks.py: 207]:\tstep: 62890\tsteps/sec: 1.22528\tloss: 0.36973\tlr:[7.2941666e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:49:09,414 [ hooks.py: 207]:\tstep: 62900\tsteps/sec: 1.15672\tloss: 0.24687\tlr:[7.287377e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:49:17,922 [ hooks.py: 207]:\tstep: 62910\tsteps/sec: 1.13918\tloss: 0.26240\tlr:[7.2805856e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:49:26,280 [ hooks.py: 207]:\tstep: 62920\tsteps/sec: 1.20826\tloss: 1.35508\tlr:[7.273793e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:49:34,251 [ hooks.py: 207]:\tstep: 62930\tsteps/sec: 1.23670\tloss: 0.36777\tlr:[7.267005e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:49:45,083 [ hooks.py: 207]:\tstep: 62940\tsteps/sec: 0.96108\tloss: 0.19771\tlr:[7.260213e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:49:51,566 [ hooks.py: 207]:\tstep: 62950\tsteps/sec: 1.45475\tloss: 0.20112\tlr:[7.253424e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:00,848 [ hooks.py: 207]:\tstep: 62960\tsteps/sec: 1.05932\tloss: 0.42210\tlr:[7.246632e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:10,599 [ hooks.py: 207]:\tstep: 62970\tsteps/sec: 1.03989\tloss: 0.65159\tlr:[7.239842e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:19,594 [ hooks.py: 207]:\tstep: 62980\tsteps/sec: 1.15414\tloss: 0.43558\tlr:[7.233051e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:27,406 [ hooks.py: 207]:\tstep: 62990\tsteps/sec: 1.22966\tloss: 0.15914\tlr:[7.226262e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:35,747 [ hooks.py: 207]:\tstep: 63000\tsteps/sec: 1.24001\tloss: 0.70746\tlr:[7.2194703e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:35,748 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:35,752 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:35,752 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:50:35,752 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:35,752 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:50:35,752 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-19 15:58:37,769 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,770 [ trainer.py: 58]:\t{'acc': 0.7855421686746988, 'loss': 0.6381406978441354}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,771 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,771 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:58:37,772 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:58:37,772 [ trainer.py: 70]:\tacc\t0.7855421686746988\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:58:37,772 [ trainer.py: 70]:\tloss\t0.6381406978441354\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 15:58:37,772 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,772 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6381406978441354}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,772 [ exporter.py: 64]:\t[Best Exporter]: skip step 63000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,772 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6381406978441354}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,772 [ exporter.py: 104]:\t[Best Exporter]: skip step 63000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:37,773 [monitored_executor.py: 122]:\tsaving step 63000 to ./output/model_63000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:41,351 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_60000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:49,599 [ hooks.py: 207]:\tstep: 63010\tsteps/sec: 0.02024\tloss: 0.39277\tlr:[7.2126813e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:58:57,409 [ hooks.py: 207]:\tstep: 63020\tsteps/sec: 1.27474\tloss: 0.53070\tlr:[7.2058892e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:59:05,827 [ hooks.py: 207]:\tstep: 63030\tsteps/sec: 1.21267\tloss: 0.32906\tlr:[7.1990994e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:59:15,646 [ hooks.py: 207]:\tstep: 63040\tsteps/sec: 1.02379\tloss: 0.25452\tlr:[7.192309e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:59:24,665 [ hooks.py: 207]:\tstep: 63050\tsteps/sec: 1.12370\tloss: 0.71762\tlr:[7.1855193e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:59:33,169 [ hooks.py: 207]:\tstep: 63060\tsteps/sec: 1.13321\tloss: 0.10727\tlr:[7.178727e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:59:43,059 [ hooks.py: 207]:\tstep: 63070\tsteps/sec: 1.03894\tloss: 0.18139\tlr:[7.1719355e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:59:51,442 [ hooks.py: 207]:\tstep: 63080\tsteps/sec: 1.17265\tloss: 0.05662\tlr:[7.1651457e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 15:59:59,137 [ hooks.py: 207]:\tstep: 63090\tsteps/sec: 1.25210\tloss: 0.12545\tlr:[7.1583536e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:00:09,960 [ hooks.py: 207]:\tstep: 63100\tsteps/sec: 0.96598\tloss: 0.55868\tlr:[7.1515656e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:00:18,601 [ hooks.py: 207]:\tstep: 63110\tsteps/sec: 1.12152\tloss: 0.47127\tlr:[7.144773e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:00:25,470 [ hooks.py: 207]:\tstep: 63120\tsteps/sec: 1.50244\tloss: 0.32647\tlr:[7.1379845e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:00:36,646 [ hooks.py: 207]:\tstep: 63130\tsteps/sec: 0.84876\tloss: 0.32349\tlr:[7.1311924e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:00:46,038 [ hooks.py: 207]:\tstep: 63140\tsteps/sec: 1.13001\tloss: 0.10717\tlr:[7.1244035e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:00:52,882 [ hooks.py: 207]:\tstep: 63150\tsteps/sec: 1.36300\tloss: 0.08465\tlr:[7.117612e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:03,766 [ hooks.py: 207]:\tstep: 63160\tsteps/sec: 0.94689\tloss: 0.16663\tlr:[7.110822e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:13,093 [ hooks.py: 207]:\tstep: 63170\tsteps/sec: 1.07699\tloss: 0.59557\tlr:[7.104031e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:19,189 [ hooks.py: 207]:\tstep: 63180\tsteps/sec: 1.64479\tloss: 0.46495\tlr:[7.097241e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:27,595 [ hooks.py: 207]:\tstep: 63190\tsteps/sec: 1.18514\tloss: 0.18724\tlr:[7.09045e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:34,722 [ hooks.py: 207]:\tstep: 63200\tsteps/sec: 1.38231\tloss: 0.32479\tlr:[7.08366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:43,106 [ hooks.py: 207]:\tstep: 63210\tsteps/sec: 1.14707\tloss: 0.43287\tlr:[7.0768683e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:52,073 [ hooks.py: 207]:\tstep: 63220\tsteps/sec: 1.16905\tloss: 0.49432\tlr:[7.070076e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:01:58,909 [ hooks.py: 207]:\tstep: 63230\tsteps/sec: 1.46609\tloss: 0.59536\tlr:[7.063288e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:02:06,793 [ hooks.py: 207]:\tstep: 63240\tsteps/sec: 1.20852\tloss: 0.15614\tlr:[7.056495e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:02:15,200 [ hooks.py: 207]:\tstep: 63250\tsteps/sec: 1.28850\tloss: 0.17268\tlr:[7.049707e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:02:24,364 [ hooks.py: 207]:\tstep: 63260\tsteps/sec: 1.08277\tloss: 0.58079\tlr:[7.042915e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:02:31,337 [ hooks.py: 207]:\tstep: 63270\tsteps/sec: 1.43924\tloss: 0.21530\tlr:[7.036127e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:02:40,166 [ hooks.py: 207]:\tstep: 63280\tsteps/sec: 1.15840\tloss: 0.19085\tlr:[7.0293336e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:02:48,083 [ hooks.py: 207]:\tstep: 63290\tsteps/sec: 1.21032\tloss: 0.25235\tlr:[7.022546e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:02:56,031 [ hooks.py: 207]:\tstep: 63300\tsteps/sec: 1.28792\tloss: 0.58152\tlr:[7.0157525e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:03:02,976 [ hooks.py: 207]:\tstep: 63310\tsteps/sec: 1.32904\tloss: 0.08231\tlr:[7.0089645e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:03:10,325 [ hooks.py: 207]:\tstep: 63320\tsteps/sec: 1.40587\tloss: 0.55186\tlr:[7.0021733e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:03:18,900 [ hooks.py: 207]:\tstep: 63330\tsteps/sec: 1.17859\tloss: 0.27034\tlr:[6.9953835e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:03:27,179 [ hooks.py: 207]:\tstep: 63340\tsteps/sec: 1.20053\tloss: 0.51611\tlr:[6.9885914e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:03:34,927 [ hooks.py: 207]:\tstep: 63350\tsteps/sec: 1.32991\tloss: 0.68917\tlr:[6.981802e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:03:44,593 [ hooks.py: 207]:\tstep: 63360\tsteps/sec: 1.03223\tloss: 0.05129\tlr:[6.97501e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:03:55,068 [ hooks.py: 207]:\tstep: 63370\tsteps/sec: 0.93574\tloss: 1.11105\tlr:[6.9682187e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:04:04,093 [ hooks.py: 207]:\tstep: 63380\tsteps/sec: 1.08093\tloss: 0.27360\tlr:[6.961429e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:04:13,711 [ hooks.py: 207]:\tstep: 63390\tsteps/sec: 1.06444\tloss: 0.38453\tlr:[6.9546377e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:04:22,078 [ hooks.py: 207]:\tstep: 63400\tsteps/sec: 1.20232\tloss: 0.21754\tlr:[6.9478488e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:04:31,713 [ hooks.py: 207]:\tstep: 63410\tsteps/sec: 1.01393\tloss: 0.21907\tlr:[6.9410567e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:04:41,449 [ hooks.py: 207]:\tstep: 63420\tsteps/sec: 1.13655\tloss: 0.40273\tlr:[6.9342673e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:04:51,466 [ hooks.py: 207]:\tstep: 63430\tsteps/sec: 0.93981\tloss: 0.43359\tlr:[6.927476e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:04:59,968 [ hooks.py: 207]:\tstep: 63440\tsteps/sec: 1.20108\tloss: 0.39042\tlr:[6.920687e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:05:07,049 [ hooks.py: 207]:\tstep: 63450\tsteps/sec: 1.35432\tloss: 0.21983\tlr:[6.913894e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:05:14,111 [ hooks.py: 207]:\tstep: 63460\tsteps/sec: 1.44999\tloss: 0.43013\tlr:[6.907106e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:05:22,382 [ hooks.py: 207]:\tstep: 63470\tsteps/sec: 1.17993\tloss: 0.44908\tlr:[6.900315e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:05:31,137 [ hooks.py: 207]:\tstep: 63480\tsteps/sec: 1.18603\tloss: 0.54312\tlr:[6.893525e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:05:37,677 [ hooks.py: 207]:\tstep: 63490\tsteps/sec: 1.41602\tloss: 0.26764\tlr:[6.886733e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:05:47,505 [ hooks.py: 207]:\tstep: 63500\tsteps/sec: 1.05527\tloss: 0.16155\tlr:[6.879944e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:05:53,872 [ hooks.py: 207]:\tstep: 63510\tsteps/sec: 1.48854\tloss: 0.29138\tlr:[6.8731515e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:06:05,097 [ hooks.py: 207]:\tstep: 63520\tsteps/sec: 0.93970\tloss: 0.07716\tlr:[6.8663603e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 16:06:12,270 [ hooks.py: 207]:\tstep: 63530\tsteps/sec: 1.34703\tloss: 0.47536\tlr:[6.8595714e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:06:18,568 [ hooks.py: 207]:\tstep: 63540\tsteps/sec: 1.59749\tloss: 0.33341\tlr:[6.852779e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:06:25,636 [ hooks.py: 207]:\tstep: 63550\tsteps/sec: 1.43217\tloss: 0.39516\tlr:[6.8459904e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:06:33,835 [ hooks.py: 207]:\tstep: 63560\tsteps/sec: 1.20128\tloss: 0.59586\tlr:[6.839199e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:06:40,066 [ hooks.py: 207]:\tstep: 63570\tsteps/sec: 1.66774\tloss: 0.43919\tlr:[6.8324093e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:06:48,323 [ hooks.py: 207]:\tstep: 63580\tsteps/sec: 1.17915\tloss: 0.45378\tlr:[6.8256168e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:06:56,469 [ hooks.py: 207]:\tstep: 63590\tsteps/sec: 1.20415\tloss: 0.15302\tlr:[6.8188288e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:07:06,468 [ hooks.py: 207]:\tstep: 63600\tsteps/sec: 1.02171\tloss: 0.12257\tlr:[6.8120366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:07:12,882 [ hooks.py: 207]:\tstep: 63610\tsteps/sec: 1.45291\tloss: 0.55761\tlr:[6.8052477e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:07:22,396 [ hooks.py: 207]:\tstep: 63620\tsteps/sec: 1.08722\tloss: 0.22781\tlr:[6.7984556e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:07:31,260 [ hooks.py: 207]:\tstep: 63630\tsteps/sec: 1.17023\tloss: 0.33892\tlr:[6.7916667e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:07:38,128 [ hooks.py: 207]:\tstep: 63640\tsteps/sec: 1.40197\tloss: 0.23890\tlr:[6.7848755e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:07:48,268 [ hooks.py: 207]:\tstep: 63650\tsteps/sec: 0.95449\tloss: 0.20942\tlr:[6.7780857e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:07:56,437 [ hooks.py: 207]:\tstep: 63660\tsteps/sec: 1.23043\tloss: 0.60871\tlr:[6.771293e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:08:05,832 [ hooks.py: 207]:\tstep: 63670\tsteps/sec: 1.13060\tloss: 0.17208\tlr:[6.764502e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:08:15,437 [ hooks.py: 207]:\tstep: 63680\tsteps/sec: 1.04572\tloss: 0.47084\tlr:[6.757713e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:08:23,068 [ hooks.py: 207]:\tstep: 63690\tsteps/sec: 1.24659\tloss: 0.16236\tlr:[6.750921e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:08:32,096 [ hooks.py: 207]:\tstep: 63700\tsteps/sec: 1.11920\tloss: 0.58826\tlr:[6.744132e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:08:41,927 [ hooks.py: 207]:\tstep: 63710\tsteps/sec: 1.09549\tloss: 0.15376\tlr:[6.7373394e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:08:50,366 [ hooks.py: 207]:\tstep: 63720\tsteps/sec: 1.12729\tloss: 0.93917\tlr:[6.730551e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:08:57,601 [ hooks.py: 207]:\tstep: 63730\tsteps/sec: 1.32265\tloss: 0.29514\tlr:[6.7237593e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:09:06,475 [ hooks.py: 207]:\tstep: 63740\tsteps/sec: 1.15454\tloss: 0.75315\tlr:[6.7169703e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:09:16,223 [ hooks.py: 207]:\tstep: 63750\tsteps/sec: 1.02440\tloss: 0.83745\tlr:[6.7101782e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:09:24,954 [ hooks.py: 207]:\tstep: 63760\tsteps/sec: 1.15621\tloss: 0.60176\tlr:[6.7033893e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:09:32,509 [ hooks.py: 207]:\tstep: 63770\tsteps/sec: 1.22920\tloss: 0.12732\tlr:[6.696597e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:09:41,406 [ hooks.py: 207]:\tstep: 63780\tsteps/sec: 1.29106\tloss: 0.17743\tlr:[6.6898083e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:09:48,367 [ hooks.py: 207]:\tstep: 63790\tsteps/sec: 1.23977\tloss: 0.43547\tlr:[6.6830157e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:09:57,146 [ hooks.py: 207]:\tstep: 63800\tsteps/sec: 1.18071\tloss: 0.07353\tlr:[6.6762245e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:10:03,876 [ hooks.py: 207]:\tstep: 63810\tsteps/sec: 1.41450\tloss: 0.33681\tlr:[6.6694356e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:10:12,538 [ hooks.py: 207]:\tstep: 63820\tsteps/sec: 1.18509\tloss: 0.17615\tlr:[6.6626435e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:10:21,397 [ hooks.py: 207]:\tstep: 63830\tsteps/sec: 1.17437\tloss: 0.51637\tlr:[6.6558555e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:10:28,944 [ hooks.py: 207]:\tstep: 63840\tsteps/sec: 1.28140\tloss: 0.13252\tlr:[6.6490634e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:10:38,640 [ hooks.py: 207]:\tstep: 63850\tsteps/sec: 1.03484\tloss: 0.12768\tlr:[6.6422735e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:10:49,357 [ hooks.py: 207]:\tstep: 63860\tsteps/sec: 0.93727\tloss: 0.68982\tlr:[6.635481e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:10:59,681 [ hooks.py: 207]:\tstep: 63870\tsteps/sec: 0.98042\tloss: 0.17334\tlr:[6.6286925e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:11:07,839 [ hooks.py: 207]:\tstep: 63880\tsteps/sec: 1.19473\tloss: 0.21185\tlr:[6.621901e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:11:16,120 [ hooks.py: 207]:\tstep: 63890\tsteps/sec: 1.24163\tloss: 0.12872\tlr:[6.615112e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:11:25,527 [ hooks.py: 207]:\tstep: 63900\tsteps/sec: 1.07068\tloss: 0.05218\tlr:[6.60832e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:11:35,107 [ hooks.py: 207]:\tstep: 63910\tsteps/sec: 1.01413\tloss: 0.69691\tlr:[6.60153e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:11:43,561 [ hooks.py: 207]:\tstep: 63920\tsteps/sec: 1.19414\tloss: 0.37902\tlr:[6.5947397e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:11:53,159 [ hooks.py: 207]:\tstep: 63930\tsteps/sec: 1.02827\tloss: 0.08037\tlr:[6.58795e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:01,239 [ hooks.py: 207]:\tstep: 63940\tsteps/sec: 1.28974\tloss: 0.24487\tlr:[6.5811582e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:08,359 [ hooks.py: 207]:\tstep: 63950\tsteps/sec: 1.28578\tloss: 0.36450\tlr:[6.574366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:17,728 [ hooks.py: 207]:\tstep: 63960\tsteps/sec: 1.06864\tloss: 0.44580\tlr:[6.567577e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:27,094 [ hooks.py: 207]:\tstep: 63970\tsteps/sec: 1.11432\tloss: 0.35555\tlr:[6.560785e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:35,322 [ hooks.py: 207]:\tstep: 63980\tsteps/sec: 1.25113\tloss: 0.32052\tlr:[6.5539953e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:44,199 [ hooks.py: 207]:\tstep: 63990\tsteps/sec: 1.04755\tloss: 0.19930\tlr:[6.5472036e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:53,606 [ hooks.py: 207]:\tstep: 64000\tsteps/sec: 1.12854\tloss: 0.54475\tlr:[6.5404147e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:53,606 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:53,606 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:53,607 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:12:53,607 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:53,607 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:12:53,607 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:22:26,946 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,948 [ trainer.py: 58]:\t{'acc': 0.7811244979919679, 'loss': 0.645810658350969}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,948 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,949 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:22:26,950 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:22:26,950 [ trainer.py: 70]:\tacc\t0.7811244979919679\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:22:26,950 [ trainer.py: 70]:\tloss\t0.645810658350969\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:22:26,950 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,951 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7811244979919679, 'loss': 0.645810658350969}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,951 [ exporter.py: 64]:\t[Best Exporter]: skip step 64000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,951 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7811244979919679, 'loss': 0.645810658350969}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,951 [ exporter.py: 104]:\t[Best Exporter]: skip step 64000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:26,952 [monitored_executor.py: 122]:\tsaving step 64000 to ./output/model_64000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 16:22:31,181 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_61000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:40,541 [ hooks.py: 207]:\tstep: 64010\tsteps/sec: 0.01706\tloss: 0.19184\tlr:[6.5336226e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:22:53,649 [ hooks.py: 207]:\tstep: 64020\tsteps/sec: 0.72077\tloss: 0.25320\tlr:[6.526835e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:23:02,807 [ hooks.py: 207]:\tstep: 64030\tsteps/sec: 1.18589\tloss: 0.28793\tlr:[6.5200434e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:23:12,186 [ hooks.py: 207]:\tstep: 64040\tsteps/sec: 1.00144\tloss: 0.05996\tlr:[6.513254e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:23:24,318 [ hooks.py: 207]:\tstep: 64050\tsteps/sec: 0.83280\tloss: 0.06053\tlr:[6.5064614e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:23:35,270 [ hooks.py: 207]:\tstep: 64060\tsteps/sec: 0.95182\tloss: 0.20706\tlr:[6.4996725e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:23:44,033 [ hooks.py: 207]:\tstep: 64070\tsteps/sec: 1.09986\tloss: 0.34036\tlr:[6.4928813e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:23:54,485 [ hooks.py: 207]:\tstep: 64080\tsteps/sec: 0.95729\tloss: 0.45153\tlr:[6.4860915e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:24:06,951 [ hooks.py: 207]:\tstep: 64090\tsteps/sec: 1.02648\tloss: 0.24059\tlr:[6.479299e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:24:16,100 [ hooks.py: 207]:\tstep: 64100\tsteps/sec: 0.83918\tloss: 0.41655\tlr:[6.4725077e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:24:26,538 [ hooks.py: 207]:\tstep: 64110\tsteps/sec: 1.00726\tloss: 0.09971\tlr:[6.465719e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:24:37,044 [ hooks.py: 207]:\tstep: 64120\tsteps/sec: 0.92074\tloss: 0.11499\tlr:[6.4589267e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:24:48,313 [ hooks.py: 207]:\tstep: 64130\tsteps/sec: 0.87223\tloss: 0.11634\tlr:[6.4521378e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:24:58,434 [ hooks.py: 207]:\tstep: 64140\tsteps/sec: 0.95535\tloss: 0.33227\tlr:[6.4453457e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:25:08,701 [ hooks.py: 207]:\tstep: 64150\tsteps/sec: 1.02297\tloss: 0.66613\tlr:[6.4385567e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:25:21,532 [ hooks.py: 207]:\tstep: 64160\tsteps/sec: 0.79577\tloss: 0.05915\tlr:[6.4317655e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:25:31,075 [ hooks.py: 207]:\tstep: 64170\tsteps/sec: 1.06995\tloss: 0.48529\tlr:[6.424976e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:25:41,860 [ hooks.py: 207]:\tstep: 64180\tsteps/sec: 0.91338\tloss: 0.40108\tlr:[6.418184e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:25:53,170 [ hooks.py: 207]:\tstep: 64190\tsteps/sec: 0.96658\tloss: 0.27233\tlr:[6.411395e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:26:03,685 [ hooks.py: 207]:\tstep: 64200\tsteps/sec: 0.83756\tloss: 0.27706\tlr:[6.404603e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:26:13,090 [ hooks.py: 207]:\tstep: 64210\tsteps/sec: 1.08289\tloss: 0.22136\tlr:[6.397813e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:26:25,902 [ hooks.py: 207]:\tstep: 64220\tsteps/sec: 0.79286\tloss: 0.29544\tlr:[6.391022e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:26:36,394 [ hooks.py: 207]:\tstep: 64230\tsteps/sec: 0.93803\tloss: 0.16973\tlr:[6.3842326e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:26:44,520 [ hooks.py: 207]:\tstep: 64240\tsteps/sec: 1.20788\tloss: 0.18005\tlr:[6.3774405e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:26:52,038 [ hooks.py: 207]:\tstep: 64250\tsteps/sec: 1.37186\tloss: 0.84436\tlr:[6.370648e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:27:01,876 [ hooks.py: 207]:\tstep: 64260\tsteps/sec: 1.07211\tloss: 0.35512\tlr:[6.3638604e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:27:12,228 [ hooks.py: 207]:\tstep: 64270\tsteps/sec: 0.94202\tloss: 0.15439\tlr:[6.357068e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:27:23,054 [ hooks.py: 207]:\tstep: 64280\tsteps/sec: 0.87466\tloss: 0.50245\tlr:[6.3502794e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:27:34,587 [ hooks.py: 207]:\tstep: 64290\tsteps/sec: 0.88855\tloss: 0.19136\tlr:[6.343487e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:27:43,318 [ hooks.py: 207]:\tstep: 64300\tsteps/sec: 1.14869\tloss: 0.04967\tlr:[6.3366983e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:27:54,722 [ hooks.py: 207]:\tstep: 64310\tsteps/sec: 0.88329\tloss: 0.29666\tlr:[6.3299067e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:28:03,868 [ hooks.py: 207]:\tstep: 64320\tsteps/sec: 1.06454\tloss: 0.29381\tlr:[6.323117e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:28:13,754 [ hooks.py: 207]:\tstep: 64330\tsteps/sec: 0.94232\tloss: 0.17196\tlr:[6.3163257e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:28:22,717 [ hooks.py: 207]:\tstep: 64340\tsteps/sec: 1.15626\tloss: 0.07257\tlr:[6.309536e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:28:29,543 [ hooks.py: 207]:\tstep: 64350\tsteps/sec: 1.45298\tloss: 0.27039\tlr:[6.3027446e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:28:38,410 [ hooks.py: 207]:\tstep: 64360\tsteps/sec: 1.14190\tloss: 0.44060\tlr:[6.2959557e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:28:48,039 [ hooks.py: 207]:\tstep: 64370\tsteps/sec: 1.03359\tloss: 0.21909\tlr:[6.289163e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:28:54,675 [ hooks.py: 207]:\tstep: 64380\tsteps/sec: 1.50862\tloss: 0.63834\tlr:[6.2823747e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:29:05,789 [ hooks.py: 207]:\tstep: 64390\tsteps/sec: 0.92512\tloss: 0.19988\tlr:[6.275582e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:29:13,019 [ hooks.py: 207]:\tstep: 64400\tsteps/sec: 1.30798\tloss: 0.44378\tlr:[6.268791e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:29:20,728 [ hooks.py: 207]:\tstep: 64410\tsteps/sec: 1.31394\tloss: 0.10854\tlr:[6.262002e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:29:28,477 [ hooks.py: 207]:\tstep: 64420\tsteps/sec: 1.25849\tloss: 0.35300\tlr:[6.2552094e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:29:36,477 [ hooks.py: 207]:\tstep: 64430\tsteps/sec: 1.32706\tloss: 0.46133\tlr:[6.2484205e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:29:44,115 [ hooks.py: 207]:\tstep: 64440\tsteps/sec: 1.26895\tloss: 0.17579\tlr:[6.241629e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:29:55,277 [ hooks.py: 207]:\tstep: 64450\tsteps/sec: 0.90822\tloss: 0.44730\tlr:[6.23484e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:30:03,635 [ hooks.py: 207]:\tstep: 64460\tsteps/sec: 1.13034\tloss: 0.31040\tlr:[6.2280474e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:30:13,186 [ hooks.py: 207]:\tstep: 64470\tsteps/sec: 1.08700\tloss: 0.39547\tlr:[6.221259e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:30:21,872 [ hooks.py: 207]:\tstep: 64480\tsteps/sec: 1.18818\tloss: 0.30765\tlr:[6.214467e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:30:30,407 [ hooks.py: 207]:\tstep: 64490\tsteps/sec: 1.16940\tloss: 0.14312\tlr:[6.207678e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:30:37,672 [ hooks.py: 207]:\tstep: 64500\tsteps/sec: 1.32824\tloss: 0.31482\tlr:[6.200886e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:30:44,588 [ hooks.py: 207]:\tstep: 64510\tsteps/sec: 1.44017\tloss: 0.24403\tlr:[6.194097e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:30:53,037 [ hooks.py: 207]:\tstep: 64520\tsteps/sec: 1.18239\tloss: 0.55305\tlr:[6.1873047e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:02,842 [ hooks.py: 207]:\tstep: 64530\tsteps/sec: 1.04114\tloss: 0.47132\tlr:[6.1805163e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:09,941 [ hooks.py: 207]:\tstep: 64540\tsteps/sec: 1.38995\tloss: 0.74266\tlr:[6.1737246e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:19,424 [ hooks.py: 207]:\tstep: 64550\tsteps/sec: 1.05276\tloss: 0.36198\tlr:[6.166932e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:27,560 [ hooks.py: 207]:\tstep: 64560\tsteps/sec: 1.16829\tloss: 0.56109\tlr:[6.160143e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:35,555 [ hooks.py: 207]:\tstep: 64570\tsteps/sec: 1.35490\tloss: 0.34566\tlr:[6.153351e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:42,800 [ hooks.py: 207]:\tstep: 64580\tsteps/sec: 1.26713\tloss: 0.22911\tlr:[6.1465626e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:51,672 [ hooks.py: 207]:\tstep: 64590\tsteps/sec: 1.23828\tloss: 0.27531\tlr:[6.1397695e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:31:59,801 [ hooks.py: 207]:\tstep: 64600\tsteps/sec: 1.13332\tloss: 0.21581\tlr:[6.1329815e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:32:10,704 [ hooks.py: 207]:\tstep: 64610\tsteps/sec: 0.96308\tloss: 0.10539\tlr:[6.1261894e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:32:19,009 [ hooks.py: 207]:\tstep: 64620\tsteps/sec: 1.17393\tloss: 0.61727\tlr:[6.1194e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:32:26,906 [ hooks.py: 207]:\tstep: 64630\tsteps/sec: 1.31254\tloss: 0.16558\tlr:[6.112609e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:32:35,910 [ hooks.py: 207]:\tstep: 64640\tsteps/sec: 1.11509\tloss: 0.18538\tlr:[6.1058195e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 16:32:43,029 [ hooks.py: 207]:\tstep: 64650\tsteps/sec: 1.34009\tloss: 0.47543\tlr:[6.0990274e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:32:51,669 [ hooks.py: 207]:\tstep: 64660\tsteps/sec: 1.21822\tloss: 0.13184\tlr:[6.0922384e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:32:58,536 [ hooks.py: 207]:\tstep: 64670\tsteps/sec: 1.32077\tloss: 0.10852\tlr:[6.085447e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:33:08,074 [ hooks.py: 207]:\tstep: 64680\tsteps/sec: 1.07531\tloss: 0.74292\tlr:[6.0786583e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:33:16,384 [ hooks.py: 207]:\tstep: 64690\tsteps/sec: 1.25224\tloss: 0.34294\tlr:[6.071866e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:33:28,311 [ hooks.py: 207]:\tstep: 64700\tsteps/sec: 0.81849\tloss: 0.20825\tlr:[6.0650736e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:33:37,933 [ hooks.py: 207]:\tstep: 64710\tsteps/sec: 1.04445\tloss: 0.46196\tlr:[6.0582847e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:33:45,462 [ hooks.py: 207]:\tstep: 64720\tsteps/sec: 1.36115\tloss: 0.11521\tlr:[6.0514926e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:33:53,426 [ hooks.py: 207]:\tstep: 64730\tsteps/sec: 1.20893\tloss: 0.10382\tlr:[6.044704e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:34:02,045 [ hooks.py: 207]:\tstep: 64740\tsteps/sec: 1.15933\tloss: 0.38658\tlr:[6.037912e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:34:12,085 [ hooks.py: 207]:\tstep: 64750\tsteps/sec: 1.03909\tloss: 0.64769\tlr:[6.0311236e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:34:18,653 [ hooks.py: 207]:\tstep: 64760\tsteps/sec: 1.43681\tloss: 0.38069\tlr:[6.024331e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:34:27,449 [ hooks.py: 207]:\tstep: 64770\tsteps/sec: 1.09624\tloss: 0.42798\tlr:[6.017542e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:34:36,128 [ hooks.py: 207]:\tstep: 64780\tsteps/sec: 1.19269\tloss: 0.47181\tlr:[6.01075e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:34:46,766 [ hooks.py: 207]:\tstep: 64790\tsteps/sec: 0.97925\tloss: 0.15826\tlr:[6.003961e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:34:55,222 [ hooks.py: 207]:\tstep: 64800\tsteps/sec: 1.18653\tloss: 0.14964\tlr:[5.9971694e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:35:05,253 [ hooks.py: 207]:\tstep: 64810\tsteps/sec: 0.97020\tloss: 0.44286\tlr:[5.9903805e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:35:13,946 [ hooks.py: 207]:\tstep: 64820\tsteps/sec: 1.14959\tloss: 0.12066\tlr:[5.983588e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:35:23,572 [ hooks.py: 207]:\tstep: 64830\tsteps/sec: 1.03746\tloss: 0.30527\tlr:[5.9767963e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:35:30,734 [ hooks.py: 207]:\tstep: 64840\tsteps/sec: 1.42726\tloss: 0.19397\tlr:[5.970007e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:35:39,212 [ hooks.py: 207]:\tstep: 64850\tsteps/sec: 1.11625\tloss: 0.14804\tlr:[5.9632152e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:35:47,509 [ hooks.py: 207]:\tstep: 64860\tsteps/sec: 1.24739\tloss: 0.24640\tlr:[5.956427e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:35:55,772 [ hooks.py: 207]:\tstep: 64870\tsteps/sec: 1.29413\tloss: 0.22642\tlr:[5.949635e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:36:02,729 [ hooks.py: 207]:\tstep: 64880\tsteps/sec: 1.35292\tloss: 0.51028\tlr:[5.9428457e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:36:12,715 [ hooks.py: 207]:\tstep: 64890\tsteps/sec: 0.99568\tloss: 0.10980\tlr:[5.936053e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:36:18,310 [ hooks.py: 207]:\tstep: 64900\tsteps/sec: 1.66906\tloss: 0.68356\tlr:[5.9292647e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:36:28,200 [ hooks.py: 207]:\tstep: 64910\tsteps/sec: 1.03724\tloss: 0.54765\tlr:[5.922472e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:36:36,770 [ hooks.py: 207]:\tstep: 64920\tsteps/sec: 1.16103\tloss: 0.26710\tlr:[5.9156837e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:36:47,522 [ hooks.py: 207]:\tstep: 64930\tsteps/sec: 0.95202\tloss: 0.46830\tlr:[5.9088916e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:36:56,476 [ hooks.py: 207]:\tstep: 64940\tsteps/sec: 1.19279\tloss: 0.63734\tlr:[5.9021027e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:04,776 [ hooks.py: 207]:\tstep: 64950\tsteps/sec: 1.08485\tloss: 0.15143\tlr:[5.8953115e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:13,460 [ hooks.py: 207]:\tstep: 64960\tsteps/sec: 1.18576\tloss: 0.49951\tlr:[5.8885216e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:21,926 [ hooks.py: 207]:\tstep: 64970\tsteps/sec: 1.21416\tloss: 0.48908\tlr:[5.8817304e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:31,785 [ hooks.py: 207]:\tstep: 64980\tsteps/sec: 1.01171\tloss: 0.75993\tlr:[5.8749374e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:41,700 [ hooks.py: 207]:\tstep: 64990\tsteps/sec: 1.02494\tloss: 0.69253\tlr:[5.8681494e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:49,053 [ hooks.py: 207]:\tstep: 65000\tsteps/sec: 1.24591\tloss: 0.11480\tlr:[5.8613573e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:49,055 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:49,055 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:49,055 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:37:49,055 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:49,055 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:37:49,055 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:45:58,117 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,118 [ trainer.py: 58]:\t{'acc': 0.791566265060241, 'loss': 0.6337882993886104}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,119 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,119 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:45:58,119 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:45:58,119 [ trainer.py: 70]:\tacc\t0.791566265060241\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:45:58,121 [ trainer.py: 70]:\tloss\t0.6337882993886104\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 16:45:58,121 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,121 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.791566265060241, 'loss': 0.6337882993886104}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,121 [ exporter.py: 64]:\t[Best Exporter]: skip step 65000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,121 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.791566265060241, 'loss': 0.6337882993886104}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,121 [ exporter.py: 104]:\t[Best Exporter]: skip step 65000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:45:58,122 [monitored_executor.py: 122]:\tsaving step 65000 to ./output/model_65000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:46:01,269 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_62000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:46:09,812 [ hooks.py: 207]:\tstep: 65010\tsteps/sec: 0.01999\tloss: 0.25345\tlr:[5.854568e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:46:18,263 [ hooks.py: 207]:\tstep: 65020\tsteps/sec: 1.16637\tloss: 0.15067\tlr:[5.8477763e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:46:28,041 [ hooks.py: 207]:\tstep: 65030\tsteps/sec: 1.03259\tloss: 0.15582\tlr:[5.8409873e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:46:36,114 [ hooks.py: 207]:\tstep: 65040\tsteps/sec: 1.22517\tloss: 0.11347\tlr:[5.8341957e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:46:46,100 [ hooks.py: 207]:\tstep: 65050\tsteps/sec: 0.99809\tloss: 0.39042\tlr:[5.827406e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:46:55,146 [ hooks.py: 207]:\tstep: 65060\tsteps/sec: 1.12584\tloss: 0.41773\tlr:[5.8206138e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:47:02,369 [ hooks.py: 207]:\tstep: 65070\tsteps/sec: 1.32102\tloss: 0.03816\tlr:[5.813825e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:47:13,879 [ hooks.py: 207]:\tstep: 65080\tsteps/sec: 0.89191\tloss: 0.18070\tlr:[5.807033e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:47:21,146 [ hooks.py: 207]:\tstep: 65090\tsteps/sec: 1.42923\tloss: 0.18847\tlr:[5.8002447e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:47:27,698 [ hooks.py: 207]:\tstep: 65100\tsteps/sec: 1.38655\tloss: 0.09467\tlr:[5.7934526e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 16:47:37,275 [ hooks.py: 207]:\tstep: 65110\tsteps/sec: 1.10736\tloss: 0.10911\tlr:[5.7866637e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:47:45,927 [ hooks.py: 207]:\tstep: 65120\tsteps/sec: 1.16428\tloss: 0.08541\tlr:[5.779871e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:47:54,016 [ hooks.py: 207]:\tstep: 65130\tsteps/sec: 1.16103\tloss: 0.46092\tlr:[5.77308e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:48:03,703 [ hooks.py: 207]:\tstep: 65140\tsteps/sec: 1.06463\tloss: 0.38660\tlr:[5.766291e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:48:11,913 [ hooks.py: 207]:\tstep: 65150\tsteps/sec: 1.24660\tloss: 0.54374\tlr:[5.759499e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:48:21,283 [ hooks.py: 207]:\tstep: 65160\tsteps/sec: 1.07599\tloss: 0.24417\tlr:[5.7527095e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:48:30,914 [ hooks.py: 207]:\tstep: 65170\tsteps/sec: 1.03182\tloss: 0.30809\tlr:[5.7459174e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:48:39,748 [ hooks.py: 207]:\tstep: 65180\tsteps/sec: 1.12107\tloss: 0.42272\tlr:[5.7391294e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:48:48,878 [ hooks.py: 207]:\tstep: 65190\tsteps/sec: 1.07662\tloss: 1.18702\tlr:[5.732337e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:48:57,671 [ hooks.py: 207]:\tstep: 65200\tsteps/sec: 1.14932\tloss: 0.66926\tlr:[5.7255475e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:49:08,710 [ hooks.py: 207]:\tstep: 65210\tsteps/sec: 0.89694\tloss: 0.47132\tlr:[5.718756e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:49:16,800 [ hooks.py: 207]:\tstep: 65220\tsteps/sec: 1.28516\tloss: 0.39726\tlr:[5.7119664e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:49:24,598 [ hooks.py: 207]:\tstep: 65230\tsteps/sec: 1.24328\tloss: 0.31766\tlr:[5.705175e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:49:33,398 [ hooks.py: 207]:\tstep: 65240\tsteps/sec: 1.08147\tloss: 0.11583\tlr:[5.698386e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:49:41,940 [ hooks.py: 207]:\tstep: 65250\tsteps/sec: 1.27159\tloss: 0.25090\tlr:[5.6915937e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:49:51,659 [ hooks.py: 207]:\tstep: 65260\tsteps/sec: 1.05253\tloss: 0.32676\tlr:[5.6848053e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:02,408 [ hooks.py: 207]:\tstep: 65270\tsteps/sec: 0.90609\tloss: 0.37156\tlr:[5.6780127e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:09,125 [ hooks.py: 207]:\tstep: 65280\tsteps/sec: 1.45368\tloss: 0.02301\tlr:[5.6712215e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:17,268 [ hooks.py: 207]:\tstep: 65290\tsteps/sec: 1.20836\tloss: 0.51616\tlr:[5.6644326e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:25,473 [ hooks.py: 207]:\tstep: 65300\tsteps/sec: 1.23878\tloss: 0.23604\tlr:[5.6576405e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:32,649 [ hooks.py: 207]:\tstep: 65310\tsteps/sec: 1.42273\tloss: 0.59263\tlr:[5.650851e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:41,689 [ hooks.py: 207]:\tstep: 65320\tsteps/sec: 1.10218\tloss: 0.27899\tlr:[5.6440595e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:49,724 [ hooks.py: 207]:\tstep: 65330\tsteps/sec: 1.23929\tloss: 0.56800\tlr:[5.6372705e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:50:56,833 [ hooks.py: 207]:\tstep: 65340\tsteps/sec: 1.47431\tloss: 0.24518\tlr:[5.6304784e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:51:07,562 [ hooks.py: 207]:\tstep: 65350\tsteps/sec: 0.91485\tloss: 0.49841\tlr:[5.623689e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:51:14,921 [ hooks.py: 207]:\tstep: 65360\tsteps/sec: 1.32718\tloss: 0.15249\tlr:[5.6168974e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:51:23,075 [ hooks.py: 207]:\tstep: 65370\tsteps/sec: 1.17441\tloss: 0.29427\tlr:[5.610109e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:51:32,939 [ hooks.py: 207]:\tstep: 65380\tsteps/sec: 1.04536\tloss: 0.42766\tlr:[5.6033164e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:51:43,516 [ hooks.py: 207]:\tstep: 65390\tsteps/sec: 0.94713\tloss: 0.51838\tlr:[5.5965284e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:51:52,110 [ hooks.py: 207]:\tstep: 65400\tsteps/sec: 1.23346\tloss: 0.22527\tlr:[5.5897367e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:52:01,709 [ hooks.py: 207]:\tstep: 65410\tsteps/sec: 1.02208\tloss: 0.21760\tlr:[5.5829473e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:52:08,521 [ hooks.py: 207]:\tstep: 65420\tsteps/sec: 1.40941\tloss: 0.05195\tlr:[5.576155e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:52:15,826 [ hooks.py: 207]:\tstep: 65430\tsteps/sec: 1.38160\tloss: 0.45335\tlr:[5.569363e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:52:25,323 [ hooks.py: 207]:\tstep: 65440\tsteps/sec: 1.04630\tloss: 0.43351\tlr:[5.5625737e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:52:33,870 [ hooks.py: 207]:\tstep: 65450\tsteps/sec: 1.16499\tloss: 0.09099\tlr:[5.555782e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:52:41,969 [ hooks.py: 207]:\tstep: 65460\tsteps/sec: 1.29950\tloss: 0.30909\tlr:[5.548993e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:52:49,611 [ hooks.py: 207]:\tstep: 65470\tsteps/sec: 1.19785\tloss: 0.37516\tlr:[5.5422006e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:53:01,753 [ hooks.py: 207]:\tstep: 65480\tsteps/sec: 0.86336\tloss: 0.48213\tlr:[5.5354117e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:53:09,843 [ hooks.py: 207]:\tstep: 65490\tsteps/sec: 1.20249\tloss: 0.06004\tlr:[5.52862e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:53:18,826 [ hooks.py: 207]:\tstep: 65500\tsteps/sec: 1.07087\tloss: 0.08931\tlr:[5.521831e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:53:27,236 [ hooks.py: 207]:\tstep: 65510\tsteps/sec: 1.22311\tloss: 0.50536\tlr:[5.515039e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:53:36,016 [ hooks.py: 207]:\tstep: 65520\tsteps/sec: 1.16733\tloss: 0.15537\tlr:[5.50825e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:53:44,420 [ hooks.py: 207]:\tstep: 65530\tsteps/sec: 1.17159\tloss: 0.56714\tlr:[5.5014584e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:53:53,966 [ hooks.py: 207]:\tstep: 65540\tsteps/sec: 1.03718\tloss: 0.29617\tlr:[5.4946695e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:54:03,091 [ hooks.py: 207]:\tstep: 65550\tsteps/sec: 1.08683\tloss: 0.29062\tlr:[5.4878774e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:54:11,468 [ hooks.py: 207]:\tstep: 65560\tsteps/sec: 1.22217\tloss: 0.75829\tlr:[5.481088e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:54:19,996 [ hooks.py: 207]:\tstep: 65570\tsteps/sec: 1.17662\tloss: 0.19704\tlr:[5.474296e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:54:28,391 [ hooks.py: 207]:\tstep: 65580\tsteps/sec: 1.17197\tloss: 0.27587\tlr:[5.4675047e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:54:37,592 [ hooks.py: 207]:\tstep: 65590\tsteps/sec: 1.12502\tloss: 0.20382\tlr:[5.460716e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:54:45,555 [ hooks.py: 207]:\tstep: 65600\tsteps/sec: 1.23155\tloss: 0.48023\tlr:[5.4539237e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:54:54,799 [ hooks.py: 207]:\tstep: 65610\tsteps/sec: 1.09629\tloss: 0.36736\tlr:[5.4471348e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:55:02,675 [ hooks.py: 207]:\tstep: 65620\tsteps/sec: 1.27323\tloss: 0.29366\tlr:[5.4403426e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:55:11,600 [ hooks.py: 207]:\tstep: 65630\tsteps/sec: 1.12856\tloss: 0.58176\tlr:[5.4335533e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:55:20,418 [ hooks.py: 207]:\tstep: 65640\tsteps/sec: 1.11280\tloss: 0.26371\tlr:[5.426762e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:55:30,349 [ hooks.py: 207]:\tstep: 65650\tsteps/sec: 1.00509\tloss: 0.16252\tlr:[5.4199727e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:55:38,482 [ hooks.py: 207]:\tstep: 65660\tsteps/sec: 1.22133\tloss: 0.83979\tlr:[5.4131806e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:55:47,192 [ hooks.py: 207]:\tstep: 65670\tsteps/sec: 1.18313\tloss: 0.32028\tlr:[5.406392e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:55:57,215 [ hooks.py: 207]:\tstep: 65680\tsteps/sec: 1.02768\tloss: 0.24520\tlr:[5.3996e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:56:06,522 [ hooks.py: 207]:\tstep: 65690\tsteps/sec: 1.04076\tloss: 0.21421\tlr:[5.392811e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:56:13,402 [ hooks.py: 207]:\tstep: 65700\tsteps/sec: 1.32315\tloss: 0.21222\tlr:[5.386019e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:56:23,496 [ hooks.py: 207]:\tstep: 65710\tsteps/sec: 0.99373\tloss: 0.40474\tlr:[5.37923e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:56:33,199 [ hooks.py: 207]:\tstep: 65720\tsteps/sec: 1.07350\tloss: 0.26267\tlr:[5.372438e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:56:40,050 [ hooks.py: 207]:\tstep: 65730\tsteps/sec: 1.48923\tloss: 0.33016\tlr:[5.3656463e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:56:51,308 [ hooks.py: 207]:\tstep: 65740\tsteps/sec: 0.90852\tloss: 0.44412\tlr:[5.3588574e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:56:59,287 [ hooks.py: 207]:\tstep: 65750\tsteps/sec: 1.20047\tloss: 0.19471\tlr:[5.3520653e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 16:57:09,052 [ hooks.py: 207]:\tstep: 65760\tsteps/sec: 1.00423\tloss: 0.10752\tlr:[5.345277e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:57:17,115 [ hooks.py: 207]:\tstep: 65770\tsteps/sec: 1.31973\tloss: 0.27704\tlr:[5.3384842e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:57:25,328 [ hooks.py: 207]:\tstep: 65780\tsteps/sec: 1.16732\tloss: 0.54889\tlr:[5.331696e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:57:37,057 [ hooks.py: 207]:\tstep: 65790\tsteps/sec: 0.87454\tloss: 0.24315\tlr:[5.3249037e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:57:45,660 [ hooks.py: 207]:\tstep: 65800\tsteps/sec: 1.13300\tloss: 0.17173\tlr:[5.3181147e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:57:56,569 [ hooks.py: 207]:\tstep: 65810\tsteps/sec: 0.96880\tloss: 0.32215\tlr:[5.3113226e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:58:04,349 [ hooks.py: 207]:\tstep: 65820\tsteps/sec: 1.17974\tloss: 0.26579\tlr:[5.3045333e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:58:12,964 [ hooks.py: 207]:\tstep: 65830\tsteps/sec: 1.18915\tloss: 0.20616\tlr:[5.2977416e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:58:19,813 [ hooks.py: 207]:\tstep: 65840\tsteps/sec: 1.43143\tloss: 0.98617\tlr:[5.2909527e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:58:28,113 [ hooks.py: 207]:\tstep: 65850\tsteps/sec: 1.21393\tloss: 0.32651\tlr:[5.2841606e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:58:36,644 [ hooks.py: 207]:\tstep: 65860\tsteps/sec: 1.15394\tloss: 0.34997\tlr:[5.2773717e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:58:44,758 [ hooks.py: 207]:\tstep: 65870\tsteps/sec: 1.24297\tloss: 0.10103\tlr:[5.2705795e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:58:53,878 [ hooks.py: 207]:\tstep: 65880\tsteps/sec: 1.05868\tloss: 0.24189\tlr:[5.2637874e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:59:01,841 [ hooks.py: 207]:\tstep: 65890\tsteps/sec: 1.35241\tloss: 0.42465\tlr:[5.256999e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:59:11,950 [ hooks.py: 207]:\tstep: 65900\tsteps/sec: 0.99944\tloss: 0.56716\tlr:[5.250207e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:59:19,954 [ hooks.py: 207]:\tstep: 65910\tsteps/sec: 1.15429\tloss: 0.45536\tlr:[5.243418e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:59:31,256 [ hooks.py: 207]:\tstep: 65920\tsteps/sec: 0.92574\tloss: 0.18589\tlr:[5.236626e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:59:39,203 [ hooks.py: 207]:\tstep: 65930\tsteps/sec: 1.17690\tloss: 0.05122\tlr:[5.229837e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:59:48,245 [ hooks.py: 207]:\tstep: 65940\tsteps/sec: 1.15762\tloss: 0.42503\tlr:[5.223045e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 16:59:57,640 [ hooks.py: 207]:\tstep: 65950\tsteps/sec: 1.13372\tloss: 0.36502\tlr:[5.216256e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:07,963 [ hooks.py: 207]:\tstep: 65960\tsteps/sec: 0.92485\tloss: 0.28775\tlr:[5.2094642e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:16,225 [ hooks.py: 207]:\tstep: 65970\tsteps/sec: 1.22706\tloss: 0.49289\tlr:[5.2026753e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:23,404 [ hooks.py: 207]:\tstep: 65980\tsteps/sec: 1.35588\tloss: 0.45511\tlr:[5.195883e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:31,657 [ hooks.py: 207]:\tstep: 65990\tsteps/sec: 1.23285\tloss: 0.76408\tlr:[5.189094e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:40,277 [ hooks.py: 207]:\tstep: 66000\tsteps/sec: 1.14594\tloss: 0.18502\tlr:[5.182302e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:40,278 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:40,278 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:40,278 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:00:40,278 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:40,278 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:00:40,278 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:08:47,049 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,052 [ trainer.py: 58]:\t{'acc': 0.7855421686746988, 'loss': 0.6664952558154861}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,053 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,053 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:08:47,053 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:08:47,054 [ trainer.py: 70]:\tacc\t0.7855421686746988\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:08:47,054 [ trainer.py: 70]:\tloss\t0.6664952558154861\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:08:47,054 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,054 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6664952558154861}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,054 [ exporter.py: 64]:\t[Best Exporter]: skip step 66000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,054 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7855421686746988, 'loss': 0.6664952558154861}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,055 [ exporter.py: 104]:\t[Best Exporter]: skip step 66000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:47,055 [monitored_executor.py: 122]:\tsaving step 66000 to ./output/model_66000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:08:50,333 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_63000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:09:00,345 [ hooks.py: 207]:\tstep: 66010\tsteps/sec: 0.01999\tloss: 0.32327\tlr:[5.175511e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:09:09,698 [ hooks.py: 207]:\tstep: 66020\tsteps/sec: 1.06028\tloss: 0.40271\tlr:[5.1687216e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:09:17,698 [ hooks.py: 207]:\tstep: 66030\tsteps/sec: 1.29285\tloss: 0.79710\tlr:[5.1619295e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:09:26,688 [ hooks.py: 207]:\tstep: 66040\tsteps/sec: 1.11697\tloss: 0.16325\tlr:[5.15514e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:09:37,562 [ hooks.py: 207]:\tstep: 66050\tsteps/sec: 0.91952\tloss: 0.23544\tlr:[5.1483476e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:09:45,179 [ hooks.py: 207]:\tstep: 66060\tsteps/sec: 1.29326\tloss: 0.43532\tlr:[5.1415595e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:09:54,220 [ hooks.py: 207]:\tstep: 66070\tsteps/sec: 1.09289\tloss: 0.53608\tlr:[5.1347674e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:10:04,311 [ hooks.py: 207]:\tstep: 66080\tsteps/sec: 0.98467\tloss: 0.28504\tlr:[5.127978e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:10:11,807 [ hooks.py: 207]:\tstep: 66090\tsteps/sec: 1.34174\tloss: 0.47527\tlr:[5.121187e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:10:19,824 [ hooks.py: 207]:\tstep: 66100\tsteps/sec: 1.25759\tloss: 0.53127\tlr:[5.1143975e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:10:26,663 [ hooks.py: 207]:\tstep: 66110\tsteps/sec: 1.49849\tloss: 0.37126\tlr:[5.1076054e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:10:34,378 [ hooks.py: 207]:\tstep: 66120\tsteps/sec: 1.21970\tloss: 0.40648\tlr:[5.100817e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:10:44,908 [ hooks.py: 207]:\tstep: 66130\tsteps/sec: 0.97806\tloss: 0.87652\tlr:[5.0940243e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:10:53,740 [ hooks.py: 207]:\tstep: 66140\tsteps/sec: 1.13449\tloss: 0.41801\tlr:[5.0872354e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:11:01,100 [ hooks.py: 207]:\tstep: 66150\tsteps/sec: 1.32645\tloss: 0.19199\tlr:[5.0804438e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:11:10,888 [ hooks.py: 207]:\tstep: 66160\tsteps/sec: 1.03708\tloss: 0.15736\tlr:[5.073652e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:11:19,309 [ hooks.py: 207]:\tstep: 66170\tsteps/sec: 1.18207\tloss: 0.16872\tlr:[5.066863e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:11:27,423 [ hooks.py: 207]:\tstep: 66180\tsteps/sec: 1.21811\tloss: 0.54458\tlr:[5.060071e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:11:38,116 [ hooks.py: 207]:\tstep: 66190\tsteps/sec: 0.92321\tloss: 0.43358\tlr:[5.053282e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:11:49,298 [ hooks.py: 207]:\tstep: 66200\tsteps/sec: 0.91298\tloss: 0.14570\tlr:[5.0464896e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:11:58,560 [ hooks.py: 207]:\tstep: 66210\tsteps/sec: 1.08601\tloss: 0.19244\tlr:[5.0397016e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 17:12:07,150 [ hooks.py: 207]:\tstep: 66220\tsteps/sec: 1.19141\tloss: 0.30734\tlr:[5.032909e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:12:15,703 [ hooks.py: 207]:\tstep: 66230\tsteps/sec: 1.10977\tloss: 0.39956\tlr:[5.02612e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:12:24,078 [ hooks.py: 207]:\tstep: 66240\tsteps/sec: 1.22809\tloss: 0.38660\tlr:[5.0193285e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:12:32,687 [ hooks.py: 207]:\tstep: 66250\tsteps/sec: 1.20520\tloss: 0.29214\tlr:[5.0125395e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:12:42,009 [ hooks.py: 207]:\tstep: 66260\tsteps/sec: 1.07665\tloss: 0.78000\tlr:[5.005748e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:12:52,418 [ hooks.py: 207]:\tstep: 66270\tsteps/sec: 0.97152\tloss: 0.54624\tlr:[4.9989585e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:13:01,326 [ hooks.py: 207]:\tstep: 66280\tsteps/sec: 1.05164\tloss: 0.72213\tlr:[4.992167e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:13:10,472 [ hooks.py: 207]:\tstep: 66290\tsteps/sec: 1.12553\tloss: 0.05599\tlr:[4.985378e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:13:19,078 [ hooks.py: 207]:\tstep: 66300\tsteps/sec: 1.10079\tloss: 0.57797\tlr:[4.9785854e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:13:29,904 [ hooks.py: 207]:\tstep: 66310\tsteps/sec: 0.96190\tloss: 0.08941\tlr:[4.9717933e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:13:38,690 [ hooks.py: 207]:\tstep: 66320\tsteps/sec: 1.14039\tloss: 0.21768\tlr:[4.965005e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:13:46,578 [ hooks.py: 207]:\tstep: 66330\tsteps/sec: 1.28558\tloss: 0.20249\tlr:[4.958213e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:13:54,482 [ hooks.py: 207]:\tstep: 66340\tsteps/sec: 1.18897\tloss: 0.47982\tlr:[4.9514233e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:14:04,574 [ hooks.py: 207]:\tstep: 66350\tsteps/sec: 1.02277\tloss: 0.35912\tlr:[4.944631e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:14:11,706 [ hooks.py: 207]:\tstep: 66360\tsteps/sec: 1.46035\tloss: 0.08720\tlr:[4.937843e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:14:19,696 [ hooks.py: 207]:\tstep: 66370\tsteps/sec: 1.17356\tloss: 0.41494\tlr:[4.93105e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:14:28,406 [ hooks.py: 207]:\tstep: 66380\tsteps/sec: 1.14082\tloss: 0.37657\tlr:[4.9242617e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:14:39,606 [ hooks.py: 207]:\tstep: 66390\tsteps/sec: 0.92350\tloss: 0.21806\tlr:[4.91747e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:14:47,274 [ hooks.py: 207]:\tstep: 66400\tsteps/sec: 1.34035\tloss: 0.20855\tlr:[4.9106807e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:14:58,272 [ hooks.py: 207]:\tstep: 66410\tsteps/sec: 0.90634\tloss: 0.20628\tlr:[4.9038886e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:15:06,673 [ hooks.py: 207]:\tstep: 66420\tsteps/sec: 1.18492\tloss: 0.45624\tlr:[4.8970996e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:15:15,258 [ hooks.py: 207]:\tstep: 66430\tsteps/sec: 1.10695\tloss: 0.33200\tlr:[4.890308e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:15:25,562 [ hooks.py: 207]:\tstep: 66440\tsteps/sec: 0.99192\tloss: 0.20676\tlr:[4.883519e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:15:34,243 [ hooks.py: 207]:\tstep: 66450\tsteps/sec: 1.11738\tloss: 0.48054\tlr:[4.876727e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:15:44,377 [ hooks.py: 207]:\tstep: 66460\tsteps/sec: 1.01232\tloss: 0.28003\tlr:[4.8699358e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:15:52,568 [ hooks.py: 207]:\tstep: 66470\tsteps/sec: 1.27158\tloss: 0.35987\tlr:[4.8631464e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:16:03,186 [ hooks.py: 207]:\tstep: 66480\tsteps/sec: 0.93607\tloss: 0.27459\tlr:[4.8563543e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:16:11,991 [ hooks.py: 207]:\tstep: 66490\tsteps/sec: 1.14481\tloss: 0.43767\tlr:[4.8495654e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:16:19,635 [ hooks.py: 207]:\tstep: 66500\tsteps/sec: 1.20875\tloss: 0.16567\tlr:[4.8427737e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:16:27,986 [ hooks.py: 207]:\tstep: 66510\tsteps/sec: 1.28780\tloss: 0.16132\tlr:[4.835985e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:16:35,179 [ hooks.py: 207]:\tstep: 66520\tsteps/sec: 1.34536\tloss: 0.27813\tlr:[4.829192e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:16:44,222 [ hooks.py: 207]:\tstep: 66530\tsteps/sec: 1.11296\tloss: 0.26686\tlr:[4.8224033e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:16:53,831 [ hooks.py: 207]:\tstep: 66540\tsteps/sec: 1.03268\tloss: 0.41914\tlr:[4.8156116e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:17:02,687 [ hooks.py: 207]:\tstep: 66550\tsteps/sec: 1.16072\tloss: 0.55851\tlr:[4.8088227e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:17:10,540 [ hooks.py: 207]:\tstep: 66560\tsteps/sec: 1.26070\tloss: 0.49789\tlr:[4.80203e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:17:20,663 [ hooks.py: 207]:\tstep: 66570\tsteps/sec: 0.94734\tloss: 0.36067\tlr:[4.7952412e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:17:29,392 [ hooks.py: 207]:\tstep: 66580\tsteps/sec: 1.14303\tloss: 0.46595\tlr:[4.78845e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:17:39,069 [ hooks.py: 207]:\tstep: 66590\tsteps/sec: 1.05129\tloss: 0.33891\tlr:[4.78166e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:17:47,797 [ hooks.py: 207]:\tstep: 66600\tsteps/sec: 1.19784\tloss: 0.54062\tlr:[4.7748686e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:17:54,732 [ hooks.py: 207]:\tstep: 66610\tsteps/sec: 1.39339\tloss: 0.43337\tlr:[4.768077e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:18:01,819 [ hooks.py: 207]:\tstep: 66620\tsteps/sec: 1.45978\tloss: 0.24682\tlr:[4.761288e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:18:12,463 [ hooks.py: 207]:\tstep: 66630\tsteps/sec: 0.91515\tloss: 0.42206\tlr:[4.754496e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:18:22,858 [ hooks.py: 207]:\tstep: 66640\tsteps/sec: 0.99146\tloss: 0.51710\tlr:[4.7477065e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:18:31,222 [ hooks.py: 207]:\tstep: 66650\tsteps/sec: 1.13687\tloss: 0.12199\tlr:[4.7409144e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:18:39,798 [ hooks.py: 207]:\tstep: 66660\tsteps/sec: 1.20486\tloss: 0.25686\tlr:[4.7341255e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:18:48,663 [ hooks.py: 207]:\tstep: 66670\tsteps/sec: 1.10330\tloss: 0.46275\tlr:[4.7273343e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:18:58,842 [ hooks.py: 207]:\tstep: 66680\tsteps/sec: 0.98702\tloss: 0.21087\tlr:[4.720545e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:19:05,832 [ hooks.py: 207]:\tstep: 66690\tsteps/sec: 1.46136\tloss: 0.11501\tlr:[4.7137537e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:19:16,674 [ hooks.py: 207]:\tstep: 66700\tsteps/sec: 0.90401\tloss: 0.16792\tlr:[4.706964e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:19:26,860 [ hooks.py: 207]:\tstep: 66710\tsteps/sec: 1.01165\tloss: 0.47584\tlr:[4.7001727e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:19:34,281 [ hooks.py: 207]:\tstep: 66720\tsteps/sec: 1.31630\tloss: 0.20509\tlr:[4.6933833e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:19:44,131 [ hooks.py: 207]:\tstep: 66730\tsteps/sec: 1.01798\tloss: 0.22407\tlr:[4.686591e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:19:53,600 [ hooks.py: 207]:\tstep: 66740\tsteps/sec: 1.06224\tloss: 0.30838\tlr:[4.6798023e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:20:01,609 [ hooks.py: 207]:\tstep: 66750\tsteps/sec: 1.27918\tloss: 0.28375\tlr:[4.67301e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:20:10,415 [ hooks.py: 207]:\tstep: 66760\tsteps/sec: 1.11273\tloss: 0.27620\tlr:[4.6662185e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:20:20,002 [ hooks.py: 207]:\tstep: 66770\tsteps/sec: 1.00463\tloss: 0.08883\tlr:[4.6594296e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:20:27,831 [ hooks.py: 207]:\tstep: 66780\tsteps/sec: 1.35387\tloss: 0.20900\tlr:[4.6526366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:20:38,773 [ hooks.py: 207]:\tstep: 66790\tsteps/sec: 0.90122\tloss: 0.29166\tlr:[4.645849e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:20:48,962 [ hooks.py: 207]:\tstep: 66800\tsteps/sec: 0.98022\tloss: 0.37242\tlr:[4.6390564e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:20:58,870 [ hooks.py: 207]:\tstep: 66810\tsteps/sec: 0.97762\tloss: 0.19245\tlr:[4.6322675e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:21:07,688 [ hooks.py: 207]:\tstep: 66820\tsteps/sec: 1.15772\tloss: 0.25212\tlr:[4.6254754e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:21:16,169 [ hooks.py: 207]:\tstep: 66830\tsteps/sec: 1.20399\tloss: 0.64680\tlr:[4.6186865e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:21:25,005 [ hooks.py: 207]:\tstep: 66840\tsteps/sec: 1.12450\tloss: 0.39813\tlr:[4.6118944e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:21:33,752 [ hooks.py: 207]:\tstep: 66850\tsteps/sec: 1.15844\tloss: 0.51871\tlr:[4.605106e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:21:40,562 [ hooks.py: 207]:\tstep: 66860\tsteps/sec: 1.47723\tloss: 0.37380\tlr:[4.598314e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 17:21:49,382 [ hooks.py: 207]:\tstep: 66870\tsteps/sec: 1.06841\tloss: 0.60915\tlr:[4.5915253e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:22:00,211 [ hooks.py: 207]:\tstep: 66880\tsteps/sec: 0.96069\tloss: 0.12000\tlr:[4.584733e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:22:09,717 [ hooks.py: 207]:\tstep: 66890\tsteps/sec: 1.04009\tloss: 0.64864\tlr:[4.577944e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:22:18,673 [ hooks.py: 207]:\tstep: 66900\tsteps/sec: 1.14133\tloss: 0.32646\tlr:[4.571152e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:22:27,428 [ hooks.py: 207]:\tstep: 66910\tsteps/sec: 1.16938\tloss: 0.39633\tlr:[4.5643596e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:22:33,357 [ hooks.py: 207]:\tstep: 66920\tsteps/sec: 1.54053\tloss: 0.14830\tlr:[4.557571e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:22:44,440 [ hooks.py: 207]:\tstep: 66930\tsteps/sec: 0.92408\tloss: 0.13374\tlr:[4.5507795e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:22:52,460 [ hooks.py: 207]:\tstep: 66940\tsteps/sec: 1.29950\tloss: 0.19681\tlr:[4.54399e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:01,915 [ hooks.py: 207]:\tstep: 66950\tsteps/sec: 1.05068\tloss: 0.34008\tlr:[4.537198e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:09,104 [ hooks.py: 207]:\tstep: 66960\tsteps/sec: 1.34654\tloss: 0.74798\tlr:[4.5304087e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:16,932 [ hooks.py: 207]:\tstep: 66970\tsteps/sec: 1.26300\tloss: 0.34360\tlr:[4.5236175e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:26,664 [ hooks.py: 207]:\tstep: 66980\tsteps/sec: 1.05358\tloss: 0.15750\tlr:[4.5168285e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:33,765 [ hooks.py: 207]:\tstep: 66990\tsteps/sec: 1.37365\tloss: 0.09995\tlr:[4.510037e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:41,940 [ hooks.py: 207]:\tstep: 67000\tsteps/sec: 1.25888\tloss: 0.37441\tlr:[4.5032475e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:41,941 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:41,941 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:41,942 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:23:41,942 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:41,942 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:23:41,942 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:31:37,467 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,468 [ trainer.py: 58]:\t{'acc': 0.7839357429718875, 'loss': 0.642334353704101}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,468 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,469 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:31:37,469 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:31:37,469 [ trainer.py: 70]:\tacc\t0.7839357429718875\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:31:37,469 [ trainer.py: 70]:\tloss\t0.642334353704101\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:31:37,469 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,469 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7839357429718875, 'loss': 0.642334353704101}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,469 [ exporter.py: 64]:\t[Best Exporter]: skip step 67000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,469 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7839357429718875, 'loss': 0.642334353704101}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,469 [ exporter.py: 104]:\t[Best Exporter]: skip step 67000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:37,470 [monitored_executor.py: 122]:\tsaving step 67000 to ./output/model_67000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:40,446 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_64000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:48,649 [ hooks.py: 207]:\tstep: 67010\tsteps/sec: 0.02054\tloss: 0.20102\tlr:[4.496456e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:31:57,597 [ hooks.py: 207]:\tstep: 67020\tsteps/sec: 1.17042\tloss: 0.47047\tlr:[4.489667e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:32:04,277 [ hooks.py: 207]:\tstep: 67030\tsteps/sec: 1.38166\tloss: 0.26785\tlr:[4.482875e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:32:11,011 [ hooks.py: 207]:\tstep: 67040\tsteps/sec: 1.48446\tloss: 0.04874\tlr:[4.476083e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:32:21,878 [ hooks.py: 207]:\tstep: 67050\tsteps/sec: 0.93436\tloss: 0.53907\tlr:[4.469294e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:32:30,049 [ hooks.py: 207]:\tstep: 67060\tsteps/sec: 1.19282\tloss: 0.55581\tlr:[4.4625017e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:32:38,660 [ hooks.py: 207]:\tstep: 67070\tsteps/sec: 1.17981\tloss: 0.35613\tlr:[4.4557128e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:32:47,472 [ hooks.py: 207]:\tstep: 67080\tsteps/sec: 1.15302\tloss: 0.11802\tlr:[4.44892e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:32:57,228 [ hooks.py: 207]:\tstep: 67090\tsteps/sec: 1.06202\tloss: 0.27558\tlr:[4.442132e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:33:05,784 [ hooks.py: 207]:\tstep: 67100\tsteps/sec: 1.09396\tloss: 0.38032\tlr:[4.43534e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:33:12,946 [ hooks.py: 207]:\tstep: 67110\tsteps/sec: 1.43585\tloss: 0.19507\tlr:[4.4285507e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:33:23,873 [ hooks.py: 207]:\tstep: 67120\tsteps/sec: 0.88491\tloss: 0.49103\tlr:[4.4217586e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:33:32,840 [ hooks.py: 207]:\tstep: 67130\tsteps/sec: 1.16022\tloss: 0.35667\tlr:[4.41497e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:33:41,591 [ hooks.py: 207]:\tstep: 67140\tsteps/sec: 1.14197\tloss: 0.38852\tlr:[4.4081776e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:33:48,612 [ hooks.py: 207]:\tstep: 67150\tsteps/sec: 1.43242\tloss: 0.15347\tlr:[4.4013887e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:33:56,977 [ hooks.py: 207]:\tstep: 67160\tsteps/sec: 1.23712\tloss: 0.63629\tlr:[4.3945965e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:34:05,308 [ hooks.py: 207]:\tstep: 67170\tsteps/sec: 1.13786\tloss: 0.40670\tlr:[4.387808e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:34:12,148 [ hooks.py: 207]:\tstep: 67180\tsteps/sec: 1.49787\tloss: 0.40175\tlr:[4.381016e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:34:23,187 [ hooks.py: 207]:\tstep: 67190\tsteps/sec: 0.91051\tloss: 0.58633\tlr:[4.3742243e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:34:29,707 [ hooks.py: 207]:\tstep: 67200\tsteps/sec: 1.40627\tloss: 0.20037\tlr:[4.367435e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:34:38,992 [ hooks.py: 207]:\tstep: 67210\tsteps/sec: 1.06916\tloss: 0.22431\tlr:[4.360643e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:34:48,729 [ hooks.py: 207]:\tstep: 67220\tsteps/sec: 1.07930\tloss: 0.31340\tlr:[4.353854e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:34:58,394 [ hooks.py: 207]:\tstep: 67230\tsteps/sec: 1.05748\tloss: 0.46000\tlr:[4.347062e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:35:06,312 [ hooks.py: 207]:\tstep: 67240\tsteps/sec: 1.21770\tloss: 0.67812\tlr:[4.3402733e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:35:14,228 [ hooks.py: 207]:\tstep: 67250\tsteps/sec: 1.26673\tloss: 0.32459\tlr:[4.3334812e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:35:21,580 [ hooks.py: 207]:\tstep: 67260\tsteps/sec: 1.32099\tloss: 0.17937\tlr:[4.3266923e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:35:30,160 [ hooks.py: 207]:\tstep: 67270\tsteps/sec: 1.19880\tloss: 0.25982\tlr:[4.3199e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:35:36,807 [ hooks.py: 207]:\tstep: 67280\tsteps/sec: 1.46428\tloss: 0.19908\tlr:[4.3131113e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:35:45,235 [ hooks.py: 207]:\tstep: 67290\tsteps/sec: 1.21477\tloss: 0.66913\tlr:[4.3063196e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:35:53,481 [ hooks.py: 207]:\tstep: 67300\tsteps/sec: 1.23701\tloss: 0.02987\tlr:[4.2995307e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:36:00,786 [ hooks.py: 207]:\tstep: 67310\tsteps/sec: 1.34032\tloss: 0.30004\tlr:[4.292738e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:36:09,818 [ hooks.py: 207]:\tstep: 67320\tsteps/sec: 1.09459\tloss: 0.68057\tlr:[4.285949e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 17:36:17,706 [ hooks.py: 207]:\tstep: 67330\tsteps/sec: 1.28257\tloss: 0.31510\tlr:[4.2791576e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:36:25,258 [ hooks.py: 207]:\tstep: 67340\tsteps/sec: 1.30898\tloss: 0.45818\tlr:[4.272366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:36:31,470 [ hooks.py: 207]:\tstep: 67350\tsteps/sec: 1.55889\tloss: 0.66250\tlr:[4.265577e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:36:41,323 [ hooks.py: 207]:\tstep: 67360\tsteps/sec: 1.05857\tloss: 0.14773\tlr:[4.2587844e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:36:49,335 [ hooks.py: 207]:\tstep: 67370\tsteps/sec: 1.16527\tloss: 0.12030\tlr:[4.251996e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:36:58,817 [ hooks.py: 207]:\tstep: 67380\tsteps/sec: 1.10968\tloss: 0.63722\tlr:[4.2452034e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:37:05,289 [ hooks.py: 207]:\tstep: 67390\tsteps/sec: 1.53403\tloss: 0.31319\tlr:[4.2384154e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:37:14,517 [ hooks.py: 207]:\tstep: 67400\tsteps/sec: 1.07993\tloss: 0.23439\tlr:[4.2316233e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:37:24,117 [ hooks.py: 207]:\tstep: 67410\tsteps/sec: 1.04077\tloss: 0.45531\tlr:[4.224834e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:37:33,649 [ hooks.py: 207]:\tstep: 67420\tsteps/sec: 1.07100\tloss: 0.58581\tlr:[4.218042e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:37:43,637 [ hooks.py: 207]:\tstep: 67430\tsteps/sec: 1.00847\tloss: 0.20669\tlr:[4.2112533e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:37:52,919 [ hooks.py: 207]:\tstep: 67440\tsteps/sec: 1.04115\tloss: 0.39293\tlr:[4.204461e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:03,346 [ hooks.py: 207]:\tstep: 67450\tsteps/sec: 0.94949\tloss: 0.63161\tlr:[4.1976728e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:10,914 [ hooks.py: 207]:\tstep: 67460\tsteps/sec: 1.36523\tloss: 0.61395\tlr:[4.1908806e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:19,656 [ hooks.py: 207]:\tstep: 67470\tsteps/sec: 1.12499\tloss: 0.48043\tlr:[4.184091e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:27,521 [ hooks.py: 207]:\tstep: 67480\tsteps/sec: 1.31710\tloss: 0.11965\tlr:[4.1772996e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:35,073 [ hooks.py: 207]:\tstep: 67490\tsteps/sec: 1.24262\tloss: 0.16906\tlr:[4.1705075e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:42,878 [ hooks.py: 207]:\tstep: 67500\tsteps/sec: 1.37171\tloss: 0.60059\tlr:[4.1637186e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:50,712 [ hooks.py: 207]:\tstep: 67510\tsteps/sec: 1.25466\tloss: 0.49169\tlr:[4.1569265e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:38:59,796 [ hooks.py: 207]:\tstep: 67520\tsteps/sec: 1.08584\tloss: 0.24940\tlr:[4.150137e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:39:07,783 [ hooks.py: 207]:\tstep: 67530\tsteps/sec: 1.30163\tloss: 0.24316\tlr:[4.1433454e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:39:15,184 [ hooks.py: 207]:\tstep: 67540\tsteps/sec: 1.25789\tloss: 0.11058\tlr:[4.136557e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:39:23,133 [ hooks.py: 207]:\tstep: 67550\tsteps/sec: 1.28046\tloss: 0.14652\tlr:[4.129765e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:39:31,324 [ hooks.py: 207]:\tstep: 67560\tsteps/sec: 1.26209\tloss: 0.41536\tlr:[4.122976e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:39:39,353 [ hooks.py: 207]:\tstep: 67570\tsteps/sec: 1.23395\tloss: 0.08353\tlr:[4.1161843e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:39:46,771 [ hooks.py: 207]:\tstep: 67580\tsteps/sec: 1.36438\tloss: 0.31152\tlr:[4.109395e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:39:55,612 [ hooks.py: 207]:\tstep: 67590\tsteps/sec: 1.15038\tloss: 0.38426\tlr:[4.1026033e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:02,515 [ hooks.py: 207]:\tstep: 67600\tsteps/sec: 1.37765\tloss: 0.53871\tlr:[4.095814e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:12,214 [ hooks.py: 207]:\tstep: 67610\tsteps/sec: 1.06314\tloss: 0.84090\tlr:[4.089022e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:20,183 [ hooks.py: 207]:\tstep: 67620\tsteps/sec: 1.22912\tloss: 0.46602\tlr:[4.0822324e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:27,313 [ hooks.py: 207]:\tstep: 67630\tsteps/sec: 1.43373\tloss: 0.48877\tlr:[4.0754408e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:35,195 [ hooks.py: 207]:\tstep: 67640\tsteps/sec: 1.23835\tloss: 0.34876\tlr:[4.0686486e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:43,427 [ hooks.py: 207]:\tstep: 67650\tsteps/sec: 1.23836\tloss: 0.29655\tlr:[4.06186e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:50,851 [ hooks.py: 207]:\tstep: 67660\tsteps/sec: 1.33928\tloss: 0.13084\tlr:[4.0550676e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:40:58,371 [ hooks.py: 207]:\tstep: 67670\tsteps/sec: 1.34513\tloss: 0.07404\tlr:[4.048279e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:41:05,363 [ hooks.py: 207]:\tstep: 67680\tsteps/sec: 1.33596\tloss: 0.13576\tlr:[4.041487e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:41:14,293 [ hooks.py: 207]:\tstep: 67690\tsteps/sec: 1.16618\tloss: 0.30428\tlr:[4.034698e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:41:21,962 [ hooks.py: 207]:\tstep: 67700\tsteps/sec: 1.33341\tloss: 0.12818\tlr:[4.0279065e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:41:28,717 [ hooks.py: 207]:\tstep: 67710\tsteps/sec: 1.38349\tloss: 0.20121\tlr:[4.0211175e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:41:37,730 [ hooks.py: 207]:\tstep: 67720\tsteps/sec: 1.18933\tloss: 0.29231\tlr:[4.014325e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:41:44,714 [ hooks.py: 207]:\tstep: 67730\tsteps/sec: 1.35271\tloss: 0.38610\tlr:[4.0075365e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:41:52,805 [ hooks.py: 207]:\tstep: 67740\tsteps/sec: 1.29106\tloss: 0.07303\tlr:[4.0007444e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:02,153 [ hooks.py: 207]:\tstep: 67750\tsteps/sec: 1.03907\tloss: 0.20761\tlr:[3.9939555e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:09,079 [ hooks.py: 207]:\tstep: 67760\tsteps/sec: 1.44168\tloss: 0.18636\tlr:[3.9871634e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:16,457 [ hooks.py: 207]:\tstep: 67770\tsteps/sec: 1.35770\tloss: 0.44351\tlr:[3.9803745e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:23,011 [ hooks.py: 207]:\tstep: 67780\tsteps/sec: 1.49248\tloss: 0.21785\tlr:[3.973582e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:34,435 [ hooks.py: 207]:\tstep: 67790\tsteps/sec: 0.88023\tloss: 0.09179\tlr:[3.9667907e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:42,727 [ hooks.py: 207]:\tstep: 67800\tsteps/sec: 1.25363\tloss: 0.69532\tlr:[3.960002e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:49,593 [ hooks.py: 207]:\tstep: 67810\tsteps/sec: 1.40269\tloss: 0.64965\tlr:[3.95321e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:42:59,175 [ hooks.py: 207]:\tstep: 67820\tsteps/sec: 1.04276\tloss: 0.26039\tlr:[3.9464207e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:43:07,223 [ hooks.py: 207]:\tstep: 67830\tsteps/sec: 1.23570\tloss: 0.42417\tlr:[3.939629e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:43:15,337 [ hooks.py: 207]:\tstep: 67840\tsteps/sec: 1.19507\tloss: 0.27155\tlr:[3.9328397e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:43:25,147 [ hooks.py: 207]:\tstep: 67850\tsteps/sec: 1.01995\tloss: 0.24515\tlr:[3.926048e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:43:34,636 [ hooks.py: 207]:\tstep: 67860\tsteps/sec: 1.09563\tloss: 0.11263\tlr:[3.919259e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:43:42,795 [ hooks.py: 207]:\tstep: 67870\tsteps/sec: 1.26625\tloss: 0.28191\tlr:[3.9124666e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:43:50,754 [ hooks.py: 207]:\tstep: 67880\tsteps/sec: 1.22394\tloss: 0.49007\tlr:[3.9056777e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:44:00,342 [ hooks.py: 207]:\tstep: 67890\tsteps/sec: 1.02500\tloss: 0.49623\tlr:[3.898886e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:44:07,067 [ hooks.py: 207]:\tstep: 67900\tsteps/sec: 1.54480\tloss: 0.10890\tlr:[3.892097e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:44:17,188 [ hooks.py: 207]:\tstep: 67910\tsteps/sec: 0.95011\tloss: 0.34580\tlr:[3.885305e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:44:27,410 [ hooks.py: 207]:\tstep: 67920\tsteps/sec: 1.01995\tloss: 0.20574\tlr:[3.8785165e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:44:34,843 [ hooks.py: 207]:\tstep: 67930\tsteps/sec: 1.25796\tloss: 0.47502\tlr:[3.871725e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:44:42,846 [ hooks.py: 207]:\tstep: 67940\tsteps/sec: 1.28399\tloss: 0.14894\tlr:[3.864932e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:44:51,631 [ hooks.py: 207]:\tstep: 67950\tsteps/sec: 1.12132\tloss: 0.07260\tlr:[3.8581434e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:01,371 [ hooks.py: 207]:\tstep: 67960\tsteps/sec: 1.02496\tloss: 0.26957\tlr:[3.8513513e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:11,864 [ hooks.py: 207]:\tstep: 67970\tsteps/sec: 0.98898\tloss: 0.29598\tlr:[3.8445623e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 17:45:18,887 [ hooks.py: 207]:\tstep: 67980\tsteps/sec: 1.32611\tloss: 0.12101\tlr:[3.8377702e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:28,354 [ hooks.py: 207]:\tstep: 67990\tsteps/sec: 1.11869\tloss: 0.92341\tlr:[3.8309818e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:37,174 [ hooks.py: 207]:\tstep: 68000\tsteps/sec: 1.13625\tloss: 0.42549\tlr:[3.8241897e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:37,174 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:37,175 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:37,175 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:45:37,175 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:37,175 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:45:37,175 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:53:28,741 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,742 [ trainer.py: 58]:\t{'acc': 0.7919678714859437, 'loss': 0.6658163778483868}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,742 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,743 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:53:28,743 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:53:28,743 [ trainer.py: 70]:\tacc\t0.7919678714859437\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:53:28,744 [ trainer.py: 70]:\tloss\t0.6658163778483868\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 17:53:28,744 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,744 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7919678714859437, 'loss': 0.6658163778483868}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,744 [ exporter.py: 64]:\t[Best Exporter]: skip step 68000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,744 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7919678714859437, 'loss': 0.6658163778483868}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,744 [ exporter.py: 104]:\t[Best Exporter]: skip step 68000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:28,745 [monitored_executor.py: 122]:\tsaving step 68000 to ./output/model_68000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:31,952 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_65000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:38,898 [ hooks.py: 207]:\tstep: 68010\tsteps/sec: 0.02076\tloss: 0.22388\tlr:[3.8174007e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:45,468 [ hooks.py: 207]:\tstep: 68020\tsteps/sec: 1.46752\tloss: 0.41102\tlr:[3.8106082e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:53:54,170 [ hooks.py: 207]:\tstep: 68030\tsteps/sec: 1.18060\tloss: 0.22383\tlr:[3.8038193e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:54:01,192 [ hooks.py: 207]:\tstep: 68040\tsteps/sec: 1.34561\tloss: 0.23382\tlr:[3.7970278e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:54:10,698 [ hooks.py: 207]:\tstep: 68050\tsteps/sec: 1.07975\tloss: 0.50565\tlr:[3.7902387e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:54:17,531 [ hooks.py: 207]:\tstep: 68060\tsteps/sec: 1.48417\tloss: 0.17348\tlr:[3.7834468e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:54:28,029 [ hooks.py: 207]:\tstep: 68070\tsteps/sec: 0.92844\tloss: 0.20717\tlr:[3.7766574e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:54:38,859 [ hooks.py: 207]:\tstep: 68080\tsteps/sec: 0.94251\tloss: 0.44216\tlr:[3.7698655e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:54:48,639 [ hooks.py: 207]:\tstep: 68090\tsteps/sec: 1.03092\tloss: 0.75027\tlr:[3.7630737e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:54:57,410 [ hooks.py: 207]:\tstep: 68100\tsteps/sec: 1.17573\tloss: 0.17888\tlr:[3.756285e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:55:05,422 [ hooks.py: 207]:\tstep: 68110\tsteps/sec: 1.17791\tloss: 0.27639\tlr:[3.7494933e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:55:14,732 [ hooks.py: 207]:\tstep: 68120\tsteps/sec: 1.07969\tloss: 0.47746\tlr:[3.7427037e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:55:22,728 [ hooks.py: 207]:\tstep: 68130\tsteps/sec: 1.27393\tloss: 0.82588\tlr:[3.7359118e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:55:31,302 [ hooks.py: 207]:\tstep: 68140\tsteps/sec: 1.16009\tloss: 0.66783\tlr:[3.7291231e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:55:39,072 [ hooks.py: 207]:\tstep: 68150\tsteps/sec: 1.29386\tloss: 0.63169\tlr:[3.7223313e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:55:48,453 [ hooks.py: 207]:\tstep: 68160\tsteps/sec: 1.06890\tloss: 0.44664\tlr:[3.7155426e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:55:58,094 [ hooks.py: 207]:\tstep: 68170\tsteps/sec: 1.07638\tloss: 0.14006\tlr:[3.7087502e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:56:05,388 [ hooks.py: 207]:\tstep: 68180\tsteps/sec: 1.28017\tloss: 0.29625\tlr:[3.7019608e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:56:12,360 [ hooks.py: 207]:\tstep: 68190\tsteps/sec: 1.40813\tloss: 0.16023\tlr:[3.695169e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:56:21,581 [ hooks.py: 207]:\tstep: 68200\tsteps/sec: 1.10537\tloss: 0.08051\tlr:[3.68838e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:56:30,883 [ hooks.py: 207]:\tstep: 68210\tsteps/sec: 1.07840\tloss: 0.20344\tlr:[3.6815882e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:56:40,982 [ hooks.py: 207]:\tstep: 68220\tsteps/sec: 0.99774\tloss: 0.06116\tlr:[3.6747965e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:56:49,594 [ hooks.py: 207]:\tstep: 68230\tsteps/sec: 1.13257\tloss: 0.25226\tlr:[3.6680071e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:57:01,307 [ hooks.py: 207]:\tstep: 68240\tsteps/sec: 0.83265\tloss: 0.13307\tlr:[3.661216e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:57:10,207 [ hooks.py: 207]:\tstep: 68250\tsteps/sec: 1.12996\tloss: 0.11198\tlr:[3.6544266e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:57:21,270 [ hooks.py: 207]:\tstep: 68260\tsteps/sec: 0.93438\tloss: 0.56762\tlr:[3.6476347e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:57:29,756 [ hooks.py: 207]:\tstep: 68270\tsteps/sec: 1.11822\tloss: 0.26545\tlr:[3.6408458e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:57:40,437 [ hooks.py: 207]:\tstep: 68280\tsteps/sec: 0.97588\tloss: 0.33505\tlr:[3.6340534e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:57:47,688 [ hooks.py: 207]:\tstep: 68290\tsteps/sec: 1.41200\tloss: 0.60311\tlr:[3.6272647e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:57:54,199 [ hooks.py: 207]:\tstep: 68300\tsteps/sec: 1.52989\tloss: 0.20974\tlr:[3.6204729e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:00,889 [ hooks.py: 207]:\tstep: 68310\tsteps/sec: 1.48490\tloss: 0.44853\tlr:[3.6136842e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:10,680 [ hooks.py: 207]:\tstep: 68320\tsteps/sec: 1.00625\tloss: 0.39214\tlr:[3.6068918e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:17,828 [ hooks.py: 207]:\tstep: 68330\tsteps/sec: 1.44063\tloss: 0.47177\tlr:[3.6001034e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:24,619 [ hooks.py: 207]:\tstep: 68340\tsteps/sec: 1.40317\tloss: 0.21566\tlr:[3.593311e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:33,335 [ hooks.py: 207]:\tstep: 68350\tsteps/sec: 1.19434\tloss: 0.31788\tlr:[3.5865216e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:43,175 [ hooks.py: 207]:\tstep: 68360\tsteps/sec: 1.01924\tloss: 0.20704\tlr:[3.5797298e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:50,174 [ hooks.py: 207]:\tstep: 68370\tsteps/sec: 1.38235\tloss: 0.05439\tlr:[3.5729377e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:58:59,121 [ hooks.py: 207]:\tstep: 68380\tsteps/sec: 1.14513\tloss: 0.52698\tlr:[3.5661492e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:59:05,833 [ hooks.py: 207]:\tstep: 68390\tsteps/sec: 1.44163\tloss: 0.20478\tlr:[3.5593573e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:59:14,131 [ hooks.py: 207]:\tstep: 68400\tsteps/sec: 1.23373\tloss: 0.20282\tlr:[3.552568e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:59:21,374 [ hooks.py: 207]:\tstep: 68410\tsteps/sec: 1.34077\tloss: 0.42538\tlr:[3.5457763e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:59:31,096 [ hooks.py: 207]:\tstep: 68420\tsteps/sec: 1.03753\tloss: 0.24842\tlr:[3.5389874e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:59:40,682 [ hooks.py: 207]:\tstep: 68430\tsteps/sec: 1.04260\tloss: 0.33695\tlr:[3.532195e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 17:59:49,541 [ hooks.py: 207]:\tstep: 68440\tsteps/sec: 1.11969\tloss: 0.50851\tlr:[3.5254063e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 17:59:55,569 [ hooks.py: 207]:\tstep: 68450\tsteps/sec: 1.66522\tloss: 0.26588\tlr:[3.5186144e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:00:04,868 [ hooks.py: 207]:\tstep: 68460\tsteps/sec: 1.07128\tloss: 0.45799\tlr:[3.5118255e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:00:13,616 [ hooks.py: 207]:\tstep: 68470\tsteps/sec: 1.18009\tloss: 0.37046\tlr:[3.5050336e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:00:20,122 [ hooks.py: 207]:\tstep: 68480\tsteps/sec: 1.41613\tloss: 0.44788\tlr:[3.4982445e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:00:28,876 [ hooks.py: 207]:\tstep: 68490\tsteps/sec: 1.21166\tloss: 0.14049\tlr:[3.4914522e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:00:38,307 [ hooks.py: 207]:\tstep: 68500\tsteps/sec: 1.06450\tloss: 0.63551\tlr:[3.4846632e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:00:47,899 [ hooks.py: 207]:\tstep: 68510\tsteps/sec: 1.00505\tloss: 0.12621\tlr:[3.4778718e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:00:58,527 [ hooks.py: 207]:\tstep: 68520\tsteps/sec: 0.97865\tloss: 0.30740\tlr:[3.4710797e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:01:05,456 [ hooks.py: 207]:\tstep: 68530\tsteps/sec: 1.38449\tloss: 0.40896\tlr:[3.4642903e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:01:14,036 [ hooks.py: 207]:\tstep: 68540\tsteps/sec: 1.19220\tloss: 0.11977\tlr:[3.4574985e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:01:22,779 [ hooks.py: 207]:\tstep: 68550\tsteps/sec: 1.11013\tloss: 0.07110\tlr:[3.4507098e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:01:33,158 [ hooks.py: 207]:\tstep: 68560\tsteps/sec: 0.99743\tloss: 0.31444\tlr:[3.4439179e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:01:42,255 [ hooks.py: 207]:\tstep: 68570\tsteps/sec: 1.07552\tloss: 0.37198\tlr:[3.437129e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:01:49,963 [ hooks.py: 207]:\tstep: 68580\tsteps/sec: 1.23036\tloss: 0.61719\tlr:[3.430337e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:01:58,869 [ hooks.py: 207]:\tstep: 68590\tsteps/sec: 1.15671\tloss: 0.61751\tlr:[3.423548e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:02:07,425 [ hooks.py: 207]:\tstep: 68600\tsteps/sec: 1.20567\tloss: 0.47296\tlr:[3.4167558e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:02:15,002 [ hooks.py: 207]:\tstep: 68610\tsteps/sec: 1.27423\tloss: 0.32397\tlr:[3.4099671e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:02:23,699 [ hooks.py: 207]:\tstep: 68620\tsteps/sec: 1.18176\tloss: 0.12302\tlr:[3.4031755e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:02:31,681 [ hooks.py: 207]:\tstep: 68630\tsteps/sec: 1.24358\tloss: 0.14491\tlr:[3.396386e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:02:39,390 [ hooks.py: 207]:\tstep: 68640\tsteps/sec: 1.29107\tloss: 0.37142\tlr:[3.3895942e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:02:49,093 [ hooks.py: 207]:\tstep: 68650\tsteps/sec: 1.04709\tloss: 0.39765\tlr:[3.3828048e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:02:58,399 [ hooks.py: 207]:\tstep: 68660\tsteps/sec: 1.07820\tloss: 0.41228\tlr:[3.3760134e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:03:06,276 [ hooks.py: 207]:\tstep: 68670\tsteps/sec: 1.23220\tloss: 0.21014\tlr:[3.3692213e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:03:14,943 [ hooks.py: 207]:\tstep: 68680\tsteps/sec: 1.10414\tloss: 0.21105\tlr:[3.3624324e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:03:24,293 [ hooks.py: 207]:\tstep: 68690\tsteps/sec: 1.13111\tloss: 0.58316\tlr:[3.3556407e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:03:33,289 [ hooks.py: 207]:\tstep: 68700\tsteps/sec: 1.12964\tloss: 0.52109\tlr:[3.3488516e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:03:40,560 [ hooks.py: 207]:\tstep: 68710\tsteps/sec: 1.34746\tloss: 0.35651\tlr:[3.3420595e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:03:47,338 [ hooks.py: 207]:\tstep: 68720\tsteps/sec: 1.47160\tloss: 0.07425\tlr:[3.33527e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:03:56,840 [ hooks.py: 207]:\tstep: 68730\tsteps/sec: 1.10452\tloss: 0.13525\tlr:[3.3284787e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:04:03,648 [ hooks.py: 207]:\tstep: 68740\tsteps/sec: 1.38789\tloss: 0.40366\tlr:[3.32169e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:04:12,451 [ hooks.py: 207]:\tstep: 68750\tsteps/sec: 1.14877\tloss: 0.29086\tlr:[3.3148976e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:04:19,659 [ hooks.py: 207]:\tstep: 68760\tsteps/sec: 1.35565\tloss: 0.64472\tlr:[3.3081092e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:04:26,415 [ hooks.py: 207]:\tstep: 68770\tsteps/sec: 1.48753\tloss: 0.11580\tlr:[3.3013168e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:04:34,961 [ hooks.py: 207]:\tstep: 68780\tsteps/sec: 1.15897\tloss: 0.41789\tlr:[3.2945275e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:04:41,946 [ hooks.py: 207]:\tstep: 68790\tsteps/sec: 1.45343\tloss: 0.66152\tlr:[3.2877358e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:04:51,209 [ hooks.py: 207]:\tstep: 68800\tsteps/sec: 1.06740\tloss: 0.23750\tlr:[3.2809464e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:05:01,631 [ hooks.py: 207]:\tstep: 68810\tsteps/sec: 0.97052\tloss: 0.25218\tlr:[3.274155e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:05:08,459 [ hooks.py: 207]:\tstep: 68820\tsteps/sec: 1.40982\tloss: 0.23661\tlr:[3.2673627e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:05:17,569 [ hooks.py: 207]:\tstep: 68830\tsteps/sec: 1.12848\tloss: 0.28949\tlr:[3.2605744e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:05:26,726 [ hooks.py: 207]:\tstep: 68840\tsteps/sec: 1.04187\tloss: 0.23827\tlr:[3.2537826e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:05:35,098 [ hooks.py: 207]:\tstep: 68850\tsteps/sec: 1.23673\tloss: 0.03634\tlr:[3.2469932e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:05:43,542 [ hooks.py: 207]:\tstep: 68860\tsteps/sec: 1.22311\tloss: 0.20863\tlr:[3.240201e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:05:50,540 [ hooks.py: 207]:\tstep: 68870\tsteps/sec: 1.38551\tloss: 0.11737\tlr:[3.233412e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:06:01,217 [ hooks.py: 207]:\tstep: 68880\tsteps/sec: 0.94155\tloss: 0.47562\tlr:[3.2266203e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:06:09,293 [ hooks.py: 207]:\tstep: 68890\tsteps/sec: 1.25293\tloss: 0.28008\tlr:[3.2198313e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:06:17,767 [ hooks.py: 207]:\tstep: 68900\tsteps/sec: 1.19226\tloss: 0.64501\tlr:[3.2130397e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:06:27,056 [ hooks.py: 207]:\tstep: 68910\tsteps/sec: 1.05145\tloss: 0.26610\tlr:[3.2062503e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:06:34,248 [ hooks.py: 207]:\tstep: 68920\tsteps/sec: 1.42496\tloss: 0.36381\tlr:[3.1994584e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:06:44,020 [ hooks.py: 207]:\tstep: 68930\tsteps/sec: 1.02672\tloss: 0.61144\tlr:[3.192669e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:06:51,029 [ hooks.py: 207]:\tstep: 68940\tsteps/sec: 1.40407\tloss: 0.39315\tlr:[3.1858774e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:00,202 [ hooks.py: 207]:\tstep: 68950\tsteps/sec: 1.07842\tloss: 0.39995\tlr:[3.1790883e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:08,517 [ hooks.py: 207]:\tstep: 68960\tsteps/sec: 1.23117\tloss: 0.73210\tlr:[3.1722966e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:18,042 [ hooks.py: 207]:\tstep: 68970\tsteps/sec: 1.04971\tloss: 0.22812\tlr:[3.1655047e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:25,040 [ hooks.py: 207]:\tstep: 68980\tsteps/sec: 1.38783\tloss: 0.34259\tlr:[3.1587156e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:33,601 [ hooks.py: 207]:\tstep: 68990\tsteps/sec: 1.20523\tloss: 0.52481\tlr:[3.1519237e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:41,110 [ hooks.py: 207]:\tstep: 69000\tsteps/sec: 1.31487\tloss: 0.41374\tlr:[3.1451348e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:41,114 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:41,115 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:41,115 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:07:41,115 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:41,115 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:07:41,115 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:15:32,570 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,571 [ trainer.py: 58]:\t{'acc': 0.7891566265060241, 'loss': 0.666722267651214}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,572 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,572 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:15:32,573 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:15:32,573 [ trainer.py: 70]:\tacc\t0.7891566265060241\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:15:32,573 [ trainer.py: 70]:\tloss\t0.666722267651214\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:15:32,573 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,573 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7891566265060241, 'loss': 0.666722267651214}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,573 [ exporter.py: 64]:\t[Best Exporter]: skip step 69000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,573 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7891566265060241, 'loss': 0.666722267651214}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,574 [ exporter.py: 104]:\t[Best Exporter]: skip step 69000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:32,574 [monitored_executor.py: 122]:\tsaving step 69000 to ./output/model_69000\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 18:15:35,813 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_66000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:43,928 [ hooks.py: 207]:\tstep: 69010\tsteps/sec: 0.02072\tloss: 0.43396\tlr:[3.1383424e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:15:53,464 [ hooks.py: 207]:\tstep: 69020\tsteps/sec: 1.04594\tloss: 0.26155\tlr:[3.1315535e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:16:01,890 [ hooks.py: 207]:\tstep: 69030\tsteps/sec: 1.16934\tloss: 0.20066\tlr:[3.1247623e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:16:08,987 [ hooks.py: 207]:\tstep: 69040\tsteps/sec: 1.40887\tloss: 0.37093\tlr:[3.1179727e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:16:19,657 [ hooks.py: 207]:\tstep: 69050\tsteps/sec: 0.92859\tloss: 0.50699\tlr:[3.1111806e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:16:26,934 [ hooks.py: 207]:\tstep: 69060\tsteps/sec: 1.37248\tloss: 0.05908\tlr:[3.104392e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:16:35,069 [ hooks.py: 207]:\tstep: 69070\tsteps/sec: 1.17380\tloss: 0.36816\tlr:[3.0975996e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:16:44,725 [ hooks.py: 207]:\tstep: 69080\tsteps/sec: 1.08547\tloss: 0.27419\tlr:[3.090811e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:16:53,208 [ hooks.py: 207]:\tstep: 69090\tsteps/sec: 1.15995\tloss: 0.16516\tlr:[3.084019e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:03,484 [ hooks.py: 207]:\tstep: 69100\tsteps/sec: 1.00612\tloss: 0.49615\tlr:[3.0772308e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:11,238 [ hooks.py: 207]:\tstep: 69110\tsteps/sec: 1.26166\tloss: 0.44410\tlr:[3.0704377e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:18,617 [ hooks.py: 207]:\tstep: 69120\tsteps/sec: 1.37377\tloss: 0.43954\tlr:[3.063646e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:27,200 [ hooks.py: 207]:\tstep: 69130\tsteps/sec: 1.13360\tloss: 0.31716\tlr:[3.0568572e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:34,700 [ hooks.py: 207]:\tstep: 69140\tsteps/sec: 1.36377\tloss: 0.23839\tlr:[3.050065e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:43,307 [ hooks.py: 207]:\tstep: 69150\tsteps/sec: 1.17297\tloss: 0.37618\tlr:[3.043276e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:50,555 [ hooks.py: 207]:\tstep: 69160\tsteps/sec: 1.33006\tloss: 0.37732\tlr:[3.0364838e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:17:59,302 [ hooks.py: 207]:\tstep: 69170\tsteps/sec: 1.18563\tloss: 0.29230\tlr:[3.029695e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:18:07,408 [ hooks.py: 207]:\tstep: 69180\tsteps/sec: 1.21148\tloss: 0.13154\tlr:[3.0229035e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:18:15,970 [ hooks.py: 207]:\tstep: 69190\tsteps/sec: 1.16085\tloss: 0.39136\tlr:[3.016115e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:18:23,890 [ hooks.py: 207]:\tstep: 69200\tsteps/sec: 1.27377\tloss: 0.74366\tlr:[3.0093229e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:18:32,787 [ hooks.py: 207]:\tstep: 69210\tsteps/sec: 1.13968\tloss: 0.60270\tlr:[3.0025333e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:18:40,374 [ hooks.py: 207]:\tstep: 69220\tsteps/sec: 1.30499\tloss: 0.35871\tlr:[2.995742e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:18:50,425 [ hooks.py: 207]:\tstep: 69230\tsteps/sec: 0.97315\tloss: 0.38897\tlr:[2.9889525e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:18:59,195 [ hooks.py: 207]:\tstep: 69240\tsteps/sec: 1.18851\tloss: 0.21734\tlr:[2.9821604e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:19:06,249 [ hooks.py: 207]:\tstep: 69250\tsteps/sec: 1.31108\tloss: 0.12991\tlr:[2.975369e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:19:15,176 [ hooks.py: 207]:\tstep: 69260\tsteps/sec: 1.15199\tloss: 0.44655\tlr:[2.9685796e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:19:23,083 [ hooks.py: 207]:\tstep: 69270\tsteps/sec: 1.30355\tloss: 0.74970\tlr:[2.9617877e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:19:30,470 [ hooks.py: 207]:\tstep: 69280\tsteps/sec: 1.26143\tloss: 0.13764\tlr:[2.954999e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:19:38,901 [ hooks.py: 207]:\tstep: 69290\tsteps/sec: 1.26511\tloss: 0.07328\tlr:[2.948207e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:19:48,634 [ hooks.py: 207]:\tstep: 69300\tsteps/sec: 1.02815\tloss: 0.36015\tlr:[2.9414182e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:19:56,613 [ hooks.py: 207]:\tstep: 69310\tsteps/sec: 1.24777\tloss: 0.53475\tlr:[2.9346265e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:20:05,678 [ hooks.py: 207]:\tstep: 69320\tsteps/sec: 1.04784\tloss: 0.11402\tlr:[2.927837e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:20:15,947 [ hooks.py: 207]:\tstep: 69330\tsteps/sec: 1.00787\tloss: 0.47586\tlr:[2.9210448e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:20:25,679 [ hooks.py: 207]:\tstep: 69340\tsteps/sec: 1.02400\tloss: 0.22228\tlr:[2.9142557e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:20:33,241 [ hooks.py: 207]:\tstep: 69350\tsteps/sec: 1.30227\tloss: 0.40650\tlr:[2.9074645e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:20:44,157 [ hooks.py: 207]:\tstep: 69360\tsteps/sec: 0.93978\tloss: 0.21835\tlr:[2.9006753e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:20:52,997 [ hooks.py: 207]:\tstep: 69370\tsteps/sec: 1.12800\tloss: 0.37392\tlr:[2.893883e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:01,529 [ hooks.py: 207]:\tstep: 69380\tsteps/sec: 1.19010\tloss: 0.15964\tlr:[2.8870943e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:09,572 [ hooks.py: 207]:\tstep: 69390\tsteps/sec: 1.16195\tloss: 0.51634\tlr:[2.8803026e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:18,863 [ hooks.py: 207]:\tstep: 69400\tsteps/sec: 1.09371\tloss: 0.54927\tlr:[2.8735103e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:26,550 [ hooks.py: 207]:\tstep: 69410\tsteps/sec: 1.33657\tloss: 0.25355\tlr:[2.8667212e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:33,413 [ hooks.py: 207]:\tstep: 69420\tsteps/sec: 1.48214\tloss: 0.23778\tlr:[2.85993e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:41,226 [ hooks.py: 207]:\tstep: 69430\tsteps/sec: 1.27903\tloss: 0.34748\tlr:[2.8531408e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:49,781 [ hooks.py: 207]:\tstep: 69440\tsteps/sec: 1.15699\tloss: 0.46220\tlr:[2.8463485e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:21:59,406 [ hooks.py: 207]:\tstep: 69450\tsteps/sec: 1.04849\tloss: 0.37812\tlr:[2.8395596e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:22:10,534 [ hooks.py: 207]:\tstep: 69460\tsteps/sec: 0.88526\tloss: 0.15827\tlr:[2.8327681e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:22:18,719 [ hooks.py: 207]:\tstep: 69470\tsteps/sec: 1.25780\tloss: 0.56966\tlr:[2.825979e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:22:28,191 [ hooks.py: 207]:\tstep: 69480\tsteps/sec: 1.05023\tloss: 0.13816\tlr:[2.8191864e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:22:35,626 [ hooks.py: 207]:\tstep: 69490\tsteps/sec: 1.25669\tloss: 0.22111\tlr:[2.8123982e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:22:46,029 [ hooks.py: 207]:\tstep: 69500\tsteps/sec: 0.99435\tloss: 0.31143\tlr:[2.8056056e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:22:56,808 [ hooks.py: 207]:\tstep: 69510\tsteps/sec: 0.96789\tloss: 0.32457\tlr:[2.798817e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:05,589 [ hooks.py: 207]:\tstep: 69520\tsteps/sec: 1.08552\tloss: 0.58860\tlr:[2.7920248e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:12,227 [ hooks.py: 207]:\tstep: 69530\tsteps/sec: 1.51075\tloss: 0.49897\tlr:[2.7852357e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:20,789 [ hooks.py: 207]:\tstep: 69540\tsteps/sec: 1.14980\tloss: 0.45163\tlr:[2.7784438e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:28,759 [ hooks.py: 207]:\tstep: 69550\tsteps/sec: 1.31008\tloss: 0.27966\tlr:[2.771652e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:35,911 [ hooks.py: 207]:\tstep: 69560\tsteps/sec: 1.31158\tloss: 0.23526\tlr:[2.764863e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:43,490 [ hooks.py: 207]:\tstep: 69570\tsteps/sec: 1.40785\tloss: 0.24586\tlr:[2.7580713e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:49,903 [ hooks.py: 207]:\tstep: 69580\tsteps/sec: 1.51532\tloss: 0.05757\tlr:[2.751282e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:23:57,612 [ hooks.py: 207]:\tstep: 69590\tsteps/sec: 1.30060\tloss: 0.12532\tlr:[2.7444903e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:24:06,008 [ hooks.py: 207]:\tstep: 69600\tsteps/sec: 1.18137\tloss: 0.33248\tlr:[2.737701e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:24:13,293 [ hooks.py: 207]:\tstep: 69610\tsteps/sec: 1.39655\tloss: 0.45171\tlr:[2.7309093e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:24:21,598 [ hooks.py: 207]:\tstep: 69620\tsteps/sec: 1.20052\tloss: 0.33192\tlr:[2.7241201e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:24:29,448 [ hooks.py: 207]:\tstep: 69630\tsteps/sec: 1.21876\tloss: 0.47517\tlr:[2.7173282e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:24:40,231 [ hooks.py: 207]:\tstep: 69640\tsteps/sec: 0.96307\tloss: 0.19376\tlr:[2.7105395e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 18:24:46,313 [ hooks.py: 207]:\tstep: 69650\tsteps/sec: 1.60173\tloss: 0.24585\tlr:[2.7037474e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:24:56,738 [ hooks.py: 207]:\tstep: 69660\tsteps/sec: 1.00341\tloss: 0.23581\tlr:[2.696959e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:25:07,053 [ hooks.py: 207]:\tstep: 69670\tsteps/sec: 0.93074\tloss: 0.10374\tlr:[2.690167e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:25:15,705 [ hooks.py: 207]:\tstep: 69680\tsteps/sec: 1.13727\tloss: 0.38865\tlr:[2.6833773e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:25:23,783 [ hooks.py: 207]:\tstep: 69690\tsteps/sec: 1.24340\tloss: 0.86303\tlr:[2.676586e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:25:32,547 [ hooks.py: 207]:\tstep: 69700\tsteps/sec: 1.11535\tloss: 0.37006\tlr:[2.669794e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:25:44,186 [ hooks.py: 207]:\tstep: 69710\tsteps/sec: 0.90168\tloss: 0.38788\tlr:[2.6630048e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:25:51,005 [ hooks.py: 207]:\tstep: 69720\tsteps/sec: 1.42622\tloss: 0.27406\tlr:[2.6562132e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:25:59,545 [ hooks.py: 207]:\tstep: 69730\tsteps/sec: 1.18872\tloss: 0.31359\tlr:[2.6494242e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:26:07,192 [ hooks.py: 207]:\tstep: 69740\tsteps/sec: 1.27774\tloss: 0.54699\tlr:[2.6426321e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:26:14,453 [ hooks.py: 207]:\tstep: 69750\tsteps/sec: 1.41037\tloss: 0.21511\tlr:[2.635843e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:26:24,433 [ hooks.py: 207]:\tstep: 69760\tsteps/sec: 0.99690\tloss: 0.47812\tlr:[2.6290513e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:26:34,549 [ hooks.py: 207]:\tstep: 69770\tsteps/sec: 0.95535\tloss: 0.17624\tlr:[2.6222622e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:26:44,311 [ hooks.py: 207]:\tstep: 69780\tsteps/sec: 1.06579\tloss: 0.44998\tlr:[2.61547e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:26:51,619 [ hooks.py: 207]:\tstep: 69790\tsteps/sec: 1.35562\tloss: 0.39003\tlr:[2.6086807e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:27:01,279 [ hooks.py: 207]:\tstep: 69800\tsteps/sec: 1.01979\tloss: 0.14218\tlr:[2.6018893e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:27:10,342 [ hooks.py: 207]:\tstep: 69810\tsteps/sec: 1.11079\tloss: 0.22020\tlr:[2.5951e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:27:21,220 [ hooks.py: 207]:\tstep: 69820\tsteps/sec: 0.92740\tloss: 0.43077\tlr:[2.588308e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:27:27,902 [ hooks.py: 207]:\tstep: 69830\tsteps/sec: 1.44204\tloss: 0.13331\tlr:[2.581519e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:27:37,999 [ hooks.py: 207]:\tstep: 69840\tsteps/sec: 1.02721\tloss: 0.38707\tlr:[2.574727e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:27:45,908 [ hooks.py: 207]:\tstep: 69850\tsteps/sec: 1.18599\tloss: 0.18335\tlr:[2.5679349e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:27:53,879 [ hooks.py: 207]:\tstep: 69860\tsteps/sec: 1.34775\tloss: 0.36082\tlr:[2.5611462e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:03,620 [ hooks.py: 207]:\tstep: 69870\tsteps/sec: 1.02412\tloss: 0.45223\tlr:[2.5543543e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:12,571 [ hooks.py: 207]:\tstep: 69880\tsteps/sec: 1.09972\tloss: 0.46439\tlr:[2.5475654e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:19,583 [ hooks.py: 207]:\tstep: 69890\tsteps/sec: 1.46306\tloss: 0.28913\tlr:[2.5407737e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:26,600 [ hooks.py: 207]:\tstep: 69900\tsteps/sec: 1.38383\tloss: 1.08019\tlr:[2.5339846e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:33,390 [ hooks.py: 207]:\tstep: 69910\tsteps/sec: 1.50957\tloss: 0.21875\tlr:[2.5271922e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:41,664 [ hooks.py: 207]:\tstep: 69920\tsteps/sec: 1.20884\tloss: 0.09489\tlr:[2.5204038e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:49,173 [ hooks.py: 207]:\tstep: 69930\tsteps/sec: 1.33417\tloss: 0.37709\tlr:[2.5136112e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:28:56,331 [ hooks.py: 207]:\tstep: 69940\tsteps/sec: 1.45238\tloss: 0.29592\tlr:[2.5068227e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:04,999 [ hooks.py: 207]:\tstep: 69950\tsteps/sec: 1.09538\tloss: 0.41287\tlr:[2.5000306e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:12,884 [ hooks.py: 207]:\tstep: 69960\tsteps/sec: 1.28754\tloss: 0.39735\tlr:[2.4932417e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:20,176 [ hooks.py: 207]:\tstep: 69970\tsteps/sec: 1.35041\tloss: 0.47305\tlr:[2.48645e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:28,902 [ hooks.py: 207]:\tstep: 69980\tsteps/sec: 1.12778\tloss: 0.54124\tlr:[2.4796605e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:40,561 [ hooks.py: 207]:\tstep: 69990\tsteps/sec: 0.87641\tloss: 0.68620\tlr:[2.4728688e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:47,016 [ hooks.py: 207]:\tstep: 70000\tsteps/sec: 1.45823\tloss: 0.72198\tlr:[2.4660774e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:47,017 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:47,017 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:47,017 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:29:47,017 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:47,017 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:29:47,017 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:37:38,487 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,488 [ trainer.py: 58]:\t{'acc': 0.7755020080321285, 'loss': 0.6884883861415662}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,488 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,489 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:37:38,489 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:37:38,489 [ trainer.py: 70]:\tacc\t0.7755020080321285\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:37:38,489 [ trainer.py: 70]:\tloss\t0.6884883861415662\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:37:38,490 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,490 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7755020080321285, 'loss': 0.6884883861415662}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,490 [ exporter.py: 64]:\t[Best Exporter]: skip step 70000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,490 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7755020080321285, 'loss': 0.6884883861415662}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,490 [ exporter.py: 104]:\t[Best Exporter]: skip step 70000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:38,491 [monitored_executor.py: 122]:\tsaving step 70000 to ./output/model_70000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:41,705 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_67000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:51,006 [ hooks.py: 207]:\tstep: 70010\tsteps/sec: 0.02067\tloss: 0.33961\tlr:[2.459288e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:37:58,463 [ hooks.py: 207]:\tstep: 70020\tsteps/sec: 1.38808\tloss: 0.13523\tlr:[2.4524963e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:38:07,639 [ hooks.py: 207]:\tstep: 70030\tsteps/sec: 1.06598\tloss: 0.48423\tlr:[2.4457072e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:38:14,603 [ hooks.py: 207]:\tstep: 70040\tsteps/sec: 1.38892\tloss: 0.19816\tlr:[2.4389153e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:38:23,541 [ hooks.py: 207]:\tstep: 70050\tsteps/sec: 1.16924\tloss: 0.33358\tlr:[2.4321262e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:38:30,509 [ hooks.py: 207]:\tstep: 70060\tsteps/sec: 1.41943\tloss: 0.94050\tlr:[2.425334e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:38:38,426 [ hooks.py: 207]:\tstep: 70070\tsteps/sec: 1.22340\tloss: 0.04030\tlr:[2.418545e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:38:46,169 [ hooks.py: 207]:\tstep: 70080\tsteps/sec: 1.30827\tloss: 0.19291\tlr:[2.4117537e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:38:54,321 [ hooks.py: 207]:\tstep: 70090\tsteps/sec: 1.27970\tloss: 0.67914\tlr:[2.4049646e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:39:00,996 [ hooks.py: 207]:\tstep: 70100\tsteps/sec: 1.44982\tloss: 0.16179\tlr:[2.3981722e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 18:39:08,820 [ hooks.py: 207]:\tstep: 70110\tsteps/sec: 1.32007\tloss: 0.48423\tlr:[2.3913833e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:39:15,443 [ hooks.py: 207]:\tstep: 70120\tsteps/sec: 1.36515\tloss: 0.20177\tlr:[2.3845917e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:39:25,828 [ hooks.py: 207]:\tstep: 70130\tsteps/sec: 1.00254\tloss: 0.05138\tlr:[2.3778027e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:39:35,504 [ hooks.py: 207]:\tstep: 70140\tsteps/sec: 1.03330\tloss: 0.38898\tlr:[2.3710104e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:39:42,525 [ hooks.py: 207]:\tstep: 70150\tsteps/sec: 1.40049\tloss: 0.30644\tlr:[2.3642187e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:39:51,114 [ hooks.py: 207]:\tstep: 70160\tsteps/sec: 1.19148\tloss: 0.49378\tlr:[2.3574294e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:39:58,572 [ hooks.py: 207]:\tstep: 70170\tsteps/sec: 1.37589\tloss: 0.36464\tlr:[2.3506377e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:40:05,936 [ hooks.py: 207]:\tstep: 70180\tsteps/sec: 1.30119\tloss: 0.17615\tlr:[2.3438486e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:40:13,239 [ hooks.py: 207]:\tstep: 70190\tsteps/sec: 1.40915\tloss: 0.29007\tlr:[2.337057e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:40:22,544 [ hooks.py: 207]:\tstep: 70200\tsteps/sec: 1.06598\tloss: 0.10020\tlr:[2.3302678e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:40:30,781 [ hooks.py: 207]:\tstep: 70210\tsteps/sec: 1.21645\tloss: 0.11911\tlr:[2.3234759e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:40:38,536 [ hooks.py: 207]:\tstep: 70220\tsteps/sec: 1.30717\tloss: 0.89531\tlr:[2.3166865e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:40:46,693 [ hooks.py: 207]:\tstep: 70230\tsteps/sec: 1.20367\tloss: 0.11142\tlr:[2.3098946e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:40:54,812 [ hooks.py: 207]:\tstep: 70240\tsteps/sec: 1.25728\tloss: 0.14349\tlr:[2.3031062e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:41:03,568 [ hooks.py: 207]:\tstep: 70250\tsteps/sec: 1.15249\tloss: 0.77220\tlr:[2.296314e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:41:10,711 [ hooks.py: 207]:\tstep: 70260\tsteps/sec: 1.34690\tloss: 0.24822\tlr:[2.289525e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:41:18,701 [ hooks.py: 207]:\tstep: 70270\tsteps/sec: 1.23324\tloss: 0.20428\tlr:[2.2827328e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:41:26,818 [ hooks.py: 207]:\tstep: 70280\tsteps/sec: 1.25856\tloss: 0.42432\tlr:[2.2759446e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:41:36,750 [ hooks.py: 207]:\tstep: 70290\tsteps/sec: 1.00874\tloss: 0.67860\tlr:[2.269152e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:41:45,272 [ hooks.py: 207]:\tstep: 70300\tsteps/sec: 1.18463\tloss: 0.08876\tlr:[2.2623606e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:41:51,666 [ hooks.py: 207]:\tstep: 70310\tsteps/sec: 1.47906\tloss: 0.13814\tlr:[2.2555714e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:01,816 [ hooks.py: 207]:\tstep: 70320\tsteps/sec: 1.00729\tloss: 0.43756\tlr:[2.2487789e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:09,883 [ hooks.py: 207]:\tstep: 70330\tsteps/sec: 1.26564\tloss: 0.25626\tlr:[2.2419906e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:18,122 [ hooks.py: 207]:\tstep: 70340\tsteps/sec: 1.20762\tloss: 0.18657\tlr:[2.2351983e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:26,715 [ hooks.py: 207]:\tstep: 70350\tsteps/sec: 1.15544\tloss: 0.19864\tlr:[2.228409e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:33,915 [ hooks.py: 207]:\tstep: 70360\tsteps/sec: 1.42625\tloss: 0.10509\tlr:[2.2216175e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:41,309 [ hooks.py: 207]:\tstep: 70370\tsteps/sec: 1.32003\tloss: 0.22263\tlr:[2.2148283e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:49,396 [ hooks.py: 207]:\tstep: 70380\tsteps/sec: 1.20872\tloss: 0.53890\tlr:[2.2080367e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:42:58,980 [ hooks.py: 207]:\tstep: 70390\tsteps/sec: 1.08170\tloss: 0.48350\tlr:[2.2012478e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:43:05,490 [ hooks.py: 207]:\tstep: 70400\tsteps/sec: 1.45973\tloss: 0.78569\tlr:[2.1944556e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:43:14,357 [ hooks.py: 207]:\tstep: 70410\tsteps/sec: 1.16867\tloss: 0.15164\tlr:[2.1876663e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:43:23,830 [ hooks.py: 207]:\tstep: 70420\tsteps/sec: 1.05552\tloss: 0.15496\tlr:[2.1808742e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:43:32,356 [ hooks.py: 207]:\tstep: 70430\tsteps/sec: 1.14750\tloss: 0.12121\tlr:[2.1740823e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:43:40,602 [ hooks.py: 207]:\tstep: 70440\tsteps/sec: 1.22300\tloss: 0.17593\tlr:[2.167294e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:43:51,640 [ hooks.py: 207]:\tstep: 70450\tsteps/sec: 0.90323\tloss: 0.53221\tlr:[2.1605017e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:43:59,012 [ hooks.py: 207]:\tstep: 70460\tsteps/sec: 1.37944\tloss: 0.12729\tlr:[2.1537128e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:44:08,208 [ hooks.py: 207]:\tstep: 70470\tsteps/sec: 1.08108\tloss: 0.85897\tlr:[2.1469211e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:44:17,098 [ hooks.py: 207]:\tstep: 70480\tsteps/sec: 1.11128\tloss: 0.28690\tlr:[2.1401318e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:44:25,619 [ hooks.py: 207]:\tstep: 70490\tsteps/sec: 1.17155\tloss: 0.64336\tlr:[2.1333399e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:44:34,612 [ hooks.py: 207]:\tstep: 70500\tsteps/sec: 1.13238\tloss: 0.22335\tlr:[2.1265507e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:44:43,037 [ hooks.py: 207]:\tstep: 70510\tsteps/sec: 1.17574\tloss: 0.24773\tlr:[2.1197593e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:44:51,274 [ hooks.py: 207]:\tstep: 70520\tsteps/sec: 1.20177\tloss: 0.26414\tlr:[2.1129702e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:45:01,097 [ hooks.py: 207]:\tstep: 70530\tsteps/sec: 1.01645\tloss: 0.30488\tlr:[2.106178e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:45:10,436 [ hooks.py: 207]:\tstep: 70540\tsteps/sec: 1.07807\tloss: 0.17386\tlr:[2.0993893e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:45:18,297 [ hooks.py: 207]:\tstep: 70550\tsteps/sec: 1.30104\tloss: 0.28729\tlr:[2.0925975e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:45:25,640 [ hooks.py: 207]:\tstep: 70560\tsteps/sec: 1.31524\tloss: 0.16483\tlr:[2.0858083e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:45:33,379 [ hooks.py: 207]:\tstep: 70570\tsteps/sec: 1.26980\tloss: 0.13825\tlr:[2.0790162e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:45:43,538 [ hooks.py: 207]:\tstep: 70580\tsteps/sec: 1.02061\tloss: 0.31967\tlr:[2.0722246e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:45:53,334 [ hooks.py: 207]:\tstep: 70590\tsteps/sec: 1.02562\tloss: 0.76709\tlr:[2.0654354e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:01,873 [ hooks.py: 207]:\tstep: 70600\tsteps/sec: 1.15228\tloss: 0.30121\tlr:[2.0586438e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:10,259 [ hooks.py: 207]:\tstep: 70610\tsteps/sec: 1.19377\tloss: 0.29762\tlr:[2.0518542e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:16,974 [ hooks.py: 207]:\tstep: 70620\tsteps/sec: 1.48457\tloss: 0.16799\tlr:[2.0450627e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:24,520 [ hooks.py: 207]:\tstep: 70630\tsteps/sec: 1.30610\tloss: 0.07882\tlr:[2.0382734e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:32,993 [ hooks.py: 207]:\tstep: 70640\tsteps/sec: 1.19584\tloss: 0.44732\tlr:[2.0314812e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:40,202 [ hooks.py: 207]:\tstep: 70650\tsteps/sec: 1.43605\tloss: 0.21432\tlr:[2.0246923e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:49,947 [ hooks.py: 207]:\tstep: 70660\tsteps/sec: 1.01010\tloss: 0.20403\tlr:[2.0179007e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:46:59,542 [ hooks.py: 207]:\tstep: 70670\tsteps/sec: 1.00510\tloss: 0.29915\tlr:[2.0111117e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:47:09,470 [ hooks.py: 207]:\tstep: 70680\tsteps/sec: 1.04508\tloss: 0.35344\tlr:[2.0043199e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:47:18,697 [ hooks.py: 207]:\tstep: 70690\tsteps/sec: 1.07835\tloss: 0.42058\tlr:[1.9975305e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:47:27,805 [ hooks.py: 207]:\tstep: 70700\tsteps/sec: 1.08594\tloss: 0.69764\tlr:[1.9907386e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:47:35,524 [ hooks.py: 207]:\tstep: 70710\tsteps/sec: 1.30429\tloss: 0.17868\tlr:[1.9839501e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:47:41,990 [ hooks.py: 207]:\tstep: 70720\tsteps/sec: 1.45766\tloss: 0.57427\tlr:[1.9771578e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:47:51,042 [ hooks.py: 207]:\tstep: 70730\tsteps/sec: 1.10908\tloss: 0.13747\tlr:[1.970366e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:48:00,536 [ hooks.py: 207]:\tstep: 70740\tsteps/sec: 1.07798\tloss: 0.19603\tlr:[1.9635768e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:48:09,754 [ hooks.py: 207]:\tstep: 70750\tsteps/sec: 1.11062\tloss: 0.28520\tlr:[1.9567851e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 18:48:19,602 [ hooks.py: 207]:\tstep: 70760\tsteps/sec: 1.02028\tloss: 0.19459\tlr:[1.9499957e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:48:26,576 [ hooks.py: 207]:\tstep: 70770\tsteps/sec: 1.34450\tloss: 0.09340\tlr:[1.943204e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:48:35,034 [ hooks.py: 207]:\tstep: 70780\tsteps/sec: 1.24020\tloss: 0.22852\tlr:[1.9364152e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:48:42,013 [ hooks.py: 207]:\tstep: 70790\tsteps/sec: 1.34408\tloss: 0.42199\tlr:[1.929623e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:48:51,953 [ hooks.py: 207]:\tstep: 70800\tsteps/sec: 1.04898\tloss: 0.16016\tlr:[1.9228341e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:48:59,617 [ hooks.py: 207]:\tstep: 70810\tsteps/sec: 1.25368\tloss: 0.43412\tlr:[1.9160423e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:49:09,510 [ hooks.py: 207]:\tstep: 70820\tsteps/sec: 1.05437\tloss: 0.50612\tlr:[1.909253e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:49:19,228 [ hooks.py: 207]:\tstep: 70830\tsteps/sec: 1.03807\tloss: 0.11558\tlr:[1.9024612e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:49:29,078 [ hooks.py: 207]:\tstep: 70840\tsteps/sec: 1.01656\tloss: 0.36728\tlr:[1.8956722e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:49:36,996 [ hooks.py: 207]:\tstep: 70850\tsteps/sec: 1.23511\tloss: 0.30101\tlr:[1.8888807e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:49:45,084 [ hooks.py: 207]:\tstep: 70860\tsteps/sec: 1.25538\tloss: 0.61179\tlr:[1.8820916e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:49:53,908 [ hooks.py: 207]:\tstep: 70870\tsteps/sec: 1.13891\tloss: 0.07770\tlr:[1.8752997e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:02,019 [ hooks.py: 207]:\tstep: 70880\tsteps/sec: 1.20198\tloss: 0.43557\tlr:[1.8685076e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:10,307 [ hooks.py: 207]:\tstep: 70890\tsteps/sec: 1.20422\tloss: 0.57134\tlr:[1.8617186e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:19,608 [ hooks.py: 207]:\tstep: 70900\tsteps/sec: 1.09683\tloss: 0.63238\tlr:[1.8549268e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:28,310 [ hooks.py: 207]:\tstep: 70910\tsteps/sec: 1.10021\tloss: 0.28047\tlr:[1.8481377e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:36,242 [ hooks.py: 207]:\tstep: 70920\tsteps/sec: 1.27085\tloss: 0.80424\tlr:[1.8413454e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:43,433 [ hooks.py: 207]:\tstep: 70930\tsteps/sec: 1.44787\tloss: 0.49874\tlr:[1.8345568e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:50,484 [ hooks.py: 207]:\tstep: 70940\tsteps/sec: 1.39566\tloss: 0.14481\tlr:[1.8277649e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:50:57,439 [ hooks.py: 207]:\tstep: 70950\tsteps/sec: 1.47541\tloss: 0.17724\tlr:[1.8209759e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:05,502 [ hooks.py: 207]:\tstep: 70960\tsteps/sec: 1.20822\tloss: 0.30071\tlr:[1.814184e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:13,413 [ hooks.py: 207]:\tstep: 70970\tsteps/sec: 1.28272\tloss: 0.26815\tlr:[1.8073952e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:20,278 [ hooks.py: 207]:\tstep: 70980\tsteps/sec: 1.45037\tloss: 0.28918\tlr:[1.8006032e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:29,193 [ hooks.py: 207]:\tstep: 70990\tsteps/sec: 1.09268\tloss: 0.48885\tlr:[1.7938137e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:37,491 [ hooks.py: 207]:\tstep: 71000\tsteps/sec: 1.17253\tloss: 0.37832\tlr:[1.7870219e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:37,492 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:37,492 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:37,492 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:51:37,492 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:37,492 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:51:37,492 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:59:30,365 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,366 [ trainer.py: 58]:\t{'acc': 0.7763052208835342, 'loss': 0.6724040948618681}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,366 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,366 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:59:30,367 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:59:30,367 [ trainer.py: 70]:\tacc\t0.7763052208835342\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:59:30,367 [ trainer.py: 70]:\tloss\t0.6724040948618681\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 18:59:30,367 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,367 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7763052208835342, 'loss': 0.6724040948618681}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,367 [ exporter.py: 64]:\t[Best Exporter]: skip step 71000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,367 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7763052208835342, 'loss': 0.6724040948618681}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,367 [ exporter.py: 104]:\t[Best Exporter]: skip step 71000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:30,368 [monitored_executor.py: 122]:\tsaving step 71000 to ./output/model_71000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:33,543 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_68000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:45,521 [ hooks.py: 207]:\tstep: 71010\tsteps/sec: 0.02051\tloss: 0.46104\tlr:[1.7802331e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 18:59:54,221 [ hooks.py: 207]:\tstep: 71020\tsteps/sec: 1.16379\tloss: 0.64305\tlr:[1.7734411e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:00:02,091 [ hooks.py: 207]:\tstep: 71030\tsteps/sec: 1.20847\tloss: 0.26862\tlr:[1.7666489e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:00:14,034 [ hooks.py: 207]:\tstep: 71040\tsteps/sec: 0.86632\tloss: 0.41078\tlr:[1.7598601e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:00:22,970 [ hooks.py: 207]:\tstep: 71050\tsteps/sec: 1.09433\tloss: 0.33438\tlr:[1.7530683e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:00:33,995 [ hooks.py: 207]:\tstep: 71060\tsteps/sec: 0.89612\tloss: 0.19697\tlr:[1.7462789e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:00:43,595 [ hooks.py: 207]:\tstep: 71070\tsteps/sec: 1.07992\tloss: 0.12687\tlr:[1.7394874e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:00:52,412 [ hooks.py: 207]:\tstep: 71080\tsteps/sec: 1.11782\tloss: 0.70718\tlr:[1.732698e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:00,846 [ hooks.py: 207]:\tstep: 71090\tsteps/sec: 1.18336\tloss: 0.34188\tlr:[1.7259061e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:09,778 [ hooks.py: 207]:\tstep: 71100\tsteps/sec: 1.10314\tloss: 0.11596\tlr:[1.7191174e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:18,724 [ hooks.py: 207]:\tstep: 71110\tsteps/sec: 1.10797\tloss: 0.15404\tlr:[1.7123257e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:27,937 [ hooks.py: 207]:\tstep: 71120\tsteps/sec: 1.11945\tloss: 0.31345\tlr:[1.7055364e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:36,351 [ hooks.py: 207]:\tstep: 71130\tsteps/sec: 1.16497\tloss: 0.43709\tlr:[1.6987448e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:43,642 [ hooks.py: 207]:\tstep: 71140\tsteps/sec: 1.42327\tloss: 0.22603\tlr:[1.6919558e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:51,768 [ hooks.py: 207]:\tstep: 71150\tsteps/sec: 1.21745\tloss: 0.12730\tlr:[1.6851637e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:01:59,367 [ hooks.py: 207]:\tstep: 71160\tsteps/sec: 1.34348\tloss: 0.11531\tlr:[1.6783746e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:02:07,007 [ hooks.py: 207]:\tstep: 71170\tsteps/sec: 1.21864\tloss: 0.06287\tlr:[1.6715828e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:02:16,420 [ hooks.py: 207]:\tstep: 71180\tsteps/sec: 1.06113\tloss: 0.69976\tlr:[1.6647905e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:02:26,499 [ hooks.py: 207]:\tstep: 71190\tsteps/sec: 1.02790\tloss: 0.47661\tlr:[1.6580017e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:02:34,702 [ hooks.py: 207]:\tstep: 71200\tsteps/sec: 1.23780\tloss: 0.53797\tlr:[1.6512101e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:02:45,844 [ hooks.py: 207]:\tstep: 71210\tsteps/sec: 0.89290\tloss: 0.26415\tlr:[1.6444205e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 19:02:52,376 [ hooks.py: 207]:\tstep: 71220\tsteps/sec: 1.53085\tloss: 0.31228\tlr:[1.6376288e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:01,072 [ hooks.py: 207]:\tstep: 71230\tsteps/sec: 1.14046\tloss: 0.58361\tlr:[1.6308398e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:10,830 [ hooks.py: 207]:\tstep: 71240\tsteps/sec: 1.03266\tloss: 0.32787\tlr:[1.6240483e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:18,082 [ hooks.py: 207]:\tstep: 71250\tsteps/sec: 1.36665\tloss: 0.18804\tlr:[1.6172587e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:25,211 [ hooks.py: 207]:\tstep: 71260\tsteps/sec: 1.44060\tloss: 0.16565\tlr:[1.6104669e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:33,882 [ hooks.py: 207]:\tstep: 71270\tsteps/sec: 1.14833\tloss: 0.39796\tlr:[1.6036782e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:40,978 [ hooks.py: 207]:\tstep: 71280\tsteps/sec: 1.41604\tloss: 0.71083\tlr:[1.5968862e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:48,881 [ hooks.py: 207]:\tstep: 71290\tsteps/sec: 1.26245\tloss: 0.33417\tlr:[1.5900969e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:03:58,321 [ hooks.py: 207]:\tstep: 71300\tsteps/sec: 1.01301\tloss: 0.30077\tlr:[1.5833053e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:04:08,555 [ hooks.py: 207]:\tstep: 71310\tsteps/sec: 1.00802\tloss: 0.30216\tlr:[1.5765165e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:04:16,908 [ hooks.py: 207]:\tstep: 71320\tsteps/sec: 1.19564\tloss: 0.33459\tlr:[1.5697244e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:04:24,620 [ hooks.py: 207]:\tstep: 71330\tsteps/sec: 1.26368\tloss: 0.24587\tlr:[1.5629323e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:04:34,052 [ hooks.py: 207]:\tstep: 71340\tsteps/sec: 1.10340\tloss: 0.30955\tlr:[1.5561432e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:04:40,630 [ hooks.py: 207]:\tstep: 71350\tsteps/sec: 1.51713\tloss: 0.36248\tlr:[1.5493513e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:04:48,927 [ hooks.py: 207]:\tstep: 71360\tsteps/sec: 1.20155\tloss: 0.04064\tlr:[1.5425626e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:04:57,068 [ hooks.py: 207]:\tstep: 71370\tsteps/sec: 1.20447\tloss: 0.49665\tlr:[1.5357706e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:05:06,460 [ hooks.py: 207]:\tstep: 71380\tsteps/sec: 1.04641\tloss: 0.18869\tlr:[1.5289813e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:05:15,543 [ hooks.py: 207]:\tstep: 71390\tsteps/sec: 1.13269\tloss: 0.49384\tlr:[1.5221892e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:05:23,831 [ hooks.py: 207]:\tstep: 71400\tsteps/sec: 1.21402\tloss: 0.04827\tlr:[1.5154004e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:05:31,708 [ hooks.py: 207]:\tstep: 71410\tsteps/sec: 1.24212\tloss: 0.55557\tlr:[1.5086089e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:05:41,347 [ hooks.py: 207]:\tstep: 71420\tsteps/sec: 1.01222\tloss: 0.45566\tlr:[1.5018195e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:05:52,443 [ hooks.py: 207]:\tstep: 71430\tsteps/sec: 0.92721\tloss: 0.07934\tlr:[1.4950276e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:06:00,841 [ hooks.py: 207]:\tstep: 71440\tsteps/sec: 1.17034\tloss: 0.32638\tlr:[1.4882388e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:06:09,635 [ hooks.py: 207]:\tstep: 71450\tsteps/sec: 1.17977\tloss: 0.09739\tlr:[1.4814466e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:06:17,624 [ hooks.py: 207]:\tstep: 71460\tsteps/sec: 1.23428\tloss: 0.35369\tlr:[1.4746547e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:06:27,381 [ hooks.py: 207]:\tstep: 71470\tsteps/sec: 1.01960\tloss: 0.34230\tlr:[1.4678658e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:06:35,744 [ hooks.py: 207]:\tstep: 71480\tsteps/sec: 1.18898\tloss: 0.69260\tlr:[1.4610738e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:06:45,266 [ hooks.py: 207]:\tstep: 71490\tsteps/sec: 1.05942\tloss: 0.64256\tlr:[1.4542849e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:06:53,093 [ hooks.py: 207]:\tstep: 71500\tsteps/sec: 1.25524\tloss: 0.26073\tlr:[1.4474931e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:02,678 [ hooks.py: 207]:\tstep: 71510\tsteps/sec: 1.04189\tloss: 0.12736\tlr:[1.4407041e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:09,711 [ hooks.py: 207]:\tstep: 71520\tsteps/sec: 1.39152\tloss: 0.27165\tlr:[1.4339122e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:18,763 [ hooks.py: 207]:\tstep: 71530\tsteps/sec: 1.15431\tloss: 0.40736\tlr:[1.427123e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:26,852 [ hooks.py: 207]:\tstep: 71540\tsteps/sec: 1.20866\tloss: 0.23251\tlr:[1.4203314e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:33,279 [ hooks.py: 207]:\tstep: 71550\tsteps/sec: 1.54901\tloss: 0.66206\tlr:[1.4135421e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:39,726 [ hooks.py: 207]:\tstep: 71560\tsteps/sec: 1.54926\tloss: 0.24771\tlr:[1.4067504e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:47,214 [ hooks.py: 207]:\tstep: 71570\tsteps/sec: 1.35553\tloss: 0.33606\tlr:[1.3999613e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:07:57,383 [ hooks.py: 207]:\tstep: 71580\tsteps/sec: 1.01149\tloss: 0.08966\tlr:[1.3931694e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:08:03,698 [ hooks.py: 207]:\tstep: 71590\tsteps/sec: 1.46143\tloss: 0.10947\tlr:[1.3863803e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:08:12,244 [ hooks.py: 207]:\tstep: 71600\tsteps/sec: 1.15886\tloss: 0.19354\tlr:[1.3795882e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:08:20,729 [ hooks.py: 207]:\tstep: 71610\tsteps/sec: 1.23846\tloss: 0.21609\tlr:[1.3727963e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:08:27,579 [ hooks.py: 207]:\tstep: 71620\tsteps/sec: 1.43493\tloss: 0.67268\tlr:[1.3660076e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:08:36,486 [ hooks.py: 207]:\tstep: 71630\tsteps/sec: 1.07596\tloss: 0.22436\tlr:[1.3592156e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:08:47,149 [ hooks.py: 207]:\tstep: 71640\tsteps/sec: 0.97259\tloss: 0.45436\tlr:[1.3524266e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:08:56,643 [ hooks.py: 207]:\tstep: 71650\tsteps/sec: 1.08324\tloss: 0.35726\tlr:[1.3456345e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:09:05,258 [ hooks.py: 207]:\tstep: 71660\tsteps/sec: 1.14675\tloss: 0.58837\tlr:[1.3388455e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:09:13,357 [ hooks.py: 207]:\tstep: 71670\tsteps/sec: 1.21265\tloss: 0.23362\tlr:[1.3320539e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:09:20,014 [ hooks.py: 207]:\tstep: 71680\tsteps/sec: 1.47036\tloss: 0.38244\tlr:[1.3252647e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:09:27,447 [ hooks.py: 207]:\tstep: 71690\tsteps/sec: 1.30423\tloss: 0.09181\tlr:[1.3184726e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:09:36,660 [ hooks.py: 207]:\tstep: 71700\tsteps/sec: 1.11504\tloss: 0.88497\tlr:[1.3116837e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:09:44,527 [ hooks.py: 207]:\tstep: 71710\tsteps/sec: 1.32854\tloss: 0.42166\tlr:[1.3048918e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:09:52,425 [ hooks.py: 207]:\tstep: 71720\tsteps/sec: 1.23942\tloss: 0.06075\tlr:[1.298103e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:01,000 [ hooks.py: 207]:\tstep: 71730\tsteps/sec: 1.11800\tloss: 0.21633\tlr:[1.2913109e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:09,999 [ hooks.py: 207]:\tstep: 71740\tsteps/sec: 1.15913\tloss: 0.16822\tlr:[1.2845218e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:17,925 [ hooks.py: 207]:\tstep: 71750\tsteps/sec: 1.24978\tloss: 0.16055\tlr:[1.2777299e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:24,907 [ hooks.py: 207]:\tstep: 71760\tsteps/sec: 1.47709\tloss: 0.48799\tlr:[1.2709382e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:34,325 [ hooks.py: 207]:\tstep: 71770\tsteps/sec: 1.05837\tloss: 0.32123\tlr:[1.2641492e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:42,492 [ hooks.py: 207]:\tstep: 71780\tsteps/sec: 1.20306\tloss: 0.36639\tlr:[1.257357e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:50,236 [ hooks.py: 207]:\tstep: 71790\tsteps/sec: 1.30951\tloss: 0.32042\tlr:[1.2505682e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:10:59,088 [ hooks.py: 207]:\tstep: 71800\tsteps/sec: 1.13815\tloss: 0.51701\tlr:[1.2437762e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:11:06,609 [ hooks.py: 207]:\tstep: 71810\tsteps/sec: 1.27803\tloss: 0.15927\tlr:[1.236987e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:11:17,397 [ hooks.py: 207]:\tstep: 71820\tsteps/sec: 0.94064\tloss: 0.25227\tlr:[1.2301953e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:11:25,727 [ hooks.py: 207]:\tstep: 71830\tsteps/sec: 1.19572\tloss: 0.18691\tlr:[1.2234062e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:11:35,340 [ hooks.py: 207]:\tstep: 71840\tsteps/sec: 1.05923\tloss: 0.49882\tlr:[1.2166146e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:11:42,706 [ hooks.py: 207]:\tstep: 71850\tsteps/sec: 1.25807\tloss: 0.07363\tlr:[1.2098254e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:11:51,239 [ hooks.py: 207]:\tstep: 71860\tsteps/sec: 1.17599\tloss: 0.51325\tlr:[1.2030335e-06]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 19:12:00,760 [ hooks.py: 207]:\tstep: 71870\tsteps/sec: 1.09053\tloss: 0.40092\tlr:[1.1962443e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:12:09,116 [ hooks.py: 207]:\tstep: 71880\tsteps/sec: 1.21829\tloss: 0.23057\tlr:[1.1894525e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:12:17,831 [ hooks.py: 207]:\tstep: 71890\tsteps/sec: 1.12545\tloss: 0.37460\tlr:[1.1826635e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:12:28,069 [ hooks.py: 207]:\tstep: 71900\tsteps/sec: 0.97774\tloss: 0.03305\tlr:[1.1758718e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:12:37,589 [ hooks.py: 207]:\tstep: 71910\tsteps/sec: 1.01325\tloss: 0.63095\tlr:[1.1690797e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:12:46,377 [ hooks.py: 207]:\tstep: 71920\tsteps/sec: 1.14207\tloss: 0.17375\tlr:[1.1622907e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:12:54,242 [ hooks.py: 207]:\tstep: 71930\tsteps/sec: 1.30417\tloss: 0.67174\tlr:[1.1554987e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:02,212 [ hooks.py: 207]:\tstep: 71940\tsteps/sec: 1.26538\tloss: 0.13068\tlr:[1.14871e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:10,228 [ hooks.py: 207]:\tstep: 71950\tsteps/sec: 1.27783\tloss: 0.16387\tlr:[1.1419178e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:19,280 [ hooks.py: 207]:\tstep: 71960\tsteps/sec: 1.11021\tloss: 0.46107\tlr:[1.1351289e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:26,112 [ hooks.py: 207]:\tstep: 71970\tsteps/sec: 1.41329\tloss: 0.19708\tlr:[1.1283367e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:35,000 [ hooks.py: 207]:\tstep: 71980\tsteps/sec: 1.16303\tloss: 0.23692\tlr:[1.1215479e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:42,869 [ hooks.py: 207]:\tstep: 71990\tsteps/sec: 1.24472\tloss: 0.34472\tlr:[1.114756e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:50,054 [ hooks.py: 207]:\tstep: 72000\tsteps/sec: 1.35383\tloss: 0.21579\tlr:[1.1079669e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:50,055 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:50,055 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:50,055 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:13:50,055 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:50,055 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:13:50,055 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:21:43,063 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,065 [ trainer.py: 58]:\t{'acc': 0.7803212851405622, 'loss': 0.6785604520581472}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,065 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,065 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:21:43,066 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:21:43,066 [ trainer.py: 70]:\tacc\t0.7803212851405622\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:21:43,066 [ trainer.py: 70]:\tloss\t0.6785604520581472\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:21:43,066 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,066 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7803212851405622, 'loss': 0.6785604520581472}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,067 [ exporter.py: 64]:\t[Best Exporter]: skip step 72000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,067 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7803212851405622, 'loss': 0.6785604520581472}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,067 [ exporter.py: 104]:\t[Best Exporter]: skip step 72000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:43,068 [monitored_executor.py: 122]:\tsaving step 72000 to ./output/model_72000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:46,268 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_69000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:21:54,260 [ hooks.py: 207]:\tstep: 72010\tsteps/sec: 0.02065\tloss: 0.39005\tlr:[1.1011751e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:22:02,573 [ hooks.py: 207]:\tstep: 72020\tsteps/sec: 1.22870\tloss: 0.48999\tlr:[1.094386e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:22:11,383 [ hooks.py: 207]:\tstep: 72030\tsteps/sec: 1.13631\tloss: 0.26486\tlr:[1.0875942e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:22:18,757 [ hooks.py: 207]:\tstep: 72040\tsteps/sec: 1.30387\tloss: 0.23454\tlr:[1.0808051e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:22:28,678 [ hooks.py: 207]:\tstep: 72050\tsteps/sec: 1.04652\tloss: 0.24117\tlr:[1.0740131e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:22:37,021 [ hooks.py: 207]:\tstep: 72060\tsteps/sec: 1.17895\tloss: 0.50930\tlr:[1.0672212e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:22:48,014 [ hooks.py: 207]:\tstep: 72070\tsteps/sec: 0.92066\tloss: 0.48983\tlr:[1.0604323e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:22:56,559 [ hooks.py: 207]:\tstep: 72080\tsteps/sec: 1.21079\tloss: 0.20486\tlr:[1.0536402e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:23:06,493 [ hooks.py: 207]:\tstep: 72090\tsteps/sec: 0.96481\tloss: 0.84687\tlr:[1.0468515e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:23:15,486 [ hooks.py: 207]:\tstep: 72100\tsteps/sec: 1.14954\tloss: 0.49574\tlr:[1.0400594e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:23:22,451 [ hooks.py: 207]:\tstep: 72110\tsteps/sec: 1.34097\tloss: 0.21027\tlr:[1.0332705e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:23:30,138 [ hooks.py: 207]:\tstep: 72120\tsteps/sec: 1.37775\tloss: 0.04465\tlr:[1.0264786e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:23:38,493 [ hooks.py: 207]:\tstep: 72130\tsteps/sec: 1.16562\tloss: 0.23558\tlr:[1.0196896e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:23:46,552 [ hooks.py: 207]:\tstep: 72140\tsteps/sec: 1.24350\tloss: 0.11924\tlr:[1.0128977e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:23:56,842 [ hooks.py: 207]:\tstep: 72150\tsteps/sec: 1.01240\tloss: 0.52823\tlr:[1.0061087e-06]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:04,039 [ hooks.py: 207]:\tstep: 72160\tsteps/sec: 1.33719\tloss: 0.63235\tlr:[9.993167e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:12,547 [ hooks.py: 207]:\tstep: 72170\tsteps/sec: 1.17532\tloss: 0.72804\tlr:[9.925276e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:19,167 [ hooks.py: 207]:\tstep: 72180\tsteps/sec: 1.52881\tloss: 0.33837\tlr:[9.857357e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:26,915 [ hooks.py: 207]:\tstep: 72190\tsteps/sec: 1.21037\tloss: 1.00378\tlr:[9.789468e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:35,123 [ hooks.py: 207]:\tstep: 72200\tsteps/sec: 1.30536\tloss: 0.53829\tlr:[9.721549e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:43,030 [ hooks.py: 207]:\tstep: 72210\tsteps/sec: 1.18590\tloss: 0.31988\tlr:[9.65363e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:51,680 [ hooks.py: 207]:\tstep: 72220\tsteps/sec: 1.19785\tloss: 0.05462\tlr:[9.585739e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:24:59,806 [ hooks.py: 207]:\tstep: 72230\tsteps/sec: 1.26087\tloss: 0.49005\tlr:[9.5178183e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:25:06,607 [ hooks.py: 207]:\tstep: 72240\tsteps/sec: 1.35064\tloss: 0.11166\tlr:[9.4499296e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:25:14,676 [ hooks.py: 207]:\tstep: 72250\tsteps/sec: 1.35530\tloss: 0.50705\tlr:[9.38201e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:25:22,175 [ hooks.py: 207]:\tstep: 72260\tsteps/sec: 1.27891\tloss: 0.52519\tlr:[9.3141205e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:25:29,236 [ hooks.py: 207]:\tstep: 72270\tsteps/sec: 1.45395\tloss: 0.08472\tlr:[9.246203e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:25:40,861 [ hooks.py: 207]:\tstep: 72280\tsteps/sec: 0.87126\tloss: 0.15549\tlr:[9.1783124e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:25:49,595 [ hooks.py: 207]:\tstep: 72290\tsteps/sec: 1.13903\tloss: 0.21471\tlr:[9.110391e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:25:57,194 [ hooks.py: 207]:\tstep: 72300\tsteps/sec: 1.34425\tloss: 0.55093\tlr:[9.042503e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:26:05,268 [ hooks.py: 207]:\tstep: 72310\tsteps/sec: 1.21128\tloss: 0.12940\tlr:[8.9745834e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:26:12,491 [ hooks.py: 207]:\tstep: 72320\tsteps/sec: 1.35895\tloss: 0.20417\tlr:[8.906692e-07]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 19:26:19,453 [ hooks.py: 207]:\tstep: 72330\tsteps/sec: 1.38730\tloss: 0.69783\tlr:[8.8387725e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:26:27,116 [ hooks.py: 207]:\tstep: 72340\tsteps/sec: 1.28468\tloss: 0.07707\tlr:[8.7708855e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:26:36,242 [ hooks.py: 207]:\tstep: 72350\tsteps/sec: 1.12825\tloss: 0.20031\tlr:[8.7029633e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:26:44,499 [ hooks.py: 207]:\tstep: 72360\tsteps/sec: 1.23824\tloss: 0.22666\tlr:[8.6350457e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:26:50,982 [ hooks.py: 207]:\tstep: 72370\tsteps/sec: 1.54348\tloss: 0.16996\tlr:[8.567157e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:26:57,792 [ hooks.py: 207]:\tstep: 72380\tsteps/sec: 1.37465\tloss: 0.29380\tlr:[8.4992365e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:27:07,275 [ hooks.py: 207]:\tstep: 72390\tsteps/sec: 1.09511\tloss: 0.09915\tlr:[8.431346e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:27:15,940 [ hooks.py: 207]:\tstep: 72400\tsteps/sec: 1.20712\tloss: 0.15985\tlr:[8.363426e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:27:25,155 [ hooks.py: 207]:\tstep: 72410\tsteps/sec: 1.06665\tloss: 0.08165\tlr:[8.2955376e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:27:34,616 [ hooks.py: 207]:\tstep: 72420\tsteps/sec: 1.01368\tloss: 0.34269\tlr:[8.2276193e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:27:43,434 [ hooks.py: 207]:\tstep: 72430\tsteps/sec: 1.16462\tloss: 1.05116\tlr:[8.1597284e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:27:53,207 [ hooks.py: 207]:\tstep: 72440\tsteps/sec: 1.02169\tloss: 0.05399\tlr:[8.0918073e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:28:02,305 [ hooks.py: 207]:\tstep: 72450\tsteps/sec: 1.09349\tloss: 0.24426\tlr:[8.0239187e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:28:11,879 [ hooks.py: 207]:\tstep: 72460\tsteps/sec: 1.00579\tloss: 0.34747\tlr:[7.955999e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:28:22,134 [ hooks.py: 207]:\tstep: 72470\tsteps/sec: 1.01142\tloss: 0.04416\tlr:[7.88811e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:28:30,464 [ hooks.py: 207]:\tstep: 72480\tsteps/sec: 1.22072\tloss: 0.10462\tlr:[7.820191e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:28:38,576 [ hooks.py: 207]:\tstep: 72490\tsteps/sec: 1.21007\tloss: 0.15544\tlr:[7.752299e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:28:46,477 [ hooks.py: 207]:\tstep: 72500\tsteps/sec: 1.20420\tloss: 0.18765\tlr:[7.6843804e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:28:56,261 [ hooks.py: 207]:\tstep: 72510\tsteps/sec: 1.14303\tloss: 0.30890\tlr:[7.61646e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:29:03,593 [ hooks.py: 207]:\tstep: 72520\tsteps/sec: 1.26166\tloss: 0.28687\tlr:[7.548571e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:29:10,995 [ hooks.py: 207]:\tstep: 72530\tsteps/sec: 1.38683\tloss: 0.75834\tlr:[7.480652e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:29:17,631 [ hooks.py: 207]:\tstep: 72540\tsteps/sec: 1.45510\tloss: 0.57832\tlr:[7.412763e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:29:27,084 [ hooks.py: 207]:\tstep: 72550\tsteps/sec: 1.05351\tloss: 0.09364\tlr:[7.344842e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:29:36,269 [ hooks.py: 207]:\tstep: 72560\tsteps/sec: 1.10492\tloss: 0.20281\tlr:[7.276952e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:29:42,533 [ hooks.py: 207]:\tstep: 72570\tsteps/sec: 1.50329\tloss: 0.38000\tlr:[7.2090336e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:29:53,504 [ hooks.py: 207]:\tstep: 72580\tsteps/sec: 0.92431\tloss: 0.24392\tlr:[7.141143e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:30:02,681 [ hooks.py: 207]:\tstep: 72590\tsteps/sec: 1.11670\tloss: 0.43625\tlr:[7.073225e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:30:09,915 [ hooks.py: 207]:\tstep: 72600\tsteps/sec: 1.40723\tloss: 0.22634\tlr:[7.005336e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:30:17,494 [ hooks.py: 207]:\tstep: 72610\tsteps/sec: 1.28918\tloss: 0.17057\tlr:[6.937416e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:30:28,729 [ hooks.py: 207]:\tstep: 72620\tsteps/sec: 0.89276\tloss: 0.24197\tlr:[6.869526e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:30:34,975 [ hooks.py: 207]:\tstep: 72630\tsteps/sec: 1.56735\tloss: 0.14654\tlr:[6.801606e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:30:44,777 [ hooks.py: 207]:\tstep: 72640\tsteps/sec: 1.05387\tloss: 0.41068\tlr:[6.7336873e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:30:51,708 [ hooks.py: 207]:\tstep: 72650\tsteps/sec: 1.39219\tloss: 0.36577\tlr:[6.665797e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:31:01,173 [ hooks.py: 207]:\tstep: 72660\tsteps/sec: 1.07194\tloss: 0.46540\tlr:[6.597878e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:31:10,242 [ hooks.py: 207]:\tstep: 72670\tsteps/sec: 1.11958\tloss: 0.48441\tlr:[6.5299884e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:31:17,847 [ hooks.py: 207]:\tstep: 72680\tsteps/sec: 1.23147\tloss: 0.25373\tlr:[6.462069e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:31:27,449 [ hooks.py: 207]:\tstep: 72690\tsteps/sec: 1.07081\tloss: 0.47921\tlr:[6.394178e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:31:36,123 [ hooks.py: 207]:\tstep: 72700\tsteps/sec: 1.16573\tloss: 0.22886\tlr:[6.326259e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:31:44,919 [ hooks.py: 207]:\tstep: 72710\tsteps/sec: 1.16063\tloss: 0.40205\tlr:[6.2583695e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:31:52,797 [ hooks.py: 207]:\tstep: 72720\tsteps/sec: 1.23073\tloss: 0.69237\tlr:[6.1904495e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:01,922 [ hooks.py: 207]:\tstep: 72730\tsteps/sec: 1.09637\tloss: 0.47353\tlr:[6.122559e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:09,204 [ hooks.py: 207]:\tstep: 72740\tsteps/sec: 1.38671\tloss: 0.42538\tlr:[6.05464e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:16,333 [ hooks.py: 207]:\tstep: 72750\tsteps/sec: 1.30528\tloss: 0.26311\tlr:[5.986751e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:25,279 [ hooks.py: 207]:\tstep: 72760\tsteps/sec: 1.21074\tloss: 0.62845\tlr:[5.918832e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:33,578 [ hooks.py: 207]:\tstep: 72770\tsteps/sec: 1.17605\tloss: 0.25355\tlr:[5.850942e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:42,140 [ hooks.py: 207]:\tstep: 72780\tsteps/sec: 1.15952\tloss: 0.03579\tlr:[5.7830226e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:50,841 [ hooks.py: 207]:\tstep: 72790\tsteps/sec: 1.12223\tloss: 0.65040\tlr:[5.7151016e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:32:59,388 [ hooks.py: 207]:\tstep: 72800\tsteps/sec: 1.17432\tloss: 0.67301\tlr:[5.6472135e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:33:07,642 [ hooks.py: 207]:\tstep: 72810\tsteps/sec: 1.24033\tloss: 0.35867\tlr:[5.579294e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:33:14,499 [ hooks.py: 207]:\tstep: 72820\tsteps/sec: 1.49239\tloss: 0.12210\tlr:[5.511403e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:33:22,467 [ hooks.py: 207]:\tstep: 72830\tsteps/sec: 1.23431\tloss: 0.22812\tlr:[5.443483e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:33:30,808 [ hooks.py: 207]:\tstep: 72840\tsteps/sec: 1.20799\tloss: 0.17961\tlr:[5.375594e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:33:38,349 [ hooks.py: 207]:\tstep: 72850\tsteps/sec: 1.31197\tloss: 0.14618\tlr:[5.307676e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:33:49,439 [ hooks.py: 207]:\tstep: 72860\tsteps/sec: 0.91105\tloss: 0.56125\tlr:[5.239786e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:01,375 [ hooks.py: 207]:\tstep: 72870\tsteps/sec: 0.82618\tloss: 0.38646\tlr:[5.1718655e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:09,377 [ hooks.py: 207]:\tstep: 72880\tsteps/sec: 1.28517\tloss: 0.13616\tlr:[5.103977e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:17,700 [ hooks.py: 207]:\tstep: 72890\tsteps/sec: 1.17777\tloss: 0.12759\tlr:[5.036057e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:24,618 [ hooks.py: 207]:\tstep: 72900\tsteps/sec: 1.47545\tloss: 0.66137\tlr:[4.968167e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:31,875 [ hooks.py: 207]:\tstep: 72910\tsteps/sec: 1.34488\tloss: 0.20310\tlr:[4.9002466e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:38,291 [ hooks.py: 207]:\tstep: 72920\tsteps/sec: 1.58143\tloss: 0.25680\tlr:[4.8323574e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:48,255 [ hooks.py: 207]:\tstep: 72930\tsteps/sec: 0.99874\tloss: 0.73408\tlr:[4.764438e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:34:57,275 [ hooks.py: 207]:\tstep: 72940\tsteps/sec: 1.14318\tloss: 0.30939\tlr:[4.696519e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:06,492 [ hooks.py: 207]:\tstep: 72950\tsteps/sec: 1.05274\tloss: 0.35347\tlr:[4.6286297e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:13,722 [ hooks.py: 207]:\tstep: 72960\tsteps/sec: 1.39087\tloss: 0.14521\tlr:[4.5607092e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:20,705 [ hooks.py: 207]:\tstep: 72970\tsteps/sec: 1.47857\tloss: 0.17789\tlr:[4.4928203e-07]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 19:35:26,828 [ hooks.py: 207]:\tstep: 72980\tsteps/sec: 1.45911\tloss: 0.25368\tlr:[4.4248998e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:36,166 [ hooks.py: 207]:\tstep: 72990\tsteps/sec: 1.14862\tloss: 0.34600\tlr:[4.3570108e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:42,584 [ hooks.py: 207]:\tstep: 73000\tsteps/sec: 1.46793\tloss: 0.41725\tlr:[4.289091e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:42,585 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:42,585 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:42,585 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:35:42,585 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:42,585 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:35:42,585 [monitored_executor.py: 289]:\ttrain loop has hook \u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:43:41,704 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,705 [ trainer.py: 58]:\t{'acc': 0.7799196787148595, 'loss': 0.6702217317831057}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,706 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,706 [ trainer.py: 65]:\twrite to tensorboard ./output/eval_history/eval\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:43:41,707 [ trainer.py: 68]:\t*** eval res: eval ***\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:43:41,707 [ trainer.py: 70]:\tacc\t0.7799196787148595\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:43:41,707 [ trainer.py: 70]:\tloss\t0.6702217317831057\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:43:41,707 [ trainer.py: 71]:\t******************************\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,707 [ exporter.py: 50]:\tNew evaluate result: {'eval': {'acc': 0.7799196787148595, 'loss': 0.6702217317831057}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,707 [ exporter.py: 64]:\t[Best Exporter]: skip step 73000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,707 [ exporter.py: 75]:\tNew evaluate result: {'eval': {'acc': 0.7799196787148595, 'loss': 0.6702217317831057}} \n", + "old: {'eval': {'acc': 0.793574297188755, 'loss': 0.6033463256958967}}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,707 [ exporter.py: 104]:\t[Best Exporter]: skip step 73000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:41,708 [monitored_executor.py: 122]:\tsaving step 73000 to ./output/model_73000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:45,078 [monitored_executor.py: 138]:\tNo. of ckpt exceed 3, clean up: ./output/model_70000\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:43:52,660 [ hooks.py: 207]:\tstep: 73010\tsteps/sec: 0.02041\tloss: 0.14225\tlr:[4.2212008e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:44:02,401 [ hooks.py: 207]:\tstep: 73020\tsteps/sec: 1.00268\tloss: 0.43323\tlr:[4.1532817e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:44:13,223 [ hooks.py: 207]:\tstep: 73030\tsteps/sec: 0.95071\tloss: 0.35474\tlr:[4.0853925e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:44:19,683 [ hooks.py: 207]:\tstep: 73040\tsteps/sec: 1.60102\tloss: 0.34364\tlr:[4.0174726e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:44:29,698 [ hooks.py: 207]:\tstep: 73050\tsteps/sec: 0.99715\tloss: 0.18309\tlr:[3.9495833e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:44:37,671 [ hooks.py: 207]:\tstep: 73060\tsteps/sec: 1.21825\tloss: 0.16727\tlr:[3.8816637e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:44:47,529 [ hooks.py: 207]:\tstep: 73070\tsteps/sec: 1.01774\tloss: 0.10636\tlr:[3.8137736e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:44:55,325 [ hooks.py: 207]:\tstep: 73080\tsteps/sec: 1.38707\tloss: 0.58464\tlr:[3.745855e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:45:03,021 [ hooks.py: 207]:\tstep: 73090\tsteps/sec: 1.21395\tloss: 0.19951\tlr:[3.677935e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:45:11,672 [ hooks.py: 207]:\tstep: 73100\tsteps/sec: 1.15592\tloss: 0.08290\tlr:[3.6100457e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:45:19,700 [ hooks.py: 207]:\tstep: 73110\tsteps/sec: 1.21569\tloss: 0.10244\tlr:[3.542126e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:45:30,613 [ hooks.py: 207]:\tstep: 73120\tsteps/sec: 0.94941\tloss: 0.26335\tlr:[3.4742362e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:45:39,818 [ hooks.py: 207]:\tstep: 73130\tsteps/sec: 1.08133\tloss: 0.25094\tlr:[3.4063163e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:45:47,962 [ hooks.py: 207]:\tstep: 73140\tsteps/sec: 1.19418\tloss: 0.40517\tlr:[3.3384265e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:45:58,072 [ hooks.py: 207]:\tstep: 73150\tsteps/sec: 0.99539\tloss: 0.23094\tlr:[3.2705069e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:46:07,676 [ hooks.py: 207]:\tstep: 73160\tsteps/sec: 1.03405\tloss: 0.63789\tlr:[3.202618e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:46:16,738 [ hooks.py: 207]:\tstep: 73170\tsteps/sec: 1.09463\tloss: 0.08099\tlr:[3.134698e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:46:26,254 [ hooks.py: 207]:\tstep: 73180\tsteps/sec: 1.08911\tloss: 0.34100\tlr:[3.0668082e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:46:34,082 [ hooks.py: 207]:\tstep: 73190\tsteps/sec: 1.24290\tloss: 0.29189\tlr:[2.9988885e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:46:40,826 [ hooks.py: 207]:\tstep: 73200\tsteps/sec: 1.55050\tloss: 0.53902\tlr:[2.9309993e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:46:48,962 [ hooks.py: 207]:\tstep: 73210\tsteps/sec: 1.14012\tloss: 0.76479\tlr:[2.8630794e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:46:58,679 [ hooks.py: 207]:\tstep: 73220\tsteps/sec: 1.06501\tloss: 0.47421\tlr:[2.79519e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:47:09,100 [ hooks.py: 207]:\tstep: 73230\tsteps/sec: 0.96228\tloss: 0.41339\tlr:[2.7272705e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:47:18,598 [ hooks.py: 207]:\tstep: 73240\tsteps/sec: 1.05790\tloss: 0.33196\tlr:[2.659351e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:47:27,133 [ hooks.py: 207]:\tstep: 73250\tsteps/sec: 1.16271\tloss: 0.09712\tlr:[2.5914616e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:47:34,227 [ hooks.py: 207]:\tstep: 73260\tsteps/sec: 1.41426\tloss: 0.88020\tlr:[2.5235417e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:47:42,420 [ hooks.py: 207]:\tstep: 73270\tsteps/sec: 1.17365\tloss: 0.47295\tlr:[2.4556516e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:47:51,327 [ hooks.py: 207]:\tstep: 73280\tsteps/sec: 1.18645\tloss: 0.30803\tlr:[2.3877325e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:00,810 [ hooks.py: 207]:\tstep: 73290\tsteps/sec: 1.07467\tloss: 0.14186\tlr:[2.3198427e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:09,590 [ hooks.py: 207]:\tstep: 73300\tsteps/sec: 1.11633\tloss: 0.35738\tlr:[2.2519237e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:17,756 [ hooks.py: 207]:\tstep: 73310\tsteps/sec: 1.20754\tloss: 0.64980\tlr:[2.1840339e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:25,627 [ hooks.py: 207]:\tstep: 73320\tsteps/sec: 1.29733\tloss: 0.42344\tlr:[2.1161145e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:32,359 [ hooks.py: 207]:\tstep: 73330\tsteps/sec: 1.42652\tloss: 0.46392\tlr:[2.0482246e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:40,128 [ hooks.py: 207]:\tstep: 73340\tsteps/sec: 1.35782\tloss: 0.30680\tlr:[1.9803049e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:49,262 [ hooks.py: 207]:\tstep: 73350\tsteps/sec: 1.06132\tloss: 0.43727\tlr:[1.912416e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:48:57,279 [ hooks.py: 207]:\tstep: 73360\tsteps/sec: 1.22738\tloss: 0.70546\tlr:[1.8444959e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:49:06,512 [ hooks.py: 207]:\tstep: 73370\tsteps/sec: 1.09439\tloss: 0.72504\tlr:[1.7766058e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:49:15,260 [ hooks.py: 207]:\tstep: 73380\tsteps/sec: 1.17716\tloss: 0.70920\tlr:[1.7086873e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:49:22,870 [ hooks.py: 207]:\tstep: 73390\tsteps/sec: 1.22109\tloss: 0.33604\tlr:[1.6407675e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:49:32,813 [ hooks.py: 207]:\tstep: 73400\tsteps/sec: 1.00780\tloss: 0.17966\tlr:[1.5728773e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:49:41,563 [ hooks.py: 207]:\tstep: 73410\tsteps/sec: 1.15686\tloss: 0.34460\tlr:[1.5049585e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:49:50,785 [ hooks.py: 207]:\tstep: 73420\tsteps/sec: 1.12665\tloss: 0.18208\tlr:[1.4370683e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:49:58,497 [ hooks.py: 207]:\tstep: 73430\tsteps/sec: 1.27396\tloss: 0.17798\tlr:[1.3691492e-07]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[37m[DEBUG] 2019-09-19 19:50:06,399 [ hooks.py: 207]:\tstep: 73440\tsteps/sec: 1.20901\tloss: 0.37308\tlr:[1.3012594e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:50:15,982 [ hooks.py: 207]:\tstep: 73450\tsteps/sec: 1.10159\tloss: 0.57594\tlr:[1.2333399e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:50:24,809 [ hooks.py: 207]:\tstep: 73460\tsteps/sec: 1.12811\tloss: 0.19768\tlr:[1.1654502e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:50:32,882 [ hooks.py: 207]:\tstep: 73470\tsteps/sec: 1.17914\tloss: 0.25062\tlr:[1.0975307e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:50:41,837 [ hooks.py: 207]:\tstep: 73480\tsteps/sec: 1.10132\tloss: 0.13024\tlr:[1.0296407e-07]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:50:51,182 [ hooks.py: 207]:\tstep: 73490\tsteps/sec: 1.10918\tloss: 0.30766\tlr:[9.61721e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:00,698 [ hooks.py: 207]:\tstep: 73500\tsteps/sec: 1.07685\tloss: 0.11769\tlr:[8.938315e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:07,604 [ hooks.py: 207]:\tstep: 73510\tsteps/sec: 1.41962\tloss: 0.33494\tlr:[8.259121e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:16,239 [ hooks.py: 207]:\tstep: 73520\tsteps/sec: 1.18572\tloss: 0.23853\tlr:[7.580224e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:25,076 [ hooks.py: 207]:\tstep: 73530\tsteps/sec: 1.12349\tloss: 0.73388\tlr:[6.901027e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:35,795 [ hooks.py: 207]:\tstep: 73540\tsteps/sec: 0.94072\tloss: 0.34600\tlr:[6.2218334e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:42,926 [ hooks.py: 207]:\tstep: 73550\tsteps/sec: 1.34731\tloss: 0.72539\tlr:[5.5429343e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:49,626 [ hooks.py: 207]:\tstep: 73560\tsteps/sec: 1.53671\tloss: 0.60786\tlr:[4.863741e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:51:56,663 [ hooks.py: 207]:\tstep: 73570\tsteps/sec: 1.50594\tloss: 0.38916\tlr:[4.1848427e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:04,895 [ hooks.py: 207]:\tstep: 73580\tsteps/sec: 1.16404\tloss: 0.15805\tlr:[3.505648e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:12,927 [ hooks.py: 207]:\tstep: 73590\tsteps/sec: 1.18040\tloss: 0.60175\tlr:[2.8267507e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:23,301 [ hooks.py: 207]:\tstep: 73600\tsteps/sec: 0.99424\tloss: 0.81070\tlr:[2.1475559e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:30,420 [ hooks.py: 207]:\tstep: 73610\tsteps/sec: 1.43489\tloss: 0.23326\tlr:[1.4686584e-08]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:38,504 [ hooks.py: 207]:\tstep: 73620\tsteps/sec: 1.21536\tloss: 0.59705\tlr:[7.89464e-09]\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:45,562 [ hooks.py: 207]:\tstep: 73630\tsteps/sec: 1.44796\tloss: 0.25941\tlr:[1.1056669e-09]\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-19 19:52:46,398 [ hooks.py: 219]:\tStopAtStepHook called stop\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:46,399 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "[None, None, None, None, None]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# `train_and_eval` takes key-word args only\n", + "# we are now ready to train\n", + "hooks = [propeller.train.TqdmNotebookProgressBarHook(run_config.max_steps)] # to show the progress bar, you need to `pip install tqdm ipywidgets`\n", + "propeller.train_and_eval(\n", + " model_class_or_model_fn=ClassificationErnieModel, #**careful**, you should pass a Class to `train_and_eval`, propeller will try to instantiate it.\n", + " params=config, \n", + " run_config=run_config, \n", + " train_dataset=train_ds, \n", + " eval_dataset=dev_ds, \n", + " warm_start_setting=ws, \n", + " exporters=[best_exporter, best_inference_exporter],\n", + " train_hooks=hooks,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Predict" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-19 19:52:46,983 [feature_column.py: 266]:\treading raw files from ./xnli_data/test/part.0\u001b[0m\n" + ] + } + ], + "source": [ + "# after training you might want to check your model performance on test-set\n", + "# let's do this via `propeller.predict`\n", + "# keep in mind that model of best performace has been exported during thet `train_and_eval` phrase\n", + "\n", + "best_filename = [file for file in os.listdir(os.path.join(OUTPUT_DIR, 'best_model')) if 'model' in file][0]\n", + "best_model_path = os.path.join(os.path.join(OUTPUT_DIR, 'best_model'), best_filename)\n", + "true_label = [label_map[(line.strip().split(b'\\t')[-1])]for line in open('./xnli_data/test/part.0', 'rb')]\n", + "\n", + "def drop_label(sentence, segments, label): #we drop the label column here\n", + " return sentence, segments\n", + "\n", + "test_ds = feature_column.build_dataset('test', use_gz=False, data_dir='./xnli_data/test', shuffle=False, repeat=False) \\\n", + " .map(before_pad) \\\n", + " .padded_batch(BATCH, (0, 0, 0)) \\\n", + " .map(after_pad) \\\n", + " .map(drop_label)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-19 19:52:46,992 [ trainer.py: 237]:\tBuilding Predict Graph\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:46,992 [functional.py: 313]:\tTry to infer data shapes & types from generator\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:47,004 [functional.py: 326]:\tDataset `predict` has data_shapes: [(16, 82, 1), (16, 82, 1)] data_types: ['int64', 'int64']\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:47,577 [ trainer.py: 241]:\tDone\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:47,613 [ trainer.py: 247]:\tPredict with: \n", + "> Run_config: RunConfig(model_dir='./output', run_steps=None, max_steps=73631.625, save_steps=1000, eval_steps=1000, eval_max_steps=None, skip_steps=10, log_steps=10, max_ckpt=3, shit=None)\n", + "> Params: {'attention_probs_dropout_prob': 0.1, 'hidden_act': 'relu', 'hidden_dropout_prob': 0.1, 'hidden_size': 768, 'initializer_range': 0.02, 'max_position_embeddings': 513, 'num_attention_heads': 12, 'num_hidden_layers': 12, 'type_vocab_size': 2, 'vocab_size': 18000, 'sent_type_vocab_size': None, 'use_task_id': False, 'task_id': 0, 'warmup_proportion': 0.1, 'weight_decay': 0.01, 'fp16': 0, 'learning_rate': 5e-05, 'num_label': 3, 'batch_size': 32}\n", + "> Train_model_spec: ModelSpec(loss=None, predictions=[name: \"softmax_12.tmp_0\"\n", + "type {\n", + " type: LOD_TENSOR\n", + " lod_tensor {\n", + " tensor {\n", + " data_type: FP32\n", + " dims: 16\n", + " dims: 3\n", + " }\n", + " lod_level: 0\n", + " }\n", + "}\n", + "persistable: false\n", + "], metrics=None, mode=2, inference_spec=None)\n", + "\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:47,673 [monitored_executor.py: 105]:\tckpt_list in this Saver: ['model_71000', 'model_72000', 'model_73000']\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:47,674 [monitored_executor.py: 161]:\trestore from ckpt ./output/model_73000, ckpt-status: {'global_step': 73000, 'time': 1568892940.8021994}\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:48,051 [monitored_executor.py: 281]:\tfreezing program\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:48,052 [monitored_executor.py: 251]:\twill not freeze a program without loss\u001b[0m\n", + "\u001b[37m[DEBUG] 2019-09-19 19:52:48,052 [monitored_executor.py: 283]:\tdone freezing\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:48,052 [monitored_executor.py: 284]:\t********** Start Loop ************\u001b[0m\n", + "\u001b[32m[INFO] 2019-09-19 19:52:48,052 [ trainer.py: 366]:\tRunining predict from dir: {'global_step': 73000, 'time': 1568892940.8021994}\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test accuracy:0.7780439121756487\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m[INFO] 2019-09-19 20:08:45,357 [monitored_executor.py: 336]:\t********** Stop Loop ************\u001b[0m\n" + ] + } + ], + "source": [ + "result = []\n", + "learner = propeller.Learner(ClassificationErnieModel, run_config, params=config, )\n", + "for pred in learner.predict(test_ds, ckpt=-1):\n", + " result.append(np.argmax(pred))\n", + " \n", + "result, true_label = np.array(result), np.array(true_label)\n", + "\n", + "test_acc = (result == true_label).sum() / len(true_label)\n", + "print('test accuracy:%.5f' % test_acc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Serving\n", + "your model is now ready to serve! \n", + "you can open up a server by propeller with \n", + "```script\n", + "python -m propeller.tools.start_server -m /path/to/saved/model -p 8888\n", + "```\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} -- GitLab