From 330f442d4193bb7f089593b28209024de527808d Mon Sep 17 00:00:00 2001 From: minqiyang Date: Wed, 3 Oct 2018 12:52:12 +0800 Subject: [PATCH] Polish code --- 01.fit_a_line/index.cn.html | 77 ++++++++++++++++++- 01.fit_a_line/index.html | 77 ++++++++++++++++++- 03.image_classification/index.cn.html | 2 +- 03.image_classification/index.html | 2 +- 04.word2vec/index.cn.html | 3 +- 04.word2vec/index.html | 3 +- 06.understand_sentiment/index.cn.html | 2 +- 06.understand_sentiment/index.html | 2 +- 06.understand_sentiment/train_conv.py | 3 +- 06.understand_sentiment/train_dyn_rnn.py | 3 +- 06.understand_sentiment/train_stacked_lstm.py | 3 +- 07.label_semantic_roles/index.cn.html | 5 +- 07.label_semantic_roles/index.html | 5 +- 13 files changed, 170 insertions(+), 17 deletions(-) diff --git a/01.fit_a_line/index.cn.html b/01.fit_a_line/index.cn.html index c5a8ba6..c6ad4e2 100644 --- a/01.fit_a_line/index.cn.html +++ b/01.fit_a_line/index.cn.html @@ -219,6 +219,80 @@ PaddlePaddle提供了读取数据者发生器机制来读取训练数据。读 feed_order=['x', 'y'] ``` +以及一个绘画器来进行绘制： + +```python +import six +import os + + +class PlotData(object): + def __init__(self): + self.step = [] + self.value = [] + + def append(self, step, value): + self.step.append(step) + self.value.append(value) + + def reset(self): + self.step = [] + self.value = [] + + +class Ploter(object): + def __init__(self, *args): + self.__args__ = args + self.__plot_data__ = {} + for title in args: + self.__plot_data__[title] = PlotData() + # demo in notebooks will use Ploter to plot figure, but when we convert + # the ipydb to py file for testing, the import of matplotlib will make the + # script crash. So we can use `export DISABLE_PLOT=True` to disable import + # these libs + self.__disable_plot__ = os.environ.get("DISABLE_PLOT") + if not self.__plot_is_disabled__(): + import matplotlib.pyplot as plt + from IPython import display + self.plt = plt + self.display = display + + def __plot_is_disabled__(self): + return self.__disable_plot__ == "True" + + def append(self, title, step, value): + assert isinstance(title, six.string_types) + assert title in self.__plot_data__ + data = self.__plot_data__[title] + assert isinstance(data, PlotData) + data.append(step, value) + + def plot(self, path=None): + if self.__plot_is_disabled__(): + return + + titles = [] + for title in self.__args__: + data = self.__plot_data__[title] + assert isinstance(data, PlotData) + if len(data.step) > 0: + titles.append(title) + self.plt.plot(data.step, data.value) + self.plt.legend(titles, loc='upper left') + if path is None: + self.display.clear_output(wait=True) + self.display.display(self.plt.gcf()) + else: + self.plt.savefig(path) + self.plt.gcf().clear() + + def reset(self): + for key in self.__plot_data__: + data = self.__plot_data__[key] + assert isinstance(data, PlotData) + data.reset() +``` + 除此之外，可以定义一个事件响应器来处理类似`打印训练进程`的事件： ```python @@ -226,7 +300,6 @@ feed_order=['x', 'y'] params_dirname = "fit_a_line.inference.model" # Plot data -from paddle.v2.plot import Ploter train_title = "Train cost" test_title = "Test cost" plot_cost = Ploter(train_title, test_title) @@ -301,7 +374,7 @@ inferencer = fluid.contrib.inferencer.Inferencer( batch_size = 10 test_reader = paddle.batch(paddle.dataset.uci_housing.test(),batch_size=batch_size) -test_data = test_reader().next() +test_data = next(test_reader()) test_x = numpy.array([data[0] for data in test_data]).astype("float32") test_y = numpy.array([data[1] for data in test_data]).astype("float32") diff --git a/01.fit_a_line/index.html b/01.fit_a_line/index.html index f1b1076..d17a3c2 100644 --- a/01.fit_a_line/index.html +++ b/01.fit_a_line/index.html @@ -238,6 +238,80 @@ for loading the training data. A reader may return multiple columns, and we need feed_order=['x', 'y'] ``` +And a ploter to plot metrics: + +```python +import six +import os + + +class PlotData(object): + def __init__(self): + self.step = [] + self.value = [] + + def append(self, step, value): + self.step.append(step) + self.value.append(value) + + def reset(self): + self.step = [] + self.value = [] + + +class Ploter(object): + def __init__(self, *args): + self.__args__ = args + self.__plot_data__ = {} + for title in args: + self.__plot_data__[title] = PlotData() + # demo in notebooks will use Ploter to plot figure, but when we convert + # the ipydb to py file for testing, the import of matplotlib will make the + # script crash. So we can use `export DISABLE_PLOT=True` to disable import + # these libs + self.__disable_plot__ = os.environ.get("DISABLE_PLOT") + if not self.__plot_is_disabled__(): + import matplotlib.pyplot as plt + from IPython import display + self.plt = plt + self.display = display + + def __plot_is_disabled__(self): + return self.__disable_plot__ == "True" + + def append(self, title, step, value): + assert isinstance(title, six.string_types) + assert title in self.__plot_data__ + data = self.__plot_data__[title] + assert isinstance(data, PlotData) + data.append(step, value) + + def plot(self, path=None): + if self.__plot_is_disabled__(): + return + + titles = [] + for title in self.__args__: + data = self.__plot_data__[title] + assert isinstance(data, PlotData) + if len(data.step) > 0: + titles.append(title) + self.plt.plot(data.step, data.value) + self.plt.legend(titles, loc='upper left') + if path is None: + self.display.clear_output(wait=True) + self.display.display(self.plt.gcf()) + else: + self.plt.savefig(path) + self.plt.gcf().clear() + + def reset(self): + for key in self.__plot_data__: + data = self.__plot_data__[key] + assert isinstance(data, PlotData) + data.reset() +``` + Moreover, an event handler is provided to print the training progress: ```python @@ -245,7 +319,6 @@ Moreover, an event handler is provided to print the training progress: params_dirname = "fit_a_line.inference.model" # Plot data -from paddle.v2.plot import Ploter train_title = "Train cost" test_title = "Test cost" plot_cost = Ploter(train_title, test_title) @@ -323,7 +396,7 @@ inferencer = fluid.contrib.inferencer.Inferencer( batch_size = 10 test_reader = paddle.batch(paddle.dataset.uci_housing.test(),batch_size=batch_size) -test_data = test_reader().next() +test_data = next(test_reader()) test_x = numpy.array([data[0] for data in test_data]).astype("float32") test_y = numpy.array([data[1] for data in test_data]).astype("float32") diff --git a/03.image_classification/index.cn.html b/03.image_classification/index.cn.html index 0bf5043..36c0b4c 100644 --- a/03.image_classification/index.cn.html +++ b/03.image_classification/index.cn.html @@ -324,7 +324,7 @@ def layer_warp(block_func, input, ch_in, ch_out, count, stride): def resnet_cifar10(ipt, depth=32): # depth should be one of 20, 32, 44, 56, 110, 1202 assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 nStages = {16, 64, 128} conv1 = conv_bn_layer(ipt, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/03.image_classification/index.html b/03.image_classification/index.html index 8d4a444..c79db6c 100644 --- a/03.image_classification/index.html +++ b/03.image_classification/index.html @@ -324,7 +324,7 @@ Note: besides the first convolutional layer and the last fully-connected layer, def resnet_cifar10(ipt, depth=32): # depth should be one of 20, 32, 44, 56, 110, 1202 assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 nStages = {16, 64, 128} conv1 = conv_bn_layer(ipt, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/04.word2vec/index.cn.html b/04.word2vec/index.cn.html index d5e3fa5..1ee73b4 100644 --- a/04.word2vec/index.cn.html +++ b/04.word2vec/index.cn.html @@ -250,6 +250,7 @@ import numpy from functools import partial import math import os +import six import sys from __future__ import print_function ``` @@ -436,7 +437,7 @@ def infer(use_cuda, inference_program, params_dirname=None): most_possible_word_index = numpy.argmax(result[0]) print(most_possible_word_index) print([ - key for key, value in word_dict.iteritems() + key for key, value in six.iteritems(word_dict) if value == most_possible_word_index ][0]) ``` diff --git a/04.word2vec/index.html b/04.word2vec/index.html index a945728..1eb5381 100644 --- a/04.word2vec/index.html +++ b/04.word2vec/index.html @@ -263,6 +263,7 @@ import numpy from functools import partial import math import os +import six import sys from __future__ import print_function ``` @@ -454,7 +455,7 @@ def infer(use_cuda, inference_program, params_dirname=None): most_possible_word_index = numpy.argmax(result[0]) print(most_possible_word_index) print([ - key for key, value in word_dict.iteritems() + key for key, value in six.iteritems(word_dict) if value == most_possible_word_index ][0]) ``` diff --git a/06.understand_sentiment/index.cn.html b/06.understand_sentiment/index.cn.html index 66183b0..03956dc 100644 --- a/06.understand_sentiment/index.cn.html +++ b/06.understand_sentiment/index.cn.html @@ -316,7 +316,7 @@ params_dirname = "understand_sentiment_conv.inference.model" def event_handler(event): if isinstance(event, fluid.contrib.trainer.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) + event.step, event.epoch, list(map(np.array, event.metrics)))) if event.step == 10: trainer.save_params(params_dirname) diff --git a/06.understand_sentiment/index.html b/06.understand_sentiment/index.html index 49d8ee2..a8896e9 100644 --- a/06.understand_sentiment/index.html +++ b/06.understand_sentiment/index.html @@ -323,7 +323,7 @@ params_dirname = "understand_sentiment_conv.inference.model" def event_handler(event): if isinstance(event, fluid.contrib.trainer.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) + event.step, event.epoch, list(map(np.array, event.metrics)))) if event.step == 10: trainer.save_params(params_dirname) diff --git a/06.understand_sentiment/train_conv.py b/06.understand_sentiment/train_conv.py index eeef644..f9435cc 100644 --- a/06.understand_sentiment/train_conv.py +++ b/06.understand_sentiment/train_conv.py @@ -111,7 +111,8 @@ def train(use_cuda, train_program, params_dirname): event.step, avg_cost, acc)) print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, list(map(np.array, event.metrics)))) + event.step, event.epoch, list(map(np.array, + event.metrics)))) elif isinstance(event, EndEpochEvent): trainer.save_params(params_dirname) diff --git a/06.understand_sentiment/train_dyn_rnn.py b/06.understand_sentiment/train_dyn_rnn.py index 090720a..5f6f79d 100644 --- a/06.understand_sentiment/train_dyn_rnn.py +++ b/06.understand_sentiment/train_dyn_rnn.py @@ -128,7 +128,8 @@ def train(use_cuda, train_program, params_dirname): event.step, avg_cost, acc)) print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, list(map(np.array, event.metrics)))) + event.step, event.epoch, list(map(np.array, + event.metrics)))) elif isinstance(event, EndEpochEvent): trainer.save_params(params_dirname) diff --git a/06.understand_sentiment/train_stacked_lstm.py b/06.understand_sentiment/train_stacked_lstm.py index b665c6e..8304979 100644 --- a/06.understand_sentiment/train_stacked_lstm.py +++ b/06.understand_sentiment/train_stacked_lstm.py @@ -119,7 +119,8 @@ def train(use_cuda, train_program, params_dirname): event.step, avg_cost, acc)) print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, list(map(np.array, event.metrics)))) + event.step, event.epoch, list(map(np.array, + event.metrics)))) elif isinstance(event, EndEpochEvent): trainer.save_params(params_dirname) diff --git a/07.label_semantic_roles/index.cn.html b/07.label_semantic_roles/index.cn.html index 3561514..f08d42d 100644 --- a/07.label_semantic_roles/index.cn.html +++ b/07.label_semantic_roles/index.cn.html @@ -226,8 +226,9 @@ from __future__ import print_function import math, os import numpy as np import paddle -import paddle.v2.dataset.conll05 as conll05 +import paddle.dataset.conll05 as conll05 import paddle.fluid as fluid +import six import time with_gpu = os.getenv('WITH_GPU', '0') != '0' @@ -459,7 +460,7 @@ def train(use_cuda, save_dirname=None, is_local=True): start_time = time.time() batch_id = 0 - for pass_id in xrange(PASS_NUM): + for pass_id in six.moves.xrange(PASS_NUM): for data in train_data(): cost = exe.run(main_program, feed=feeder.feed(data), diff --git a/07.label_semantic_roles/index.html b/07.label_semantic_roles/index.html index 170c6b6..2319d7c 100644 --- a/07.label_semantic_roles/index.html +++ b/07.label_semantic_roles/index.html @@ -249,8 +249,9 @@ from __future__ import print_function import math, os import numpy as np import paddle -import paddle.v2.dataset.conll05 as conll05 +import paddle.dataset.conll05 as conll05 import paddle.fluid as fluid +import six import time with_gpu = os.getenv('WITH_GPU', '0') != '0' @@ -469,7 +470,7 @@ def train(use_cuda, save_dirname=None, is_local=True): start_time = time.time() batch_id = 0 - for pass_id in xrange(PASS_NUM): + for pass_id in six.moves.xrange(PASS_NUM): for data in train_data(): cost = exe.run(main_program, feed=feeder.feed(data), -- GitLab