Merge pull request #4 from alexqdh/master

add AutoDL Design

AutoDL Design/autodl.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    AutoDL definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import argparse
+import numpy as np
+import subprocess
+import paddle.fluid as fluid
+from reinforce_policy_gradient import ReinforcePolicyGradient
+from policy_model import PolicyModel
+from autodl_agent import AutoDLAgent
+import utils
+import collections
+
+
+class AutoDL(object):
+    """
+        AutoDL class
+    """
+    def __init__(self):
+        """
+            init
+        """
+        self.parse_args = self._init_parser()
+        self.bl_decay = self.parse_args.bl_decay
+        self.log_dir = self.parse_args.log_dir
+        self.early_stop = self.parse_args.early_stop
+        self.data_path = self.parse_args.data_path
+        self.num_models = self.parse_args.num_models
+        self.batch_size = self.parse_args.batch_size
+        self.chunk_size= self.parse_args.chunk_size
+
+        self._init_dir_path()
+        self.model = PolicyModel(self.parse_args)
+        algo_hyperparas = {'lr': self.parse_args.learning_rate}
+        self.algorithm = ReinforcePolicyGradient(self.model,
+                                                 hyperparas=algo_hyperparas)
+        self.autodl_agent = AutoDLAgent(self.algorithm, self.parse_args)
+        self.total_reward = 0
+
+    def _init_dir_path(self):
+        """
+            init dir path
+        """
+        utils.prepare(self.log_dir)
+        utils.prepare(self.log_dir, "actions")
+        utils.prepare(self.log_dir, "rewards")
+        utils.prepare(self.log_dir, "checkpoints")
+
+    def _init_parser(self):
+        """
+            init parser
+        """
+        parser = argparse.ArgumentParser(description='AutoDL Parser',
+                                         prog='AutoDL')
+        parser.add_argument('-v', '--version', action='version',
+                            version='%(prog)s 0.1')
+        parser.add_argument('--num_nodes', dest="num_nodes", nargs="?",
+                            type=int, const=10, default=10,
+                            help="number of nodes")
+        parser.add_argument('--num_tokens', dest="num_tokens", nargs="?",
+                            type=int, const=10, default=10,
+                            help="number of tokens")
+        parser.add_argument('--learning_rate', dest="learning_rate", nargs="?",
+                            type=float, default=1e-3,
+                            help="learning rate")
+        parser.add_argument('--batch_size', dest="batch_size", nargs="?",
+                            type=int, const=10, default=10, help="batch size")
+        parser.add_argument('--num_models', dest="num_models", nargs="?",
+                            type=int, const=32000, default=32000,
+                            help="maximum number of models sampled")
+        parser.add_argument('--early_stop', dest="early_stop", nargs="?",
+                            type=int, const=20, default=20, help="early stop")
+        parser.add_argument('--log_dir', dest="log_dir", nargs="?", type=str,
+                            const="./log", default="./log",
+                            help="directory of log")
+        parser.add_argument('--input_size', dest="input_size", nargs="?",
+                            type=int, const=10, default=10, help="input size")
+        parser.add_argument('--hidden_size', dest="hidden_size", nargs="?",
+                            type=int, const=64, default=64, help="hidden size")
+        parser.add_argument('--num_layers', dest="num_layers", nargs="?",
+                            type=int, const=2, default=2, help="num layers")
+        parser.add_argument('--bl_decay', dest="bl_decay", nargs="?",
+                            type=float, const=0.9, default=0.9,
+                            help="base line decay")
+        # inception train config
+        parser.add_argument('--data_path', dest="data_path", nargs="?",
+                            type=str, default="./cifar/pickle-cifar-10",
+                            help="path of data files")
+        parser.add_argument('--chunk_size', dest="chunk_size", nargs="?",
+                            type=int, const=100, default=100,
+                            help="chunk size")
+        parse_args = parser.parse_args()
+        return parse_args
+
+    def supervisor(self, mid):
+        """
+            execute cnn training
+            sample cmd: python -u inception_train/train.py --mid=9 \
+                    --early_stop=20 --data_path=./cifar/pickle-cifar-10
+        """
+        tokens, adjvec = utils.load_action(mid, self.log_dir)
+        cmd = ("CUDA_VISIBLE_DEVICES=1 python -u inception_train/train.py \
+               --mid=%d --early_stop=%d --logdir=%s --data_path=%s --chunk_size=%d") % \
+              (mid, self.early_stop, self.log_dir, self.data_path, self.chunk_size)
+        print("cmd:{}".format(cmd))
+        while True:
+            try:
+                subprocess.check_call(cmd, shell=True)
+                break
+            except subprocess.CalledProcessError as e:
+                print("[%s] training model #%d exits with exit code %d" %
+                      (utils.stime(), mid, e.returncode), file=sys.stderr)
+                return
+
+    def simple_run(self):
+        """
+            simple run
+        """
+        print("Simple run target is 20")
+        mid = 0
+        shadow = 0
+        is_first = True
+        while mid <= self.num_models:
+            actions_to, actions_ad = self.autodl_agent.sample()
+            rewards = np.count_nonzero(actions_to == 1, axis=1).astype("int32")
+            # moving average
+            current_mean_reward = np.mean(rewards)
+            if is_first:
+                shadow = current_mean_reward
+                is_first = False
+            else:
+                shadow = shadow * self.bl_decay \
+                        + current_mean_reward * (1 - self.bl_decay)
+            self.autodl_agent.learn((np.array(actions_to).astype("int32"),
+                                    np.array(actions_ad).astype("int32")),
+                                    rewards - shadow)
+
+            if mid % 10 == 0:
+                print('mid=%d, average rewards=%.3f' % (mid, np.mean(rewards)))
+            mid += 1
+
+    def run(self):
+        """
+            run
+        """
+        rewards = []
+        mid = 0
+        while mid <= self.num_models:
+            actions_to, actions_ad = self.autodl_agent.sample()
+
+            for action in zip(actions_to, actions_ad):
+                utils.dump_action(mid, action, self.log_dir)
+                self.supervisor(mid)
+                current_reward = utils.load_reward(mid, self.log_dir)
+                if not np.isnan(current_reward):
+                    rewards.append(current_reward.item())
+                mid += 1
+
+            if len(rewards) % self.batch_size == 0:
+                print("[%s] step = %d, average accuracy = %.3f" %
+                      (utils.stime(), self.autodl_agent.global_step,
+                       np.mean(rewards)))
+                rewards_array = np.array(rewards).astype("float32")
+                if self.total_reward == 0:
+                    self.total_reward = rewards_array.mean()
+                else:
+                    self.total_reward = self.total_reward * self.bl_decay \
+                            + (1 - self.bl_decay) * rewards_array.mean()
+                rewards_array = rewards_array - self.total_reward
+                self.autodl_agent.learn([actions_to.astype("int32"),
+                                         actions_ad.astype("int32")],
+                                        rewards_array ** 3)
+                rewards = []

AutoDL Design/autodl_agent.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    AutoDL Agent Definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import numpy as np
+import paddle.fluid as fluid
+import paddle.fluid.layers as layers
+from parl.framework.agent_base import Agent
+
+
+class AutoDLAgent(Agent):
+    """
+        AutoDLAgent
+    """
+    def __init__(self, algorithm, parse_args):
+        """
+            init
+        """
+        self.global_step = 0
+        self.parse_args = parse_args
+        self.num_nodes = self.parse_args.num_nodes
+        self.batch_size = self.parse_args.batch_size
+        super(AutoDLAgent, self).__init__(algorithm)
+        self.inputs_data = np.zeros([self.batch_size,
+                                     1]).astype('int32')
+
+    def build_program(self):
+        """
+            build program
+        """
+        self.predict_program = fluid.Program()
+        self.train_program = fluid.Program()
+        with fluid.program_guard(self.predict_program):
+            self.predict_inputs = layers.data(
+                    name='input',
+                    append_batch_size=False,
+                    shape=[self.batch_size, 1],
+                    dtype='int32')
+            self.predict_tokens, self.predict_adjvec = self.alg.define_predict(
+                    self.predict_inputs)
+
+        with fluid.program_guard(self.train_program):
+            self.train_inputs = layers.data(
+                    name='input',
+                    append_batch_size=False,
+                    shape=[self.batch_size, 1],
+                    dtype='int32')
+            self.actions_to = layers.data(
+                    name='actions_to',
+                    append_batch_size=False,
+                    shape=[self.batch_size,
+                           self.num_nodes * 2],
+                    dtype='int32')
+            self.actions_ad = layers.data(
+                    name='actions_ad',
+                    append_batch_size=False,
+                    shape=[self.batch_size,
+                           self.num_nodes * (self.num_nodes - 1)],
+                    dtype='int32')
+            self.rewards = layers.data(
+                name='rewards',
+                append_batch_size=False,
+                shape=[self.batch_size],
+                dtype='float32')
+            self.cost = self.alg.define_learn(
+                    obs=self.train_inputs, reward=self.rewards,
+                    action=[self.actions_to, self.actions_ad])
+
+    def sample(self):
+        """
+            sample
+        """
+        feed_dict = {'input': self.inputs_data}
+        [actions_to, actions_ad] = self.fluid_executor.run(
+                self.predict_program, feed=feed_dict,
+                fetch_list=[self.predict_tokens, self.predict_adjvec])
+        return actions_to, actions_ad
+
+    def learn(self, actions, reward):
+        """
+            learn
+        """
+        (actions_to, actions_ad) = actions
+        feed_dict = {'input': self.inputs_data, 'actions_to': actions_to,
+                     'actions_ad': actions_ad, 'rewards': reward}
+        cost = self.fluid_executor.run(
+                self.train_program, feed=feed_dict, fetch_list=[self.cost])[0]
+        self.global_step += 1
+        return cost

AutoDL Design/cifar/dataset_maker.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Generate pkl files from cifar10
+"""
+import os
+import cPickle as pickle
+import random
+import numpy as np
+import sys
+import argparse
+
+
+def init_parser():
+    """
+        init_parser
+    """
+    parser = argparse.ArgumentParser(description='Data generator')
+    parser.add_argument('--chunk_size', dest="chunk_size", nargs="?",
+                        type=int, default=100,
+                        help="size of chunk")
+    parser.add_argument('--input_dir', dest="input_dir", nargs="?",
+                        type=str, default='./cifar-10-batches-py',
+                        help="path of input")
+    parser.add_argument('--output_dir', dest="output_dir", nargs="?",
+                        type=str, default='./pickle-cifar-10',
+                        help="path of output")
+    parse_args, unknown_flags = parser.parse_known_args()
+    return parse_args
+
+
+def get_file_names(input_dir):
+    """
+        get all file names located in dir_path
+    """
+    sub_name = 'data_batch'
+    files = os.listdir(input_dir)
+    names = [each_item for each_item in files if sub_name in each_item]
+    return names
+
+
+def check_output_dir(output_dir):
+    """
+        check exist of output dir
+    """
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+
+def get_datasets(input_dir, chunk_size):
+    """
+        get image datasets
+        chunk_size is the number of each class
+    """
+    total_size = chunk_size * 10
+    names = get_file_names(parse_args.input_dir)
+    img_count = 0
+    datasets = []
+    class_map = {i: 0 for i in range(10)}
+    for name in names:
+        print("Reading file " + name)
+        batch = pickle.load(open(input_dir + "/" + name, 'rb'))
+        data = batch['data']
+        labels = batch.get('labels', batch.get('fine_labels', None))
+        assert labels is not None
+        data_tuples = zip(data, labels)
+        for data in data_tuples:
+            if class_map[data[1]] < chunk_size:
+                datasets.append(data)
+                class_map[data[1]] += 1
+                img_count += 1
+                if img_count >= total_size:
+                    random.shuffle(datasets)
+                    for k, v in class_map.items():
+                        print("label:{} count:{}".format(k, v))
+                    return np.array(datasets)
+    random.shuffle(datasets)
+    return np.array(datasets)
+
+
+def dump_pkl(datasets, output_dir):
+    """
+        dump_pkl
+    """
+    chunk_size = parse_args.chunk_size
+    for i in range(10):
+        sub_dataset = datasets[i * chunk_size:(i + 1) * chunk_size, :]
+        sub_dataset.dump(output_dir + "/" + 'data_batch_' + str(i) + '.pkl')
+
+
+if __name__ == "__main__":
+    parse_args = init_parser()
+    check_output_dir(parse_args.output_dir)
+    datasets = get_datasets(parse_args.input_dir, parse_args.chunk_size)
+    dump_pkl(datasets, parse_args.output_dir)

AutoDL Design/distribute_generator.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Implementation of binomial and multinomial distribution
+"""
+
+import paddle.fluid as fluid
+import functools
+import numpy as np
+
+
+def create_tmp_var(name, dtype, shape, program=None):
+    """
+        Create variable which is used to store the py_func result
+    """
+    if program is None:
+        return fluid.default_main_program().current_block().create_var(
+                name=fluid.unique_name.generate(name),
+                dtype=dtype, shape=shape)
+    else:
+        return program.current_block().create_var(
+                name=fluid.unique_name.generate(name),
+                dtype=dtype, shape=shape)
+
+
+def sigmoid(x):
+    """
+        Sigmoid
+    """
+    return (1 / (1 + np.exp(-x)))
+
+
+def softmax(x):
+    """
+        Compute softmax values for each sets of scores in x.
+    """
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum()
+
+
+def py_func_bernoulli(input):
+    """
+        Binormial python function definition
+    """
+    prob_array = sigmoid(np.array(input))
+    sample = np.random.binomial(1, prob_array)
+    return sample
+
+
+def bernoulli(input_logits, output_shape, program=None):
+    """
+        Bernoulli
+    """
+    # the output_shape is the same as input_logits
+    samples_var = create_tmp_var(name='binomial_result_var',
+                                 dtype='float32', shape=output_shape,
+                                 program=program)
+    fluid.layers.py_func(func=py_func_bernoulli, x=input_logits,
+                         out=samples_var, backward_func=None,
+                         skip_vars_in_backward_input=None)
+    return samples_var
+
+
+def py_func_multinomial(logits, num_samples_var):
+    """
+        Multinomial python function definition
+        Input:
+            input: list of [logits_array, num_samples_int]
+    """
+    def generate(x, prob_array):
+        """
+            Sample multinomial
+        """
+        sample = np.random.multinomial(1, prob_array)
+        ret = np.argmax(sample)
+        return ret
+
+    num_samples = int(np.array(num_samples_var)[0])
+    logits_array = np.array(logits)
+    if len(logits_array.shape) != 2:
+        raise Exception("Shape must be rank 2 but is rank {} \
+                        for 'multinomial/Multinomial' (op: 'Multinomial') \
+                        with input shapes:{}".format(len(logits_array.shape),
+                                                     logits_array.shape))
+    ret = np.array([])
+    for logits in logits_array:
+        prob = softmax(logits)
+        func = functools.partial(generate, prob_array=prob)
+        sample = np.zeros(num_samples)
+        sample = np.array(list(map(func, sample)))
+        ret = np.append(ret, sample)
+    ret = ret.reshape(-1, num_samples).astype("int32")
+    return ret
+
+
+def multinomial(input_logits, output_shape, num_samples, program=None):
+    """
+        Multinomial
+        input_logits's dimension is [M * D]
+        output_shape's dimension is [M * num_samples]
+    """
+    samples_var = create_tmp_var(name='multinomial_result_var',
+                                 dtype='int32', shape=output_shape,
+                                 program=program)
+    num_samples_var = fluid.layers.fill_constant(shape=[1], value=num_samples,
+                                                 dtype='int32')
+    fluid.layers.py_func(func=py_func_multinomial,
+                         x=[input_logits, num_samples_var],
+                         out=samples_var, backward_func=None,
+                         skip_vars_in_backward_input=None)
+    return samples_var

AutoDL Design/inception_train/models/inception.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Inception Definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import paddle.fluid as fluid
+from absl import flags
+import numpy as np
+import models.layers as layers
+import models.ops as _ops
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_integer("num_stages", 3, "number of stages")
+flags.DEFINE_integer("num_cells", 3, "number of cells per stage")
+flags.DEFINE_integer("width", 64, "network width")
+flags.DEFINE_integer("ratio", 4, "compression ratio")
+
+num_classes = 10
+
+ops = [
+    _ops.conv_1x1,
+    _ops.conv_3x3,
+    _ops.conv_5x5,
+    _ops.dilated_3x3,
+    _ops.conv_1x3_3x1,
+    _ops.conv_1x5_5x1,
+    _ops.maxpool_3x3,
+    _ops.maxpool_5x5,
+    _ops.avgpool_3x3,
+    _ops.avgpool_5x5,
+]
+
+
+def net(inputs, output, tokens, adjvec):
+    """
+        create net
+    """
+    num_nodes = len(tokens) // 2
+
+    def slice(vec):
+        """
+            slice vec
+        """
+        mat = np.zeros([num_nodes, num_nodes])
+
+        def pos(x):
+            """
+                pos
+            """
+            return x * (x - 1) // 2
+        for i in range(1, num_nodes):
+            mat[0:i, i] = vec[pos(i):pos(i + 1)]
+        return mat
+
+    normal_to, reduce_to = np.split(tokens, 2)
+    normal_ad, reduce_ad = map(slice, np.split(adjvec, 2))
+
+    x = layers.conv(inputs, FLAGS.width, (3, 3))
+
+    c = 1
+    for _ in range(FLAGS.num_cells):
+        x = cell(x, normal_to, normal_ad)
+        c += 1
+    for _ in range(1, FLAGS.num_stages):
+        x = cell(x, reduce_to, reduce_ad, downsample=True)
+        c += 1
+        for _ in range(1, FLAGS.num_cells):
+            x = cell(x, normal_to, normal_ad)
+            c += 1
+
+    x = layers.bn_relu(x)
+    x = layers.global_avgpool(x)
+    x = layers.dropout(x)
+    logits = layers.fully_connected(x, num_classes)
+    x = fluid.layers.softmax_with_cross_entropy(logits, output,
+                                                numeric_stable_mode=True)
+    loss = fluid.layers.reduce_mean(x)
+    accuracy = fluid.layers.accuracy(input=logits, label=output)
+    return loss, accuracy
+
+
+def cell(inputs, tokens, adjmat, downsample=False, name=None):
+    """
+        cell
+    """
+    filters = inputs.shape[1]
+    d = filters // FLAGS.ratio
+
+    num_nodes, tensors = len(adjmat), []
+    for n in range(num_nodes):
+        func = ops[tokens[n]]
+        idx, = np.nonzero(adjmat[:, n])
+        if len(idx) == 0:
+            x = layers.bn_relu(inputs)
+            x = layers.conv(x, d, (1, 1))
+            x = layers.bn_relu(x)
+            x = func(x, downsample)
+        else:
+            x = fluid.layers.sums([tensors[i] for i in idx])
+            x = layers.bn_relu(x)
+            x = func(x)
+        tensors.append(x)
+
+    free_ends, = np.where(~adjmat.any(axis=1))
+    tensors = [tensors[i] for i in free_ends]
+    filters = filters * 2 if downsample else filters
+    x = fluid.layers.concat(tensors, axis=1)
+    x = layers.conv(x, filters, (1, 1))
+    return x

AutoDL Design/inception_train/models/layers.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Layers Definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import operator
+import numpy as np
+import paddle.fluid as fluid
+from absl import flags
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_float("weight_decay", 0.0001,
+                   "weight decay")
+flags.DEFINE_float("bn_decay", 0.9,
+                   "batch norm decay")
+flags.DEFINE_float("relu_leakiness", 0.1,
+                   "relu leakiness")
+flags.DEFINE_float("dropout_rate", 0.5,
+                   "dropout rate")
+
+
+def calc_padding(img_width, stride, dilation, filter_width):
+    """
+        calculate pixels to padding in order to keep input/output size same.
+    """
+    filter_width = dilation * (filter_width - 1) + 1
+    if img_width % stride == 0:
+        pad_along_width = max(filter_width - stride, 0)
+    else:
+        pad_along_width = max(filter_width - (img_width % stride), 0)
+    return pad_along_width // 2, pad_along_width - pad_along_width // 2
+
+
+def conv(inputs,
+         filters,
+         kernel,
+         strides=None,
+         dilation=None,
+         num_groups=1,
+         conv_param=None,
+         name=None):
+    """
+        normal conv layer
+    """
+    if strides is None:
+        strides = (1, 1)
+    if dilation is None:
+        dilation = (1, 1)
+    if isinstance(kernel, (tuple, list)):
+        n = operator.mul(*kernel) * inputs.shape[1]
+    else:
+        n = kernel * kernel * inputs.shape[1]
+
+    # pad input
+    padding = (0, 0, 0, 0) \
+        + calc_padding(inputs.shape[2], strides[0], dilation[0], kernel[0]) \
+        + calc_padding(inputs.shape[3], strides[1], dilation[1], kernel[1])
+    if sum(padding) > 0:
+        inputs = fluid.layers.pad(inputs, padding, 0)
+
+    param_attr = fluid.param_attr.ParamAttr(
+        initializer=fluid.initializer.TruncatedNormal(
+            0.0, scale=np.sqrt(2.0 / n)),
+        regularizer=fluid.regularizer.L2Decay(FLAGS.weight_decay))
+
+    return fluid.layers.conv2d(
+        inputs,
+        filters,
+        kernel,
+        stride=strides,
+        padding=0,
+        dilation=dilation,
+        groups=num_groups,
+        param_attr=param_attr if conv_param is None else conv_param,
+        use_cudnn=False if num_groups == inputs.shape[1] == filters else True,
+        name=name)
+
+
+def sep(inputs, filters, kernel, strides=None, dilation=None, name=None):
+    """
+        Separable convolution layer
+    """
+    if strides is None:
+        strides = (1, 1)
+    if dilation is None:
+        dilation = (1, 1)
+    if isinstance(kernel, (tuple, list)):
+        n_depth = operator.mul(*kernel)
+    else:
+        n_depth = kernel * kernel
+    n_point = inputs.shape[1]
+
+    if isinstance(strides, (tuple, list)):
+        multiplier = strides[0]
+    else:
+        multiplier = strides
+
+    depthwise_param = fluid.param_attr.ParamAttr(
+        initializer=fluid.initializer.TruncatedNormal(
+            0.0, scale=np.sqrt(2.0 / n_depth)),
+        regularizer=fluid.regularizer.L2Decay(FLAGS.weight_decay))
+
+    pointwise_param = fluid.param_attr.ParamAttr(
+        initializer=fluid.initializer.TruncatedNormal(
+            0.0, scale=np.sqrt(2.0 / n_point)),
+        regularizer=fluid.regularizer.L2Decay(FLAGS.weight_decay))
+
+    depthwise_conv = conv(
+        inputs=inputs,
+        kernel=kernel,
+        filters=int(filters * multiplier),
+        strides=strides,
+        dilation=dilation,
+        num_groups=int(filters * multiplier),
+        conv_param=depthwise_param,
+        name='depthwise_' + name)
+
+    return conv(
+        inputs=depthwise_conv,
+        kernel=(1, 1),
+        filters=int(filters * multiplier),
+        strides=(1, 1),
+        dilation=dilation,
+        conv_param=pointwise_param,
+        name='pointwise_' + name)
+
+
+def maxpool(inputs, kernel, strides=None, name=None):
+    """
+        maxpool
+    """
+    if strides is None:
+        strides = (1, 1)
+    padding = (0, 0, 0, 0) \
+        + calc_padding(inputs.shape[2], strides[0], 1, kernel[0]) \
+        + calc_padding(inputs.shape[3], strides[1], 1, kernel[1])
+    if sum(padding) > 0:
+        inputs = fluid.layers.pad(inputs, padding, 0)
+
+    return fluid.layers.pool2d(
+            inputs, kernel, 'max', strides, pool_padding=0,
+            ceil_mode=False, name=name)
+
+
+def avgpool(inputs, kernel, strides=None, name=None):
+    """
+        avgpool
+    """
+    if strides is None:
+        strides = (1, 1)
+    padding_pixel = (0, 0, 0, 0)
+    padding_pixel += calc_padding(inputs.shape[2], strides[0], 1, kernel[0])
+    padding_pixel += calc_padding(inputs.shape[3], strides[1], 1, kernel[1])
+
+    if padding_pixel[4] == padding_pixel[5] and padding_pixel[
+            6] == padding_pixel[7]:
+        # same padding pixel num on all sides.
+        return fluid.layers.pool2d(
+            inputs,
+            kernel,
+            'avg',
+            strides,
+            pool_padding=(padding_pixel[4], padding_pixel[6]),
+            ceil_mode=False)
+    elif padding_pixel[4] + 1 == padding_pixel[5] \
+            and padding_pixel[6] + 1 == padding_pixel[7] \
+            and strides == (1, 1):
+        # different padding size: first pad then crop.
+        x = fluid.layers.pool2d(
+            inputs,
+            kernel,
+            'avg',
+            strides,
+            pool_padding=(padding_pixel[5], padding_pixel[7]),
+            ceil_mode=False)
+        x_shape = x.shape
+        return fluid.layers.crop(
+            x,
+            shape=(-1, x_shape[1], x_shape[2] - 1, x_shape[3] - 1),
+            offsets=(0, 0, 1, 1), name=name)
+    else:
+        # not support. use padding-zero and pool2d.
+        print("Warning: use zero-padding in avgpool")
+        outputs = fluid.layers.pad(inputs, padding_pixel, 0)
+        return fluid.layers.pool2d(
+                outputs, kernel, 'avg', strides, pool_padding=0,
+                ceil_mode=False, name=name)
+
+
+def global_avgpool(inputs, name=None):
+    """
+        global avgpool
+    """
+    return fluid.layers.reduce_mean(inputs, dim=[2, 3], name=name)
+
+
+def fully_connected(inputs, units, name=None):
+    """
+        fully connected
+    """
+    n = inputs.shape[1]
+    param_attr = fluid.param_attr.ParamAttr(
+        initializer=fluid.initializer.TruncatedNormal(
+            0.0, scale=np.sqrt(2.0 / n)),
+        regularizer=fluid.regularizer.L2Decay(FLAGS.weight_decay))
+
+    return fluid.layers.fc(inputs,
+                           units,
+                           param_attr=param_attr)
+
+
+def batch_norm(inputs, name=None):
+    """
+        batch norm
+    """
+    param_attr = fluid.param_attr.ParamAttr(
+        regularizer=fluid.regularizer.L2Decay(FLAGS.weight_decay))
+    bias_attr = fluid.param_attr.ParamAttr(
+        regularizer=fluid.regularizer.L2Decay(FLAGS.weight_decay))
+
+    return fluid.layers.batch_norm(
+        inputs, momentum=FLAGS.bn_decay, epsilon=0.001,
+        param_attr=param_attr,
+        bias_attr=bias_attr)
+
+
+def relu(inputs, name=None):
+    """
+        relu
+    """
+    if FLAGS.relu_leakiness:
+        return fluid.layers.leaky_relu(inputs, FLAGS.relu_leakiness, name=name)
+    return fluid.layers.relu(inputs, name=name)
+
+
+def bn_relu(inputs, name=None):
+    """
+        batch norm + rely layer
+    """
+    output = batch_norm(inputs)
+    return fluid.layers.relu(output, name=name)
+
+
+def dropout(inputs, name=None):
+    """
+        dropout layer
+    """
+    return fluid.layers.dropout(inputs, dropout_prob=FLAGS.dropout_rate,
+                                dropout_implementation='upscale_in_train',
+                                name=name)

AutoDL Design/inception_train/models/ops.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Base Ops Definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import models.layers as layers
+
+
+def conv_1x1(inputs, downsample=False):
+    """
+        conv_1x1
+    """
+    return conv_base(inputs, (1, 1), downsample=downsample)
+
+
+def conv_2x2(inputs, downsample=False):
+    """
+        conv_2x2
+    """
+    return conv_base(inputs, (2, 2), downsample=downsample)
+
+
+def conv_3x3(inputs, downsample=False):
+    """
+        conv_3x3
+    """
+    return conv_base(inputs, (3, 3), downsample=downsample)
+
+
+def conv_4x4(inputs, downsample=False):
+    """
+        conv_4x4
+    """
+    return conv_base(inputs, (4, 4), downsample=downsample)
+
+
+def conv_5x5(inputs, downsample=False):
+    """
+        conv_5x5
+    """
+    return conv_base(inputs, (5, 5), downsample=downsample)
+
+
+def dilated_2x2(inputs, downsample=False):
+    """
+        dilated_2x2
+    """
+    return conv_base(inputs, (2, 2), (2, 2), downsample)
+
+
+def dilated_3x3(inputs, downsample=False):
+    """
+        dilated_3x3
+    """
+    return conv_base(inputs, (3, 3), (2, 2), downsample)
+
+
+def conv_1x2_2x1(inputs, downsample=False):
+    """
+        conv_1x2_2x1
+    """
+    return pair_base(inputs, 2, downsample)
+
+
+def conv_1x3_3x1(inputs, downsample=False):
+    """
+        conv_1x3_3x1
+    """
+    return pair_base(inputs, 3, downsample)
+
+
+def conv_1x4_4x1(inputs, downsample=False):
+    """
+        conv_1x4_4x1
+    """
+    return pair_base(inputs, 4, downsample)
+
+
+def conv_1x5_5x1(inputs, downsample=False):
+    """
+        conv_1x5_5x1
+    """
+    return pair_base(inputs, 5, downsample)
+
+
+def sep_2x2(inputs, downsample=False):
+    """
+        sep_2x2
+    """
+    return sep_base(inputs, (2, 2), downsample=downsample)
+
+
+def sep_3x3(inputs, downsample=False):
+    """
+        sep_3x3
+    """
+    return sep_base(inputs, (3, 3), downsample=downsample)
+
+
+def sep_4x4(inputs, downsample=False):
+    """
+        sep_4x4
+    """
+    return sep_base(inputs, (4, 4), downsample=downsample)
+
+
+def sep_5x5(inputs, downsample=False):
+    """
+        sep_5x5
+    """
+    return sep_base(inputs, (5, 5), downsample=downsample)
+
+
+def maxpool_2x2(inputs, downsample=False):
+    """
+        maxpool_2x2
+    """
+    return maxpool_base(inputs, (2, 2), downsample)
+
+
+def maxpool_3x3(inputs, downsample=False):
+    """
+        maxpool_3x3
+    """
+    return maxpool_base(inputs, (3, 3), downsample)
+
+
+def maxpool_4x4(inputs, downsample=False):
+    """
+        maxpool_4x4
+    """
+    return maxpool_base(inputs, (4, 4), downsample)
+
+
+def maxpool_5x5(inputs, downsample=False):
+    """
+        maxpool_5x5
+    """
+    return maxpool_base(inputs, (5, 5), downsample)
+
+
+def avgpool_2x2(inputs, downsample=False):
+    """
+        avgpool_2x2
+    """
+    return avgpool_base(inputs, (2, 2), downsample)
+
+
+def avgpool_3x3(inputs, downsample=False):
+    """
+        avgpool_3x3
+    """
+    return avgpool_base(inputs, (3, 3), downsample)
+
+
+def avgpool_4x4(inputs, downsample=False):
+    """
+        avgpool_4x4
+    """
+    return avgpool_base(inputs, (4, 4), downsample)
+
+
+def avgpool_5x5(inputs, downsample=False):
+    """
+        avgpool_5x5
+    """
+    return avgpool_base(inputs, (5, 5), downsample)
+
+
+def conv_base(inputs, kernel, dilation=None, downsample=False):
+    """
+        conv_base
+    """
+    if dilation is None:
+        dilation = (1, 1)
+    filters = inputs.shape[1]
+    if downsample:
+        output = layers.conv(inputs, filters * 2, kernel, (2, 2))
+    else:
+        output = layers.conv(inputs, filters, kernel, dilation=dilation)
+    return output
+
+
+def pair_base(inputs, kernel, downsample=False):
+    """
+        pair_base
+    """
+    filters = inputs.shape[1]
+    if downsample:
+        output = layers.conv(inputs, filters, (1, kernel), (1, 2))
+        output = layers.conv(output, filters, (kernel, 1), (2, 1))
+        output = layers.conv(output, filters * 2, (1, 1))
+    else:
+        output = layers.conv(inputs, filters, (1, kernel))
+        output = layers.conv(output, filters, (kernel, 1))
+    return output
+
+
+def sep_base(inputs, kernel, dilation=None, downsample=False):
+    """
+        sep_base
+    """
+    if dilation is None:
+        dilation = (1, 1)
+    filters = inputs.shape[1]
+    if downsample:
+        output = layers.sep(inputs, filters * 2, kernel, (2, 2))
+    else:
+        output = layers.sep(inputs, filters, kernel, dilation=dilation)
+    return output
+
+
+def maxpool_base(inputs, kernel, downsample=False):
+    """
+        maxpool_base
+    """
+    if downsample:
+        filters = inputs.shape[1]
+        output = layers.maxpool(inputs, kernel, (2, 2))
+        output = layers.conv(output, filters * 2, (1, 1))
+    else:
+        output = layers.maxpool(inputs, kernel)
+    return output
+
+
+def avgpool_base(inputs, kernel, downsample=False):
+    """
+        avgpool_base
+    """
+    if downsample:
+        filters = inputs.shape[1]
+        output = layers.avgpool(inputs, kernel, (2, 2))
+        output = layers.conv(output, filters * 2, (1, 1))
+    else:
+        output = layers.avgpool(inputs, kernel)
+    return output

AutoDL Design/inception_train/nn.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Network Definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import cPickle as cp
+import paddle.fluid as fluid
+import paddle.fluid.layers.ops as ops
+import paddle.fluid as fluid
+from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
+import math
+from paddle.fluid.initializer import init_on_cpu
+from models import inception
+from absl import flags
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_float("lr_max", 0.1,
+                   "initial learning rate")
+flags.DEFINE_float("lr_min", 0.0001,
+                   "limiting learning rate")
+flags.DEFINE_integer("batch_size", 128,
+                     "batch size")
+flags.DEFINE_integer("T_0", 200,
+                     "number of epochs")
+flags.DEFINE_integer("chunk_size", 100,
+                     "chunk size")
+
+
+class CIFARModel(object):
+    """
+        CIFARModel class
+    """
+    def __init__(self, tokens, adjvec, im_shape):
+        """
+            CIFARModel init
+        """
+        chunk_size = FLAGS.chunk_size
+        self.batch_size = FLAGS.batch_size
+        self.tokens = tokens
+        self.adjvec = adjvec
+        self.im_shape = im_shape
+        max_step = chunk_size * 9 * FLAGS.T_0 // FLAGS.batch_size
+        test_batch = chunk_size // FLAGS.batch_size
+
+        def cosine_decay():
+            """
+                Applies cosine decay to the learning rate.
+            """
+            global_step = _decay_step_counter()
+            with init_on_cpu():
+                frac = (1 + ops.cos(global_step / max_step * math.pi)) / 2
+            return FLAGS.lr_min + (FLAGS.lr_max - FLAGS.lr_min) * frac
+
+        self.lr_strategy = cosine_decay
+
+    def fn_model(self, py_reader):
+        """
+            fn model
+        """
+        self.image, self.label = fluid.layers.read_file(py_reader)
+        self.loss, self.accuracy = inception.net(
+                self.image, self.label, self.tokens, self.adjvec)
+        return self.loss, self.accuracy
+
+    def build_input(self, image_shape, is_train):
+        """
+            build_input
+        """
+        name = 'train_reader' if is_train else 'test_reader'
+        py_reader = fluid.layers.py_reader(
+            capacity=64,
+            shapes=[[-1] + image_shape, [-1, 1]],
+            lod_levels=[0, 0],
+            dtypes=["float32", "int64"],
+            use_double_buffer=True,
+            name=name)
+        return py_reader
+
+    def build_program(self, main_prog, startup_prog, is_train):
+        """
+            build_program
+        """
+        out = []
+        with fluid.program_guard(main_prog, startup_prog):
+            py_reader = self.build_input(self.im_shape, is_train)
+            if is_train:
+                with fluid.unique_name.guard():
+                    loss, accuracy = self.fn_model(py_reader)
+                    optimizer = fluid.optimizer.Momentum(
+                        learning_rate=self.lr_strategy(),
+                        momentum=0.9,
+                        use_nesterov=True)
+                    optimizer.minimize(loss)
+                    out = [py_reader, loss, accuracy]
+            else:
+                with fluid.unique_name.guard():
+                    loss, accuracy = self.fn_model(py_reader)
+                    out = [py_reader, loss, accuracy]
+        return out

AutoDL Design/inception_train/preprocess.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Data preprocess
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import flags
+from PIL import Image
+from PIL import ImageOps
+from PIL import ImageEnhance
+import numpy as np
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_boolean("random_flip_left_right", True,
+                     "random flip left and right")
+flags.DEFINE_boolean("random_flip_up_down", False,
+                     "random flip up and down")
+flags.DEFINE_boolean("random_brightness", False,
+                     "randomly adjust brightness")
+image_size = 32
+
+
+def augmentation(sample, is_training):
+    """
+        augmentation
+    """
+    image_array = sample.reshape(3, image_size, image_size)
+    rgb_array = np.transpose(image_array, (1, 2, 0))
+    img = Image.fromarray(rgb_array, 'RGB')
+
+    if is_training:
+        # pad and crop
+        img = ImageOps.expand(img, (4, 4, 4, 4), fill=0)  # pad to 40 * 40 * 3
+        left_top = np.random.randint(9, size=2)  # rand 0 - 8
+        img = img.crop((left_top[0], left_top[1], left_top[0] + image_size,
+                        left_top[1] + image_size))
+
+        if FLAGS.random_flip_left_right:
+            if np.random.randint(2):
+                img = img.transpose(Image.FLIP_LEFT_RIGHT)
+        if FLAGS.random_flip_up_down:
+            if np.random.randint(2):
+                img = img.transpose(Image.FLIP_TOP_BOTTOM)
+        if FLAGS.random_brightness:
+            delta = np.random.uniform(-0.3, 0.3) + 1.
+            img = ImageEnhance.Brightness(img).enhance(delta)
+
+    img = np.array(img).astype(np.float32)
+
+    # per_image_standardization
+    img_float = img / 255.0
+    num_pixels = img_float.size
+    img_mean = img_float.mean()
+    img_std = img_float.std()
+    scale = np.maximum(np.sqrt(num_pixels), img_std)
+    img = (img_float - img_mean) / scale
+
+    img = np.transpose(img, (2, 0, 1))
+    return img

AutoDL Design/inception_train/reader.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rig hts Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Based on:
+# --------------------------------------------------------
+# DARTS
+# Copyright (c) 2018, Hanxiao Liu.
+# Licensed under the Apache License, Version 2.0;
+# --------------------------------------------------------
+"""
+    CIFAR-10 dataset.
+    This module will download dataset from
+    https://www.cs.toronto.edu/~kriz/cifar.html and parse train/test set into
+    paddle reader creators.
+    The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes,
+    with 6000 images per class. There are 50000 training images
+    and 10000 test images.
+"""
+
+import numpy as np
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+import random
+import utils
+import paddle.fluid as fluid
+import os
+from preprocess import augmentation
+
+
+def reader_creator_filepath(filename, sub_name, is_training,
+                            batch_size, data_list):
+    """
+        reader creator
+    """
+    dataset = []
+    for name in data_list:
+        print("Reading file " + name)
+        file_path = os.path.join(filename, name)
+        batch_data = pickle.load(open(file_path))
+        dataset.append(batch_data)
+    datasets = np.concatenate(dataset)
+    if is_training:
+        np.random.shuffle(dataset)
+
+    def read_batch(datasets, is_training):
+        """
+            read batch
+        """
+        for sample, label in datasets:
+            im = augmentation(sample, is_training)
+            yield im, [int(label)]
+
+    def reader():
+        """
+            get reader
+        """
+        batch_data = []
+        batch_label = []
+        for data, label in read_batch(datasets, is_training):
+            batch_data.append(data)
+            batch_label.append(label)
+            if len(batch_data) == batch_size:
+                batch_data = np.array(batch_data, dtype='float32')
+                batch_label = np.array(batch_label, dtype='int64')
+                batch_out = [[batch_data, batch_label]]
+                yield batch_out
+                batch_data = []
+                batch_label = []
+        if len(batch_data) != 0:
+            batch_data = np.array(batch_data, dtype='float32')
+            batch_label = np.array(batch_label, dtype='int64')
+            batch_out = [[batch_data, batch_label]]
+            yield batch_out
+            batch_data = []
+            batch_label = []
+    return reader
+
+
+def train10(data, batch_size, data_list):
+    """
+    CIFAR-10 training set creator.
+    It returns a reader creator, each sample in the reader is image pixels in
+    [0, 1] and label in [0, 9].
+    :return: Training reader creator
+    :rtype: callable
+    """
+    return reader_creator_filepath(data, 'data_batch', True,
+                                   batch_size, data_list)
+
+
+def test10(data, batch_size, data_list):
+    """
+    CIFAR-10 test set creator.
+    It returns a reader creator, each sample in the reader is image pixels in
+    [0, 1] and label in [0, 9].
+    :return: Test reader creator.
+    :rtype: callable
+    """
+    return reader_creator_filepath(data, 'test_batch', False,
+                                   batch_size, data_list)

AutoDL Design/inception_train/train.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Trainer Definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import numpy as np
+import reader
+import sys
+import os
+import time
+import paddle.fluid as fluid
+import utils
+import cPickle as cp
+from absl import flags
+from absl import app
+from nn import CIFARModel
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string("data_path",
+                    "./cifar/pickle-cifar-10",
+                    "data path")
+flags.DEFINE_string("logdir", "log",
+                    "logging directory")
+flags.DEFINE_integer("mid", 0,
+                     "model id")
+flags.DEFINE_integer("early_stop", 20,
+                     "early stop")
+
+image_size = 32
+
+
+def main(_):
+    """
+        main
+    """
+    image_shape = [3, image_size, image_size]
+    files = os.listdir(FLAGS.data_path)
+    names = [each_item for each_item in files]
+    np.random.shuffle(names)
+    train_list = names[:9]
+    test_list = names[-1]
+    tokens, adjvec = utils.load_action(FLAGS.mid)
+
+    model = CIFARModel(tokens, adjvec, image_shape)
+
+    place = fluid.CUDAPlace(0)
+    exe = fluid.Executor(place)
+
+    startup = fluid.Program()
+    train_prog = fluid.Program()
+    test_prog = fluid.Program()
+    train_vars = model.build_program(train_prog, startup, True)
+    test_vars = model.build_program(test_prog, startup, False)
+    exe.run(startup)
+
+    train_accuracy, epoch_id = train(model, FLAGS.early_stop,
+                                     train_prog, train_vars, exe, train_list)
+    if epoch_id < FLAGS.early_stop:
+        utils.dump_reward(FLAGS.mid, train_accuracy)
+    else:
+        test_accuracy = test(model, test_prog, test_vars, exe, [test_list])
+        utils.dump_reward(FLAGS.mid, test_accuracy)
+
+
+def train(model, epoch_num, train_prog, train_vars, exe, data_list):
+    """
+        train
+    """
+    train_py_reader, loss_train, acc_train = train_vars
+    exec_strategy = fluid.ExecutionStrategy()
+    exec_strategy.num_threads = 1
+    build_strategy = fluid.BuildStrategy()
+    build_strategy.memory_optimize = False
+    build_strategy.enable_inplace = True
+    train_exe = fluid.ParallelExecutor(
+        main_program=train_prog,
+        use_cuda=True,
+        loss_name=loss_train.name,
+        exec_strategy=exec_strategy,
+        build_strategy=build_strategy)
+
+    train_reader = reader.train10(FLAGS.data_path, FLAGS.batch_size, data_list)
+    train_py_reader.decorate_paddle_reader(train_reader)
+
+    train_fetch_list = [loss_train, acc_train]
+    epoch_start_time = time.time()
+    for epoch_id in range(epoch_num):
+        train_py_reader.start()
+        epoch_end_time = time.time()
+        if epoch_id > 0:
+            print("Epoch {}, total time {}".format(epoch_id - 1, epoch_end_time
+                                                   - epoch_start_time))
+        epoch_start_time = epoch_end_time
+        epoch_end_time
+        start_time = time.time()
+        step_id = 0
+        try:
+            while True:
+                prev_start_time = start_time
+                start_time = time.time()
+                loss_v, acc_v = train_exe.run(
+                    fetch_list=[v.name for v in train_fetch_list])
+                if np.isnan(np.array(loss_v).mean()):
+                    format_str = "[%s] jobs done, step = %d, loss = nan"
+                    print(format_str % (utils.stime(), step_id))
+                    return np.array(acc_v).mean(), epoch_id
+                print("Epoch {}, Step {}, loss {}, acc {}, time {}".format(
+                        epoch_id, step_id, np.array(loss_v).mean(),
+                        np.array(acc_v).mean(), start_time - prev_start_time))
+                step_id += 1
+                sys.stdout.flush()
+        except fluid.core.EOFException:
+            train_py_reader.reset()
+    return np.array(acc_v).mean(), epoch_id
+
+
+def test(model, test_prog, test_vars, exe, data_list):
+    """
+        test
+    """
+    test_py_reader, loss_test, acc_test = test_vars
+    test_prog = test_prog.clone(for_test=True)
+    objs = utils.AvgrageMeter()
+
+    test_reader = reader.test10(FLAGS.data_path, FLAGS.batch_size, data_list)
+    test_py_reader.decorate_paddle_reader(test_reader)
+
+    test_py_reader.start()
+    test_fetch_list = [acc_test]
+    test_start_time = time.time()
+    step_id = 0
+    try:
+        while True:
+            prev_test_start_time = test_start_time
+            test_start_time = time.time()
+            acc_v, = exe.run(
+                test_prog, fetch_list=test_fetch_list)
+            objs.update(np.array(acc_v), np.array(acc_v).shape[0])
+            step_id += 1
+    except fluid.core.EOFException:
+        test_py_reader.reset()
+    print("test acc {0}".format(objs.avg))
+    return objs.avg
+
+
+if __name__ == '__main__':
+    app.run(main)

AutoDL Design/inception_train/utils.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+"""
+    Utils Definition
+"""
+
+import os
+import pickle
+import time
+from absl import flags
+
+FLAGS = flags.FLAGS
+
+
+def stime():
+    """
+        stime
+    """
+    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
+
+
+def load_action(mid):
+    """
+        load action by mid
+    """
+    filename = os.path.join(FLAGS.logdir, "actions", "%d.pkl" % mid)
+    return pickle.load(open(filename, "rb"))
+
+
+def dump_action(mid, action):
+    """
+        dump action
+    """
+    filename = os.path.join(FLAGS.logdir, "actions", "%d.pkl" % mid)
+    pickle.dump(action, open(filename, "wb"))
+
+
+def dump_reward(mid, reward):
+    """
+        dump reward
+    """
+    filename = os.path.join(FLAGS.logdir, "rewards", "%d.pkl" % mid)
+    pickle.dump(reward, open(filename, "wb"))
+
+
+class AvgrageMeter(object):
+    """
+        AvgrageMeter for test
+    """
+    def __init__(self):
+        """
+            init
+        """
+        self.reset()
+
+    def reset(self):
+        """
+            reset
+        """
+        self.avg = 0
+        self.sum = 0
+        self.cnt = 0
+
+    def update(self, val, n=1):
+        """
+            update
+        """
+        self.sum += val * n
+        self.cnt += n
+        self.avg = self.sum / self.cnt

AutoDL Design/main.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    AutoDL main definition
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import traceback
+import autodl
+
+if __name__ == "__main__":
+    try:
+        autodl_exe = autodl.AutoDL()
+        autodl_exe.run()
+    except Exception as e:
+        print(str(e))
+        traceback.print_exc()

AutoDL Design/policy_model.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    PolicyModel definition
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import numpy as np
+import paddle.fluid as fluid
+from parl.framework.model_base import Model
+import distribute_generator
+
+
+class LstmUnit(object):
+    """
+        implemetation of lstm unit
+    """
+    def __init__(self, input_size, hidden_size, num_layers=1,
+                 init_scale=0.1):
+        """
+            init
+        """
+        self.weight_1_arr = []
+        self.bias_1_arr = []
+        for i in range(num_layers):
+            weight_1 = fluid.layers.create_parameter(
+                    [input_size + hidden_size, hidden_size * 4],
+                    dtype="float32",
+                    name="fc_weight1_" + str(i),
+                    default_initializer=fluid.initializer.UniformInitializer(
+                            low=-init_scale,
+                            high=init_scale))
+            input_size = hidden_size
+            self.weight_1_arr.append(weight_1)
+            bias_1 = fluid.layers.create_parameter(
+                [hidden_size * 4],
+                dtype="float32",
+                name="fc_bias1_" + str(i),
+                default_initializer=fluid.initializer.Constant(0.0))
+            self.bias_1_arr.append(bias_1)
+
+        self.num_layers = num_layers
+        self.hidden_size = hidden_size
+
+    def lstm_step(self, inputs, hidden, cell):
+        """
+            lstm step
+        """
+        hidden_array = []
+        cell_array = []
+        for i in range(self.num_layers):
+            hidden_temp = fluid.layers.slice(hidden, axes=[0], starts=[i],
+                                             ends=[i + 1])
+            hidden_temp = fluid.layers.reshape(hidden_temp,
+                                               shape=[-1, self.hidden_size])
+            hidden_array.append(hidden_temp)
+
+            cell_temp = fluid.layers.slice(cell, axes=[0], starts=[i],
+                                           ends=[i + 1])
+            cell_temp = fluid.layers.reshape(cell_temp,
+                                             shape=[-1, self.hidden_size])
+            cell_array.append(cell_temp)
+
+        last_hidden_array = []
+        step_input = inputs
+        for k in range(self.num_layers):
+            pre_hidden = hidden_array[k]
+            pre_cell = cell_array[k]
+            weight = self.weight_1_arr[k]
+            bias = self.bias_1_arr[k]
+
+            nn = fluid.layers.concat([step_input, pre_hidden], 1)
+            gate_input = fluid.layers.matmul(x=nn, y=weight)
+
+            gate_input = fluid.layers.elementwise_add(gate_input, bias)
+            i, j, f, o = fluid.layers.split(gate_input, num_or_sections=4,
+                                            dim=-1)
+
+            c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid(i) \
+                * fluid.layers.tanh(j)
+            m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o)
+
+            hidden_array[k] = m
+            cell_array[k] = c
+            step_input = m
+
+        last_hidden = fluid.layers.concat(hidden_array, axis=0)
+        last_hidden = fluid.layers.reshape(last_hidden, shape=[
+                self.num_layers, -1, self.hidden_size])
+
+        last_cell = fluid.layers.concat(cell_array, axis=0)
+        last_cell = fluid.layers.reshape(
+                last_cell,
+                shape=[self.num_layers, -1, self.hidden_size])
+        return step_input, last_hidden, last_cell
+
+    def __call__(self, inputs, hidden, cell):
+        """
+            lstm step call
+        """
+        return self.lstm_step(inputs, hidden, cell)
+
+
+class PolicyModel(Model):
+    """
+        PolicyModel
+    """
+    def __init__(self, parser_args):
+        """
+            construct rnn net
+        """
+        self.parser_args = parser_args
+
+    def policy(self, inputs):
+        """
+            policy function is used by `define_predict` in PolicyGradient
+        """
+        [tokens, softmax, adjvec, sigmoid] = self.build_rnn(inputs)
+        return [tokens, softmax, adjvec, sigmoid]
+
+    def build_rnn(self, inputs):
+        """
+            build rnn net
+        """
+        batch_size = self.parser_args.batch_size
+        input_size = self.parser_args.input_size
+        hidden_size = self.parser_args.hidden_size
+        num_layers = self.parser_args.num_layers
+        num_nodes = self.parser_args.num_nodes
+        num_tokens = self.parser_args.num_tokens
+
+        depth = max(num_nodes - 1, num_tokens)
+        lstm_unit = LstmUnit(input_size, hidden_size, num_layers)
+
+        def encode_token(inp):
+            """
+                encode token
+            """
+            token = fluid.layers.assign(inp)
+            token.stop_gradient = True
+            token = fluid.layers.one_hot(token, depth)
+            return token
+
+        def encode_adj(adj, step):
+            """
+                encode adj
+            """
+            adj = fluid.layers.cast(adj, dtype='float32')
+            adj_pad = fluid.layers.pad(x=adj, paddings=[0, 0, 0, depth - step],
+                                       pad_value=0.0)
+            return adj_pad
+
+        def decode_token(hidden):
+            """
+                decode token
+            """
+            initiallizer = fluid.initializer.TruncatedNormalInitializer(
+                    scale=np.sqrt(2.0 / self.parser_args.hidden_size))
+            param_attr = fluid.ParamAttr(initializer=initiallizer)
+            logits = fluid.layers.fc(hidden, num_tokens, param_attr=param_attr)
+            temper = 5.0
+            tanh_c = 2.5
+            logits = fluid.layers.tanh(logits / temper) * tanh_c
+            token = distribute_generator.multinomial(logits,
+                                                     [batch_size, 1], 1)
+            return token, fluid.layers.unsqueeze(logits, axes=[1])
+
+        def decode_adj(hidden, step):
+            """
+                decode adj
+            """
+            initiallizer = fluid.initializer.TruncatedNormalInitializer(
+                    scale=np.sqrt(2.0 / self.parser_args.hidden_size))
+            param_attr = fluid.ParamAttr(initializer=initiallizer)
+            logits = fluid.layers.fc(hidden, step, param_attr=param_attr)
+            temper = 5.0
+            tanh_c = 2.5
+            logits = fluid.layers.tanh(logits / temper) * tanh_c
+            adj = distribute_generator.bernoulli(logits,
+                                                 output_shape=logits.shape)
+            return adj, logits
+
+        tokens = []
+        softmax = []
+        adjvec = []
+        sigmoid = []
+
+        def rnn_block(hidden, last_hidden, last_cell):
+            """
+                rnn block
+            """
+            last_output, last_hidden, last_cell = lstm_unit(
+                    hidden, last_hidden, last_cell)
+            token, logits = decode_token(last_output)
+            tokens.append(token)
+            softmax.append(logits)
+
+            for step in range(1, num_nodes):
+                token_vec = encode_token(token)
+                last_output, last_hidden, last_cell = lstm_unit(
+                        token_vec, last_hidden, last_cell)
+                adj, logits = decode_adj(last_output, step)
+                adjvec.append(adj)
+                sigmoid.append(logits)
+                adj_vec = encode_adj(adj, step)
+                last_output, last_hidden, last_cell = lstm_unit(
+                        adj_vec, last_hidden, last_cell)
+                token, logits = decode_token(last_output)
+                tokens.append(token)
+                softmax.append(logits)
+            return token, last_hidden, last_cell
+        init_hidden = fluid.layers.fill_constant(
+                shape=[num_layers, batch_size, hidden_size],
+                value=0.0, dtype='float32')
+        init_cell = fluid.layers.fill_constant(
+                shape=[num_layers, batch_size, hidden_size],
+                value=0.0, dtype='float32')
+
+        hidden = encode_adj(inputs, 1)
+        token, last_hidden, last_cell = rnn_block(hidden, init_hidden,
+                                                  init_cell)
+        hidden = encode_token(token)
+        token, last_hidden, last_cell = rnn_block(hidden, last_hidden,
+                                                  last_cell)
+        token_out = fluid.layers.concat(tokens, axis=1)
+        softmax_out = fluid.layers.concat(softmax, axis=1)
+        adjvec_out = fluid.layers.concat(adjvec, axis=1)
+        sigmoid_out = fluid.layers.concat(sigmoid, axis=1)
+        return [token_out, softmax_out, adjvec_out, sigmoid_out]

AutoDL Design/reinforce_policy_gradient.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    AutoDL definition
+"""
+import paddle.fluid as fluid
+from parl.framework.algorithm_base import Algorithm
+import paddle.fluid.layers as layers
+import os
+import sys
+
+
+class ReinforcePolicyGradient(Algorithm):
+    """
+        Implement REINFORCE policy gradient for autoDL
+    """
+    def __init__(self, model, hyperparas):
+        """
+        """
+        Algorithm.__init__(self, model, hyperparas)
+        self.model = model
+        self.lr = hyperparas['lr']
+
+    def define_predict(self, obs):
+        """
+            use policy model self.model to predict the action probability
+            obs is `inputs`
+        """
+        with fluid.unique_name.guard():
+            [tokens, softmax, adjvec, sigmoid] = self.model.policy(obs)
+            return tokens, adjvec
+
+    def define_learn(self, obs, action, reward):
+        """
+            update policy model self.model with policy gradient algorithm
+            obs is `inputs`
+        """
+        tokens = action[0]
+        adjvec = action[1]
+        with fluid.unique_name.guard():
+            [_, softmax, _, sigmoid] = self.model.policy(obs)
+            reshape_softmax = layers.reshape(
+                    softmax,
+                    [-1, self.model.parser_args.num_tokens])
+            reshape_tokens = layers.reshape(tokens, [-1, 1])
+            reshape_tokens.stop_gradient = True
+            raw_neglogp_to = layers.softmax_with_cross_entropy(
+                    soft_label=False,
+                    logits=reshape_softmax,
+                    label=fluid.layers.cast(x=reshape_tokens, dtype="int64"))
+
+            action_to_shape_sec = self.model.parser_args.num_nodes * 2
+            neglogp_to = layers.reshape(fluid.layers.cast(
+                                            raw_neglogp_to, dtype="float32"),
+                                        [-1, action_to_shape_sec])
+
+            adjvec = layers.cast(x=adjvec, dtype='float32')
+            neglogp_ad = layers.sigmoid_cross_entropy_with_logits(
+                    x=sigmoid, label=adjvec)
+
+            neglogp = layers.elementwise_add(
+                    x=layers.reduce_sum(neglogp_to, dim=1),
+                    y=layers.reduce_sum(neglogp_ad, dim=1))
+            reward = layers.cast(reward, dtype="float32")
+            cost = layers.reduce_mean(
+                    fluid.layers.elementwise_mul(x=neglogp, y=reward))
+            optimizer = fluid.optimizer.Adam(learning_rate=self.lr)
+            train_op = optimizer.minimize(cost)
+            return cost

AutoDL Design/run.sh

+export FLAGS_fraction_of_gpu_memory_to_use=0.98
+export FLAGS_eager_delete_tensor_gb=0.0
+export FLAGS_fast_eager_deletion_mode=1
+CUDA_VISIBLE_DEVICES=0 python -u main.py > main.log 2>&1 &

AutoDL Design/simple_main.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    AutoDL main definition
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+import traceback
+import autodl
+
+if __name__ == "__main__":
+    try:
+        autodl_exe = autodl.AutoDL()
+        autodl_exe.simple_run()
+    except Exception as e:
+        print(str(e))
+        traceback.print_exc()

AutoDL Design/utils.py

+#!/usr/bin/env python
+# -*- encoding:utf-8 -*-
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    AutoDL definition
+"""
+
+import os
+import time
+import pickle
+
+
+def stime():
+    """
+        stime
+    """
+    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
+
+
+def prepare(log_dir, category=""):
+    """
+        prepare directory
+    """
+    subdir = os.path.join(log_dir, category)
+    if not os.path.exists(subdir):
+        os.mkdir(subdir)
+
+
+def dump_action(mid, action, log_dir):
+    """
+        dump action
+    """
+    filename = os.path.join(log_dir, "actions", "%d.pkl" % mid)
+    pickle.dump(action, open(filename, "wb"))
+
+
+def load_action(mid, log_dir):
+    """
+        load action
+    """
+    filename = os.path.join(log_dir, "actions", "%d.pkl" % mid)
+    return pickle.load(open(filename, "rb"))
+
+
+def dump_reward(mid, reward, log_dir):
+    """
+        dump reward
+    """
+    filename = os.path.join(log_dir, "rewards", "%d.pkl" % mid)
+    pickle.dump(reward, open(filename, "wb"))
+
+
+def load_reward(mid, log_dir):
+    """
+        load reward
+    """
+    filename = os.path.join(log_dir, "rewards", "%d.pkl" % mid)
+    return pickle.load(open(filename, "rb"))