Merge branch 'develop' into 'develop'

merge

See merge request !19

models/rank/tagspace/__init__.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

models/rank/tagspace/config.yaml

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+train:
+  trainer:
+    # for cluster training
+    strategy: "async"
+
+  epochs: 10
+  workspace: "fleetrec.models.rank.tagspace"
+
+  reader:
+    batch_size: 5
+    class: "{workspace}/reader.py"
+    train_data_path: "{workspace}/train_data"
+
+  model:
+    models: "{workspace}/model.py"
+    hyper_parameters:
+        vocab_text_size: 11447
+        vocab_tag_size: 4
+        emb_dim: 10
+        hid_dim: 1000
+        win_size: 5
+        margin: 0.1
+        neg_size: 3
+        num_devices: 1
+
+
+  save:
+    increment:
+      dirname: "increment"
+      epoch_interval: 1
+      save_last: True
+    inference:
+      dirname: "inference"
+      epoch_interval: 100
+      save_last: True
+

models/rank/tagspace/model.py

+import paddle.fluid as fluid
+import math
+
+from fleetrec.core.utils import envs
+from fleetrec.core.model import Model as ModelBase
+
+import paddle.fluid as fluid
+import paddle.fluid.layers.nn as nn
+import paddle.fluid.layers.tensor as tensor
+import paddle.fluid.layers.control_flow as cf
+
+class Model(ModelBase):
+    def __init__(self, config):
+        ModelBase.__init__(self, config)
+        self.cost = None
+        self.metrics = {}
+        self.vocab_text_size = 11447#envs.get_global_env("vocab_text_size", None, self._namespace)
+        self.vocab_tag_size = 4#envs.get_global_env("vocab_tag_size", None, self._namespace)
+        self.emb_dim = 10#envs.get_global_env("emb_dim", None, self._namespace)
+        self.hid_dim = 1000#envs.get_global_env("hid_dim", None, self._namespace)
+        self.win_size = 5#envs.get_global_env("win_size", None, self._namespace)
+        self.margin = 0.1#envs.get_global_env("margin", None, self._namespace)
+        self.neg_size = 3#envs.get_global_env("neg_size", None, self._namespace)
+        print self.emb_dim
+
+    def train_net(self):
+        """ network definition """
+        text = fluid.data(name="text", shape=[None, 1], lod_level=1, dtype='int64')
+        pos_tag = fluid.data(
+            name="pos_tag", shape=[None, 1], lod_level=1, dtype='int64')
+        neg_tag = fluid.data(
+            name="neg_tag", shape=[None, 1], lod_level=1, dtype='int64')
+
+        self._data_var = [text, pos_tag, neg_tag]
+
+        text_emb = fluid.embedding(
+            input=text, size=[self.vocab_text_size, self.emb_dim], param_attr="text_emb")
+        text_emb = fluid.layers.squeeze(input=text_emb, axes=[1])
+        pos_tag_emb = fluid.embedding(
+            input=pos_tag, size=[self.vocab_tag_size, self.emb_dim], param_attr="tag_emb")
+        pos_tag_emb = fluid.layers.squeeze(input=pos_tag_emb, axes=[1])
+        neg_tag_emb = fluid.embedding(
+            input=neg_tag, size=[self.vocab_tag_size, self.emb_dim], param_attr="tag_emb")
+        neg_tag_emb = fluid.layers.squeeze(input=neg_tag_emb, axes=[1])
+
+        conv_1d = fluid.nets.sequence_conv_pool(
+            input=text_emb,
+            num_filters=self.hid_dim,
+            filter_size=self.win_size,
+            act="tanh",
+            pool_type="max",
+            param_attr="cnn")
+        text_hid = fluid.layers.fc(input=conv_1d,
+                                   size=self.emb_dim,
+                                   param_attr="text_hid")
+        cos_pos = nn.cos_sim(pos_tag_emb, text_hid)
+        mul_text_hid = fluid.layers.sequence_expand_as(x=text_hid, y=neg_tag_emb)
+        mul_cos_neg = nn.cos_sim(neg_tag_emb, mul_text_hid)
+        cos_neg_all = fluid.layers.sequence_reshape(
+            input=mul_cos_neg, new_dim=self.neg_size)
+        #choose max negtive cosine
+        cos_neg = nn.reduce_max(cos_neg_all, dim=1, keep_dim=True)
+        #calculate hinge loss
+        loss_part1 = nn.elementwise_sub(
+            tensor.fill_constant_batch_size_like(
+                input=cos_pos, shape=[-1, 1], value=self.margin, dtype='float32'),
+            cos_pos)
+        loss_part2 = nn.elementwise_add(loss_part1, cos_neg)
+        loss_part3 = nn.elementwise_max(
+            tensor.fill_constant_batch_size_like(
+                input=loss_part2, shape=[-1, 1], value=0.0, dtype='float32'),
+            loss_part2)
+        avg_cost = nn.mean(loss_part3)
+        less = tensor.cast(cf.less_than(cos_neg, cos_pos), dtype='float32')
+        correct = nn.reduce_sum(less)
+        self.cost = avg_cost
+        
+        self.metrics["correct"] = correct
+        self.metrics["cos_pos"] = cos_pos
+
+    def get_cost_op(self):
+        return self.cost
+
+    def get_metrics(self):
+        return self.metrics
+
+    def optimizer(self):
+        learning_rate = 0.01#envs.get_global_env("hyper_parameters.base_lr", None, self._namespace)
+        sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=learning_rate)
+        #sgd_optimizer.minimize(avg_cost)
+        return sgd_optimizer
+
+
+    def infer_net(self, parameter_list):
+        self.train_net()

models/rank/tagspace/reader.py

+import re
+import sys
+import collections
+import os
+import six
+import time
+import numpy as np
+import paddle.fluid as fluid
+import paddle
+import csv
+import io
+
+from fleetrec.core.reader import Reader
+from fleetrec.core.utils import envs
+
+class TrainReader(Reader):
+    def init(self):
+        pass
+
+    def _process_line(self, l): 
+        tag_size = 4
+        neg_size = 3
+        l = l.strip().split(",")
+        pos_index = int(l[0])
+        pos_tag = []
+        pos_tag.append(pos_index)
+        text_raw = l[1].split()
+        text = [int(w) for w in text_raw]
+        neg_tag = []
+        max_iter = 100
+        now_iter = 0
+        sum_n = 0
+        while (sum_n < neg_size):
+            now_iter += 1
+            if now_iter > max_iter:
+                print("error : only one class")
+                sys.exit(0)
+            rand_i = np.random.randint(0, tag_size)
+            if rand_i != pos_index:
+                neg_index = rand_i
+                neg_tag.append(neg_index)
+                sum_n += 1
+       # if n > 0 and len(text) > n:
+       #    #yield None
+       #    return None, None, None
+        return  text, pos_tag, neg_tag
+
+    def generate_sample(self, line):
+        def data_iter():
+            text, pos_tag, neg_tag = self._process_line(line)
+            if text is None:
+                yield None
+                return
+            yield [('text', text), ('pos_tag', pos_tag), ('neg_tag', neg_tag)]
+        return data_iter

models/rank/text_classification/config.yaml

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+train:
+  trainer:
+    # for cluster training
+    strategy: "async"
+
+  epochs: 10
+  workspace: "fleetrec.models.rank.text_classification"
+
+  reader:
+    batch_size: 5
+    class: "{workspace}/reader.py"
+    train_data_path: "{workspace}/train_data"
+
+  model:
+    models: "{workspace}/model.py"
+
+  save:
+    increment:
+      dirname: "increment"
+      epoch_interval: 1
+      save_last: True
+    inference:
+      dirname: "inference"
+      epoch_interval: 100
+      save_last: True
+

models/rank/text_classification/model.py

+import paddle.fluid as fluid
+import math
+
+from fleetrec.core.utils import envs
+from fleetrec.core.model import Model as ModelBase
+
+import paddle.fluid as fluid
+import paddle.fluid.layers.nn as nn
+import paddle.fluid.layers.tensor as tensor
+import paddle.fluid.layers.control_flow as cf
+
+class Model(ModelBase):
+    def __init__(self, config):
+        ModelBase.__init__(self, config)
+
+    def train_net(self):
+        """ network definition """
+       
+        data = fluid.data(name="input", shape=[None, max_len], dtype='int64')
+        label = fluid.data(name="label", shape=[None, 1], dtype='int64')
+        seq_len = fluid.data(name="seq_len", shape=[None], dtype='int64')
+        # embedding layer
+        emb = fluid.embedding(input=data, size=[dict_dim, emb_dim])
+        emb = fluid.layers.sequence_unpad(emb, length=seq_len)
+        # convolution layer
+        conv = fluid.nets.sequence_conv_pool(
+            input=emb,
+            num_filters=cnn_dim,
+            filter_size=cnn_filter_size,
+            act="tanh",
+            pool_type="max")
+
+        # full connect layer
+        fc_1 = fluid.layers.fc(input=[conv], size=hid_dim)
+        # softmax layer
+        prediction = fluid.layers.fc(input=[fc_1], size=class_dim, act="softmax")
+        #if is_prediction:
+        #    return prediction
+        cost = fluid.layers.cross_entropy(input=prediction, label=label)
+        avg_cost = fluid.layers.mean(x=cost)
+        acc = fluid.layers.accuracy(input=prediction, label=label) 
+
+        self.cost = avg_cost
+        self.metrics["acc"] = cos_pos
+
+    def get_cost_op(self):
+        return self.cost
+
+    def get_metrics(self):
+        return self.metrics
+
+    def optimizer(self):
+        learning_rate = 0.01#envs.get_global_env("hyper_parameters.base_lr", None, self._namespace)
+        sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=learning_rate)
+        #sgd_optimizer.minimize(avg_cost)
+        return sgd_optimizer
+
+
+    def infer_net(self, parameter_list):
+        self.train_net()

models/rank/text_classification/reader.py

+import re
+import sys
+import collections
+import os
+import six
+import time
+import numpy as np
+import paddle.fluid as fluid
+import paddle
+import csv
+import io
+
+from fleetrec.core.reader import Reader
+from fleetrec.core.utils import envs
+
+class TrainReader(Reader):
+    def init(self):
+        pass
+
+    def _process_line(self, l): 
+        l = l.strip().split(" ")
+        data = l[0:10]
+        seq_len = l[10:11]
+        label = l[11:]
+        return  data, label, seq_len
+
+    def generate_sample(self, line):
+        def data_iter():
+            data, label, seq_len = self._process_line(line)
+            if data is None:
+                yield None
+                return
+            yield [('data', data), ('label', label), ('seq_len', seq_len)]
+        return data_iter

models/rank/text_classification/train_data/part-0

+12 27 13  0 25 52 89 20 39  4 9 1
+78 10 61 58 29 79 85 16 46 41 9 1
+81 77 44  4  5 57 43 97 42 89 6 0
+ 7 77 86  3 98 89 56 24  7 59 9 1
+65 89 99 27 65 98 16 89 42  0 3 0
+66 14 48 38 66  5 56 89 98 19 4 1
+78  7 10 20 77 16 37 43 59 23 6 1
+84 95 28 35  0 82 55 19 13 81 7 0
+34 32 98 37 43 51  6 38 20 40 9 0
+75 36 13 51 70 24 62 90 32 91 7 1
+13  5 49 21 57 21 67 85 74 14 1 0
+68 13 86 16 52 50 23 11 65 99 1 1
+15 20 75 55 15 90 54 54 15 91 9 0
+44 56 15 88 57  3 62 53 89 57 8 1
+23  8 40 25 60 33  8 69 44 88 7 1
+63 94  5 43 23 70 31 67 21 55 6 0
+44 11 64 92 10 37 30 84 19 71 5 1
+89 18 71 13 16 58 47 60 77 87 7 1
+13 48 56 39 98 53 32 93 13 91 7 0
+56 78 67 68 27 11 77 48 45 10 1 1
+52 12 14  5  2  8  3 36 33 59 6 0
+86 42 91 81  2  9 21  0 44  7 9 1
+96 27 82 55 81 30 91 41 91 58 2 1
+97 69 76 47 80 62 23 30 87 22 7 1
+42 56 25 47 42 18 80 53 15 57 7 0
+34 73 75 88 61 79 40 74 87 87 6 1
+ 7 91  9 24 42 60 76 31 10 13 4 0
+21  1 46 59 61 54 99 54 89 55 5 1
+67 21  1 29 88  5  3 85 39 22 5 1
+90 99  7  8 17 77 73  3 32 10 5 0
+30 44 26 32 37 74 90 71 42 29 9 1
+79 68  3 24 21 37 35  3 76 23 6 1
+ 3 66  7  4  2 88 94 64 47 81 6 1
+10 48 16 49 96 93 61 97 84 39 3 1
+73 28 67 59 89 92 17 24 52 71 3 1
+98  4 35 62 91  2 78 51 72 93 1 1
+37 42 96 10 48 49 84 45 59 47 5 1
+13 24  7 49 63 78 29 75 45 92 7 1
+ 1  6 95 23 38 34 85 94 33 47 6 1
+99 63 65 39 72 73 91 20 16 45 9 0
+35  8 81 24 62  0 95  0 52 46 4 1
+58 66 88 42 86 94 91  8 18 92 7 0
+12 62 56 43 99 31 63 80 11  7 4 1
+22 36  1 39 69 20 56 75 17 15 7 0
+25 97 62 50 99 98 32  2 98 75 7 1
+ 7 59 98 68 62 19 28 28 60 27 7 0
+39 63 43 45 43 11 40 81  4 25 6 0
+81 95 27 84 71 45 87 65 40 50 1 0
+82 21 69 55 71 92 52 65 90 16 3 0
+24  6  5 22 36 34 66 71  3 52 2 0
+ 5 14 66 71 49 10 52 81 32 14 1 0
+ 8 94 52 23 60 27 43 19 89 91 9 0
+26 14 36 37 28 94 46 96 11 80 8 1
+89 19 77 66 48 75 62 58 90 81 8 1
+25 43 95 21 25 81 39 79  9 74 9 0
+25  2 64 27 67 36 59 68 99 66 5 1
+13 46 41 55 89 93 79 83 32 52 6 0
+49 77 57  9 91 49 86 50 32  5 2 0
+94  7 53 54 70 69  5 51 59 91 5 1
+24 72 94 13 17 12  2 67  0 89 6 1
+70 38 19 27 38 87 72 41 98 84 6 1
+89 76 82  4 69 64 97 77 88 58 9 0
+67 41 99  1 80 38 96 24 67 59 3 1
+42 83 50 19 97 99 99 50 46 76 8 1
+43 99 63 40 93 15  3 57 11  0 1 0
+16 65 31 43 89 37 98 63 29 69 8 1
+39  5 65 45 12 82 46 87 82 93 8 0
+34 69 82 13  4 20 92 58 46 83 2 1
+46 79 87 57 87 23 72 95 37 88 8 0
+41 72 81 71 60 15 32  1  9 97 3 0
+84 98 15 78 39 82 89 74 46 32 9 0
+16 18 92 80 50 44 98 45 15 41 3 1
+74 78 81 40 17 65 38 21 27  9 1 0
+14 69 68 50 57 11 62  2 89 54 6 0
+70 29 79 29 44 56 33 27 25  4 3 1
+44 20 87 67 65 41 93 37 99 78 1 1
+93 57 87 11 33 40 21  3 47 87 9 1
+ 8  3 24 49 99 48 40 22 99 41 2 0
+19 90  9 83 93 22 36 96 44 73 7 1
+ 4 73  2 88 79 90 32 48 45 12 5 0
+24 58 34 67 85 62 84 48 14 79 5 1
+54 69 19 18 59 78 84 48 61 46 4 0
+72 69 95 26 30 74 49 30 95 61 8 0
+73 29 46 39 48 30 97 63 89 34 9 1
+51 32 44 22 70 69 91 81 74 52 3 0
+99 66 89 71 31 42  5 40 21 12 6 0
+58 26 59 56 91 49 79 57 57 74 6 1
+30 36 59 74  6 30 17  1 99 38 4 0
+43 48 77 86 67 25 38 36  3 91 4 1
+67 24 51 34 37  8 98 76 84 13 1 1
+73 47 88 15 32 99 67 26 28 89 3 1
+91 66 11 86  5 12 15 43 79 89 1 1
+15 60 43 58 61  0 62 32 98 29 9 0
+80 36 78 42 70 52  2 10 42 41 6 1
+36 16 46 34 96 39  8 21 86 54 5 1
+80 72 13  1 28 49 73 90 81 34 1 0
+73 64 86  9 94 49 44 38 47 64 2 0
+69 90 69 36 60 45 39  7 41 72 8 0
+31 86 54 82 81 77 93 99 68 63 1 1
+95 76 97 36 40 12  4 95 59 64 4 1
+88 20 64 40 27 11 96 40 41 73 6 0
+28 72 70 43 34 54 98 43 29 63 5 0
+78 72  4 47 47 38 73  8 65 40 3 1
+91 64 51 93  8 78 53 15 42 32 4 0
+34 36 45  9 16  0 51 40 90 29 2 1
+80 93 65 80 11 19 26 61 29  8 4 0
+94 11 60 36 58 98 43 90 64  1 1 0
+42 54 89 86 80 72 81 48 19 67 5 0
+81 25 30 60 59 20 75 38 75 29 6 0
+84 16 48 28 23 20 53 13 32 90 1 0
+58 31 77 68 27 88 51 97 70 93 8 1
+63 67 85  6 35 22 28 65  8  7 3 0
+54 75 93 58 98  9 15 37 61 38 6 1
+56 24 50 62 63 47  9  4 58 30 8 1
+64 91 32 68 50 90 51 86 52  6 1 1
+55 50 46 41 28  1 11 39 75  9 1 0
+23 27 98 73 25  7 89 48  7 44 4 1
+86 98 68  1 74 46 15 92 59 25 9 1
+95 86 72 13 33 60 62 83 96 84 1 0
+ 9 58 37 50 57 16 78  0 21 80 2 0
+82 94 74 42  3 60 61 93 34 22 3 1
+16 97 97 14 47 50 90 35  9 58 5 0
+70 94 82 42 85 88 59 58  6 68 9 0
+14 58 24 44  8 29 12 18 26 80 7 0
+22 23  7 82 39 28 96 92 23 40 5 1
+40 31 72 94 20 81 89  4 42  1 5 0
+57 63 71 41 28  2 39 67 90 54 6 0
+ 9 74  4 41 11 31 15 21 44 32 6 1
+31 28 66 66 61 78 72 80 82 88 3 1
+79 18  1 59 35 62  0 72 78 97 7 0
+14 19 30 63 38 37 12 15 54 15 6 1
+54 91 37 79 60 35 55 62 94 84 7 1
+10 55 78 96 45 55 35 56 54 70 6 1
+23 46 15 93 66 11 32 45 74 25 4 0
+51 55  9  9 88 59 21 66 87 12 1 1
+90 22 38 66 12  9 30 48 55 85 1 1
+39 23 82 29 57 76 79 56  3 19 2 0
+ 7 72 76 15 90 23 40 40 33 39 4 1
+60 64 34 11 18 18 38 39 53 37 1 1
+85 72 51 47 83 90 32 96 78 23 9 1
+85 51 96 31 83 70 57 65 15  0 6 0
+41 11 56 94 40  6 62 86 68 83 7 0
+34 82 44 30  2  2 94 62 41 27 6 1
+54 86 50 83 76 65  0 87 80 70 7 0
+97 50 65 78  2 90 28  5 12 56 5 1
+34 19 68 93 11  9 14 87 22 70 9 0
+63 77 27 20 20 37 65 51 29 29 9 1
+22 79 98 57 56 97 43 49  4 80 4 1
+ 6  4 35 54  4 36  1 79 85 35 6 0
+12 55 68 61 91 43 49  5 93 27 8 0
+64 22 69 16 63 20 28 60 13 35 7 1
+ 9 19 60 89 62 29 47 33  6 13 4 0
+14 15 39 86 47 75  7 70 57 60 6 1
+90 63 12 43 28 46 39 97 83 42 6 0
+49  3  3 64 59 46 30 13 61 10 2 0
+79 47 29 47 54 38 50 66 18 63 5 1
+98 67  1 22 66 32 91 77 63 33 3 0
+72 22 10 27 28 44 29 66 71  1 7 0
+20 52 19 23  9 38  1 93 83 73 5 0
+88 57 22 64 93 66 20 90 78  2 7 1
+90 86 41 28 14 25 86 73  7 21 4 0
+63 91  0 29  2 78 86 76  9 20 4 1
+ 3 57 91 37 21 85 80 99 18 79 1 1
+69 95 36  6 85 47 83 83 61 52 4 0
+72  4 34 16 59 78 56 70 27 44 9 1
+58 42  6 53 21  7 83 38 86 66 5 0
+22 86 22 21 86 22 83 38 62 19 4 0
+14 63 20 53 98 76 10 22 35 76 9 1
+16 88 13 66 37 33 11 40 61 97 2 1
+60  9 98 35 51 11 98 73 67 26 6 1
+25 48 87 93 58 58 15  9 23 13 7 1
+61 47 47 36 97 22 63 35  9 38 5 1
+94 49 41 38  0 81 59 39 13 65 3 0
+88 82 71 96 76 16 57 24 72 36 5 1
+28 46  8 95 94 86 63  1 42 63 6 0
+12 95 29 66 64 77 19 26 73 53 4 0
+19  5 52 34 13 62  6  4 25 58 5 0
+18 39 39 56 73 29  5 15 13 82 1 1
+50 66 99 67 76 25 43 12 24 67 9 0
+74 56 61 97 23 63 22 63  6 83 2 1
+10 96 13 49 43 20 58 19 99 58 7 1
+ 2 95 31  4 99 91 27 90 85 32 3 0
+41 23 20 71 41 75 75 35 16 12 3 1
+21 33 87 57 19 27 94 36 80 10 6 0
+ 8  0 25 74 14 61 86  8 42 82 9 0
+23 33 91 19 84 99 95 92 29 31 8 0
+94 94  5  6 98 23 37 65 14 25 6 1
+42 16 39 32  2 20 86 81 90 91 8 0
+72 39 20 63 88 52 65 81 77 96 4 0
+48 73 65 75 89 36 75 36 11 35 8 0
+79 74  3 29 63 20 76 46  8 82 5 0
+ 7 46 38 77 79 92 71 98 30 35 6 0
+44 69 93 31 22 68 91 70 32 86 5 0
+45 38 77 87 64 44 69 19 28 82 9 0
+93 63 92 84 22 44 51 94  4 99 9 0
+77 10 49 29 59 55 44  7 95 39 2 0
+10 85 99  9 91 29 64 14 50 24 6 1
+74  4 21 12 77 36 71 51 50 31 9 1
+66 76 28 18 23 49 33 31  6 44 1 1
+92 50 90 64 95 58 93  4 78 88 6 1
+69 79 76 47 46 26 30 40 33 58 8 1
+97 12 87 82  6 18 57 49 49 58 1 1
+70 79 55 86 29 88 55 39 17 74 5 1
+65 51 45 62 54 17 59 12 29 79 5 0
+ 5 63 82 51 54 97 54 36 57 46 3 0
+74 77 52 10 12  9 34 95  2  0 5 0
+50 20 22 89 50 70 55 98 80 50 1 0
+61 80  7  3 78 36 44 37 90 18 9 0
+81 13 55 57 88 81 66 55 18 34 2 1
+52 30 54 70 28 56 48 82 67 20 8 1
+ 0 41 15 63 27 90 12 16 56 79 3 0
+69 89 54  1 93 10 15  2 25 59 8 0
+74 99 17 93 96 82 38 77 98 85 4 0
+ 8 59 17 92 60 21 59 76 55 73 2 1
+53 56 79 19 29 94 86 96 62 39 3 1
+23 44 25 63 41 94 65 10  8 40 9 1
+ 7 18 80 43 20 70 14 59 72 17 9 0
+84 97 79 14 37 64 23 68  8 24 2 0
+63 94 98 77  8 62 10 77 63 56 4 0
+ 8 63 74 34 49 22 52 54 44 93 3 0
+94 48 92 58 82 48 53 34 96 25 2 0
+33 15  3 95 48 93  9 69 44 77 7 1
+69 72 80 77 64 24 52 21 36 49 2 0
+59 34 54 66 60 19 76 79 16 70 5 1
+ 8 83  9 91 67 79 31 20 31 88 2 0
+64 95 46 95 78 63  4 60 66 63 7 1
+10 39 78 45 36  4 89 94 68 75 7 0
+81 52 70 11 48 15 40 63 29 14 8 1
+94 49 30 14 53 12 53 42 77 82 8 1
+40 88 46 20 54 84 76 15  2 73 2 1
+71 50 79 54 17 58 30 16 17 99 1 1
+74 79 74 61 61 36 28 39 89 36 6 0
+53 45 45 23 51 32 93 26 10  8 3 0
+ 1 97  6 67 88 20 41 63 49  6 8 0
+ 3 64 41 19 41 80 75 71 69 90 8 0
+31 90 38 93 52  0 38 86 41 68 9 1
+50 94 53  9 73 59 94  7 24 57 3 0
+87 11  4 62 96  7  0 59 46 11 6 1
+77 67 56 88 45 62 10 51 86 27 6 1
+62 62 59 99 83 84 79 97 56 37 5 0
+19 55  0 37 44 44  2  7 54 50 5 1
+23 60 11 83  6 48 20 77 54 31 6 0
+27 53 52 30  3 70 57 38 47 96 5 0
+75 14  5 83 72 46 47 64 14 12 7 0
+29 95 36 63 59 49 38 44 13 15 2 1
+38  3 70 89  2 94 89 74 33  6 8 1
+28 56 49 43 83 34  7 63 36 13 7 0
+25 90 23 85 50 65 36 10 64 38 5 0
+35 94 48 38 99 71 42 39 61 75 8 1
+28 73 34 22 51  8 52 98 74 19 8 1
+12 40 65 12  7 96 73 65 12 90 5 0
+42 42 48 16 80 14 48 29 29 45 5 0
+58 20  4  0 69 99 15  4 16  4 1 1
+93 30 90  5 23 63 25 30 99 32 7 1
+91 23 20 26 84 78 58 76 58 90 5 1
+33  2 36 59 55  9 79 34 92 57 9 0
+80 63 84 73 22 40 70 94 59 34 5 0
+49 95 50 32 90 22 18 66 46 32 2 0
+47 72  3 94 33 78 87 43 11 67 5 0
+76 44 86 81 95 48 79 46 11 65 8 1
+59 51 97 75 17  5 40 59 32 62 6 0
+41 13 58  7 54 84  8 84 27 55 1 0
+24 80 44 26 86 99 68 80 81 22 9 0
+12 45 16 44 66 76 33 53  3 20 9 0
+22  3 79  6 32 38 75 66 15 25 9 1
+51 48 26 53 33 26 18 74  9 39 5 1
+35 67 89 91 29 81 23 52 19 11 6 0
+64 50 43  1 43 49 19 20 84 19 8 0
+34  4  9 77 24 61 55 82 42 76 9 0
+37 84 94 33 67 60  3 95 78  8 9 0
+82 10 54 12 47 23 78 97  6 51 5 0
+70 40 38 47  5 38 83 70 37 90 2 0
+42 21 62 27 43 47 82 80 88 49 4 0
+68 68 67 12 38 13 32 30 93 27 3 1
+ 5 44 98 28  5 81 20 56 10 34 9 1
+40 46 11 33 73 62 68 70 66 85 4 0
+ 9 46 11 84  6 31 18 89 66 32 1 1
+ 6 78 44 98 77 29 69 39 62 78 1 0
+47 90 18  0  3  8 12 20 51 75 4 1
+21 29 74 19 12 29 41 22 63 47 8 1
+22 59 64 62 18 89 19 92 87  8 8 0
+ 6 21 24 58 14 53 18 93 62 15 8 0
+20 33 88 25 37 52  1 72 74 11 2 0
+90 49 28 53 28 80 22 81  0 46 9 0
+87 31 51 27 15 31 68 93  5  4 7 1
+21 72 60  2 24 79 22 24 77 61 9 0
+20  4  6 40 28 14 16 78 58 99 7 1
+80 35 98 20 91 35 47 29  3 19 2 1
+57 21 24 61 60 39 83 34 53  2 2 0
+74 86 78 78 18 44 20 94 85 71 4 1
+27 48 44 92 10 18 74 54 25 85 2 0
+74 77 28 75 74 91 69 36 95 68 7 0
+32 84 17 18 55 79 59 57 21 69 2 1
+69 77 40 98 83 40  4 66 39 83 1 1
+63 24 32 39 75 92 81 49  2 51 5 1
+35 40 84 71  3 16 82 91 44 52 8 0
+21 78 66  4 57 27 21 89  4 34 7 1
+94 18 57 49 88 26 29 76 56 67 6 0
+14 91 71 30  5 36 28 74 16 73 3 1
+93 36 43 46 77 44 59 19 56 84 3 0
+11 16  2 67 11 96 20 91 20 59 2 1
+72 79 26 99 90 71 56 46 35 99 3 0
+29 87 20 40 13 14 14 40 61 27 6 0
+41 64 28 51 56 52 87 67 37 91 6 1
+33 14  5 30 99 54 27 80 54 55 4 1
+60 44 73 91 71 53 54 95 59 81 6 0
+69 33 11 83  4 53 34 39 43 84 1 0
+73 31 19  4 50 20 66 73 94 88 4 0
+30 49 41 76  5 21 88 69 76  3 2 0
+18 50 27 76 67 38 87 16 52 87 5 1
+33 36 80  8 43 82 89 76 37  3 5 0
+98 21 61 24 58 13  9 85 56 74 1 1
+84 27 50 96  9 56 30 31 85 65 1 1
+65 74 40  2  8 40 18 57 30 38 1 1
+76 44 64  6 10 32 84 70 74 24 1 1
+14 29 59 34 27  8  0 37 27 68 3 0
+ 6 47  5 77 15 41 93 49 59 83 4 1
+39 88 43 89 32 98 82  0  5 12 9 0
+78 79 30 26 58  6  9 58 37 65 8 1
+25 28 66 41 70 87 76 62 29 39 7 1