From 5590948a39889a8dab7b3e280b384349d669fd3e Mon Sep 17 00:00:00 2001 From: tangwei Date: Tue, 31 Mar 2020 15:35:15 +0800 Subject: [PATCH] add copyright --- __init__.py | 13 +++++ metrics/auc_metrics.py | 13 +++++ metrics/base.py | 17 +++++-- models/__init__.py | 13 +++++ models/base.py | 17 +++++-- models/ctr_dnn/model.py | 14 ++++++ models/ctr_dnn/reader.py | 13 +++++ models/layers.py | 14 ++++++ reader/__init__.py | 13 +++++ reader/data_loader.py | 13 +++++ reader/dataset.py | 17 +++++-- trainer/cluster_training_local.py | 77 +++++++++++++++++++++++++++++ trainer/cluster_training_offline.py | 13 +++++ trainer/ctr_trainer.py | 17 +++++-- trainer/trainer.py | 17 +++++-- utils/__init__.py | 13 +++++ utils/envs.py | 15 ++++++ utils/fs.py | 17 +++++-- utils/table.py | 17 +++++-- utils/util.py | 17 +++++-- 20 files changed, 336 insertions(+), 24 deletions(-) create mode 100644 trainer/cluster_training_local.py create mode 100644 trainer/cluster_training_offline.py diff --git a/__init__.py b/__init__.py index e69de29b..abf198b9 100755 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/metrics/auc_metrics.py b/metrics/auc_metrics.py index 788c3567..4c937bde 100644 --- a/metrics/auc_metrics.py +++ b/metrics/auc_metrics.py @@ -1,3 +1,16 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import math import numpy as np diff --git a/metrics/base.py b/metrics/base.py index 251422f7..469fd56c 100644 --- a/metrics/base.py +++ b/metrics/base.py @@ -1,6 +1,17 @@ -""" -Do metric jobs. calculate AUC, MSE, COCP ... -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc diff --git a/models/__init__.py b/models/__init__.py index e69de29b..abf198b9 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/base.py b/models/base.py index d651567d..47b48c79 100644 --- a/models/base.py +++ b/models/base.py @@ -1,6 +1,17 @@ -""" -Model Net: analyse layer config, and parse to Paddle Pragram -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import copy import yaml diff --git a/models/ctr_dnn/model.py b/models/ctr_dnn/model.py index 31018026..35d1a52a 100644 --- a/models/ctr_dnn/model.py +++ b/models/ctr_dnn/model.py @@ -1,3 +1,17 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import paddle.fluid as fluid diff --git a/models/ctr_dnn/reader.py b/models/ctr_dnn/reader.py index 93f5f9cf..7806fc46 100644 --- a/models/ctr_dnn/reader.py +++ b/models/ctr_dnn/reader.py @@ -1,3 +1,16 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from ...utils import envs diff --git a/models/layers.py b/models/layers.py index b214cc64..6f79e64d 100644 --- a/models/layers.py +++ b/models/layers.py @@ -1,3 +1,17 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.fluid as fluid from .base import Layer diff --git a/reader/__init__.py b/reader/__init__.py index e69de29b..abf198b9 100644 --- a/reader/__init__.py +++ b/reader/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/reader/data_loader.py b/reader/data_loader.py index e69de29b..abf198b9 100644 --- a/reader/data_loader.py +++ b/reader/data_loader.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/reader/dataset.py b/reader/dataset.py index c4171488..66450aed 100755 --- a/reader/dataset.py +++ b/reader/dataset.py @@ -1,6 +1,17 @@ -""" -Define Dataset -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import copy import yaml diff --git a/trainer/cluster_training_local.py b/trainer/cluster_training_local.py new file mode 100644 index 00000000..f90d7b18 --- /dev/null +++ b/trainer/cluster_training_local.py @@ -0,0 +1,77 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import os +import time +import numpy as np +import logging +import paddle.fluid as fluid +from network import CTR +from argument import params_args + +logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger("fluid") +logger.setLevel(logging.INFO) + + +def get_dataset(inputs, params): + dataset = fluid.DatasetFactory().create_dataset() + dataset.set_use_var(inputs) + dataset.set_pipe_command("python dataset_generator.py") + dataset.set_batch_size(params.batch_size) + dataset.set_thread(int(params.cpu_num)) + file_list = [ + str(params.train_files_path) + "/%s" % x + for x in os.listdir(params.train_files_path) + ] + dataset.set_filelist(file_list) + logger.info("file list: {}".format(file_list)) + return dataset + + +def train(params): + ctr_model = CTR() + inputs = ctr_model.input_data(params) + avg_cost, auc_var, batch_auc_var = ctr_model.net(inputs, params) + optimizer = fluid.optimizer.Adam(params.learning_rate) + optimizer.minimize(avg_cost) + fluid.default_main_program() + exe = fluid.Executor(fluid.CPUPlace()) + exe.run(fluid.default_startup_program()) + dataset = get_dataset(inputs, params) + + logger.info("Training Begin") + for epoch in range(params.epochs): + start_time = time.time() + exe.train_from_dataset(program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[auc_var], + fetch_info=["Epoch {} auc ".format(epoch)], + print_period=100, + debug=False) + end_time = time.time() + logger.info("epoch %d finished, use time=%d\n" % + ((epoch), end_time - start_time)) + + if params.test: + model_path = (str(params.model_path) + "/" + "epoch_" + str(epoch)) + fluid.io.save_persistables(executor=exe, dirname=model_path) + + logger.info("Train Success!") + + +if __name__ == "__main__": + params = params_args() + train(params) \ No newline at end of file diff --git a/trainer/cluster_training_offline.py b/trainer/cluster_training_offline.py new file mode 100644 index 00000000..abf198b9 --- /dev/null +++ b/trainer/cluster_training_offline.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/trainer/ctr_trainer.py b/trainer/ctr_trainer.py index 62444dbc..2448ff4f 100755 --- a/trainer/ctr_trainer.py +++ b/trainer/ctr_trainer.py @@ -1,6 +1,17 @@ -""" -A paddle trainer Adapt to Ctr -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import sys import copy diff --git a/trainer/trainer.py b/trainer/trainer.py index 331cf348..e56ed0d7 100755 --- a/trainer/trainer.py +++ b/trainer/trainer.py @@ -1,6 +1,17 @@ -""" -Define A Trainer Base -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import time diff --git a/utils/__init__.py b/utils/__init__.py index e69de29b..abf198b9 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/utils/envs.py b/utils/envs.py index e68683e2..51e5b641 100644 --- a/utils/envs.py +++ b/utils/envs.py @@ -1,3 +1,18 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import os diff --git a/utils/fs.py b/utils/fs.py index f8d6bb98..b4f129a9 100755 --- a/utils/fs.py +++ b/utils/fs.py @@ -1,6 +1,17 @@ -""" -util for file_system io -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import time from paddle.fluid.incubate.fleet.utils.hdfs import HDFSClient diff --git a/utils/table.py b/utils/table.py index 6d3ccb3a..7c86d0f0 100755 --- a/utils/table.py +++ b/utils/table.py @@ -1,6 +1,17 @@ -""" -Construct ParamTable Meta -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy import yaml diff --git a/utils/util.py b/utils/util.py index 7f0e3ca8..950aa59c 100755 --- a/utils/util.py +++ b/utils/util.py @@ -1,6 +1,17 @@ -""" -Util lib -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import time import datetime -- GitLab