diff --git a/__init__.py b/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..abf198b97e6e818e1fbe59006f98492640bcee54 100755 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/metrics/auc_metrics.py b/metrics/auc_metrics.py index 788c3567ce5bcfbf5618a5349ae4c348b3ecb7bb..4c937bdef2599402de5228208e679d120346e8d1 100644 --- a/metrics/auc_metrics.py +++ b/metrics/auc_metrics.py @@ -1,3 +1,16 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import math import numpy as np diff --git a/metrics/base.py b/metrics/base.py index 251422f7fd1ad91469de59bc54d8f56e9ed81029..469fd56c8cb97c8e21739b402b4b89daab57bde9 100644 --- a/metrics/base.py +++ b/metrics/base.py @@ -1,6 +1,17 @@ -""" -Do metric jobs. calculate AUC, MSE, COCP ... -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc diff --git a/models/__init__.py b/models/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..abf198b97e6e818e1fbe59006f98492640bcee54 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/models/base.py b/models/base.py index d651567d811675a0a692168f477db6570509df1d..47b48c79b72d6d49310a07c6a9c9c6767b4e6dd5 100644 --- a/models/base.py +++ b/models/base.py @@ -1,6 +1,17 @@ -""" -Model Net: analyse layer config, and parse to Paddle Pragram -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import copy import yaml diff --git a/models/ctr_dnn/model.py b/models/ctr_dnn/model.py index 31018026370d5eb1ce8a26290f05bb8b60d44257..35d1a52a99743c2e4128849fad488d0c26f8d656 100644 --- a/models/ctr_dnn/model.py +++ b/models/ctr_dnn/model.py @@ -1,3 +1,17 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import paddle.fluid as fluid diff --git a/models/ctr_dnn/reader.py b/models/ctr_dnn/reader.py index 93f5f9cfdd2993b83c5d90eb49febc9d2b66eb18..7806fc4604d60b09c1a3786c31dc2dfaf6d64621 100644 --- a/models/ctr_dnn/reader.py +++ b/models/ctr_dnn/reader.py @@ -1,3 +1,16 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from ...utils import envs diff --git a/models/layers.py b/models/layers.py index b214cc647eda168ab77b7f4c2330b085b5bc8b35..6f79e64dec1df2c0e6cc4d70ac0aceda682708d4 100644 --- a/models/layers.py +++ b/models/layers.py @@ -1,3 +1,17 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle.fluid as fluid from .base import Layer diff --git a/reader/__init__.py b/reader/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..abf198b97e6e818e1fbe59006f98492640bcee54 100644 --- a/reader/__init__.py +++ b/reader/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/reader/data_loader.py b/reader/data_loader.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..abf198b97e6e818e1fbe59006f98492640bcee54 100644 --- a/reader/data_loader.py +++ b/reader/data_loader.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/reader/dataset.py b/reader/dataset.py index c417148893d7facc50dd5a0ee9aa87d1178a56c2..66450aed28222606cb4aa00a80d64354252d5572 100755 --- a/reader/dataset.py +++ b/reader/dataset.py @@ -1,6 +1,17 @@ -""" -Define Dataset -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import copy import yaml diff --git a/trainer/cluster_training_local.py b/trainer/cluster_training_local.py new file mode 100644 index 0000000000000000000000000000000000000000..f90d7b181da33274f4075a693571c9dcad0abefa --- /dev/null +++ b/trainer/cluster_training_local.py @@ -0,0 +1,77 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import os +import time +import numpy as np +import logging +import paddle.fluid as fluid +from network import CTR +from argument import params_args + +logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger("fluid") +logger.setLevel(logging.INFO) + + +def get_dataset(inputs, params): + dataset = fluid.DatasetFactory().create_dataset() + dataset.set_use_var(inputs) + dataset.set_pipe_command("python dataset_generator.py") + dataset.set_batch_size(params.batch_size) + dataset.set_thread(int(params.cpu_num)) + file_list = [ + str(params.train_files_path) + "/%s" % x + for x in os.listdir(params.train_files_path) + ] + dataset.set_filelist(file_list) + logger.info("file list: {}".format(file_list)) + return dataset + + +def train(params): + ctr_model = CTR() + inputs = ctr_model.input_data(params) + avg_cost, auc_var, batch_auc_var = ctr_model.net(inputs, params) + optimizer = fluid.optimizer.Adam(params.learning_rate) + optimizer.minimize(avg_cost) + fluid.default_main_program() + exe = fluid.Executor(fluid.CPUPlace()) + exe.run(fluid.default_startup_program()) + dataset = get_dataset(inputs, params) + + logger.info("Training Begin") + for epoch in range(params.epochs): + start_time = time.time() + exe.train_from_dataset(program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[auc_var], + fetch_info=["Epoch {} auc ".format(epoch)], + print_period=100, + debug=False) + end_time = time.time() + logger.info("epoch %d finished, use time=%d\n" % + ((epoch), end_time - start_time)) + + if params.test: + model_path = (str(params.model_path) + "/" + "epoch_" + str(epoch)) + fluid.io.save_persistables(executor=exe, dirname=model_path) + + logger.info("Train Success!") + + +if __name__ == "__main__": + params = params_args() + train(params) \ No newline at end of file diff --git a/trainer/cluster_training_offline.py b/trainer/cluster_training_offline.py new file mode 100644 index 0000000000000000000000000000000000000000..abf198b97e6e818e1fbe59006f98492640bcee54 --- /dev/null +++ b/trainer/cluster_training_offline.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/trainer/ctr_trainer.py b/trainer/ctr_trainer.py index 62444dbc48350554d3e7d77f301b95bda718521f..2448ff4fae27dd3603debde68e580b7e6b2504ac 100755 --- a/trainer/ctr_trainer.py +++ b/trainer/ctr_trainer.py @@ -1,6 +1,17 @@ -""" -A paddle trainer Adapt to Ctr -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import sys import copy diff --git a/trainer/trainer.py b/trainer/trainer.py index 331cf3481536cd01a53f5abd914a567be5922f60..e56ed0d755596696781f00a846d0b562a0742c03 100755 --- a/trainer/trainer.py +++ b/trainer/trainer.py @@ -1,6 +1,17 @@ -""" -Define A Trainer Base -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc import time diff --git a/utils/__init__.py b/utils/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..abf198b97e6e818e1fbe59006f98492640bcee54 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/utils/envs.py b/utils/envs.py index e68683e2ea87942aa6fda4c78754c8180825c2e2..51e5b641075310a8992aae8cb34682f2ba79f0a6 100644 --- a/utils/envs.py +++ b/utils/envs.py @@ -1,3 +1,18 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import os diff --git a/utils/fs.py b/utils/fs.py index f8d6bb98d36ff036001b6bdeba81d7406e9313e0..b4f129a92777387fd12c2f265d0534f83dfc2062 100755 --- a/utils/fs.py +++ b/utils/fs.py @@ -1,6 +1,17 @@ -""" -util for file_system io -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import time from paddle.fluid.incubate.fleet.utils.hdfs import HDFSClient diff --git a/utils/table.py b/utils/table.py index 6d3ccb3ae85a45e9f538d7055921bc8dcda0ad6a..7c86d0f03edb47a7835ab7364ebe318985d091e2 100755 --- a/utils/table.py +++ b/utils/table.py @@ -1,6 +1,17 @@ -""" -Construct ParamTable Meta -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy import yaml diff --git a/utils/util.py b/utils/util.py index 7f0e3ca858b0d758207adba26b358a6823e30bca..950aa59c4786328c90a711baebab752b51b78b17 100755 --- a/utils/util.py +++ b/utils/util.py @@ -1,6 +1,17 @@ -""" -Util lib -""" +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import time import datetime