diff --git a/demo/user_define_trainer.py b/demo/user_define_trainer.py index 068fd6907b348702c2dafc2354c6d984fa3ec70e..4765ed36f79bd66ec0991c873f532d758e13083a 100644 --- a/demo/user_define_trainer.py +++ b/demo/user_define_trainer.py @@ -14,8 +14,8 @@ import paddle.fluid as fluid -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer -from fleetrec.core.utils import envs +from fleet_rec.core.trainers.transpiler_trainer import TranspileTrainer +from fleet_rec.core.utils import envs class UserDefineTrainer(TranspileTrainer): diff --git a/fleetrec/__init__.py b/fleet_rec/__init__.py similarity index 100% rename from fleetrec/__init__.py rename to fleet_rec/__init__.py diff --git a/fleetrec/check.py b/fleet_rec/check.py similarity index 100% rename from fleetrec/check.py rename to fleet_rec/check.py diff --git a/fleetrec/core/__init__.py b/fleet_rec/core/__init__.py similarity index 100% rename from fleetrec/core/__init__.py rename to fleet_rec/core/__init__.py diff --git a/fleetrec/core/engine/__init__.py b/fleet_rec/core/engine/__init__.py similarity index 100% rename from fleetrec/core/engine/__init__.py rename to fleet_rec/core/engine/__init__.py diff --git a/fleetrec/core/engine/engine.py b/fleet_rec/core/engine/engine.py similarity index 100% rename from fleetrec/core/engine/engine.py rename to fleet_rec/core/engine/engine.py diff --git a/fleetrec/core/engine/local_cluster_engine.py b/fleet_rec/core/engine/local_cluster_engine.py similarity index 98% rename from fleetrec/core/engine/local_cluster_engine.py rename to fleet_rec/core/engine/local_cluster_engine.py index e6e108d13bf917bda8eccdd72fa63e3abd4e37f5..643cb75716cac556920a9532456be411b175d821 100644 --- a/fleetrec/core/engine/local_cluster_engine.py +++ b/fleet_rec/core/engine/local_cluster_engine.py @@ -19,7 +19,7 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine +from fleet_rec.core.engine.engine import Engine class LocalClusterEngine(Engine): diff --git a/fleetrec/core/engine/local_mpi_engine.py b/fleet_rec/core/engine/local_mpi_engine.py similarity index 97% rename from fleetrec/core/engine/local_mpi_engine.py rename to fleet_rec/core/engine/local_mpi_engine.py index 84ed6c307ab270b17f8307fd8252fbef3b197474..9d68f478090b0f4e7595408a2f3c64bcd5adf87e 100644 --- a/fleetrec/core/engine/local_mpi_engine.py +++ b/fleet_rec/core/engine/local_mpi_engine.py @@ -19,7 +19,7 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine +from fleet_rec.core.engine.engine import Engine class LocalMPIEngine(Engine): diff --git a/fleetrec/core/factory.py b/fleet_rec/core/factory.py similarity index 98% rename from fleetrec/core/factory.py rename to fleet_rec/core/factory.py index 631bb1f6bfbc9603fd796cdba12f573050994f54..7ecaa2dd8f9f651d24746ddeca818fdd2a33dd97 100644 --- a/fleetrec/core/factory.py +++ b/fleet_rec/core/factory.py @@ -17,7 +17,7 @@ import sys import yaml -from fleetrec.core.utils import envs +from fleet_rec.core.utils import envs trainer_abs = os.path.join(os.path.dirname(os.path.abspath(__file__)), "trainers") trainers = {} diff --git a/fleetrec/core/layer.py b/fleet_rec/core/layer.py similarity index 100% rename from fleetrec/core/layer.py rename to fleet_rec/core/layer.py diff --git a/fleetrec/core/metric.py b/fleet_rec/core/metric.py similarity index 100% rename from fleetrec/core/metric.py rename to fleet_rec/core/metric.py diff --git a/fleetrec/core/metrics/__init__.py b/fleet_rec/core/metrics/__init__.py similarity index 100% rename from fleetrec/core/metrics/__init__.py rename to fleet_rec/core/metrics/__init__.py diff --git a/fleetrec/core/metrics/auc_metrics.py b/fleet_rec/core/metrics/auc_metrics.py similarity index 99% rename from fleetrec/core/metrics/auc_metrics.py rename to fleet_rec/core/metrics/auc_metrics.py index f2f68890c79ef35123c131e9064b99e28d8e0aea..46fdf594e21c8651f2f1b82716e837c8ab4b252c 100644 --- a/fleetrec/core/metrics/auc_metrics.py +++ b/fleet_rec/core/metrics/auc_metrics.py @@ -15,7 +15,7 @@ import math import numpy as np import paddle.fluid as fluid -from fleetrec.core.metric import Metric +from fleet_rec.core.metric import Metric class AUCMetric(Metric): diff --git a/fleetrec/core/model.py b/fleet_rec/core/model.py similarity index 95% rename from fleetrec/core/model.py rename to fleet_rec/core/model.py index 9f940b6c0a204a0d1270f9ea46933583dc6307e0..dbd1e5c3624ab312b6605a5299f2927bde6074c9 100644 --- a/fleetrec/core/model.py +++ b/fleet_rec/core/model.py @@ -1,5 +1,5 @@ import abc -from fleetrec.core.utils import envs +from fleet_rec.core.utils import envs class Model(object): """R diff --git a/fleetrec/core/modules/__init__.py b/fleet_rec/core/modules/__init__.py similarity index 100% rename from fleetrec/core/modules/__init__.py rename to fleet_rec/core/modules/__init__.py diff --git a/fleetrec/core/modules/coding/__init__.py b/fleet_rec/core/modules/coding/__init__.py similarity index 100% rename from fleetrec/core/modules/coding/__init__.py rename to fleet_rec/core/modules/coding/__init__.py diff --git a/fleetrec/core/modules/coding/layers.py b/fleet_rec/core/modules/coding/layers.py similarity index 100% rename from fleetrec/core/modules/coding/layers.py rename to fleet_rec/core/modules/coding/layers.py diff --git a/fleetrec/core/modules/modul/__init__.py b/fleet_rec/core/modules/modul/__init__.py similarity index 100% rename from fleetrec/core/modules/modul/__init__.py rename to fleet_rec/core/modules/modul/__init__.py diff --git a/fleetrec/core/modules/modul/build.py b/fleet_rec/core/modules/modul/build.py similarity index 99% rename from fleetrec/core/modules/modul/build.py rename to fleet_rec/core/modules/modul/build.py index 51b83e5f3ddc1b51ad3c990701cd38625772629e..7fad447170bdcb45414b4c6d0675377e47ced84e 100644 --- a/fleetrec/core/modules/modul/build.py +++ b/fleet_rec/core/modules/modul/build.py @@ -3,8 +3,8 @@ import copy import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet -from fleetrec.core.model import Model -from fleetrec.core.utils import table +from fleet_rec.core.model import Model +from fleet_rec.core.utils import table def create(config): diff --git a/fleetrec/core/modules/modul/layers.py b/fleet_rec/core/modules/modul/layers.py similarity index 99% rename from fleetrec/core/modules/modul/layers.py rename to fleet_rec/core/modules/modul/layers.py index 84a08ee808377175a7b378a95f06181e7fca5173..46f1667f8a1536beff578f33b7633905f5041db3 100644 --- a/fleetrec/core/modules/modul/layers.py +++ b/fleet_rec/core/modules/modul/layers.py @@ -13,7 +13,7 @@ # limitations under the License. import paddle.fluid as fluid -from fleetrec.core.layer import Layer +from fleet_rec.core.layer import Layer class EmbeddingInputLayer(Layer): diff --git a/fleetrec/core/reader.py b/fleet_rec/core/reader.py similarity index 97% rename from fleetrec/core/reader.py rename to fleet_rec/core/reader.py index 3aa96d2ccf9a14a3c7910dc4d82b8e6ca717f18c..aeba3afd930eb3219517e032956da3711d882d52 100644 --- a/fleetrec/core/reader.py +++ b/fleet_rec/core/reader.py @@ -19,7 +19,7 @@ import os import paddle.fluid.incubate.data_generator as dg import yaml -from fleetrec.core.utils import envs +from fleet_rec.core.utils import envs class Reader(dg.MultiSlotDataGenerator): diff --git a/fleetrec/core/trainer.py b/fleet_rec/core/trainer.py similarity index 98% rename from fleetrec/core/trainer.py rename to fleet_rec/core/trainer.py index 06b5800ee49fdf51be51d9ca1e9e304c46d174cd..baf0249b34b23df0121d2b396305c10e8c3c740a 100755 --- a/fleetrec/core/trainer.py +++ b/fleet_rec/core/trainer.py @@ -20,7 +20,7 @@ import time import yaml from paddle import fluid -from fleetrec.core.utils import envs +from fleet_rec.core.utils import envs class Trainer(object): diff --git a/fleetrec/core/trainers/__init__.py b/fleet_rec/core/trainers/__init__.py similarity index 100% rename from fleetrec/core/trainers/__init__.py rename to fleet_rec/core/trainers/__init__.py diff --git a/fleetrec/core/trainers/cluster_trainer.py b/fleet_rec/core/trainers/cluster_trainer.py similarity index 97% rename from fleetrec/core/trainers/cluster_trainer.py rename to fleet_rec/core/trainers/cluster_trainer.py index 4a204bcee87cb5396380073b26ec2e14a7691540..439cacfc4c8fa4c42928992858b1460aa96d27ba 100644 --- a/fleetrec/core/trainers/cluster_trainer.py +++ b/fleet_rec/core/trainers/cluster_trainer.py @@ -23,8 +23,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import f from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer +from fleet_rec.core.utils import envs +from fleet_rec.core.trainers.transpiler_trainer import TranspileTrainer class ClusterTrainer(TranspileTrainer): diff --git a/fleetrec/core/trainers/ctr_coding_trainer.py b/fleet_rec/core/trainers/ctr_coding_trainer.py similarity index 98% rename from fleetrec/core/trainers/ctr_coding_trainer.py rename to fleet_rec/core/trainers/ctr_coding_trainer.py index ae6b63ceb94e87f006fb5a8b342bf298954bf3d2..6436aedc41a2664ecbfaf3e610a20f0ff319765c 100755 --- a/fleetrec/core/trainers/ctr_coding_trainer.py +++ b/fleet_rec/core/trainers/ctr_coding_trainer.py @@ -18,8 +18,8 @@ import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainer import Trainer +from fleet_rec.core.utils import envs +from fleet_rec.core.trainer import Trainer class CtrPaddleTrainer(Trainer): diff --git a/fleetrec/core/trainers/ctr_modul_trainer.py b/fleet_rec/core/trainers/ctr_modul_trainer.py similarity index 98% rename from fleetrec/core/trainers/ctr_modul_trainer.py rename to fleet_rec/core/trainers/ctr_modul_trainer.py index e72715e74634791c2c90890bc6b8b8f0a0e56c71..0d80a938346c3225fe7738261e1f6c0a32ec3b80 100755 --- a/fleetrec/core/trainers/ctr_modul_trainer.py +++ b/fleet_rec/core/trainers/ctr_modul_trainer.py @@ -24,12 +24,12 @@ from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker -from fleetrec.core.utils import fs as fs -from fleetrec.core.utils import util as util -from fleetrec.core.metrics.auc_metrics import AUCMetric -from fleetrec.core.modules.modul import build as model_basic -from fleetrec.core.utils import dataset -from fleetrec.core.trainer import Trainer +from fleet_rec.core.utils import fs as fs +from fleet_rec.core.utils import util as util +from fleet_rec.core.metrics.auc_metrics import AUCMetric +from fleet_rec.core.modules.modul import build as model_basic +from fleet_rec.core.utils import dataset +from fleet_rec.core.trainer import Trainer def wroker_numric_opt(value, env, opt): diff --git a/fleetrec/core/trainers/single_trainer.py b/fleet_rec/core/trainers/single_trainer.py similarity index 97% rename from fleetrec/core/trainers/single_trainer.py rename to fleet_rec/core/trainers/single_trainer.py index 989297a04f6d031588ec58d606f9e74a0d5e4556..c8eabdf9380705d9cfd79268cffff76b9e6f5e39 100644 --- a/fleetrec/core/trainers/single_trainer.py +++ b/fleet_rec/core/trainers/single_trainer.py @@ -20,8 +20,8 @@ from __future__ import print_function import logging import paddle.fluid as fluid -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer -from fleetrec.core.utils import envs +from fleet_rec.core.trainers.transpiler_trainer import TranspileTrainer +from fleet_rec.core.utils import envs import numpy as np logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/fleetrec/core/trainers/transpiler_trainer.py b/fleet_rec/core/trainers/transpiler_trainer.py similarity index 97% rename from fleetrec/core/trainers/transpiler_trainer.py rename to fleet_rec/core/trainers/transpiler_trainer.py index 7589286dad8be5481c27123a931615adf57f6efa..f93bcc691d91d16b788c601e2ba3cffa95707b3d 100644 --- a/fleetrec/core/trainers/transpiler_trainer.py +++ b/fleet_rec/core/trainers/transpiler_trainer.py @@ -20,9 +20,9 @@ import os import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from fleetrec.core.trainer import Trainer -from fleetrec.core.utils import envs -from fleetrec.core.utils import dataloader_instance +from fleet_rec.core.trainer import Trainer +from fleet_rec.core.utils import envs +from fleet_rec.core.utils import dataloader_instance class TranspileTrainer(Trainer): diff --git a/fleetrec/core/utils/__init__.py b/fleet_rec/core/utils/__init__.py similarity index 100% rename from fleetrec/core/utils/__init__.py rename to fleet_rec/core/utils/__init__.py diff --git a/fleetrec/core/utils/dataloader_instance.py b/fleet_rec/core/utils/dataloader_instance.py similarity index 94% rename from fleetrec/core/utils/dataloader_instance.py rename to fleet_rec/core/utils/dataloader_instance.py index 8c43c610a29394b3b334d755bc95d21fb0f514ce..ce87601224c1d93d93700030e256d4e6a9ad18d6 100644 --- a/fleetrec/core/utils/dataloader_instance.py +++ b/fleet_rec/core/utils/dataloader_instance.py @@ -16,8 +16,8 @@ from __future__ import print_function import os import sys -from fleetrec.core.utils.envs import lazy_instance -from fleetrec.core.utils.envs import get_global_env +from fleet_rec.core.utils.envs import lazy_instance +from fleet_rec.core.utils.envs import get_global_env def dataloader(readerclass, train, yaml_file): diff --git a/fleetrec/core/utils/dataset.py b/fleet_rec/core/utils/dataset.py similarity index 98% rename from fleetrec/core/utils/dataset.py rename to fleet_rec/core/utils/dataset.py index 92cbe8911a327f07d667e0e9f6d35bcb18140914..a979b364f42a7f6bd9e8dafdb1dbf2d14579f725 100755 --- a/fleetrec/core/utils/dataset.py +++ b/fleet_rec/core/utils/dataset.py @@ -18,8 +18,8 @@ import datetime import paddle.fluid as fluid -from fleetrec.core.utils import fs as fs -from fleetrec.core.utils import util as util +from fleet_rec.core.utils import fs as fs +from fleet_rec.core.utils import util as util class Dataset(object): diff --git a/fleetrec/core/utils/dataset_instance.py b/fleet_rec/core/utils/dataset_instance.py similarity index 95% rename from fleetrec/core/utils/dataset_instance.py rename to fleet_rec/core/utils/dataset_instance.py index 91b6af94e122c882bc7bdbbed48a6f8475f02b50..bb72f7722aedae8280377bf23e8088fba3e467eb 100644 --- a/fleetrec/core/utils/dataset_instance.py +++ b/fleet_rec/core/utils/dataset_instance.py @@ -14,7 +14,7 @@ from __future__ import print_function import sys -from fleetrec.core.utils.envs import lazy_instance +from fleet_rec.core.utils.envs import lazy_instance if len(sys.argv) != 4: raise ValueError("reader only accept 3 argument: 1. reader_class 2.train/evaluate 3.yaml_abs_path") diff --git a/fleetrec/core/utils/envs.py b/fleet_rec/core/utils/envs.py similarity index 100% rename from fleetrec/core/utils/envs.py rename to fleet_rec/core/utils/envs.py diff --git a/fleetrec/core/utils/fs.py b/fleet_rec/core/utils/fs.py similarity index 100% rename from fleetrec/core/utils/fs.py rename to fleet_rec/core/utils/fs.py diff --git a/fleetrec/core/utils/table.py b/fleet_rec/core/utils/table.py similarity index 100% rename from fleetrec/core/utils/table.py rename to fleet_rec/core/utils/table.py diff --git a/fleetrec/core/utils/util.py b/fleet_rec/core/utils/util.py similarity index 99% rename from fleetrec/core/utils/util.py rename to fleet_rec/core/utils/util.py index b410df1d8bb31fe7e613925e2a731d61924d8835..041793ef754fb67d30713e4ff29564d44e8eac76 100755 --- a/fleetrec/core/utils/util.py +++ b/fleet_rec/core/utils/util.py @@ -17,7 +17,7 @@ import time import datetime from paddle import fluid -from fleetrec.core.utils import fs as fs +from fleet_rec.core.utils import fs as fs def save_program_proto(path, program=None): diff --git a/fleetrec/run.py b/fleet_rec/run.py similarity index 95% rename from fleetrec/run.py rename to fleet_rec/run.py index a2fe242871b2c7f2ec2e107c4bf8e51c1598c19c..469750f60a80ee9a38cf247eb085be08e74a861e 100644 --- a/fleetrec/run.py +++ b/fleet_rec/run.py @@ -3,9 +3,9 @@ import os import subprocess import yaml -from fleetrec.core.factory import TrainerFactory -from fleetrec.core.utils import envs -from fleetrec.core.utils import util +from fleet_rec.core.factory import TrainerFactory +from fleet_rec.core.utils import envs +from fleet_rec.core.utils import util engines = {} device = ["CPU", "GPU"] @@ -123,7 +123,7 @@ def cluster_mpi_engine(args): def local_cluster_engine(args): print("launch cluster engine with cluster to run model: {}".format(args.model)) - from fleetrec.core.engine.local_cluster_engine import LocalClusterEngine + from fleet_rec.core.engine.local_cluster_engine import LocalClusterEngine cluster_envs = {} cluster_envs["server_num"] = 1 @@ -148,7 +148,7 @@ def local_cluster_engine(args): def local_mpi_engine(args): print("launch cluster engine with cluster to run model: {}".format(args.model)) - from fleetrec.core.engine.local_mpi_engine import LocalMPIEngine + from fleet_rec.core.engine.local_mpi_engine import LocalMPIEngine print("use 1X1 MPI ClusterTraining at localhost to run model: {}".format(args.model)) diff --git a/fleetrec/tests/__init__.py b/fleet_rec/tests/__init__.py similarity index 100% rename from fleetrec/tests/__init__.py rename to fleet_rec/tests/__init__.py diff --git a/models/rank/criteo_reader.py b/models/rank/criteo_reader.py index c4930c8ddc628c13f5c958743673bd42c507161f..88cc3b088fc452469d887627318bf7b0d2acb6c9 100644 --- a/models/rank/criteo_reader.py +++ b/models/rank/criteo_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from fleet_rec.core.reader import Reader +from fleet_rec.core.utils import envs class TrainReader(Reader): diff --git a/models/rank/dnn/model.py b/models/rank/dnn/model.py index a4e6cf6868ccac57e457c9c58cab2b8ee2310d94..d6a1fbae7dc5f891ce5e24d63faacc3ab5be6b49 100644 --- a/models/rank/dnn/model.py +++ b/models/rank/dnn/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from fleet_rec.core.utils import envs +from fleet_rec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/setup.py b/setup.py index cb7327df400be587b999f9041f9b3b4d0266d717..7a4ac3fa42a7ebf306b97d43c8385d92671d4cf7 100644 --- a/setup.py +++ b/setup.py @@ -10,13 +10,13 @@ import shutil if sys.version_info.major == 2: requires = [ "paddlepaddle == 1.7.2", -# "netron >= 0.0.0", + # "netron >= 0.0.0", "pyyaml >= 5.1.1" ] else: requires = [ "paddlepaddle >= 0.0.0", -# "netron >= 0.0.0", + # "netron >= 0.0.0", "pyyaml >= 5.1.1" ] @@ -40,12 +40,14 @@ def build(dirname): print(dirname) package_dir = os.path.dirname(os.path.abspath(__file__)) run_cmd("cp -r {}/* {}".format(package_dir, dirname)) + run_cmd("mkdir {}".format(os.path.join(dirname, "fleetrec"))) + run_cmd("mv {}/* {}".format(os.path.join(dirname, "fleet_rec"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "demo"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "doc"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "models"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "tools"), os.path.join(dirname, "fleetrec"))) - packages = find_packages(dirname) + packages = find_packages(dirname, include=('fleetrec.*')) package_dir = {'': dirname} setup(