From d14987ed94f3cc111002258804790cd0474606cf Mon Sep 17 00:00:00 2001 From: tangwei Date: Thu, 9 Apr 2020 23:16:56 +0800 Subject: [PATCH] update to fleet_rec --- examples/ctr-dnn_train_cluster.yaml | 6 +++--- examples/ctr-dnn_train_single.yaml | 6 +++--- examples/train.py | 2 +- models/ctr_dnn/model.py | 2 +- trainer/cluster_trainer.py | 11 ++++------- trainer/ctr_trainer.py | 10 +++++----- trainer/factory.py | 16 ++++++++-------- trainer/local_engine.py | 2 +- utils/envs.py | 2 +- 9 files changed, 27 insertions(+), 30 deletions(-) diff --git a/examples/ctr-dnn_train_cluster.yaml b/examples/ctr-dnn_train_cluster.yaml index 7239e582..d24bf882 100644 --- a/examples/ctr-dnn_train_cluster.yaml +++ b/examples/ctr-dnn_train_cluster.yaml @@ -41,11 +41,11 @@ train: reader: mode: "dataset" batch_size: 2 - pipe_command: "python /paddle/eleps/models/ctr_dnn/dataset.py" - train_data_path: "/paddle/eleps/models/ctr_dnn/data/train" + pipe_command: "python /paddle/fleet_rec/models/ctr_dnn/dataset.py" + train_data_path: "/paddle/fleet_rec/models/ctr_dnn/data/train" model: - models: "eleps.models.ctr_dnn.model" + models: "fleet_rec.models.ctr_dnn.model" hyper_parameters: sparse_inputs_slots: 27 sparse_feature_number: 1000001 diff --git a/examples/ctr-dnn_train_single.yaml b/examples/ctr-dnn_train_single.yaml index 2009714b..1dc61903 100644 --- a/examples/ctr-dnn_train_single.yaml +++ b/examples/ctr-dnn_train_single.yaml @@ -35,11 +35,11 @@ train: reader: mode: "dataset" batch_size: 2 - pipe_command: "python /paddle/eleps/models/ctr_dnn/dataset.py" - train_data_path: "/paddle/eleps/models/ctr_dnn/data/train" + pipe_command: "python /paddle/fleet_rec/models/ctr_dnn/dataset.py" + train_data_path: "/paddle/fleet_rec/models/ctr_dnn/data/train" model: - models: "eleps.models.ctr_dnn.model" + models: "fleet_rec.models.ctr_dnn.model" hyper_parameters: sparse_inputs_slots: 27 sparse_feature_number: 1000001 diff --git a/examples/train.py b/examples/train.py index ac7fdd30..4d84b871 100644 --- a/examples/train.py +++ b/examples/train.py @@ -26,7 +26,7 @@ import os -from eleps.trainer.factory import TrainerFactory +from fleet_rec.trainer.factory import TrainerFactory if __name__ == "__main__": diff --git a/models/ctr_dnn/model.py b/models/ctr_dnn/model.py index e62ef34c..d144cc06 100644 --- a/models/ctr_dnn/model.py +++ b/models/ctr_dnn/model.py @@ -15,7 +15,7 @@ import math import paddle.fluid as fluid -from eleps.utils import envs +from fleet_rec.utils import envs class Train(object): diff --git a/trainer/cluster_trainer.py b/trainer/cluster_trainer.py index 65f9efa3..0ecc1eb6 100644 --- a/trainer/cluster_trainer.py +++ b/trainer/cluster_trainer.py @@ -17,23 +17,20 @@ Training use fluid with one node only. """ from __future__ import print_function -import os import logging -import paddle.fluid as fluid - -from .trainer import Trainer -from ..utils import envs +import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker +from ..utils import envs +from .transpiler_trainer import TranspileTrainer + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger("fluid") logger.setLevel(logging.INFO) -from .transpiler_trainer import TranspileTrainer - class ClusterTrainerWithDataloader(TranspileTrainer): pass diff --git a/trainer/ctr_trainer.py b/trainer/ctr_trainer.py index bc885128..b58200c9 100755 --- a/trainer/ctr_trainer.py +++ b/trainer/ctr_trainer.py @@ -23,11 +23,11 @@ import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker -from eleps.utils import fs as fs -from eleps.utils import util as util -from eleps.metrics.auc_metrics import AUCMetric -from eleps.models import base as model_basic -from eleps.reader import dataset +from fleet_rec.utils import fs as fs +from fleet_rec.utils import util as util +from fleet_rec.metrics.auc_metrics import AUCMetric +from fleet_rec.models import base as model_basic +from fleet_rec.reader import dataset from .trainer import Trainer diff --git a/trainer/factory.py b/trainer/factory.py index dc590ce5..3301c13c 100644 --- a/trainer/factory.py +++ b/trainer/factory.py @@ -29,16 +29,16 @@ import sys import yaml -from eleps.trainer.single_trainer import SingleTrainerWithDataloader -from eleps.trainer.single_trainer import SingleTrainerWithDataset +from fleet_rec.trainer.single_trainer import SingleTrainerWithDataloader +from fleet_rec.trainer.single_trainer import SingleTrainerWithDataset -from eleps.trainer.cluster_trainer import ClusterTrainerWithDataloader -from eleps.trainer.cluster_trainer import ClusterTrainerWithDataset +from fleet_rec.trainer.cluster_trainer import ClusterTrainerWithDataloader +from fleet_rec.trainer.cluster_trainer import ClusterTrainerWithDataset -from eleps.trainer.local_engine import Launch -from eleps.trainer.ctr_trainer import CtrPaddleTrainer +from fleet_rec.trainer.local_engine import Launch +from fleet_rec.trainer.ctr_trainer import CtrPaddleTrainer -from eleps.utils import envs +from fleet_rec.utils import envs def str2bool(v): @@ -103,7 +103,7 @@ class TrainerFactory(object): with open(config, 'r') as rb: _config = yaml.load(rb.read(), Loader=yaml.FullLoader) else: - raise ValueError("eleps's config only support yaml") + raise ValueError("fleet_rec's config only support yaml") envs.set_global_envs(_config) mode = envs.get_global_env("train.trainer") diff --git a/trainer/local_engine.py b/trainer/local_engine.py index 07be9ac4..fb883549 100644 --- a/trainer/local_engine.py +++ b/trainer/local_engine.py @@ -38,7 +38,7 @@ def start_procs(args, yaml): user_endpoints_ips = [x.split(":")[0] for x in user_endpoints.split(",")] user_endpoints_port = [x.split(":")[1] for x in user_endpoints.split(",")] - factory = "eleps.trainer.factory" + factory = "fleet_rec.trainer.factory" cmd = [sys.executable, "-u", "-m", factory, yaml] for i in range(server_num): diff --git a/utils/envs.py b/utils/envs.py index dce663b1..0c467c28 100644 --- a/utils/envs.py +++ b/utils/envs.py @@ -70,7 +70,7 @@ def pretty_print_envs(envs, header=None): if header: draws += h_format.format(header[0], header[1]) else: - draws += h_format.format("Eleps Global Envs", "Value") + draws += h_format.format("fleet_rec Global Envs", "Value") draws += line + "\n" -- GitLab