From 06a44f67fedc82a7b56afa690afdab21941eb8ed Mon Sep 17 00:00:00 2001 From: tangwei Date: Wed, 13 May 2020 15:08:55 +0800 Subject: [PATCH] add cluster run, rename fleet-rec to paddle-rec --- .gitignore | 2 +- fleet_rec/__init__.py => __init__.py | 0 {fleet_rec/core => core}/__init__.py | 0 {fleet_rec/core => core}/engine/__init__.py | 0 .../core => core}/engine/cluster/__init__.py | 0 .../core => core}/engine/cluster/cluster.py | 6 ++-- .../core => core}/engine/cluster/master.sh | 0 {fleet_rec/core => core}/engine/engine.py | 0 .../core => core}/engine/local_cluster.py | 6 ++-- {fleet_rec/core => core}/engine/local_mpi.py | 4 +-- {fleet_rec/core => core}/factory.py | 4 +-- {fleet_rec/core => core}/layer.py | 0 {fleet_rec/core => core}/metric.py | 0 {fleet_rec/core => core}/metrics/__init__.py | 0 .../core => core}/metrics/auc_metrics.py | 2 +- {fleet_rec/core => core}/model.py | 2 +- {fleet_rec/core => core}/modules/__init__.py | 0 .../core => core}/modules/coding/__init__.py | 0 .../core => core}/modules/coding/layers.py | 0 .../core => core}/modules/modul/__init__.py | 0 .../core => core}/modules/modul/build.py | 4 +-- .../core => core}/modules/modul/layers.py | 2 +- {fleet_rec/core => core}/reader.py | 2 +- {fleet_rec/core => core}/trainer.py | 2 +- {fleet_rec/core => core}/trainers/__init__.py | 0 .../core => core}/trainers/cluster_trainer.py | 4 +-- .../trainers/ctr_coding_trainer.py | 4 +-- .../trainers/ctr_modul_trainer.py | 12 +++---- .../core => core}/trainers/single_trainer.py | 4 +-- .../trainers/tdm_cluster_trainer.py | 4 +-- .../trainers/tdm_single_trainer.py | 6 ++-- .../trainers/transpiler_trainer.py | 8 ++--- {fleet_rec/core => core}/utils/__init__.py | 0 .../utils/dataloader_instance.py | 8 ++--- {fleet_rec/core => core}/utils/dataset.py | 4 +-- .../core => core}/utils/dataset_instance.py | 2 +- {fleet_rec/core => core}/utils/envs.py | 6 ++-- {fleet_rec/core => core}/utils/fs.py | 0 {fleet_rec/core => core}/utils/table.py | 0 {fleet_rec/core => core}/utils/util.py | 2 +- fleet_rec/check.py | 8 ----- .../contentunderstanding/tagspace/config.yaml | 2 +- models/contentunderstanding/tagspace/model.py | 4 +-- .../contentunderstanding/tagspace/reader.py | 4 +-- .../text_classification/config.yaml | 2 +- .../text_classification/model.py | 4 +-- .../text_classification/reader.py | 4 +-- models/match/dssm/config.yaml | 2 +- models/match/dssm/model.py | 4 +-- .../match/dssm/synthetic_evaluate_reader.py | 4 +-- models/match/dssm/synthetic_reader.py | 4 +-- models/match/multiview-simnet/config.yaml | 4 +-- .../match/multiview-simnet/evaluate_reader.py | 4 +-- models/match/multiview-simnet/model.py | 4 +-- models/match/multiview-simnet/reader.py | 4 +-- models/multitask/esmm/config.yaml | 2 +- models/multitask/esmm/esmm_reader.py | 4 +-- models/multitask/esmm/model.py | 4 +-- models/multitask/mmoe/census_reader.py | 4 +-- models/multitask/mmoe/config.yaml | 2 +- models/multitask/mmoe/model.py | 4 +-- .../multitask/share-bottom/census_reader.py | 4 +-- models/multitask/share-bottom/config.yaml | 2 +- models/multitask/share-bottom/model.py | 4 +-- models/rank/criteo_reader.py | 4 +-- models/rank/dcn/config.yaml | 2 +- models/rank/dcn/criteo_reader.py | 4 +-- models/rank/dcn/data/download.py | 2 +- models/rank/dcn/model.py | 4 +-- models/rank/deepfm/config.yaml | 2 +- models/rank/deepfm/criteo_reader.py | 4 +-- .../rank/deepfm/data/download_preprocess.py | 2 +- models/rank/deepfm/model.py | 4 +-- models/rank/din/config.yaml | 2 +- models/rank/din/model.py | 4 +-- models/rank/din/reader.py | 4 +-- models/rank/dnn/backend.yaml | 4 +-- models/rank/dnn/config.yaml | 2 +- models/rank/dnn/model.py | 4 +-- models/rank/dnn/worker.sh | 4 +-- models/rank/wide_deep/config.yaml | 2 +- models/rank/wide_deep/model.py | 4 +-- models/rank/wide_deep/reader.py | 4 +-- models/rank/xdeepfm/config.yaml | 2 +- models/rank/xdeepfm/criteo_reader.py | 4 +-- models/rank/xdeepfm/data/download.py | 2 +- models/rank/xdeepfm/model.py | 4 +-- models/recall/gnn/config.yaml | 4 +-- models/recall/gnn/evaluate_reader.py | 4 +-- models/recall/gnn/model.py | 4 +-- models/recall/gnn/reader.py | 4 +-- models/recall/gru4rec/config.yaml | 2 +- models/recall/gru4rec/model.py | 4 +-- models/recall/gru4rec/rsc15_reader.py | 4 +-- models/recall/readme.md | 2 +- models/recall/ssr/config.yaml | 2 +- models/recall/ssr/model.py | 4 +-- models/recall/ssr/ssr_reader.py | 4 +-- models/recall/tdm/config.yaml | 4 +-- models/recall/tdm/model.py | 4 +-- models/recall/tdm/tdm_evaluate_reader.py | 2 +- models/recall/tdm/tdm_reader.py | 2 +- models/recall/word2vec/config.yaml | 4 +-- models/recall/word2vec/model.py | 4 +-- models/recall/word2vec/w2v_evaluate_reader.py | 4 +-- models/recall/word2vec/w2v_reader.py | 4 +-- readme.md | 28 +++++++-------- fleet_rec/run.py => run.py | 22 ++++++------ setup.py | 34 ++++++++++--------- {fleet_rec/tests => tests}/__init__.py | 0 110 files changed, 203 insertions(+), 209 deletions(-) rename fleet_rec/__init__.py => __init__.py (100%) rename {fleet_rec/core => core}/__init__.py (100%) rename {fleet_rec/core => core}/engine/__init__.py (100%) rename {fleet_rec/core => core}/engine/cluster/__init__.py (100%) rename {fleet_rec/core => core}/engine/cluster/cluster.py (92%) rename {fleet_rec/core => core}/engine/cluster/master.sh (100%) rename {fleet_rec/core => core}/engine/engine.py (100%) rename {fleet_rec/core => core}/engine/local_cluster.py (96%) rename {fleet_rec/core => core}/engine/local_mpi.py (95%) rename {fleet_rec/core => core}/factory.py (96%) rename {fleet_rec/core => core}/layer.py (100%) rename {fleet_rec/core => core}/metric.py (100%) rename {fleet_rec/core => core}/metrics/__init__.py (100%) rename {fleet_rec/core => core}/metrics/auc_metrics.py (99%) rename {fleet_rec/core => core}/model.py (98%) rename {fleet_rec/core => core}/modules/__init__.py (100%) rename {fleet_rec/core => core}/modules/coding/__init__.py (100%) rename {fleet_rec/core => core}/modules/coding/layers.py (100%) rename {fleet_rec/core => core}/modules/modul/__init__.py (100%) rename {fleet_rec/core => core}/modules/modul/build.py (99%) rename {fleet_rec/core => core}/modules/modul/layers.py (99%) rename {fleet_rec/core => core}/reader.py (97%) rename {fleet_rec/core => core}/trainer.py (98%) rename {fleet_rec/core => core}/trainers/__init__.py (100%) rename {fleet_rec/core => core}/trainers/cluster_trainer.py (98%) rename {fleet_rec/core => core}/trainers/ctr_coding_trainer.py (98%) rename {fleet_rec/core => core}/trainers/ctr_modul_trainer.py (98%) rename {fleet_rec/core => core}/trainers/single_trainer.py (97%) rename {fleet_rec/core => core}/trainers/tdm_cluster_trainer.py (97%) rename {fleet_rec/core => core}/trainers/tdm_single_trainer.py (96%) rename {fleet_rec/core => core}/trainers/transpiler_trainer.py (97%) rename {fleet_rec/core => core}/utils/__init__.py (100%) rename {fleet_rec/core => core}/utils/dataloader_instance.py (90%) rename {fleet_rec/core => core}/utils/dataset.py (98%) rename {fleet_rec/core => core}/utils/dataset_instance.py (94%) rename {fleet_rec/core => core}/utils/envs.py (96%) rename {fleet_rec/core => core}/utils/fs.py (100%) rename {fleet_rec/core => core}/utils/table.py (100%) rename {fleet_rec/core => core}/utils/util.py (99%) delete mode 100755 fleet_rec/check.py rename fleet_rec/run.py => run.py (92%) rename {fleet_rec/tests => tests}/__init__.py (100%) diff --git a/.gitignore b/.gitignore index cca95ea7..f06e0492 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ output .idea/ build/ dist/ -fleetrec.egg-info/ +paddlerec.egg-info/ *~ *.pyc diff --git a/fleet_rec/__init__.py b/__init__.py similarity index 100% rename from fleet_rec/__init__.py rename to __init__.py diff --git a/fleet_rec/core/__init__.py b/core/__init__.py similarity index 100% rename from fleet_rec/core/__init__.py rename to core/__init__.py diff --git a/fleet_rec/core/engine/__init__.py b/core/engine/__init__.py similarity index 100% rename from fleet_rec/core/engine/__init__.py rename to core/engine/__init__.py diff --git a/fleet_rec/core/engine/cluster/__init__.py b/core/engine/cluster/__init__.py similarity index 100% rename from fleet_rec/core/engine/cluster/__init__.py rename to core/engine/cluster/__init__.py diff --git a/fleet_rec/core/engine/cluster/cluster.py b/core/engine/cluster/cluster.py similarity index 92% rename from fleet_rec/core/engine/cluster/cluster.py rename to core/engine/cluster/cluster.py index 7386d36e..0143d73e 100644 --- a/fleet_rec/core/engine/cluster/cluster.py +++ b/core/engine/cluster/cluster.py @@ -20,9 +20,9 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine -from fleetrec.core.factory import TrainerFactory -from fleetrec.core.utils import envs +from paddlerec.core.engine.engine import Engine +from paddlerec.core.factory import TrainerFactory +from paddlerec.core.utils import envs class ClusterEngine(Engine): diff --git a/fleet_rec/core/engine/cluster/master.sh b/core/engine/cluster/master.sh similarity index 100% rename from fleet_rec/core/engine/cluster/master.sh rename to core/engine/cluster/master.sh diff --git a/fleet_rec/core/engine/engine.py b/core/engine/engine.py similarity index 100% rename from fleet_rec/core/engine/engine.py rename to core/engine/engine.py diff --git a/fleet_rec/core/engine/local_cluster.py b/core/engine/local_cluster.py similarity index 96% rename from fleet_rec/core/engine/local_cluster.py rename to core/engine/local_cluster.py index 86a8d249..fa224280 100755 --- a/fleet_rec/core/engine/local_cluster.py +++ b/core/engine/local_cluster.py @@ -19,8 +19,8 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine -from fleetrec.core.utils import envs +from paddlerec.core.engine.engine import Engine +from paddlerec.core.utils import envs class LocalClusterEngine(Engine): def start_procs(self): @@ -47,7 +47,7 @@ class LocalClusterEngine(Engine): user_endpoints_ips = [x.split(":")[0] for x in user_endpoints.split(",")] user_endpoints_port = [x.split(":")[1] for x in user_endpoints.split(",")] - factory = "fleetrec.core.factory" + factory = "paddlerec.core.factory" cmd = [sys.executable, "-u", "-m", factory, self.trainer] for i in range(server_num): diff --git a/fleet_rec/core/engine/local_mpi.py b/core/engine/local_mpi.py similarity index 95% rename from fleet_rec/core/engine/local_mpi.py rename to core/engine/local_mpi.py index 84ed6c30..29e34907 100755 --- a/fleet_rec/core/engine/local_mpi.py +++ b/core/engine/local_mpi.py @@ -19,7 +19,7 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine +from paddlerec.core.engine.engine import Engine class LocalMPIEngine(Engine): @@ -33,7 +33,7 @@ class LocalMPIEngine(Engine): procs = [] log_fns = [] - factory = "fleetrec.core.factory" + factory = "paddlerec.core.factory" cmd = "mpirun -npernode 2 -timestamp-output -tag-output".split(" ") cmd.extend([sys.executable, "-u", "-m", factory, self.trainer]) diff --git a/fleet_rec/core/factory.py b/core/factory.py similarity index 96% rename from fleet_rec/core/factory.py rename to core/factory.py index c5e6f30f..4c08f1f6 100755 --- a/fleet_rec/core/factory.py +++ b/core/factory.py @@ -17,7 +17,7 @@ import sys import yaml -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs trainer_abs = os.path.join(os.path.dirname( os.path.abspath(__file__)), "trainers") @@ -71,7 +71,7 @@ class TrainerFactory(object): with open(config, 'r') as rb: _config = yaml.load(rb.read(), Loader=yaml.FullLoader) else: - raise ValueError("fleetrec's config only support yaml") + raise ValueError("paddlerec's config only support yaml") envs.set_global_envs(_config) envs.update_workspace() diff --git a/fleet_rec/core/layer.py b/core/layer.py similarity index 100% rename from fleet_rec/core/layer.py rename to core/layer.py diff --git a/fleet_rec/core/metric.py b/core/metric.py similarity index 100% rename from fleet_rec/core/metric.py rename to core/metric.py diff --git a/fleet_rec/core/metrics/__init__.py b/core/metrics/__init__.py similarity index 100% rename from fleet_rec/core/metrics/__init__.py rename to core/metrics/__init__.py diff --git a/fleet_rec/core/metrics/auc_metrics.py b/core/metrics/auc_metrics.py similarity index 99% rename from fleet_rec/core/metrics/auc_metrics.py rename to core/metrics/auc_metrics.py index f2f68890..3c48040d 100755 --- a/fleet_rec/core/metrics/auc_metrics.py +++ b/core/metrics/auc_metrics.py @@ -15,7 +15,7 @@ import math import numpy as np import paddle.fluid as fluid -from fleetrec.core.metric import Metric +from paddlerec.core.metric import Metric class AUCMetric(Metric): diff --git a/fleet_rec/core/model.py b/core/model.py similarity index 98% rename from fleet_rec/core/model.py rename to core/model.py index 141c1577..cb937b7b 100755 --- a/fleet_rec/core/model.py +++ b/core/model.py @@ -2,7 +2,7 @@ import abc import paddle.fluid as fluid -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs class Model(object): diff --git a/fleet_rec/core/modules/__init__.py b/core/modules/__init__.py similarity index 100% rename from fleet_rec/core/modules/__init__.py rename to core/modules/__init__.py diff --git a/fleet_rec/core/modules/coding/__init__.py b/core/modules/coding/__init__.py similarity index 100% rename from fleet_rec/core/modules/coding/__init__.py rename to core/modules/coding/__init__.py diff --git a/fleet_rec/core/modules/coding/layers.py b/core/modules/coding/layers.py similarity index 100% rename from fleet_rec/core/modules/coding/layers.py rename to core/modules/coding/layers.py diff --git a/fleet_rec/core/modules/modul/__init__.py b/core/modules/modul/__init__.py similarity index 100% rename from fleet_rec/core/modules/modul/__init__.py rename to core/modules/modul/__init__.py diff --git a/fleet_rec/core/modules/modul/build.py b/core/modules/modul/build.py similarity index 99% rename from fleet_rec/core/modules/modul/build.py rename to core/modules/modul/build.py index 51b83e5f..20e7702e 100755 --- a/fleet_rec/core/modules/modul/build.py +++ b/core/modules/modul/build.py @@ -3,8 +3,8 @@ import copy import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet -from fleetrec.core.model import Model -from fleetrec.core.utils import table +from paddlerec.core.model import Model +from paddlerec.core.utils import table def create(config): diff --git a/fleet_rec/core/modules/modul/layers.py b/core/modules/modul/layers.py similarity index 99% rename from fleet_rec/core/modules/modul/layers.py rename to core/modules/modul/layers.py index 84a08ee8..f533a46d 100755 --- a/fleet_rec/core/modules/modul/layers.py +++ b/core/modules/modul/layers.py @@ -13,7 +13,7 @@ # limitations under the License. import paddle.fluid as fluid -from fleetrec.core.layer import Layer +from paddlerec.core.layer import Layer class EmbeddingInputLayer(Layer): diff --git a/fleet_rec/core/reader.py b/core/reader.py similarity index 97% rename from fleet_rec/core/reader.py rename to core/reader.py index 81da9ebf..955afa5e 100755 --- a/fleet_rec/core/reader.py +++ b/core/reader.py @@ -19,7 +19,7 @@ import os import paddle.fluid.incubate.data_generator as dg import yaml -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs class Reader(dg.MultiSlotDataGenerator): diff --git a/fleet_rec/core/trainer.py b/core/trainer.py similarity index 98% rename from fleet_rec/core/trainer.py rename to core/trainer.py index ce6edf68..df5626b7 100755 --- a/fleet_rec/core/trainer.py +++ b/core/trainer.py @@ -20,7 +20,7 @@ import time import yaml from paddle import fluid -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs class Trainer(object): diff --git a/fleet_rec/core/trainers/__init__.py b/core/trainers/__init__.py similarity index 100% rename from fleet_rec/core/trainers/__init__.py rename to core/trainers/__init__.py diff --git a/fleet_rec/core/trainers/cluster_trainer.py b/core/trainers/cluster_trainer.py similarity index 98% rename from fleet_rec/core/trainers/cluster_trainer.py rename to core/trainers/cluster_trainer.py index bc5a9173..8d71280c 100755 --- a/fleet_rec/core/trainers/cluster_trainer.py +++ b/core/trainers/cluster_trainer.py @@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distribu from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer +from paddlerec.core.utils import envs +from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer class ClusterTrainer(TranspileTrainer): diff --git a/fleet_rec/core/trainers/ctr_coding_trainer.py b/core/trainers/ctr_coding_trainer.py similarity index 98% rename from fleet_rec/core/trainers/ctr_coding_trainer.py rename to core/trainers/ctr_coding_trainer.py index fdfdd1ec..a6377c85 100755 --- a/fleet_rec/core/trainers/ctr_coding_trainer.py +++ b/core/trainers/ctr_coding_trainer.py @@ -18,8 +18,8 @@ import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainer import Trainer +from paddlerec.core.utils import envs +from paddlerec.core.trainer import Trainer class CtrPaddleTrainer(Trainer): diff --git a/fleet_rec/core/trainers/ctr_modul_trainer.py b/core/trainers/ctr_modul_trainer.py similarity index 98% rename from fleet_rec/core/trainers/ctr_modul_trainer.py rename to core/trainers/ctr_modul_trainer.py index e72715e7..8c99a5c8 100755 --- a/fleet_rec/core/trainers/ctr_modul_trainer.py +++ b/core/trainers/ctr_modul_trainer.py @@ -24,12 +24,12 @@ from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker -from fleetrec.core.utils import fs as fs -from fleetrec.core.utils import util as util -from fleetrec.core.metrics.auc_metrics import AUCMetric -from fleetrec.core.modules.modul import build as model_basic -from fleetrec.core.utils import dataset -from fleetrec.core.trainer import Trainer +from paddlerec.core.utils import fs as fs +from paddlerec.core.utils import util as util +from paddlerec.core.metrics.auc_metrics import AUCMetric +from paddlerec.core.modules.modul import build as model_basic +from paddlerec.core.utils import dataset +from paddlerec.core.trainer import Trainer def wroker_numric_opt(value, env, opt): diff --git a/fleet_rec/core/trainers/single_trainer.py b/core/trainers/single_trainer.py similarity index 97% rename from fleet_rec/core/trainers/single_trainer.py rename to core/trainers/single_trainer.py index db339893..ef304cf4 100755 --- a/fleet_rec/core/trainers/single_trainer.py +++ b/core/trainers/single_trainer.py @@ -20,8 +20,8 @@ from __future__ import print_function import logging import paddle.fluid as fluid -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer -from fleetrec.core.utils import envs +from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer +from paddlerec.core.utils import envs import numpy as np logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/fleet_rec/core/trainers/tdm_cluster_trainer.py b/core/trainers/tdm_cluster_trainer.py similarity index 97% rename from fleet_rec/core/trainers/tdm_cluster_trainer.py rename to core/trainers/tdm_cluster_trainer.py index 40a46d92..11fa2334 100755 --- a/fleet_rec/core/trainers/tdm_cluster_trainer.py +++ b/core/trainers/tdm_cluster_trainer.py @@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import f from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainers.cluster_trainer import ClusterTrainer +from paddlerec.core.utils import envs +from paddlerec.core.trainers.cluster_trainer import ClusterTrainer logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/fleet_rec/core/trainers/tdm_single_trainer.py b/core/trainers/tdm_single_trainer.py similarity index 96% rename from fleet_rec/core/trainers/tdm_single_trainer.py rename to core/trainers/tdm_single_trainer.py index 6d010999..118548ab 100755 --- a/fleet_rec/core/trainers/tdm_single_trainer.py +++ b/core/trainers/tdm_single_trainer.py @@ -21,9 +21,9 @@ from __future__ import print_function import logging import paddle.fluid as fluid -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer -from fleetrec.core.trainers.single_trainer import SingleTrainer -from fleetrec.core.utils import envs +from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer +from paddlerec.core.trainers.single_trainer import SingleTrainer +from paddlerec.core.utils import envs import numpy as np logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/fleet_rec/core/trainers/transpiler_trainer.py b/core/trainers/transpiler_trainer.py similarity index 97% rename from fleet_rec/core/trainers/transpiler_trainer.py rename to core/trainers/transpiler_trainer.py index 683b921e..c5c45135 100755 --- a/fleet_rec/core/trainers/transpiler_trainer.py +++ b/core/trainers/transpiler_trainer.py @@ -20,9 +20,9 @@ import os import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from fleetrec.core.trainer import Trainer -from fleetrec.core.utils import envs -from fleetrec.core.utils import dataloader_instance +from paddlerec.core.trainer import Trainer +from paddlerec.core.utils import envs +from paddlerec.core.utils import dataloader_instance class TranspileTrainer(Trainer): @@ -81,7 +81,7 @@ class TranspileTrainer(Trainer): pipe_cmd = "python {} {} {} {}".format( reader, reader_class, state, self._config_yaml) - if train_data_path.startswith("fleetrec::"): + if train_data_path.startswith("paddlerec::"): package_base = envs.get_runtime_environ("PACKAGE_BASE") assert package_base is not None train_data_path = os.path.join( diff --git a/fleet_rec/core/utils/__init__.py b/core/utils/__init__.py similarity index 100% rename from fleet_rec/core/utils/__init__.py rename to core/utils/__init__.py diff --git a/fleet_rec/core/utils/dataloader_instance.py b/core/utils/dataloader_instance.py similarity index 90% rename from fleet_rec/core/utils/dataloader_instance.py rename to core/utils/dataloader_instance.py index a16fde80..88b4a9f6 100755 --- a/fleet_rec/core/utils/dataloader_instance.py +++ b/core/utils/dataloader_instance.py @@ -16,9 +16,9 @@ from __future__ import print_function import os import sys -from fleetrec.core.utils.envs import lazy_instance_by_fliename -from fleetrec.core.utils.envs import get_global_env -from fleetrec.core.utils.envs import get_runtime_environ +from paddlerec.core.utils.envs import lazy_instance_by_fliename +from paddlerec.core.utils.envs import get_global_env +from paddlerec.core.utils.envs import get_runtime_environ def dataloader(readerclass, train, yaml_file): @@ -31,7 +31,7 @@ def dataloader(readerclass, train, yaml_file): namespace = "evaluate.reader" data_path = get_global_env("test_data_path", None, namespace) - if data_path.startswith("fleetrec::"): + if data_path.startswith("paddlerec::"): package_base = get_runtime_environ("PACKAGE_BASE") assert package_base is not None data_path = os.path.join(package_base, data_path.split("::")[1]) diff --git a/fleet_rec/core/utils/dataset.py b/core/utils/dataset.py similarity index 98% rename from fleet_rec/core/utils/dataset.py rename to core/utils/dataset.py index 92cbe891..0eff8da0 100755 --- a/fleet_rec/core/utils/dataset.py +++ b/core/utils/dataset.py @@ -18,8 +18,8 @@ import datetime import paddle.fluid as fluid -from fleetrec.core.utils import fs as fs -from fleetrec.core.utils import util as util +from paddlerec.core.utils import fs as fs +from paddlerec.core.utils import util as util class Dataset(object): diff --git a/fleet_rec/core/utils/dataset_instance.py b/core/utils/dataset_instance.py similarity index 94% rename from fleet_rec/core/utils/dataset_instance.py rename to core/utils/dataset_instance.py index 89e6e45d..94997b66 100755 --- a/fleet_rec/core/utils/dataset_instance.py +++ b/core/utils/dataset_instance.py @@ -14,7 +14,7 @@ from __future__ import print_function import sys -from fleetrec.core.utils.envs import lazy_instance_by_fliename +from paddlerec.core.utils.envs import lazy_instance_by_fliename if len(sys.argv) != 4: raise ValueError("reader only accept 3 argument: 1. reader_class 2.train/evaluate 3.yaml_abs_path") diff --git a/fleet_rec/core/utils/envs.py b/core/utils/envs.py similarity index 96% rename from fleet_rec/core/utils/envs.py rename to core/utils/envs.py index 93c9a68a..740b8ad5 100755 --- a/fleet_rec/core/utils/envs.py +++ b/core/utils/envs.py @@ -93,9 +93,9 @@ def update_workspace(): return # is fleet inner models - if workspace.startswith("fleetrec."): + if workspace.startswith("paddlerec."): fleet_package = get_runtime_environ("PACKAGE_BASE") - workspace_dir = workspace.split("fleetrec.")[1].replace(".", "/") + workspace_dir = workspace.split("paddlerec.")[1].replace(".", "/") path = os.path.join(fleet_package, workspace_dir) else: path = workspace @@ -127,7 +127,7 @@ def pretty_print_envs(envs, header=None): if header: draws += h_format.format(header[0], header[1]) else: - draws += h_format.format("fleetrec Global Envs", "Value") + draws += h_format.format("paddlerec Global Envs", "Value") draws += line + "\n" diff --git a/fleet_rec/core/utils/fs.py b/core/utils/fs.py similarity index 100% rename from fleet_rec/core/utils/fs.py rename to core/utils/fs.py diff --git a/fleet_rec/core/utils/table.py b/core/utils/table.py similarity index 100% rename from fleet_rec/core/utils/table.py rename to core/utils/table.py diff --git a/fleet_rec/core/utils/util.py b/core/utils/util.py similarity index 99% rename from fleet_rec/core/utils/util.py rename to core/utils/util.py index b410df1d..76965f50 100755 --- a/fleet_rec/core/utils/util.py +++ b/core/utils/util.py @@ -17,7 +17,7 @@ import time import datetime from paddle import fluid -from fleetrec.core.utils import fs as fs +from paddlerec.core.utils import fs as fs def save_program_proto(path, program=None): diff --git a/fleet_rec/check.py b/fleet_rec/check.py deleted file mode 100755 index d4177fd8..00000000 --- a/fleet_rec/check.py +++ /dev/null @@ -1,8 +0,0 @@ -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='fleet-rec check') - parser.add_argument("--model", type=str) - parser.add_argument("--engine", type=str) - - print("coming soon") diff --git a/models/contentunderstanding/tagspace/config.yaml b/models/contentunderstanding/tagspace/config.yaml index ce0283e2..e0232f3f 100644 --- a/models/contentunderstanding/tagspace/config.yaml +++ b/models/contentunderstanding/tagspace/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.tagspace" + workspace: "paddlerec.models.rank.tagspace" reader: batch_size: 5 diff --git a/models/contentunderstanding/tagspace/model.py b/models/contentunderstanding/tagspace/model.py index 46d3d752..aeb569c6 100644 --- a/models/contentunderstanding/tagspace/model.py +++ b/models/contentunderstanding/tagspace/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import paddle.fluid as fluid import paddle.fluid.layers.nn as nn diff --git a/models/contentunderstanding/tagspace/reader.py b/models/contentunderstanding/tagspace/reader.py index 7803d862..93446e71 100644 --- a/models/contentunderstanding/tagspace/reader.py +++ b/models/contentunderstanding/tagspace/reader.py @@ -10,8 +10,8 @@ import paddle import csv import io -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): def init(self): diff --git a/models/contentunderstanding/text_classification/config.yaml b/models/contentunderstanding/text_classification/config.yaml index 2104a613..9bdfffe1 100644 --- a/models/contentunderstanding/text_classification/config.yaml +++ b/models/contentunderstanding/text_classification/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.text_classification" + workspace: "paddlerec.models.rank.text_classification" reader: batch_size: 5 diff --git a/models/contentunderstanding/text_classification/model.py b/models/contentunderstanding/text_classification/model.py index 0218ab8c..b47dc446 100644 --- a/models/contentunderstanding/text_classification/model.py +++ b/models/contentunderstanding/text_classification/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import paddle.fluid as fluid import paddle.fluid.layers.nn as nn diff --git a/models/contentunderstanding/text_classification/reader.py b/models/contentunderstanding/text_classification/reader.py index 36d07b2c..efee0196 100644 --- a/models/contentunderstanding/text_classification/reader.py +++ b/models/contentunderstanding/text_classification/reader.py @@ -10,8 +10,8 @@ import paddle import csv import io -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): def init(self): diff --git a/models/match/dssm/config.yaml b/models/match/dssm/config.yaml index 95d93d78..22881bdf 100755 --- a/models/match/dssm/config.yaml +++ b/models/match/dssm/config.yaml @@ -23,7 +23,7 @@ train: strategy: "async" epochs: 4 - workspace: "fleetrec.models.match.dssm" + workspace: "paddlerec.models.match.dssm" reader: batch_size: 4 diff --git a/models/match/dssm/model.py b/models/match/dssm/model.py index 41427052..ef77a8f0 100755 --- a/models/match/dssm/model.py +++ b/models/match/dssm/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/match/dssm/synthetic_evaluate_reader.py b/models/match/dssm/synthetic_evaluate_reader.py index de497740..1be5b13d 100755 --- a/models/match/dssm/synthetic_evaluate_reader.py +++ b/models/match/dssm/synthetic_evaluate_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/match/dssm/synthetic_reader.py b/models/match/dssm/synthetic_reader.py index 6eca228f..c4cfaf6d 100755 --- a/models/match/dssm/synthetic_reader.py +++ b/models/match/dssm/synthetic_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/match/multiview-simnet/config.yaml b/models/match/multiview-simnet/config.yaml index 0fcc1fc2..8399b40d 100755 --- a/models/match/multiview-simnet/config.yaml +++ b/models/match/multiview-simnet/config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. evaluate: - workspace: "fleetrec.models.recall.multiview-simnet" + workspace: "paddlerec.models.recall.multiview-simnet" reader: batch_size: 2 class: "{workspace}/evaluate_reader.py" @@ -24,7 +24,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.multiview-simnet" + workspace: "paddlerec.models.recall.multiview-simnet" reader: batch_size: 2 diff --git a/models/match/multiview-simnet/evaluate_reader.py b/models/match/multiview-simnet/evaluate_reader.py index 63340ccd..f7752e47 100755 --- a/models/match/multiview-simnet/evaluate_reader.py +++ b/models/match/multiview-simnet/evaluate_reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/match/multiview-simnet/model.py b/models/match/multiview-simnet/model.py index c33c1003..aa8d8db6 100755 --- a/models/match/multiview-simnet/model.py +++ b/models/match/multiview-simnet/model.py @@ -19,8 +19,8 @@ import paddle.fluid.layers as layers import paddle.fluid.layers.tensor as tensor import paddle.fluid.layers.control_flow as cf -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class BowEncoder(object): """ bow-encoder """ diff --git a/models/match/multiview-simnet/reader.py b/models/match/multiview-simnet/reader.py index 34cabd41..5616547e 100755 --- a/models/match/multiview-simnet/reader.py +++ b/models/match/multiview-simnet/reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/multitask/esmm/config.yaml b/models/multitask/esmm/config.yaml index 5cc16a5d..a83c145c 100644 --- a/models/multitask/esmm/config.yaml +++ b/models/multitask/esmm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.multitask.esmm" + workspace: "paddlerec.models.multitask.esmm" reader: batch_size: 2 diff --git a/models/multitask/esmm/esmm_reader.py b/models/multitask/esmm/esmm_reader.py index a18702d3..f7aef643 100644 --- a/models/multitask/esmm/esmm_reader.py +++ b/models/multitask/esmm/esmm_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs from collections import defaultdict import numpy as np diff --git a/models/multitask/esmm/model.py b/models/multitask/esmm/model.py index 0063cdfe..6654f337 100644 --- a/models/multitask/esmm/model.py +++ b/models/multitask/esmm/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import numpy as np diff --git a/models/multitask/mmoe/census_reader.py b/models/multitask/mmoe/census_reader.py index 323c15a4..30997c63 100644 --- a/models/multitask/mmoe/census_reader.py +++ b/models/multitask/mmoe/census_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import numpy as np diff --git a/models/multitask/mmoe/config.yaml b/models/multitask/mmoe/config.yaml index ca6552d8..4f79fbd8 100644 --- a/models/multitask/mmoe/config.yaml +++ b/models/multitask/mmoe/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.multitask.mmoe" + workspace: "paddlerec.models.multitask.mmoe" reader: batch_size: 2 diff --git a/models/multitask/mmoe/model.py b/models/multitask/mmoe/model.py index 0b757d02..bdbbba09 100644 --- a/models/multitask/mmoe/model.py +++ b/models/multitask/mmoe/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/multitask/share-bottom/census_reader.py b/models/multitask/share-bottom/census_reader.py index 323c15a4..30997c63 100644 --- a/models/multitask/share-bottom/census_reader.py +++ b/models/multitask/share-bottom/census_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import numpy as np diff --git a/models/multitask/share-bottom/config.yaml b/models/multitask/share-bottom/config.yaml index 432ebdf3..f0ace882 100644 --- a/models/multitask/share-bottom/config.yaml +++ b/models/multitask/share-bottom/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.multitask.share-bottom" + workspace: "paddlerec.models.multitask.share-bottom" reader: batch_size: 2 diff --git a/models/multitask/share-bottom/model.py b/models/multitask/share-bottom/model.py index a364f1b8..7328695f 100644 --- a/models/multitask/share-bottom/model.py +++ b/models/multitask/share-bottom/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/criteo_reader.py b/models/rank/criteo_reader.py index c4930c8d..3d8fcf96 100755 --- a/models/rank/criteo_reader.py +++ b/models/rank/criteo_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/rank/dcn/config.yaml b/models/rank/dcn/config.yaml index d84adb74..3ba01f89 100755 --- a/models/rank/dcn/config.yaml +++ b/models/rank/dcn/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.dcn" + workspace: "paddlerec.models.rank.dcn" reader: batch_size: 2 diff --git a/models/rank/dcn/criteo_reader.py b/models/rank/dcn/criteo_reader.py index 4b81e1fb..a526d0fc 100755 --- a/models/rank/dcn/criteo_reader.py +++ b/models/rank/dcn/criteo_reader.py @@ -15,8 +15,8 @@ from __future__ import print_function import math import sys -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/dcn/data/download.py b/models/rank/dcn/data/download.py index b862988a..d9bcc6df 100755 --- a/models/rank/dcn/data/download.py +++ b/models/rank/dcn/data/download.py @@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") sys.path.append(TOOLS_PATH) -from fleetrec.tools.tools import download_file_and_uncompress +from paddlerec.tools.tools import download_file_and_uncompress if __name__ == '__main__': trainfile = 'train.txt' diff --git a/models/rank/dcn/model.py b/models/rank/dcn/model.py index c0395c27..5009f3ea 100755 --- a/models/rank/dcn/model.py +++ b/models/rank/dcn/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase from collections import OrderedDict class Model(ModelBase): diff --git a/models/rank/deepfm/config.yaml b/models/rank/deepfm/config.yaml index fd06406d..debaf2f0 100755 --- a/models/rank/deepfm/config.yaml +++ b/models/rank/deepfm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.deepfm" + workspace: "paddlerec.models.rank.deepfm" reader: batch_size: 2 diff --git a/models/rank/deepfm/criteo_reader.py b/models/rank/deepfm/criteo_reader.py index a4c2587a..ae942e0a 100755 --- a/models/rank/deepfm/criteo_reader.py +++ b/models/rank/deepfm/criteo_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/deepfm/data/download_preprocess.py b/models/rank/deepfm/data/download_preprocess.py index 1f114bcf..e8c94cc6 100755 --- a/models/rank/deepfm/data/download_preprocess.py +++ b/models/rank/deepfm/data/download_preprocess.py @@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") sys.path.append(TOOLS_PATH) -from fleetrec.tools.tools import download_file_and_uncompress, download_file +from paddlerec.tools.tools import download_file_and_uncompress, download_file if __name__ == '__main__': url = "https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz" diff --git a/models/rank/deepfm/model.py b/models/rank/deepfm/model.py index bd5e80e5..d91eef24 100755 --- a/models/rank/deepfm/model.py +++ b/models/rank/deepfm/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/din/config.yaml b/models/rank/din/config.yaml index 1244c163..bdf56b82 100755 --- a/models/rank/din/config.yaml +++ b/models/rank/din/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.din" + workspace: "paddlerec.models.rank.din" reader: batch_size: 2 diff --git a/models/rank/din/model.py b/models/rank/din/model.py index 1d7bcca3..68585c8e 100755 --- a/models/rank/din/model.py +++ b/models/rank/din/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/din/reader.py b/models/rank/din/reader.py index dfaec833..d1742bdd 100755 --- a/models/rank/din/reader.py +++ b/models/rank/din/reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import numpy as np import os import random diff --git a/models/rank/dnn/backend.yaml b/models/rank/dnn/backend.yaml index 60529faf..77384db4 100755 --- a/models/rank/dnn/backend.yaml +++ b/models/rank/dnn/backend.yaml @@ -13,7 +13,7 @@ # limitations under the License. engine: - workspace: "fleetrec.models.rank.dnn" + workspace: "paddlerec.models.rank.dnn" backend: "MPI" hdfs: @@ -24,7 +24,7 @@ engine: package: build_script: "{workspace}/package.sh" python: "/home/tangwei/fleet_rec_env/cpython-2.7.11-ucs4" - paddlerec: "/home/tangwei/fleet_rec_env/FleetRec" + paddlerec: "/home/tangwei/fleet_rec_env/PaddleRec" submit: hpc: "/home/tangwei/Plines/client/smart_client_khan/" diff --git a/models/rank/dnn/config.yaml b/models/rank/dnn/config.yaml index 66f5053a..ba2a5ac3 100755 --- a/models/rank/dnn/config.yaml +++ b/models/rank/dnn/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.dnn" + workspace: "paddlerec.models.rank.dnn" reader: batch_size: 2 diff --git a/models/rank/dnn/model.py b/models/rank/dnn/model.py index 71f88d62..7a5ed0d4 100755 --- a/models/rank/dnn/model.py +++ b/models/rank/dnn/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/dnn/worker.sh b/models/rank/dnn/worker.sh index 3ca2a1f0..9daf7488 100644 --- a/models/rank/dnn/worker.sh +++ b/models/rank/dnn/worker.sh @@ -56,7 +56,7 @@ function env_prepare() { python -c "print('heheda')" - mpirun -npernode 1 python/bin/python -m pip uninstall -y fleet-rec + mpirun -npernode 1 python/bin/python -m pip uninstall -y paddle-rec mpirun -npernode 1 python/bin/python -m pip install whl/fleet_rec-0.0.2-py2-none-any.whl --index-url=http://pip.baidu.com/pypi/simple --trusted-host pip.baidu.com check_error } @@ -64,7 +64,7 @@ function env_prepare() { function run() { echo "run" g_run_stage="run" - mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python/bin/python -u -m fleetrec.run -m fleetrec.models.rank.dnn --engine cluster --role worker + mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python/bin/python -u -m paddlerec.run -m paddlerec.models.rank.dnn --engine cluster --role worker } function main() { diff --git a/models/rank/wide_deep/config.yaml b/models/rank/wide_deep/config.yaml index 4608715a..916561da 100755 --- a/models/rank/wide_deep/config.yaml +++ b/models/rank/wide_deep/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.wide_deep" + workspace: "paddlerec.models.rank.wide_deep" reader: batch_size: 2 diff --git a/models/rank/wide_deep/model.py b/models/rank/wide_deep/model.py index c260c901..7efaccf0 100755 --- a/models/rank/wide_deep/model.py +++ b/models/rank/wide_deep/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/wide_deep/reader.py b/models/rank/wide_deep/reader.py index acb6d3cc..945dd4a5 100755 --- a/models/rank/wide_deep/reader.py +++ b/models/rank/wide_deep/reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/xdeepfm/config.yaml b/models/rank/xdeepfm/config.yaml index a93b61fb..029ef394 100755 --- a/models/rank/xdeepfm/config.yaml +++ b/models/rank/xdeepfm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.xdeepfm" + workspace: "paddlerec.models.rank.xdeepfm" reader: batch_size: 2 diff --git a/models/rank/xdeepfm/criteo_reader.py b/models/rank/xdeepfm/criteo_reader.py index 1b5e4041..ee2dac5d 100755 --- a/models/rank/xdeepfm/criteo_reader.py +++ b/models/rank/xdeepfm/criteo_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/xdeepfm/data/download.py b/models/rank/xdeepfm/data/download.py index d0483ea3..4afd1ce2 100755 --- a/models/rank/xdeepfm/data/download.py +++ b/models/rank/xdeepfm/data/download.py @@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") sys.path.append(TOOLS_PATH) -from fleetrec.tools.tools import download_file_and_uncompress, download_file +from paddlerec.tools.tools import download_file_and_uncompress, download_file if __name__ == '__main__': url_train = "https://paddlerec.bj.bcebos.com/xdeepfm%2Ftr" diff --git a/models/rank/xdeepfm/model.py b/models/rank/xdeepfm/model.py index 6619c78b..3a68f843 100755 --- a/models/rank/xdeepfm/model.py +++ b/models/rank/xdeepfm/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/gnn/config.yaml b/models/recall/gnn/config.yaml index 19eeb9e4..50c6d401 100755 --- a/models/recall/gnn/config.yaml +++ b/models/recall/gnn/config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. evaluate: - workspace: "fleetrec.models.recall.gnn" + workspace: "paddlerec.models.recall.gnn" reader: batch_size: 50 class: "{workspace}/evaluate_reader.py" @@ -24,7 +24,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.gnn" + workspace: "paddlerec.models.recall.gnn" reader: batch_size: 100 diff --git a/models/recall/gnn/evaluate_reader.py b/models/recall/gnn/evaluate_reader.py index 11343398..10d2f8e9 100755 --- a/models/recall/gnn/evaluate_reader.py +++ b/models/recall/gnn/evaluate_reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/recall/gnn/model.py b/models/recall/gnn/model.py index e63945e8..ac628ec6 100755 --- a/models/recall/gnn/model.py +++ b/models/recall/gnn/model.py @@ -17,8 +17,8 @@ import math import paddle.fluid as fluid import paddle.fluid.layers as layers -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/gnn/reader.py b/models/recall/gnn/reader.py index 89150d5d..7e943379 100755 --- a/models/recall/gnn/reader.py +++ b/models/recall/gnn/reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/recall/gru4rec/config.yaml b/models/recall/gru4rec/config.yaml index 35e57e01..71c212ad 100644 --- a/models/recall/gru4rec/config.yaml +++ b/models/recall/gru4rec/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.recall.gru4rec" + workspace: "paddlerec.models.recall.gru4rec" reader: batch_size: 5 diff --git a/models/recall/gru4rec/model.py b/models/recall/gru4rec/model.py index 5b461d76..3f4bff27 100644 --- a/models/recall/gru4rec/model.py +++ b/models/recall/gru4rec/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/gru4rec/rsc15_reader.py b/models/recall/gru4rec/rsc15_reader.py index fbfc4b91..eeb2441d 100644 --- a/models/recall/gru4rec/rsc15_reader.py +++ b/models/recall/gru4rec/rsc15_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/recall/readme.md b/models/recall/readme.md index 79b95fb4..bb41222b 100755 --- a/models/recall/readme.md +++ b/models/recall/readme.md @@ -35,7 +35,7 @@ sh data_process.sh ``` ### 训练 ```shell -python -m fleetrec.run -m config.yaml -d cpu -e single +python -m paddlerec.run -m config.yaml -d cpu -e single ``` ### 预测 diff --git a/models/recall/ssr/config.yaml b/models/recall/ssr/config.yaml index 640c93b9..b6bcbffc 100644 --- a/models/recall/ssr/config.yaml +++ b/models/recall/ssr/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.recall.ssr" + workspace: "paddlerec.models.recall.ssr" reader: batch_size: 5 diff --git a/models/recall/ssr/model.py b/models/recall/ssr/model.py index 5fd94cf8..b2fa5c89 100644 --- a/models/recall/ssr/model.py +++ b/models/recall/ssr/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import paddle.fluid.layers.tensor as tensor import paddle.fluid.layers.io as io import paddle.fluid.layers.control_flow as cf diff --git a/models/recall/ssr/ssr_reader.py b/models/recall/ssr/ssr_reader.py index ba81e01f..e2a35265 100644 --- a/models/recall/ssr/ssr_reader.py +++ b/models/recall/ssr/ssr_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import random diff --git a/models/recall/tdm/config.yaml b/models/recall/tdm/config.yaml index bd870490..80f0678b 100755 --- a/models/recall/tdm/config.yaml +++ b/models/recall/tdm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.tdm" + workspace: "paddlerec.models.recall.tdm" reader: batch_size: 32 @@ -73,7 +73,7 @@ train: save_last: True evaluate: - workspace: "fleetrec.models.recall.tdm" + workspace: "paddlerec.models.recall.tdm" reader: batch_size: 1 class: "{workspace}/tdm_evaluate_reader.py" diff --git a/models/recall/tdm/model.py b/models/recall/tdm/model.py index ee96b260..0fc1adf9 100755 --- a/models/recall/tdm/model.py +++ b/models/recall/tdm/model.py @@ -17,8 +17,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/tdm/tdm_evaluate_reader.py b/models/recall/tdm/tdm_evaluate_reader.py index 915389ba..cb324266 100644 --- a/models/recall/tdm/tdm_evaluate_reader.py +++ b/models/recall/tdm/tdm_evaluate_reader.py @@ -17,7 +17,7 @@ from __future__ import print_function -from fleetrec.core.reader import Reader +from paddlerec.core.reader import Reader class EvaluateReader(Reader): diff --git a/models/recall/tdm/tdm_reader.py b/models/recall/tdm/tdm_reader.py index 17413249..5a24fbb4 100755 --- a/models/recall/tdm/tdm_reader.py +++ b/models/recall/tdm/tdm_reader.py @@ -17,7 +17,7 @@ from __future__ import print_function -from fleetrec.core.reader import Reader +from paddlerec.core.reader import Reader class TrainReader(Reader): diff --git a/models/recall/word2vec/config.yaml b/models/recall/word2vec/config.yaml index a3591e73..fd48ef05 100755 --- a/models/recall/word2vec/config.yaml +++ b/models/recall/word2vec/config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. evaluate: - workspace: "fleetrec.models.recall.word2vec" + workspace: "paddlerec.models.recall.word2vec" reader: batch_size: 50 class: "{workspace}/w2v_evaluate_reader.py" @@ -25,7 +25,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.word2vec" + workspace: "paddlerec.models.recall.word2vec" reader: batch_size: 100 diff --git a/models/recall/word2vec/model.py b/models/recall/word2vec/model.py index 7899710a..ea8e0f5e 100755 --- a/models/recall/word2vec/model.py +++ b/models/recall/word2vec/model.py @@ -16,8 +16,8 @@ import math import numpy as np import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/word2vec/w2v_evaluate_reader.py b/models/recall/word2vec/w2v_evaluate_reader.py index df6de931..6a5c1e83 100755 --- a/models/recall/word2vec/w2v_evaluate_reader.py +++ b/models/recall/word2vec/w2v_evaluate_reader.py @@ -14,8 +14,8 @@ import numpy as np import io import six -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/recall/word2vec/w2v_reader.py b/models/recall/word2vec/w2v_reader.py index 4857a6b0..24cf2e30 100755 --- a/models/recall/word2vec/w2v_reader.py +++ b/models/recall/word2vec/w2v_reader.py @@ -13,8 +13,8 @@ # limitations under the License. import numpy as np import io -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class NumpyRandomInt(object): diff --git a/readme.md b/readme.md index 6a37e1d3..4b88c468 100644 --- a/readme.md +++ b/readme.md @@ -74,7 +74,7 @@ PadlleRec以预置模型为核心,具备以下特点： - 安装方法一： ```bash - python -m pip install fleet-rec + python -m pip install paddle-rec ``` - 安装方法二 @@ -88,8 +88,8 @@ PadlleRec以预置模型为核心,具备以下特点： * 源码安装Fleet-Rec ``` - git clone https://github.com/seiriosPlus/FleetRec/ - cd FleetRec + git clone https://github.com/PaddlePaddle/PaddleRec/ + cd PaddleRec python setup.py install ``` @@ -99,16 +99,16 @@ PadlleRec以预置模型为核心,具备以下特点： ##### 单机训练 ```bash -cd FleetRec +cd PaddleRec -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d cpu \ -e single # 使用GPU资源进行训练 -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d gpu \ -e single ``` @@ -116,10 +116,10 @@ python -m fleetrec.run \ ##### 本地模拟分布式训练 ```bash -cd FleetRec +cd PaddleRec # 使用CPU资源进行训练 -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d cpu \ -e local_cluster ``` @@ -127,10 +127,10 @@ python -m fleetrec.run \ ##### 集群提交分布式训练<需要用户预先配置好集群环境，本提交命令不包含提交客户端> ```bash -cd FleetRec +cd PaddleRec -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d cpu \ -e cluster ``` diff --git a/fleet_rec/run.py b/run.py similarity index 92% rename from fleet_rec/run.py rename to run.py index aa8d5965..5552682b 100755 --- a/fleet_rec/run.py +++ b/run.py @@ -5,9 +5,9 @@ import tempfile import yaml -from fleetrec.core.factory import TrainerFactory -from fleetrec.core.utils import envs -from fleetrec.core.utils import util +from paddlerec.core.factory import TrainerFactory +from paddlerec.core.utils import envs +from paddlerec.core.utils import util engines = {} device = ["CPU", "GPU"] @@ -117,9 +117,9 @@ def cluster_engine(args): return # is fleet inner models - if workspace.startswith("fleetrec."): + if workspace.startswith("paddlerec."): fleet_package = envs.get_runtime_environ("PACKAGE_BASE") - workspace_dir = workspace.split("fleetrec.")[1].replace(".", "/") + workspace_dir = workspace.split("paddlerec.")[1].replace(".", "/") path = os.path.join(fleet_package, workspace_dir) else: path = workspace @@ -130,7 +130,7 @@ def cluster_engine(args): cluster_envs[name] = value def master(): - from fleetrec.core.engine.cluster.cluster import ClusterEngine + from paddlerec.core.engine.cluster.cluster import ClusterEngine with open(args.backend, 'r') as rb: _envs = yaml.load(rb.read(), Loader=yaml.FullLoader) @@ -179,7 +179,7 @@ def cluster_mpi_engine(args): def local_cluster_engine(args): - from fleetrec.core.engine.local_cluster import LocalClusterEngine + from paddlerec.core.engine.local_cluster import LocalClusterEngine trainer = get_trainer_prefix(args) + "ClusterTrainer" cluster_envs = {} @@ -205,7 +205,7 @@ def local_cluster_engine(args): def local_mpi_engine(args): print("launch cluster engine with cluster to run model: {}".format(args.model)) - from fleetrec.core.engine.local_mpi import LocalMPIEngine + from paddlerec.core.engine.local_mpi import LocalMPIEngine print("use 1X1 MPI ClusterTraining at localhost to run model: {}".format(args.model)) @@ -227,9 +227,9 @@ def local_mpi_engine(args): def get_abs_model(model): - if model.startswith("fleetrec."): + if model.startswith("paddlerec."): fleet_base = envs.get_runtime_environ("PACKAGE_BASE") - workspace_dir = model.split("fleetrec.")[1].replace(".", "/") + workspace_dir = model.split("paddlerec.")[1].replace(".", "/") path = os.path.join(fleet_base, workspace_dir, "config.yaml") else: if not os.path.isfile(model): @@ -239,7 +239,7 @@ def get_abs_model(model): if __name__ == "__main__": - parser = argparse.ArgumentParser(description='fleet-rec run') + parser = argparse.ArgumentParser(description='paddle-rec run') parser.add_argument("-m", "--model", type=str) parser.add_argument("-e", "--engine", type=str, choices=["single", "local_cluster", "cluster", diff --git a/setup.py b/setup.py index 3dd1a1e9..f3e97f97 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -setup for fleet-rec. +setup for paddle-rec. """ import os from setuptools import setup, find_packages @@ -12,12 +12,12 @@ requires = [ ] about = {} -about["__title__"] = "fleet-rec" +about["__title__"] = "paddle-rec" about["__version__"] = "0.0.2" -about["__description__"] = "fleet-rec" -about["__author__"] = "seiriosPlus" -about["__author_email__"] = "tangwei12@baidu.com" -about["__url__"] = "https://github.com/seiriosPlus/FleetRec" +about["__description__"] = "paddle-rec" +about["__author__"] = "paddle-dev" +about["__author_email__"] = "paddle-dev@baidu.com" +about["__url__"] = "https://github.com/PaddlePaddle/PaddleRec" readme = "..." @@ -30,22 +30,24 @@ def run_cmd(command): def build(dirname): package_dir = os.path.dirname(os.path.abspath(__file__)) run_cmd("cp -r {}/* {}".format(package_dir, dirname)) - run_cmd("mkdir {}".format(os.path.join(dirname, "fleetrec"))) - run_cmd("mv {}/* {}".format(os.path.join(dirname, "fleet_rec"), os.path.join(dirname, "fleetrec"))) - run_cmd("mv {} {}".format(os.path.join(dirname, "doc"), os.path.join(dirname, "fleetrec"))) - run_cmd("mv {} {}".format(os.path.join(dirname, "models"), os.path.join(dirname, "fleetrec"))) - run_cmd("mv {} {}".format(os.path.join(dirname, "tools"), os.path.join(dirname, "fleetrec"))) + run_cmd("mkdir {}".format(os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "core"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "doc"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "models"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "tests"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "tools"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "*.py"), os.path.join(dirname, "paddlerec"))) - packages = find_packages(dirname, include=('fleetrec.*')) + packages = find_packages(dirname, include=('paddlerec.*')) package_dir = {'': dirname} package_data = {} models_copy = ['data/*.txt', 'data/*/*.txt', '*.yaml', '*.sh', 'tree/*.npy', 'tree/*.txt'] engine_copy = ['*/*.sh'] for package in packages: - if package.startswith("fleetrec.models."): + if package.startswith("paddlerec.models."): package_data[package] = models_copy - if package.startswith("fleetrec.core.engine"): + if package.startswith("paddlerec.core.engine"): package_data[package] = engine_copy setup( @@ -73,11 +75,11 @@ print(''' \033[32m _ _ _ _ _ _ _ _ _ / \ / \ / \ / \ / \ / \ / \ / \ / \ -( F | L | E | E | T | - | R | E | C ) +( P | A | D | D | L | E | - | R | E | C ) \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \033[0m \033[34m Installation Complete. Congratulations! -How to use it ? Please visit our webside: https://github.com/seiriosPlus/FleetRec +How to use it ? Please visit our webside: https://github.com/PaddlePaddle/PaddleRec \033[0m ''') diff --git a/fleet_rec/tests/__init__.py b/tests/__init__.py similarity index 100% rename from fleet_rec/tests/__init__.py rename to tests/__init__.py -- GitLab