diff --git a/.gitignore b/.gitignore index cca95ea7cc11c4ac25907f7646d14fe27bcd597c..f06e04924034fd11e2a935140ef4789666e0be80 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ output .idea/ build/ dist/ -fleetrec.egg-info/ +paddlerec.egg-info/ *~ *.pyc diff --git a/fleet_rec/__init__.py b/__init__.py similarity index 100% rename from fleet_rec/__init__.py rename to __init__.py diff --git a/fleet_rec/core/__init__.py b/core/__init__.py similarity index 100% rename from fleet_rec/core/__init__.py rename to core/__init__.py diff --git a/fleet_rec/core/engine/__init__.py b/core/engine/__init__.py similarity index 100% rename from fleet_rec/core/engine/__init__.py rename to core/engine/__init__.py diff --git a/fleet_rec/core/engine/cluster/__init__.py b/core/engine/cluster/__init__.py similarity index 100% rename from fleet_rec/core/engine/cluster/__init__.py rename to core/engine/cluster/__init__.py diff --git a/fleet_rec/core/engine/cluster/cluster.py b/core/engine/cluster/cluster.py similarity index 92% rename from fleet_rec/core/engine/cluster/cluster.py rename to core/engine/cluster/cluster.py index 7386d36e6d2a26376e115e3ab5f1fb3058c542ec..0143d73edbe6dc750149f3221d07cae11a9b3a61 100644 --- a/fleet_rec/core/engine/cluster/cluster.py +++ b/core/engine/cluster/cluster.py @@ -20,9 +20,9 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine -from fleetrec.core.factory import TrainerFactory -from fleetrec.core.utils import envs +from paddlerec.core.engine.engine import Engine +from paddlerec.core.factory import TrainerFactory +from paddlerec.core.utils import envs class ClusterEngine(Engine): diff --git a/fleet_rec/core/engine/cluster/master.sh b/core/engine/cluster/master.sh similarity index 100% rename from fleet_rec/core/engine/cluster/master.sh rename to core/engine/cluster/master.sh diff --git a/fleet_rec/core/engine/engine.py b/core/engine/engine.py similarity index 100% rename from fleet_rec/core/engine/engine.py rename to core/engine/engine.py diff --git a/fleet_rec/core/engine/local_cluster.py b/core/engine/local_cluster.py similarity index 96% rename from fleet_rec/core/engine/local_cluster.py rename to core/engine/local_cluster.py index 86a8d2497035c8a791f06b22137b067af9923ae1..fa2242804a2ba2b7a64758e6c026d500edbc8b58 100755 --- a/fleet_rec/core/engine/local_cluster.py +++ b/core/engine/local_cluster.py @@ -19,8 +19,8 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine -from fleetrec.core.utils import envs +from paddlerec.core.engine.engine import Engine +from paddlerec.core.utils import envs class LocalClusterEngine(Engine): def start_procs(self): @@ -47,7 +47,7 @@ class LocalClusterEngine(Engine): user_endpoints_ips = [x.split(":")[0] for x in user_endpoints.split(",")] user_endpoints_port = [x.split(":")[1] for x in user_endpoints.split(",")] - factory = "fleetrec.core.factory" + factory = "paddlerec.core.factory" cmd = [sys.executable, "-u", "-m", factory, self.trainer] for i in range(server_num): diff --git a/fleet_rec/core/engine/local_mpi.py b/core/engine/local_mpi.py similarity index 95% rename from fleet_rec/core/engine/local_mpi.py rename to core/engine/local_mpi.py index 84ed6c307ab270b17f8307fd8252fbef3b197474..29e34907845a286a49b66286eefa54a1367512ce 100755 --- a/fleet_rec/core/engine/local_mpi.py +++ b/core/engine/local_mpi.py @@ -19,7 +19,7 @@ import sys import os import copy -from fleetrec.core.engine.engine import Engine +from paddlerec.core.engine.engine import Engine class LocalMPIEngine(Engine): @@ -33,7 +33,7 @@ class LocalMPIEngine(Engine): procs = [] log_fns = [] - factory = "fleetrec.core.factory" + factory = "paddlerec.core.factory" cmd = "mpirun -npernode 2 -timestamp-output -tag-output".split(" ") cmd.extend([sys.executable, "-u", "-m", factory, self.trainer]) diff --git a/fleet_rec/core/factory.py b/core/factory.py similarity index 96% rename from fleet_rec/core/factory.py rename to core/factory.py index c5e6f30f41e854101e902ee31f7313f9ae400c1a..4c08f1f6bbd70cc65011e8430e3acf039d7b6c8f 100755 --- a/fleet_rec/core/factory.py +++ b/core/factory.py @@ -17,7 +17,7 @@ import sys import yaml -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs trainer_abs = os.path.join(os.path.dirname( os.path.abspath(__file__)), "trainers") @@ -71,7 +71,7 @@ class TrainerFactory(object): with open(config, 'r') as rb: _config = yaml.load(rb.read(), Loader=yaml.FullLoader) else: - raise ValueError("fleetrec's config only support yaml") + raise ValueError("paddlerec's config only support yaml") envs.set_global_envs(_config) envs.update_workspace() diff --git a/fleet_rec/core/layer.py b/core/layer.py similarity index 100% rename from fleet_rec/core/layer.py rename to core/layer.py diff --git a/fleet_rec/core/metric.py b/core/metric.py similarity index 100% rename from fleet_rec/core/metric.py rename to core/metric.py diff --git a/fleet_rec/core/metrics/__init__.py b/core/metrics/__init__.py similarity index 100% rename from fleet_rec/core/metrics/__init__.py rename to core/metrics/__init__.py diff --git a/fleet_rec/core/metrics/auc_metrics.py b/core/metrics/auc_metrics.py similarity index 99% rename from fleet_rec/core/metrics/auc_metrics.py rename to core/metrics/auc_metrics.py index f2f68890c79ef35123c131e9064b99e28d8e0aea..3c48040db29fc31846780f6f4e871168ac65f3f8 100755 --- a/fleet_rec/core/metrics/auc_metrics.py +++ b/core/metrics/auc_metrics.py @@ -15,7 +15,7 @@ import math import numpy as np import paddle.fluid as fluid -from fleetrec.core.metric import Metric +from paddlerec.core.metric import Metric class AUCMetric(Metric): diff --git a/fleet_rec/core/model.py b/core/model.py similarity index 98% rename from fleet_rec/core/model.py rename to core/model.py index 141c15777ad15b4837cca1f6f4a43e5696a7b05f..cb937b7b7fd2d3708c1c483ece2fad21dae2a388 100755 --- a/fleet_rec/core/model.py +++ b/core/model.py @@ -2,7 +2,7 @@ import abc import paddle.fluid as fluid -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs class Model(object): diff --git a/fleet_rec/core/modules/__init__.py b/core/modules/__init__.py similarity index 100% rename from fleet_rec/core/modules/__init__.py rename to core/modules/__init__.py diff --git a/fleet_rec/core/modules/coding/__init__.py b/core/modules/coding/__init__.py similarity index 100% rename from fleet_rec/core/modules/coding/__init__.py rename to core/modules/coding/__init__.py diff --git a/fleet_rec/core/modules/coding/layers.py b/core/modules/coding/layers.py similarity index 100% rename from fleet_rec/core/modules/coding/layers.py rename to core/modules/coding/layers.py diff --git a/fleet_rec/core/modules/modul/__init__.py b/core/modules/modul/__init__.py similarity index 100% rename from fleet_rec/core/modules/modul/__init__.py rename to core/modules/modul/__init__.py diff --git a/fleet_rec/core/modules/modul/build.py b/core/modules/modul/build.py similarity index 99% rename from fleet_rec/core/modules/modul/build.py rename to core/modules/modul/build.py index 51b83e5f3ddc1b51ad3c990701cd38625772629e..20e7702ecb220c939ed919f77747f8c768f4c5d7 100755 --- a/fleet_rec/core/modules/modul/build.py +++ b/core/modules/modul/build.py @@ -3,8 +3,8 @@ import copy import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet -from fleetrec.core.model import Model -from fleetrec.core.utils import table +from paddlerec.core.model import Model +from paddlerec.core.utils import table def create(config): diff --git a/fleet_rec/core/modules/modul/layers.py b/core/modules/modul/layers.py similarity index 99% rename from fleet_rec/core/modules/modul/layers.py rename to core/modules/modul/layers.py index 84a08ee808377175a7b378a95f06181e7fca5173..f533a46db24da5e6feb3e8723ff3265529cfbcb3 100755 --- a/fleet_rec/core/modules/modul/layers.py +++ b/core/modules/modul/layers.py @@ -13,7 +13,7 @@ # limitations under the License. import paddle.fluid as fluid -from fleetrec.core.layer import Layer +from paddlerec.core.layer import Layer class EmbeddingInputLayer(Layer): diff --git a/fleet_rec/core/reader.py b/core/reader.py similarity index 97% rename from fleet_rec/core/reader.py rename to core/reader.py index 81da9ebf82f5bfa9d409a9a17169f19b13c7716b..955afa5e2f9a4d430912ad798de95df3160a3a69 100755 --- a/fleet_rec/core/reader.py +++ b/core/reader.py @@ -19,7 +19,7 @@ import os import paddle.fluid.incubate.data_generator as dg import yaml -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs class Reader(dg.MultiSlotDataGenerator): diff --git a/fleet_rec/core/trainer.py b/core/trainer.py similarity index 98% rename from fleet_rec/core/trainer.py rename to core/trainer.py index ce6edf68a10d75b83d5901d6214351a6506a7093..df5626b7a5317b9fd7f108964adf11afe0c59c0e 100755 --- a/fleet_rec/core/trainer.py +++ b/core/trainer.py @@ -20,7 +20,7 @@ import time import yaml from paddle import fluid -from fleetrec.core.utils import envs +from paddlerec.core.utils import envs class Trainer(object): diff --git a/fleet_rec/core/trainers/__init__.py b/core/trainers/__init__.py similarity index 100% rename from fleet_rec/core/trainers/__init__.py rename to core/trainers/__init__.py diff --git a/fleet_rec/core/trainers/cluster_trainer.py b/core/trainers/cluster_trainer.py similarity index 98% rename from fleet_rec/core/trainers/cluster_trainer.py rename to core/trainers/cluster_trainer.py index bc5a91732febcc470554ecfa07cf5fb11518546f..8d71280c2998ae3ff2b89b0c4c5422e2b70dd61b 100755 --- a/fleet_rec/core/trainers/cluster_trainer.py +++ b/core/trainers/cluster_trainer.py @@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distribu from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer +from paddlerec.core.utils import envs +from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer class ClusterTrainer(TranspileTrainer): diff --git a/fleet_rec/core/trainers/ctr_coding_trainer.py b/core/trainers/ctr_coding_trainer.py similarity index 98% rename from fleet_rec/core/trainers/ctr_coding_trainer.py rename to core/trainers/ctr_coding_trainer.py index fdfdd1ecb2f40747cc8cb61e3f8ce0e3c1c19d4c..a6377c853853d3f1ab53c55b9a5253cf0ff8d5b7 100755 --- a/fleet_rec/core/trainers/ctr_coding_trainer.py +++ b/core/trainers/ctr_coding_trainer.py @@ -18,8 +18,8 @@ import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainer import Trainer +from paddlerec.core.utils import envs +from paddlerec.core.trainer import Trainer class CtrPaddleTrainer(Trainer): diff --git a/fleet_rec/core/trainers/ctr_modul_trainer.py b/core/trainers/ctr_modul_trainer.py similarity index 98% rename from fleet_rec/core/trainers/ctr_modul_trainer.py rename to core/trainers/ctr_modul_trainer.py index e72715e74634791c2c90890bc6b8b8f0a0e56c71..8c99a5c885ed1b8a577556c52d56c4a8ad04944c 100755 --- a/fleet_rec/core/trainers/ctr_modul_trainer.py +++ b/core/trainers/ctr_modul_trainer.py @@ -24,12 +24,12 @@ from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker -from fleetrec.core.utils import fs as fs -from fleetrec.core.utils import util as util -from fleetrec.core.metrics.auc_metrics import AUCMetric -from fleetrec.core.modules.modul import build as model_basic -from fleetrec.core.utils import dataset -from fleetrec.core.trainer import Trainer +from paddlerec.core.utils import fs as fs +from paddlerec.core.utils import util as util +from paddlerec.core.metrics.auc_metrics import AUCMetric +from paddlerec.core.modules.modul import build as model_basic +from paddlerec.core.utils import dataset +from paddlerec.core.trainer import Trainer def wroker_numric_opt(value, env, opt): diff --git a/fleet_rec/core/trainers/single_trainer.py b/core/trainers/single_trainer.py similarity index 97% rename from fleet_rec/core/trainers/single_trainer.py rename to core/trainers/single_trainer.py index db3398932e4cefbaf5fb5c211db87bc029019b6f..ef304cf46ee5775949910431603e34466015971d 100755 --- a/fleet_rec/core/trainers/single_trainer.py +++ b/core/trainers/single_trainer.py @@ -20,8 +20,8 @@ from __future__ import print_function import logging import paddle.fluid as fluid -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer -from fleetrec.core.utils import envs +from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer +from paddlerec.core.utils import envs import numpy as np logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/fleet_rec/core/trainers/tdm_cluster_trainer.py b/core/trainers/tdm_cluster_trainer.py similarity index 97% rename from fleet_rec/core/trainers/tdm_cluster_trainer.py rename to core/trainers/tdm_cluster_trainer.py index 40a46d92fd7aae583c313be9e27865d700b6ca73..11fa23340c7587995df2b39181cd0bc9007982a1 100755 --- a/fleet_rec/core/trainers/tdm_cluster_trainer.py +++ b/core/trainers/tdm_cluster_trainer.py @@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import f from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker -from fleetrec.core.utils import envs -from fleetrec.core.trainers.cluster_trainer import ClusterTrainer +from paddlerec.core.utils import envs +from paddlerec.core.trainers.cluster_trainer import ClusterTrainer logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/fleet_rec/core/trainers/tdm_single_trainer.py b/core/trainers/tdm_single_trainer.py similarity index 96% rename from fleet_rec/core/trainers/tdm_single_trainer.py rename to core/trainers/tdm_single_trainer.py index 6d010999d0f71f7634ee6913c911461b6180e89d..118548abd62fb0c3a22d2e43265afd028f64cfe0 100755 --- a/fleet_rec/core/trainers/tdm_single_trainer.py +++ b/core/trainers/tdm_single_trainer.py @@ -21,9 +21,9 @@ from __future__ import print_function import logging import paddle.fluid as fluid -from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer -from fleetrec.core.trainers.single_trainer import SingleTrainer -from fleetrec.core.utils import envs +from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer +from paddlerec.core.trainers.single_trainer import SingleTrainer +from paddlerec.core.utils import envs import numpy as np logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") diff --git a/fleet_rec/core/trainers/transpiler_trainer.py b/core/trainers/transpiler_trainer.py similarity index 97% rename from fleet_rec/core/trainers/transpiler_trainer.py rename to core/trainers/transpiler_trainer.py index 683b921e33de481e26cc1ff93a8300ddf1475d51..c5c4513572bc59c2f30ee8d599743b9aa1011930 100755 --- a/fleet_rec/core/trainers/transpiler_trainer.py +++ b/core/trainers/transpiler_trainer.py @@ -20,9 +20,9 @@ import os import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from fleetrec.core.trainer import Trainer -from fleetrec.core.utils import envs -from fleetrec.core.utils import dataloader_instance +from paddlerec.core.trainer import Trainer +from paddlerec.core.utils import envs +from paddlerec.core.utils import dataloader_instance class TranspileTrainer(Trainer): @@ -81,7 +81,7 @@ class TranspileTrainer(Trainer): pipe_cmd = "python {} {} {} {}".format( reader, reader_class, state, self._config_yaml) - if train_data_path.startswith("fleetrec::"): + if train_data_path.startswith("paddlerec::"): package_base = envs.get_runtime_environ("PACKAGE_BASE") assert package_base is not None train_data_path = os.path.join( diff --git a/fleet_rec/core/utils/__init__.py b/core/utils/__init__.py similarity index 100% rename from fleet_rec/core/utils/__init__.py rename to core/utils/__init__.py diff --git a/fleet_rec/core/utils/dataloader_instance.py b/core/utils/dataloader_instance.py similarity index 90% rename from fleet_rec/core/utils/dataloader_instance.py rename to core/utils/dataloader_instance.py index a16fde803d5fc1467ee5d266215da4938ff8c579..88b4a9f63662861f41ab2971f4fe751bf5c65ce3 100755 --- a/fleet_rec/core/utils/dataloader_instance.py +++ b/core/utils/dataloader_instance.py @@ -16,9 +16,9 @@ from __future__ import print_function import os import sys -from fleetrec.core.utils.envs import lazy_instance_by_fliename -from fleetrec.core.utils.envs import get_global_env -from fleetrec.core.utils.envs import get_runtime_environ +from paddlerec.core.utils.envs import lazy_instance_by_fliename +from paddlerec.core.utils.envs import get_global_env +from paddlerec.core.utils.envs import get_runtime_environ def dataloader(readerclass, train, yaml_file): @@ -31,7 +31,7 @@ def dataloader(readerclass, train, yaml_file): namespace = "evaluate.reader" data_path = get_global_env("test_data_path", None, namespace) - if data_path.startswith("fleetrec::"): + if data_path.startswith("paddlerec::"): package_base = get_runtime_environ("PACKAGE_BASE") assert package_base is not None data_path = os.path.join(package_base, data_path.split("::")[1]) diff --git a/fleet_rec/core/utils/dataset.py b/core/utils/dataset.py similarity index 98% rename from fleet_rec/core/utils/dataset.py rename to core/utils/dataset.py index 92cbe8911a327f07d667e0e9f6d35bcb18140914..0eff8da071920b9931ce073ff94abe253baa31a0 100755 --- a/fleet_rec/core/utils/dataset.py +++ b/core/utils/dataset.py @@ -18,8 +18,8 @@ import datetime import paddle.fluid as fluid -from fleetrec.core.utils import fs as fs -from fleetrec.core.utils import util as util +from paddlerec.core.utils import fs as fs +from paddlerec.core.utils import util as util class Dataset(object): diff --git a/fleet_rec/core/utils/dataset_instance.py b/core/utils/dataset_instance.py similarity index 94% rename from fleet_rec/core/utils/dataset_instance.py rename to core/utils/dataset_instance.py index 89e6e45d2c53bb033e0e5fc6e436b149d76cc7c2..94997b669a5579cae4a403422c92fe277fa046b6 100755 --- a/fleet_rec/core/utils/dataset_instance.py +++ b/core/utils/dataset_instance.py @@ -14,7 +14,7 @@ from __future__ import print_function import sys -from fleetrec.core.utils.envs import lazy_instance_by_fliename +from paddlerec.core.utils.envs import lazy_instance_by_fliename if len(sys.argv) != 4: raise ValueError("reader only accept 3 argument: 1. reader_class 2.train/evaluate 3.yaml_abs_path") diff --git a/fleet_rec/core/utils/envs.py b/core/utils/envs.py similarity index 96% rename from fleet_rec/core/utils/envs.py rename to core/utils/envs.py index 93c9a68af6ba31906b0a387fc52bd2e943ab010a..740b8ad5b23848f9e559d9090e3f738e54b6234b 100755 --- a/fleet_rec/core/utils/envs.py +++ b/core/utils/envs.py @@ -93,9 +93,9 @@ def update_workspace(): return # is fleet inner models - if workspace.startswith("fleetrec."): + if workspace.startswith("paddlerec."): fleet_package = get_runtime_environ("PACKAGE_BASE") - workspace_dir = workspace.split("fleetrec.")[1].replace(".", "/") + workspace_dir = workspace.split("paddlerec.")[1].replace(".", "/") path = os.path.join(fleet_package, workspace_dir) else: path = workspace @@ -127,7 +127,7 @@ def pretty_print_envs(envs, header=None): if header: draws += h_format.format(header[0], header[1]) else: - draws += h_format.format("fleetrec Global Envs", "Value") + draws += h_format.format("paddlerec Global Envs", "Value") draws += line + "\n" diff --git a/fleet_rec/core/utils/fs.py b/core/utils/fs.py similarity index 100% rename from fleet_rec/core/utils/fs.py rename to core/utils/fs.py diff --git a/fleet_rec/core/utils/table.py b/core/utils/table.py similarity index 100% rename from fleet_rec/core/utils/table.py rename to core/utils/table.py diff --git a/fleet_rec/core/utils/util.py b/core/utils/util.py similarity index 99% rename from fleet_rec/core/utils/util.py rename to core/utils/util.py index b410df1d8bb31fe7e613925e2a731d61924d8835..76965f50ccf298dd63bd67e1c74d4f3d23c2fe72 100755 --- a/fleet_rec/core/utils/util.py +++ b/core/utils/util.py @@ -17,7 +17,7 @@ import time import datetime from paddle import fluid -from fleetrec.core.utils import fs as fs +from paddlerec.core.utils import fs as fs def save_program_proto(path, program=None): diff --git a/fleet_rec/check.py b/fleet_rec/check.py deleted file mode 100755 index d4177fd8516e4f210d1897601fb88f4b11773875..0000000000000000000000000000000000000000 --- a/fleet_rec/check.py +++ /dev/null @@ -1,8 +0,0 @@ -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='fleet-rec check') - parser.add_argument("--model", type=str) - parser.add_argument("--engine", type=str) - - print("coming soon") diff --git a/models/contentunderstanding/tagspace/config.yaml b/models/contentunderstanding/tagspace/config.yaml index ce0283e24e0298d20f6d1b5de7200009eb9c9f35..e0232f3faf221be04a6f95a6b0d011d612d0e35c 100644 --- a/models/contentunderstanding/tagspace/config.yaml +++ b/models/contentunderstanding/tagspace/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.tagspace" + workspace: "paddlerec.models.rank.tagspace" reader: batch_size: 5 diff --git a/models/contentunderstanding/tagspace/model.py b/models/contentunderstanding/tagspace/model.py index 46d3d7529db7a689d2769642014e25ea6e3b8142..aeb569c6b712fe10e698c2def30ae17004ac7e9a 100644 --- a/models/contentunderstanding/tagspace/model.py +++ b/models/contentunderstanding/tagspace/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import paddle.fluid as fluid import paddle.fluid.layers.nn as nn diff --git a/models/contentunderstanding/tagspace/reader.py b/models/contentunderstanding/tagspace/reader.py index 7803d862ed4f55890e810073f5a561e81be4f5e8..93446e712f204c7caaf59c10d116e89e818ae78f 100644 --- a/models/contentunderstanding/tagspace/reader.py +++ b/models/contentunderstanding/tagspace/reader.py @@ -10,8 +10,8 @@ import paddle import csv import io -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): def init(self): diff --git a/models/contentunderstanding/text_classification/config.yaml b/models/contentunderstanding/text_classification/config.yaml index 2104a6131523fe20540e243e1b42d82ae81a800f..9bdfffe18f78b725f14a01ceedba9b6ec8e140da 100644 --- a/models/contentunderstanding/text_classification/config.yaml +++ b/models/contentunderstanding/text_classification/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.text_classification" + workspace: "paddlerec.models.rank.text_classification" reader: batch_size: 5 diff --git a/models/contentunderstanding/text_classification/model.py b/models/contentunderstanding/text_classification/model.py index 0218ab8c48c1508db623a27a73e611ba20257a36..b47dc44686f7424789ea6318e2f492faeaa5cfff 100644 --- a/models/contentunderstanding/text_classification/model.py +++ b/models/contentunderstanding/text_classification/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import paddle.fluid as fluid import paddle.fluid.layers.nn as nn diff --git a/models/contentunderstanding/text_classification/reader.py b/models/contentunderstanding/text_classification/reader.py index 36d07b2c553e452a23b8d53ce4553a8f5245b84a..efee0196833064db236c0a88458ea1ce34094313 100644 --- a/models/contentunderstanding/text_classification/reader.py +++ b/models/contentunderstanding/text_classification/reader.py @@ -10,8 +10,8 @@ import paddle import csv import io -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): def init(self): diff --git a/models/match/dssm/config.yaml b/models/match/dssm/config.yaml index 95d93d7843e77c407e6dc335301027b99e7c34b4..22881bdf906178a87f4b89091589523533eac934 100755 --- a/models/match/dssm/config.yaml +++ b/models/match/dssm/config.yaml @@ -23,7 +23,7 @@ train: strategy: "async" epochs: 4 - workspace: "fleetrec.models.match.dssm" + workspace: "paddlerec.models.match.dssm" reader: batch_size: 4 diff --git a/models/match/dssm/model.py b/models/match/dssm/model.py index 414270524bc13987a2ad7f724cc6a78d3b19cbcf..ef77a8f01ebb2e0cf13551001ecb8f071f1ace2f 100755 --- a/models/match/dssm/model.py +++ b/models/match/dssm/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/match/dssm/synthetic_evaluate_reader.py b/models/match/dssm/synthetic_evaluate_reader.py index de49774084607a4d37406ad8667c8476a040f42b..1be5b13d48e17dfca1a0c11736074e80aadcbdc7 100755 --- a/models/match/dssm/synthetic_evaluate_reader.py +++ b/models/match/dssm/synthetic_evaluate_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/match/dssm/synthetic_reader.py b/models/match/dssm/synthetic_reader.py index 6eca228fd24da3603f69a822b81370f4a49fba13..c4cfaf6dab61b4122a86e946dc004847c70465fb 100755 --- a/models/match/dssm/synthetic_reader.py +++ b/models/match/dssm/synthetic_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/match/multiview-simnet/config.yaml b/models/match/multiview-simnet/config.yaml index 0fcc1fc2f18ed2588d4f9eb235a1dd619e99f337..8399b40d4074f64d63e74e9cc0573ac6b422082c 100755 --- a/models/match/multiview-simnet/config.yaml +++ b/models/match/multiview-simnet/config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. evaluate: - workspace: "fleetrec.models.recall.multiview-simnet" + workspace: "paddlerec.models.recall.multiview-simnet" reader: batch_size: 2 class: "{workspace}/evaluate_reader.py" @@ -24,7 +24,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.multiview-simnet" + workspace: "paddlerec.models.recall.multiview-simnet" reader: batch_size: 2 diff --git a/models/match/multiview-simnet/evaluate_reader.py b/models/match/multiview-simnet/evaluate_reader.py index 63340ccd003589d6e4411f08ed8ffa554ee170fa..f7752e47825e358aee6ab5bc8bcd6b6d71fb539e 100755 --- a/models/match/multiview-simnet/evaluate_reader.py +++ b/models/match/multiview-simnet/evaluate_reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/match/multiview-simnet/model.py b/models/match/multiview-simnet/model.py index c33c10033c55a2d95ce14ac7755eceba8a3a7dd1..aa8d8db676155a3839496aa8096acc35cb784e90 100755 --- a/models/match/multiview-simnet/model.py +++ b/models/match/multiview-simnet/model.py @@ -19,8 +19,8 @@ import paddle.fluid.layers as layers import paddle.fluid.layers.tensor as tensor import paddle.fluid.layers.control_flow as cf -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class BowEncoder(object): """ bow-encoder """ diff --git a/models/match/multiview-simnet/reader.py b/models/match/multiview-simnet/reader.py index 34cabd415617bbbc5d4cfc942c5a48406e228d3d..5616547ee167707dd8d1cf6ab81d57f96152e023 100755 --- a/models/match/multiview-simnet/reader.py +++ b/models/match/multiview-simnet/reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/multitask/esmm/config.yaml b/models/multitask/esmm/config.yaml index 5cc16a5d14ee077d962db1de39d7f4f405a43649..a83c145c9779661a6bb35fb177ec5a53aa4ad56e 100644 --- a/models/multitask/esmm/config.yaml +++ b/models/multitask/esmm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.multitask.esmm" + workspace: "paddlerec.models.multitask.esmm" reader: batch_size: 2 diff --git a/models/multitask/esmm/esmm_reader.py b/models/multitask/esmm/esmm_reader.py index a18702d39ae922ffd881c11966df0dfe8b713f51..f7aef643cd3db5c01c1cc3a632a99fa3fa342658 100644 --- a/models/multitask/esmm/esmm_reader.py +++ b/models/multitask/esmm/esmm_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs from collections import defaultdict import numpy as np diff --git a/models/multitask/esmm/model.py b/models/multitask/esmm/model.py index 0063cdfefd7abebb315750d13fa723207fb5eb08..6654f3379d80fc944046be4840a43c4b31a03a20 100644 --- a/models/multitask/esmm/model.py +++ b/models/multitask/esmm/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import numpy as np diff --git a/models/multitask/mmoe/census_reader.py b/models/multitask/mmoe/census_reader.py index 323c15a40a29070285b89351612b1efdf162cd2c..30997c632302c2a6e963099c9e2679978c9d9b61 100644 --- a/models/multitask/mmoe/census_reader.py +++ b/models/multitask/mmoe/census_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import numpy as np diff --git a/models/multitask/mmoe/config.yaml b/models/multitask/mmoe/config.yaml index ca6552d8045761792b362fa3345e886cde396abb..4f79fbd8981346e59c1904d80fcc9aa4c67bc3ac 100644 --- a/models/multitask/mmoe/config.yaml +++ b/models/multitask/mmoe/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.multitask.mmoe" + workspace: "paddlerec.models.multitask.mmoe" reader: batch_size: 2 diff --git a/models/multitask/mmoe/model.py b/models/multitask/mmoe/model.py index 0b757d0270e7f5159bf4154e66b8e321fb9d9885..bdbbba090a5e236b23c12d9bbfc3cc4a3c13b173 100644 --- a/models/multitask/mmoe/model.py +++ b/models/multitask/mmoe/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/multitask/share-bottom/census_reader.py b/models/multitask/share-bottom/census_reader.py index 323c15a40a29070285b89351612b1efdf162cd2c..30997c632302c2a6e963099c9e2679978c9d9b61 100644 --- a/models/multitask/share-bottom/census_reader.py +++ b/models/multitask/share-bottom/census_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import numpy as np diff --git a/models/multitask/share-bottom/config.yaml b/models/multitask/share-bottom/config.yaml index 432ebdf331f098580dfad2019c05af3e4e47553d..f0ace882c3531ce05bee8af954220b84070f918b 100644 --- a/models/multitask/share-bottom/config.yaml +++ b/models/multitask/share-bottom/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.multitask.share-bottom" + workspace: "paddlerec.models.multitask.share-bottom" reader: batch_size: 2 diff --git a/models/multitask/share-bottom/model.py b/models/multitask/share-bottom/model.py index a364f1b82b9f4a6849e67cd1ff34fa8d179b475c..7328695f2aa4cad154b84aa7c298c93d26d961cb 100644 --- a/models/multitask/share-bottom/model.py +++ b/models/multitask/share-bottom/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/criteo_reader.py b/models/rank/criteo_reader.py index c4930c8ddc628c13f5c958743673bd42c507161f..3d8fcf96a4be04f60fb9928e4db2190b306813ff 100755 --- a/models/rank/criteo_reader.py +++ b/models/rank/criteo_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/rank/dcn/config.yaml b/models/rank/dcn/config.yaml index d84adb748ae460512a5c31fa898adfda1a88da56..3ba01f89d82bab93ac8176788134802b911cc645 100755 --- a/models/rank/dcn/config.yaml +++ b/models/rank/dcn/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.dcn" + workspace: "paddlerec.models.rank.dcn" reader: batch_size: 2 diff --git a/models/rank/dcn/criteo_reader.py b/models/rank/dcn/criteo_reader.py index 4b81e1fb66b6fac2ead7583ad93451e4822077d2..a526d0fcbf2ce303dcc537b1ed8a6b0f50b73506 100755 --- a/models/rank/dcn/criteo_reader.py +++ b/models/rank/dcn/criteo_reader.py @@ -15,8 +15,8 @@ from __future__ import print_function import math import sys -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/dcn/data/download.py b/models/rank/dcn/data/download.py index b862988aef1802d8bfeb2554b7dce812e941f9e9..d9bcc6df296068cfd5cd9fc1c91165f11b580d04 100755 --- a/models/rank/dcn/data/download.py +++ b/models/rank/dcn/data/download.py @@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") sys.path.append(TOOLS_PATH) -from fleetrec.tools.tools import download_file_and_uncompress +from paddlerec.tools.tools import download_file_and_uncompress if __name__ == '__main__': trainfile = 'train.txt' diff --git a/models/rank/dcn/model.py b/models/rank/dcn/model.py index c0395c27318cda335dd8271e28df1f8b0a9193b9..5009f3eac9c9f48db5604b2e1a7e631306e35950 100755 --- a/models/rank/dcn/model.py +++ b/models/rank/dcn/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase from collections import OrderedDict class Model(ModelBase): diff --git a/models/rank/deepfm/config.yaml b/models/rank/deepfm/config.yaml index fd06406db481fdfb8e00aefecda7ca42a3c89353..debaf2f0c3f5af5bd406661cfeb0137c46cb64c6 100755 --- a/models/rank/deepfm/config.yaml +++ b/models/rank/deepfm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.deepfm" + workspace: "paddlerec.models.rank.deepfm" reader: batch_size: 2 diff --git a/models/rank/deepfm/criteo_reader.py b/models/rank/deepfm/criteo_reader.py index a4c2587a507cd1744383c725e5a237950abffd42..ae942e0a82eec7d90b0dab618310dc900985a4ee 100755 --- a/models/rank/deepfm/criteo_reader.py +++ b/models/rank/deepfm/criteo_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/deepfm/data/download_preprocess.py b/models/rank/deepfm/data/download_preprocess.py index 1f114bcfb7ec7ce5c93e676bb0467026eedf6f33..e8c94cc64728a5e3ae38a29bf419fc90b55df597 100755 --- a/models/rank/deepfm/data/download_preprocess.py +++ b/models/rank/deepfm/data/download_preprocess.py @@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") sys.path.append(TOOLS_PATH) -from fleetrec.tools.tools import download_file_and_uncompress, download_file +from paddlerec.tools.tools import download_file_and_uncompress, download_file if __name__ == '__main__': url = "https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz" diff --git a/models/rank/deepfm/model.py b/models/rank/deepfm/model.py index bd5e80e56de81cce0c92379a05797ca81027ac23..d91eef24b7fcf4d8e61f4d465ebcc2519db38678 100755 --- a/models/rank/deepfm/model.py +++ b/models/rank/deepfm/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/din/config.yaml b/models/rank/din/config.yaml index 1244c1639aae1065cde0c9e109a38b728af9ea3e..bdf56b825e54bfbe161fa5e711d7401a25e3b8ee 100755 --- a/models/rank/din/config.yaml +++ b/models/rank/din/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.din" + workspace: "paddlerec.models.rank.din" reader: batch_size: 2 diff --git a/models/rank/din/model.py b/models/rank/din/model.py index 1d7bcca35064912ca4596fed9755755413c54c3b..68585c8e512005aa942d088baeaa7be7b91c0cd4 100755 --- a/models/rank/din/model.py +++ b/models/rank/din/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/din/reader.py b/models/rank/din/reader.py index dfaec833986d161b6706e101854afb3da00fa2e2..d1742bddd8bbb5b98bdecaad0f43f438701cc480 100755 --- a/models/rank/din/reader.py +++ b/models/rank/din/reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import numpy as np import os import random diff --git a/models/rank/dnn/backend.yaml b/models/rank/dnn/backend.yaml index 60529faf9fc02bd7cb7a182dc0e2ff2a12ad8af2..77384db41d4ab9163ec7b3dcc196358341f7a51a 100755 --- a/models/rank/dnn/backend.yaml +++ b/models/rank/dnn/backend.yaml @@ -13,7 +13,7 @@ # limitations under the License. engine: - workspace: "fleetrec.models.rank.dnn" + workspace: "paddlerec.models.rank.dnn" backend: "MPI" hdfs: @@ -24,7 +24,7 @@ engine: package: build_script: "{workspace}/package.sh" python: "/home/tangwei/fleet_rec_env/cpython-2.7.11-ucs4" - paddlerec: "/home/tangwei/fleet_rec_env/FleetRec" + paddlerec: "/home/tangwei/fleet_rec_env/PaddleRec" submit: hpc: "/home/tangwei/Plines/client/smart_client_khan/" diff --git a/models/rank/dnn/config.yaml b/models/rank/dnn/config.yaml index 66f5053a0f2364a9038e6b9b5f3ccaad6481d6ca..ba2a5ac307c98193923391b22bb48d3a0aabc00f 100755 --- a/models/rank/dnn/config.yaml +++ b/models/rank/dnn/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.dnn" + workspace: "paddlerec.models.rank.dnn" reader: batch_size: 2 diff --git a/models/rank/dnn/model.py b/models/rank/dnn/model.py index 71f88d627213d8ddab2f3ecaa97e105553d4a99a..7a5ed0d421d851852bbe80ae9461c4dc6bbc6627 100755 --- a/models/rank/dnn/model.py +++ b/models/rank/dnn/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/dnn/worker.sh b/models/rank/dnn/worker.sh index 3ca2a1f07dd07bd45599f23a60dd4c47f2c72700..9daf7488d86116460ec4857aa7a752b6af07abd0 100644 --- a/models/rank/dnn/worker.sh +++ b/models/rank/dnn/worker.sh @@ -56,7 +56,7 @@ function env_prepare() { python -c "print('heheda')" - mpirun -npernode 1 python/bin/python -m pip uninstall -y fleet-rec + mpirun -npernode 1 python/bin/python -m pip uninstall -y paddle-rec mpirun -npernode 1 python/bin/python -m pip install whl/fleet_rec-0.0.2-py2-none-any.whl --index-url=http://pip.baidu.com/pypi/simple --trusted-host pip.baidu.com check_error } @@ -64,7 +64,7 @@ function env_prepare() { function run() { echo "run" g_run_stage="run" - mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python/bin/python -u -m fleetrec.run -m fleetrec.models.rank.dnn --engine cluster --role worker + mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python/bin/python -u -m paddlerec.run -m paddlerec.models.rank.dnn --engine cluster --role worker } function main() { diff --git a/models/rank/wide_deep/config.yaml b/models/rank/wide_deep/config.yaml index 4608715acc8ea8e1f02c2b623edd129c2453dd31..916561dadf610e42462b007462506e6b4f9108e4 100755 --- a/models/rank/wide_deep/config.yaml +++ b/models/rank/wide_deep/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.wide_deep" + workspace: "paddlerec.models.rank.wide_deep" reader: batch_size: 2 diff --git a/models/rank/wide_deep/model.py b/models/rank/wide_deep/model.py index c260c90180b017eb777198203b6552d84730a038..7efaccf0b6dc3933488089b70df074cbc50ac790 100755 --- a/models/rank/wide_deep/model.py +++ b/models/rank/wide_deep/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/rank/wide_deep/reader.py b/models/rank/wide_deep/reader.py index acb6d3cc21ce553f4f876622b3f0a3f749f619c0..945dd4a5084ea0eb0ef4b9cd21d640d95f29c5e1 100755 --- a/models/rank/wide_deep/reader.py +++ b/models/rank/wide_deep/reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/xdeepfm/config.yaml b/models/rank/xdeepfm/config.yaml index a93b61fbce0d835b478c86f0e572bc5c88ab6138..029ef394adc69717e6ad5579a7f95848035d7b71 100755 --- a/models/rank/xdeepfm/config.yaml +++ b/models/rank/xdeepfm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 10 - workspace: "fleetrec.models.rank.xdeepfm" + workspace: "paddlerec.models.rank.xdeepfm" reader: batch_size: 2 diff --git a/models/rank/xdeepfm/criteo_reader.py b/models/rank/xdeepfm/criteo_reader.py index 1b5e4041625228f5ebaa0fefe9f2ada566a5cecb..ee2dac5da5739cfbf0b832ba5d09ecd9d878b442 100755 --- a/models/rank/xdeepfm/criteo_reader.py +++ b/models/rank/xdeepfm/criteo_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs try: import cPickle as pickle except ImportError: diff --git a/models/rank/xdeepfm/data/download.py b/models/rank/xdeepfm/data/download.py index d0483ea3f0d5ddfeb1ad5123bd91cf2d5b6e1331..4afd1ce28ec1ba99006414c6b5116178b8b28142 100755 --- a/models/rank/xdeepfm/data/download.py +++ b/models/rank/xdeepfm/data/download.py @@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") sys.path.append(TOOLS_PATH) -from fleetrec.tools.tools import download_file_and_uncompress, download_file +from paddlerec.tools.tools import download_file_and_uncompress, download_file if __name__ == '__main__': url_train = "https://paddlerec.bj.bcebos.com/xdeepfm%2Ftr" diff --git a/models/rank/xdeepfm/model.py b/models/rank/xdeepfm/model.py index 6619c78bc718674f0efd12cf841efbe85f3cd729..3a68f8439cde5879facc5d9b24538b2adc4bc740 100755 --- a/models/rank/xdeepfm/model.py +++ b/models/rank/xdeepfm/model.py @@ -1,8 +1,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/gnn/config.yaml b/models/recall/gnn/config.yaml index 19eeb9e4f8fcbf473e2ef399801fe4d2f85a468d..50c6d401153a607a88d5eba713fc439303aad868 100755 --- a/models/recall/gnn/config.yaml +++ b/models/recall/gnn/config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. evaluate: - workspace: "fleetrec.models.recall.gnn" + workspace: "paddlerec.models.recall.gnn" reader: batch_size: 50 class: "{workspace}/evaluate_reader.py" @@ -24,7 +24,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.gnn" + workspace: "paddlerec.models.recall.gnn" reader: batch_size: 100 diff --git a/models/recall/gnn/evaluate_reader.py b/models/recall/gnn/evaluate_reader.py index 113433986b79530aad1c8da8aae84e5d7ad3d60d..10d2f8e9a6ed70e3540b0b1f9038685a9a15525a 100755 --- a/models/recall/gnn/evaluate_reader.py +++ b/models/recall/gnn/evaluate_reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/recall/gnn/model.py b/models/recall/gnn/model.py index e63945e8a424257e742c911e2ae2444d234a0805..ac628ec6209e9489ac20550a7837b60c9e8e3771 100755 --- a/models/recall/gnn/model.py +++ b/models/recall/gnn/model.py @@ -17,8 +17,8 @@ import math import paddle.fluid as fluid import paddle.fluid.layers as layers -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/gnn/reader.py b/models/recall/gnn/reader.py index 89150d5d2faf241f132614ef825a409d600c7ec3..7e9433797458f795e40b881dd324521a0bce1de4 100755 --- a/models/recall/gnn/reader.py +++ b/models/recall/gnn/reader.py @@ -15,8 +15,8 @@ import numpy as np import io import copy import random -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/recall/gru4rec/config.yaml b/models/recall/gru4rec/config.yaml index 35e57e01fddf030905a4ca0ae3a94ac782ea1bb3..71c212adbb6c200d74d121a517af17d287e2c17d 100644 --- a/models/recall/gru4rec/config.yaml +++ b/models/recall/gru4rec/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.recall.gru4rec" + workspace: "paddlerec.models.recall.gru4rec" reader: batch_size: 5 diff --git a/models/recall/gru4rec/model.py b/models/recall/gru4rec/model.py index 5b461d76b17bae5535bc945c1d4c9e988eb7e1a8..3f4bff278e6af584c3a3952282a99c25b1fe0023 100644 --- a/models/recall/gru4rec/model.py +++ b/models/recall/gru4rec/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/gru4rec/rsc15_reader.py b/models/recall/gru4rec/rsc15_reader.py index fbfc4b91c749eb0c603ac335932103e2dabe83b0..eeb2441dd697bce4b3d3dc51bc64f1e2bfc0d2ab 100644 --- a/models/recall/gru4rec/rsc15_reader.py +++ b/models/recall/gru4rec/rsc15_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class TrainReader(Reader): diff --git a/models/recall/readme.md b/models/recall/readme.md index 79b95fb41dbc11e0add4889d3939d65fea136254..bb41222b788391cee8526baa702700c9accc0aa0 100755 --- a/models/recall/readme.md +++ b/models/recall/readme.md @@ -35,7 +35,7 @@ sh data_process.sh ``` ### 训练 ```shell -python -m fleetrec.run -m config.yaml -d cpu -e single +python -m paddlerec.run -m config.yaml -d cpu -e single ``` ### 预测 diff --git a/models/recall/ssr/config.yaml b/models/recall/ssr/config.yaml index 640c93b9dfa9f10cb2de4092bc7d94c45ba45b85..b6bcbffce6e144c14c22b5a72064887be1a7e025 100644 --- a/models/recall/ssr/config.yaml +++ b/models/recall/ssr/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 3 - workspace: "fleetrec.models.recall.ssr" + workspace: "paddlerec.models.recall.ssr" reader: batch_size: 5 diff --git a/models/recall/ssr/model.py b/models/recall/ssr/model.py index 5fd94cf8763d79e4cc7edd92019aae1e8bf716a2..b2fa5c89a3b859252cdfcd11b2cc558b2ee9b0aa 100644 --- a/models/recall/ssr/model.py +++ b/models/recall/ssr/model.py @@ -15,8 +15,8 @@ import math import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase import paddle.fluid.layers.tensor as tensor import paddle.fluid.layers.io as io import paddle.fluid.layers.control_flow as cf diff --git a/models/recall/ssr/ssr_reader.py b/models/recall/ssr/ssr_reader.py index ba81e01f7b84a462938383a141eeb2bfe3f816d1..e2a352650cb6d5bb09fda78d27c3925f5ce670a5 100644 --- a/models/recall/ssr/ssr_reader.py +++ b/models/recall/ssr/ssr_reader.py @@ -13,8 +13,8 @@ # limitations under the License. from __future__ import print_function -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs import random diff --git a/models/recall/tdm/config.yaml b/models/recall/tdm/config.yaml index bd870490f79646e7e250a65bda302d726ff61473..80f0678bca1c8d8277e20fa6f999dc434bbd97d4 100755 --- a/models/recall/tdm/config.yaml +++ b/models/recall/tdm/config.yaml @@ -18,7 +18,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.tdm" + workspace: "paddlerec.models.recall.tdm" reader: batch_size: 32 @@ -73,7 +73,7 @@ train: save_last: True evaluate: - workspace: "fleetrec.models.recall.tdm" + workspace: "paddlerec.models.recall.tdm" reader: batch_size: 1 class: "{workspace}/tdm_evaluate_reader.py" diff --git a/models/recall/tdm/model.py b/models/recall/tdm/model.py index ee96b260c10e85901242ac527f653e40573fefb9..0fc1adf9cd04286cdb31cc3ba08715574dbee18b 100755 --- a/models/recall/tdm/model.py +++ b/models/recall/tdm/model.py @@ -17,8 +17,8 @@ import paddle.fluid as fluid import math -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/tdm/tdm_evaluate_reader.py b/models/recall/tdm/tdm_evaluate_reader.py index 915389ba3328e3e0ba60052532f4f93eca8faa4f..cb32426685c16a2917c61c4be4d928892cb5c238 100644 --- a/models/recall/tdm/tdm_evaluate_reader.py +++ b/models/recall/tdm/tdm_evaluate_reader.py @@ -17,7 +17,7 @@ from __future__ import print_function -from fleetrec.core.reader import Reader +from paddlerec.core.reader import Reader class EvaluateReader(Reader): diff --git a/models/recall/tdm/tdm_reader.py b/models/recall/tdm/tdm_reader.py index 17413249c2adc54d450129c76ac0761e27ba27a4..5a24fbb40b350f843e37876de7707b69c07f5555 100755 --- a/models/recall/tdm/tdm_reader.py +++ b/models/recall/tdm/tdm_reader.py @@ -17,7 +17,7 @@ from __future__ import print_function -from fleetrec.core.reader import Reader +from paddlerec.core.reader import Reader class TrainReader(Reader): diff --git a/models/recall/word2vec/config.yaml b/models/recall/word2vec/config.yaml index a3591e73d447a1a51286fba295bc2eea26a097d0..fd48ef056a1dad7bcb0b992a7f3619e86d9f1cfc 100755 --- a/models/recall/word2vec/config.yaml +++ b/models/recall/word2vec/config.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. evaluate: - workspace: "fleetrec.models.recall.word2vec" + workspace: "paddlerec.models.recall.word2vec" reader: batch_size: 50 class: "{workspace}/w2v_evaluate_reader.py" @@ -25,7 +25,7 @@ train: strategy: "async" epochs: 2 - workspace: "fleetrec.models.recall.word2vec" + workspace: "paddlerec.models.recall.word2vec" reader: batch_size: 100 diff --git a/models/recall/word2vec/model.py b/models/recall/word2vec/model.py index 7899710a19740926fc43691276600611b515d1fb..ea8e0f5e81e5554668ba9ad81cc7f80718a0b3b9 100755 --- a/models/recall/word2vec/model.py +++ b/models/recall/word2vec/model.py @@ -16,8 +16,8 @@ import math import numpy as np import paddle.fluid as fluid -from fleetrec.core.utils import envs -from fleetrec.core.model import Model as ModelBase +from paddlerec.core.utils import envs +from paddlerec.core.model import Model as ModelBase class Model(ModelBase): diff --git a/models/recall/word2vec/w2v_evaluate_reader.py b/models/recall/word2vec/w2v_evaluate_reader.py index df6de9315360c1db0e6a2e4b3534a950390b6659..6a5c1e83d61e64b35ef8492ccf55399bcf1acee6 100755 --- a/models/recall/word2vec/w2v_evaluate_reader.py +++ b/models/recall/word2vec/w2v_evaluate_reader.py @@ -14,8 +14,8 @@ import numpy as np import io import six -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class EvaluateReader(Reader): diff --git a/models/recall/word2vec/w2v_reader.py b/models/recall/word2vec/w2v_reader.py index 4857a6b06e78f05503349ebaeee17f6d44d423a4..24cf2e304e1cbf81af622dde30262959cb74e74f 100755 --- a/models/recall/word2vec/w2v_reader.py +++ b/models/recall/word2vec/w2v_reader.py @@ -13,8 +13,8 @@ # limitations under the License. import numpy as np import io -from fleetrec.core.reader import Reader -from fleetrec.core.utils import envs +from paddlerec.core.reader import Reader +from paddlerec.core.utils import envs class NumpyRandomInt(object): diff --git a/readme.md b/readme.md index 6a37e1d30c5582a8ff6aa6236ec961b1ae8fa403..4b88c468ce6f0e05f4095dda6decf6ae8d80a3ef 100644 --- a/readme.md +++ b/readme.md @@ -74,7 +74,7 @@ PadlleRec以预置模型为核心,具备以下特点： - 安装方法一： ```bash - python -m pip install fleet-rec + python -m pip install paddle-rec ``` - 安装方法二 @@ -88,8 +88,8 @@ PadlleRec以预置模型为核心,具备以下特点： * 源码安装Fleet-Rec ``` - git clone https://github.com/seiriosPlus/FleetRec/ - cd FleetRec + git clone https://github.com/PaddlePaddle/PaddleRec/ + cd PaddleRec python setup.py install ``` @@ -99,16 +99,16 @@ PadlleRec以预置模型为核心,具备以下特点： ##### 单机训练 ```bash -cd FleetRec +cd PaddleRec -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d cpu \ -e single # 使用GPU资源进行训练 -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d gpu \ -e single ``` @@ -116,10 +116,10 @@ python -m fleetrec.run \ ##### 本地模拟分布式训练 ```bash -cd FleetRec +cd PaddleRec # 使用CPU资源进行训练 -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d cpu \ -e local_cluster ``` @@ -127,10 +127,10 @@ python -m fleetrec.run \ ##### 集群提交分布式训练<需要用户预先配置好集群环境，本提交命令不包含提交客户端> ```bash -cd FleetRec +cd PaddleRec -python -m fleetrec.run \ - -m fleetrec.models.rank.dnn \ +python -m paddlerec.run \ + -m paddlerec.models.rank.dnn \ -d cpu \ -e cluster ``` diff --git a/fleet_rec/run.py b/run.py similarity index 92% rename from fleet_rec/run.py rename to run.py index aa8d59655971851238df32d89c2484afc5c3fd1b..5552682b9733520f52a3c910af1c2f85b6780266 100755 --- a/fleet_rec/run.py +++ b/run.py @@ -5,9 +5,9 @@ import tempfile import yaml -from fleetrec.core.factory import TrainerFactory -from fleetrec.core.utils import envs -from fleetrec.core.utils import util +from paddlerec.core.factory import TrainerFactory +from paddlerec.core.utils import envs +from paddlerec.core.utils import util engines = {} device = ["CPU", "GPU"] @@ -117,9 +117,9 @@ def cluster_engine(args): return # is fleet inner models - if workspace.startswith("fleetrec."): + if workspace.startswith("paddlerec."): fleet_package = envs.get_runtime_environ("PACKAGE_BASE") - workspace_dir = workspace.split("fleetrec.")[1].replace(".", "/") + workspace_dir = workspace.split("paddlerec.")[1].replace(".", "/") path = os.path.join(fleet_package, workspace_dir) else: path = workspace @@ -130,7 +130,7 @@ def cluster_engine(args): cluster_envs[name] = value def master(): - from fleetrec.core.engine.cluster.cluster import ClusterEngine + from paddlerec.core.engine.cluster.cluster import ClusterEngine with open(args.backend, 'r') as rb: _envs = yaml.load(rb.read(), Loader=yaml.FullLoader) @@ -179,7 +179,7 @@ def cluster_mpi_engine(args): def local_cluster_engine(args): - from fleetrec.core.engine.local_cluster import LocalClusterEngine + from paddlerec.core.engine.local_cluster import LocalClusterEngine trainer = get_trainer_prefix(args) + "ClusterTrainer" cluster_envs = {} @@ -205,7 +205,7 @@ def local_cluster_engine(args): def local_mpi_engine(args): print("launch cluster engine with cluster to run model: {}".format(args.model)) - from fleetrec.core.engine.local_mpi import LocalMPIEngine + from paddlerec.core.engine.local_mpi import LocalMPIEngine print("use 1X1 MPI ClusterTraining at localhost to run model: {}".format(args.model)) @@ -227,9 +227,9 @@ def local_mpi_engine(args): def get_abs_model(model): - if model.startswith("fleetrec."): + if model.startswith("paddlerec."): fleet_base = envs.get_runtime_environ("PACKAGE_BASE") - workspace_dir = model.split("fleetrec.")[1].replace(".", "/") + workspace_dir = model.split("paddlerec.")[1].replace(".", "/") path = os.path.join(fleet_base, workspace_dir, "config.yaml") else: if not os.path.isfile(model): @@ -239,7 +239,7 @@ def get_abs_model(model): if __name__ == "__main__": - parser = argparse.ArgumentParser(description='fleet-rec run') + parser = argparse.ArgumentParser(description='paddle-rec run') parser.add_argument("-m", "--model", type=str) parser.add_argument("-e", "--engine", type=str, choices=["single", "local_cluster", "cluster", diff --git a/setup.py b/setup.py index 3dd1a1e9686fd447d91b3099ea14506b505aa7b5..f3e97f97b1b51c6344275a4aa96583fd79004efe 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -setup for fleet-rec. +setup for paddle-rec. """ import os from setuptools import setup, find_packages @@ -12,12 +12,12 @@ requires = [ ] about = {} -about["__title__"] = "fleet-rec" +about["__title__"] = "paddle-rec" about["__version__"] = "0.0.2" -about["__description__"] = "fleet-rec" -about["__author__"] = "seiriosPlus" -about["__author_email__"] = "tangwei12@baidu.com" -about["__url__"] = "https://github.com/seiriosPlus/FleetRec" +about["__description__"] = "paddle-rec" +about["__author__"] = "paddle-dev" +about["__author_email__"] = "paddle-dev@baidu.com" +about["__url__"] = "https://github.com/PaddlePaddle/PaddleRec" readme = "..." @@ -30,22 +30,24 @@ def run_cmd(command): def build(dirname): package_dir = os.path.dirname(os.path.abspath(__file__)) run_cmd("cp -r {}/* {}".format(package_dir, dirname)) - run_cmd("mkdir {}".format(os.path.join(dirname, "fleetrec"))) - run_cmd("mv {}/* {}".format(os.path.join(dirname, "fleet_rec"), os.path.join(dirname, "fleetrec"))) - run_cmd("mv {} {}".format(os.path.join(dirname, "doc"), os.path.join(dirname, "fleetrec"))) - run_cmd("mv {} {}".format(os.path.join(dirname, "models"), os.path.join(dirname, "fleetrec"))) - run_cmd("mv {} {}".format(os.path.join(dirname, "tools"), os.path.join(dirname, "fleetrec"))) + run_cmd("mkdir {}".format(os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "core"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "doc"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "models"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "tests"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "tools"), os.path.join(dirname, "paddlerec"))) + run_cmd("mv {} {}".format(os.path.join(dirname, "*.py"), os.path.join(dirname, "paddlerec"))) - packages = find_packages(dirname, include=('fleetrec.*')) + packages = find_packages(dirname, include=('paddlerec.*')) package_dir = {'': dirname} package_data = {} models_copy = ['data/*.txt', 'data/*/*.txt', '*.yaml', '*.sh', 'tree/*.npy', 'tree/*.txt'] engine_copy = ['*/*.sh'] for package in packages: - if package.startswith("fleetrec.models."): + if package.startswith("paddlerec.models."): package_data[package] = models_copy - if package.startswith("fleetrec.core.engine"): + if package.startswith("paddlerec.core.engine"): package_data[package] = engine_copy setup( @@ -73,11 +75,11 @@ print(''' \033[32m _ _ _ _ _ _ _ _ _ / \ / \ / \ / \ / \ / \ / \ / \ / \ -( F | L | E | E | T | - | R | E | C ) +( P | A | D | D | L | E | - | R | E | C ) \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \033[0m \033[34m Installation Complete. Congratulations! -How to use it ? Please visit our webside: https://github.com/seiriosPlus/FleetRec +How to use it ? Please visit our webside: https://github.com/PaddlePaddle/PaddleRec \033[0m ''') diff --git a/fleet_rec/tests/__init__.py b/tests/__init__.py similarity index 100% rename from fleet_rec/tests/__init__.py rename to tests/__init__.py