提交 06a44f67 编写于 作者: T tangwei

add cluster run, rename fleet-rec to paddle-rec

上级 1c8486fc
...@@ -3,6 +3,6 @@ output ...@@ -3,6 +3,6 @@ output
.idea/ .idea/
build/ build/
dist/ dist/
fleetrec.egg-info/ paddlerec.egg-info/
*~ *~
*.pyc *.pyc
...@@ -20,9 +20,9 @@ import sys ...@@ -20,9 +20,9 @@ import sys
import os import os
import copy import copy
from fleetrec.core.engine.engine import Engine from paddlerec.core.engine.engine import Engine
from fleetrec.core.factory import TrainerFactory from paddlerec.core.factory import TrainerFactory
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class ClusterEngine(Engine): class ClusterEngine(Engine):
......
...@@ -19,8 +19,8 @@ import sys ...@@ -19,8 +19,8 @@ import sys
import os import os
import copy import copy
from fleetrec.core.engine.engine import Engine from paddlerec.core.engine.engine import Engine
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class LocalClusterEngine(Engine): class LocalClusterEngine(Engine):
def start_procs(self): def start_procs(self):
...@@ -47,7 +47,7 @@ class LocalClusterEngine(Engine): ...@@ -47,7 +47,7 @@ class LocalClusterEngine(Engine):
user_endpoints_ips = [x.split(":")[0] for x in user_endpoints.split(",")] user_endpoints_ips = [x.split(":")[0] for x in user_endpoints.split(",")]
user_endpoints_port = [x.split(":")[1] for x in user_endpoints.split(",")] user_endpoints_port = [x.split(":")[1] for x in user_endpoints.split(",")]
factory = "fleetrec.core.factory" factory = "paddlerec.core.factory"
cmd = [sys.executable, "-u", "-m", factory, self.trainer] cmd = [sys.executable, "-u", "-m", factory, self.trainer]
for i in range(server_num): for i in range(server_num):
......
...@@ -19,7 +19,7 @@ import sys ...@@ -19,7 +19,7 @@ import sys
import os import os
import copy import copy
from fleetrec.core.engine.engine import Engine from paddlerec.core.engine.engine import Engine
class LocalMPIEngine(Engine): class LocalMPIEngine(Engine):
...@@ -33,7 +33,7 @@ class LocalMPIEngine(Engine): ...@@ -33,7 +33,7 @@ class LocalMPIEngine(Engine):
procs = [] procs = []
log_fns = [] log_fns = []
factory = "fleetrec.core.factory" factory = "paddlerec.core.factory"
cmd = "mpirun -npernode 2 -timestamp-output -tag-output".split(" ") cmd = "mpirun -npernode 2 -timestamp-output -tag-output".split(" ")
cmd.extend([sys.executable, "-u", "-m", factory, self.trainer]) cmd.extend([sys.executable, "-u", "-m", factory, self.trainer])
......
...@@ -17,7 +17,7 @@ import sys ...@@ -17,7 +17,7 @@ import sys
import yaml import yaml
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
trainer_abs = os.path.join(os.path.dirname( trainer_abs = os.path.join(os.path.dirname(
os.path.abspath(__file__)), "trainers") os.path.abspath(__file__)), "trainers")
...@@ -71,7 +71,7 @@ class TrainerFactory(object): ...@@ -71,7 +71,7 @@ class TrainerFactory(object):
with open(config, 'r') as rb: with open(config, 'r') as rb:
_config = yaml.load(rb.read(), Loader=yaml.FullLoader) _config = yaml.load(rb.read(), Loader=yaml.FullLoader)
else: else:
raise ValueError("fleetrec's config only support yaml") raise ValueError("paddlerec's config only support yaml")
envs.set_global_envs(_config) envs.set_global_envs(_config)
envs.update_workspace() envs.update_workspace()
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
import math import math
import numpy as np import numpy as np
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.metric import Metric from paddlerec.core.metric import Metric
class AUCMetric(Metric): class AUCMetric(Metric):
......
...@@ -2,7 +2,7 @@ import abc ...@@ -2,7 +2,7 @@ import abc
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class Model(object): class Model(object):
......
...@@ -3,8 +3,8 @@ import copy ...@@ -3,8 +3,8 @@ import copy
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
from fleetrec.core.model import Model from paddlerec.core.model import Model
from fleetrec.core.utils import table from paddlerec.core.utils import table
def create(config): def create(config):
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
# limitations under the License. # limitations under the License.
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.layer import Layer from paddlerec.core.layer import Layer
class EmbeddingInputLayer(Layer): class EmbeddingInputLayer(Layer):
......
...@@ -19,7 +19,7 @@ import os ...@@ -19,7 +19,7 @@ import os
import paddle.fluid.incubate.data_generator as dg import paddle.fluid.incubate.data_generator as dg
import yaml import yaml
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class Reader(dg.MultiSlotDataGenerator): class Reader(dg.MultiSlotDataGenerator):
......
...@@ -20,7 +20,7 @@ import time ...@@ -20,7 +20,7 @@ import time
import yaml import yaml
from paddle import fluid from paddle import fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class Trainer(object): class Trainer(object):
......
...@@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distribu ...@@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distribu
from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker
from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer
class ClusterTrainer(TranspileTrainer): class ClusterTrainer(TranspileTrainer):
......
...@@ -18,8 +18,8 @@ import paddle.fluid as fluid ...@@ -18,8 +18,8 @@ import paddle.fluid as fluid
from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.trainer import Trainer from paddlerec.core.trainer import Trainer
class CtrPaddleTrainer(Trainer): class CtrPaddleTrainer(Trainer):
......
...@@ -24,12 +24,12 @@ from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet ...@@ -24,12 +24,12 @@ from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet
from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker
from fleetrec.core.utils import fs as fs from paddlerec.core.utils import fs as fs
from fleetrec.core.utils import util as util from paddlerec.core.utils import util as util
from fleetrec.core.metrics.auc_metrics import AUCMetric from paddlerec.core.metrics.auc_metrics import AUCMetric
from fleetrec.core.modules.modul import build as model_basic from paddlerec.core.modules.modul import build as model_basic
from fleetrec.core.utils import dataset from paddlerec.core.utils import dataset
from fleetrec.core.trainer import Trainer from paddlerec.core.trainer import Trainer
def wroker_numric_opt(value, env, opt): def wroker_numric_opt(value, env, opt):
......
...@@ -20,8 +20,8 @@ from __future__ import print_function ...@@ -20,8 +20,8 @@ from __future__ import print_function
import logging import logging
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
import numpy as np import numpy as np
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
......
...@@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import f ...@@ -25,8 +25,8 @@ from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import f
from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory
from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import PaddleCloudRoleMaker
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.trainers.cluster_trainer import ClusterTrainer from paddlerec.core.trainers.cluster_trainer import ClusterTrainer
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
......
...@@ -21,9 +21,9 @@ from __future__ import print_function ...@@ -21,9 +21,9 @@ from __future__ import print_function
import logging import logging
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.trainers.transpiler_trainer import TranspileTrainer from paddlerec.core.trainers.transpiler_trainer import TranspileTrainer
from fleetrec.core.trainers.single_trainer import SingleTrainer from paddlerec.core.trainers.single_trainer import SingleTrainer
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
import numpy as np import numpy as np
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
......
...@@ -20,9 +20,9 @@ import os ...@@ -20,9 +20,9 @@ import os
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet
from fleetrec.core.trainer import Trainer from paddlerec.core.trainer import Trainer
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.utils import dataloader_instance from paddlerec.core.utils import dataloader_instance
class TranspileTrainer(Trainer): class TranspileTrainer(Trainer):
...@@ -81,7 +81,7 @@ class TranspileTrainer(Trainer): ...@@ -81,7 +81,7 @@ class TranspileTrainer(Trainer):
pipe_cmd = "python {} {} {} {}".format( pipe_cmd = "python {} {} {} {}".format(
reader, reader_class, state, self._config_yaml) reader, reader_class, state, self._config_yaml)
if train_data_path.startswith("fleetrec::"): if train_data_path.startswith("paddlerec::"):
package_base = envs.get_runtime_environ("PACKAGE_BASE") package_base = envs.get_runtime_environ("PACKAGE_BASE")
assert package_base is not None assert package_base is not None
train_data_path = os.path.join( train_data_path = os.path.join(
......
...@@ -16,9 +16,9 @@ from __future__ import print_function ...@@ -16,9 +16,9 @@ from __future__ import print_function
import os import os
import sys import sys
from fleetrec.core.utils.envs import lazy_instance_by_fliename from paddlerec.core.utils.envs import lazy_instance_by_fliename
from fleetrec.core.utils.envs import get_global_env from paddlerec.core.utils.envs import get_global_env
from fleetrec.core.utils.envs import get_runtime_environ from paddlerec.core.utils.envs import get_runtime_environ
def dataloader(readerclass, train, yaml_file): def dataloader(readerclass, train, yaml_file):
...@@ -31,7 +31,7 @@ def dataloader(readerclass, train, yaml_file): ...@@ -31,7 +31,7 @@ def dataloader(readerclass, train, yaml_file):
namespace = "evaluate.reader" namespace = "evaluate.reader"
data_path = get_global_env("test_data_path", None, namespace) data_path = get_global_env("test_data_path", None, namespace)
if data_path.startswith("fleetrec::"): if data_path.startswith("paddlerec::"):
package_base = get_runtime_environ("PACKAGE_BASE") package_base = get_runtime_environ("PACKAGE_BASE")
assert package_base is not None assert package_base is not None
data_path = os.path.join(package_base, data_path.split("::")[1]) data_path = os.path.join(package_base, data_path.split("::")[1])
......
...@@ -18,8 +18,8 @@ import datetime ...@@ -18,8 +18,8 @@ import datetime
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import fs as fs from paddlerec.core.utils import fs as fs
from fleetrec.core.utils import util as util from paddlerec.core.utils import util as util
class Dataset(object): class Dataset(object):
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
from __future__ import print_function from __future__ import print_function
import sys import sys
from fleetrec.core.utils.envs import lazy_instance_by_fliename from paddlerec.core.utils.envs import lazy_instance_by_fliename
if len(sys.argv) != 4: if len(sys.argv) != 4:
raise ValueError("reader only accept 3 argument: 1. reader_class 2.train/evaluate 3.yaml_abs_path") raise ValueError("reader only accept 3 argument: 1. reader_class 2.train/evaluate 3.yaml_abs_path")
......
...@@ -93,9 +93,9 @@ def update_workspace(): ...@@ -93,9 +93,9 @@ def update_workspace():
return return
# is fleet inner models # is fleet inner models
if workspace.startswith("fleetrec."): if workspace.startswith("paddlerec."):
fleet_package = get_runtime_environ("PACKAGE_BASE") fleet_package = get_runtime_environ("PACKAGE_BASE")
workspace_dir = workspace.split("fleetrec.")[1].replace(".", "/") workspace_dir = workspace.split("paddlerec.")[1].replace(".", "/")
path = os.path.join(fleet_package, workspace_dir) path = os.path.join(fleet_package, workspace_dir)
else: else:
path = workspace path = workspace
...@@ -127,7 +127,7 @@ def pretty_print_envs(envs, header=None): ...@@ -127,7 +127,7 @@ def pretty_print_envs(envs, header=None):
if header: if header:
draws += h_format.format(header[0], header[1]) draws += h_format.format(header[0], header[1])
else: else:
draws += h_format.format("fleetrec Global Envs", "Value") draws += h_format.format("paddlerec Global Envs", "Value")
draws += line + "\n" draws += line + "\n"
......
...@@ -17,7 +17,7 @@ import time ...@@ -17,7 +17,7 @@ import time
import datetime import datetime
from paddle import fluid from paddle import fluid
from fleetrec.core.utils import fs as fs from paddlerec.core.utils import fs as fs
def save_program_proto(path, program=None): def save_program_proto(path, program=None):
......
import argparse
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='fleet-rec check')
parser.add_argument("--model", type=str)
parser.add_argument("--engine", type=str)
print("coming soon")
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.tagspace" workspace: "paddlerec.models.rank.tagspace"
reader: reader:
batch_size: 5 batch_size: 5
......
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.layers.nn as nn import paddle.fluid.layers.nn as nn
......
...@@ -10,8 +10,8 @@ import paddle ...@@ -10,8 +10,8 @@ import paddle
import csv import csv
import io import io
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class TrainReader(Reader): class TrainReader(Reader):
def init(self): def init(self):
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.text_classification" workspace: "paddlerec.models.rank.text_classification"
reader: reader:
batch_size: 5 batch_size: 5
......
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.layers.nn as nn import paddle.fluid.layers.nn as nn
......
...@@ -10,8 +10,8 @@ import paddle ...@@ -10,8 +10,8 @@ import paddle
import csv import csv
import io import io
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class TrainReader(Reader): class TrainReader(Reader):
def init(self): def init(self):
......
...@@ -23,7 +23,7 @@ train: ...@@ -23,7 +23,7 @@ train:
strategy: "async" strategy: "async"
epochs: 4 epochs: 4
workspace: "fleetrec.models.match.dssm" workspace: "paddlerec.models.match.dssm"
reader: reader:
batch_size: 4 batch_size: 4
......
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
import math import math
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class EvaluateReader(Reader): class EvaluateReader(Reader):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class TrainReader(Reader): class TrainReader(Reader):
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
evaluate: evaluate:
workspace: "fleetrec.models.recall.multiview-simnet" workspace: "paddlerec.models.recall.multiview-simnet"
reader: reader:
batch_size: 2 batch_size: 2
class: "{workspace}/evaluate_reader.py" class: "{workspace}/evaluate_reader.py"
...@@ -24,7 +24,7 @@ train: ...@@ -24,7 +24,7 @@ train:
strategy: "async" strategy: "async"
epochs: 2 epochs: 2
workspace: "fleetrec.models.recall.multiview-simnet" workspace: "paddlerec.models.recall.multiview-simnet"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -15,8 +15,8 @@ import numpy as np ...@@ -15,8 +15,8 @@ import numpy as np
import io import io
import copy import copy
import random import random
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class EvaluateReader(Reader): class EvaluateReader(Reader):
......
...@@ -19,8 +19,8 @@ import paddle.fluid.layers as layers ...@@ -19,8 +19,8 @@ import paddle.fluid.layers as layers
import paddle.fluid.layers.tensor as tensor import paddle.fluid.layers.tensor as tensor
import paddle.fluid.layers.control_flow as cf import paddle.fluid.layers.control_flow as cf
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class BowEncoder(object): class BowEncoder(object):
""" bow-encoder """ """ bow-encoder """
......
...@@ -15,8 +15,8 @@ import numpy as np ...@@ -15,8 +15,8 @@ import numpy as np
import io import io
import copy import copy
import random import random
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class TrainReader(Reader): class TrainReader(Reader):
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 3 epochs: 3
workspace: "fleetrec.models.multitask.esmm" workspace: "paddlerec.models.multitask.esmm"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from collections import defaultdict from collections import defaultdict
import numpy as np import numpy as np
......
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
import math import math
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
import numpy as np import numpy as np
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
import numpy as np import numpy as np
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 3 epochs: 3
workspace: "fleetrec.models.multitask.mmoe" workspace: "paddlerec.models.multitask.mmoe"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
import math import math
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
import numpy as np import numpy as np
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 3 epochs: 3
workspace: "fleetrec.models.multitask.share-bottom" workspace: "paddlerec.models.multitask.share-bottom"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
import math import math
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class TrainReader(Reader): class TrainReader(Reader):
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.dcn" workspace: "paddlerec.models.rank.dcn"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -15,8 +15,8 @@ from __future__ import print_function ...@@ -15,8 +15,8 @@ from __future__ import print_function
import math import math
import sys import sys
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
try: try:
import cPickle as pickle import cPickle as pickle
except ImportError: except ImportError:
......
...@@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) ...@@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools")
sys.path.append(TOOLS_PATH) sys.path.append(TOOLS_PATH)
from fleetrec.tools.tools import download_file_and_uncompress from paddlerec.tools.tools import download_file_and_uncompress
if __name__ == '__main__': if __name__ == '__main__':
trainfile = 'train.txt' trainfile = 'train.txt'
......
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
from collections import OrderedDict from collections import OrderedDict
class Model(ModelBase): class Model(ModelBase):
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.deepfm" workspace: "paddlerec.models.rank.deepfm"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
try: try:
import cPickle as pickle import cPickle as pickle
except ImportError: except ImportError:
......
...@@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) ...@@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools")
sys.path.append(TOOLS_PATH) sys.path.append(TOOLS_PATH)
from fleetrec.tools.tools import download_file_and_uncompress, download_file from paddlerec.tools.tools import download_file_and_uncompress, download_file
if __name__ == '__main__': if __name__ == '__main__':
url = "https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz" url = "https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz"
......
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.din" workspace: "paddlerec.models.rank.din"
reader: reader:
batch_size: 2 batch_size: 2
......
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
import numpy as np import numpy as np
import os import os
import random import random
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
# limitations under the License. # limitations under the License.
engine: engine:
workspace: "fleetrec.models.rank.dnn" workspace: "paddlerec.models.rank.dnn"
backend: "MPI" backend: "MPI"
hdfs: hdfs:
...@@ -24,7 +24,7 @@ engine: ...@@ -24,7 +24,7 @@ engine:
package: package:
build_script: "{workspace}/package.sh" build_script: "{workspace}/package.sh"
python: "/home/tangwei/fleet_rec_env/cpython-2.7.11-ucs4" python: "/home/tangwei/fleet_rec_env/cpython-2.7.11-ucs4"
paddlerec: "/home/tangwei/fleet_rec_env/FleetRec" paddlerec: "/home/tangwei/fleet_rec_env/PaddleRec"
submit: submit:
hpc: "/home/tangwei/Plines/client/smart_client_khan/" hpc: "/home/tangwei/Plines/client/smart_client_khan/"
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.dnn" workspace: "paddlerec.models.rank.dnn"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
import math import math
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -56,7 +56,7 @@ function env_prepare() { ...@@ -56,7 +56,7 @@ function env_prepare() {
python -c "print('heheda')" python -c "print('heheda')"
mpirun -npernode 1 python/bin/python -m pip uninstall -y fleet-rec mpirun -npernode 1 python/bin/python -m pip uninstall -y paddle-rec
mpirun -npernode 1 python/bin/python -m pip install whl/fleet_rec-0.0.2-py2-none-any.whl --index-url=http://pip.baidu.com/pypi/simple --trusted-host pip.baidu.com mpirun -npernode 1 python/bin/python -m pip install whl/fleet_rec-0.0.2-py2-none-any.whl --index-url=http://pip.baidu.com/pypi/simple --trusted-host pip.baidu.com
check_error check_error
} }
...@@ -64,7 +64,7 @@ function env_prepare() { ...@@ -64,7 +64,7 @@ function env_prepare() {
function run() { function run() {
echo "run" echo "run"
g_run_stage="run" g_run_stage="run"
mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python/bin/python -u -m fleetrec.run -m fleetrec.models.rank.dnn --engine cluster --role worker mpirun -npernode 2 -timestamp-output -tag-output -machinefile ${PBS_NODEFILE} python/bin/python -u -m paddlerec.run -m paddlerec.models.rank.dnn --engine cluster --role worker
} }
function main() { function main() {
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.wide_deep" workspace: "paddlerec.models.rank.wide_deep"
reader: reader:
batch_size: 2 batch_size: 2
......
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
try: try:
import cPickle as pickle import cPickle as pickle
except ImportError: except ImportError:
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 10 epochs: 10
workspace: "fleetrec.models.rank.xdeepfm" workspace: "paddlerec.models.rank.xdeepfm"
reader: reader:
batch_size: 2 batch_size: 2
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
try: try:
import cPickle as pickle import cPickle as pickle
except ImportError: except ImportError:
......
...@@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) ...@@ -6,7 +6,7 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools") TOOLS_PATH = os.path.join(LOCAL_PATH, "..", "..", "tools")
sys.path.append(TOOLS_PATH) sys.path.append(TOOLS_PATH)
from fleetrec.tools.tools import download_file_and_uncompress, download_file from paddlerec.tools.tools import download_file_and_uncompress, download_file
if __name__ == '__main__': if __name__ == '__main__':
url_train = "https://paddlerec.bj.bcebos.com/xdeepfm%2Ftr" url_train = "https://paddlerec.bj.bcebos.com/xdeepfm%2Ftr"
......
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
evaluate: evaluate:
workspace: "fleetrec.models.recall.gnn" workspace: "paddlerec.models.recall.gnn"
reader: reader:
batch_size: 50 batch_size: 50
class: "{workspace}/evaluate_reader.py" class: "{workspace}/evaluate_reader.py"
...@@ -24,7 +24,7 @@ train: ...@@ -24,7 +24,7 @@ train:
strategy: "async" strategy: "async"
epochs: 2 epochs: 2
workspace: "fleetrec.models.recall.gnn" workspace: "paddlerec.models.recall.gnn"
reader: reader:
batch_size: 100 batch_size: 100
......
...@@ -15,8 +15,8 @@ import numpy as np ...@@ -15,8 +15,8 @@ import numpy as np
import io import io
import copy import copy
import random import random
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class EvaluateReader(Reader): class EvaluateReader(Reader):
......
...@@ -17,8 +17,8 @@ import math ...@@ -17,8 +17,8 @@ import math
import paddle.fluid as fluid import paddle.fluid as fluid
import paddle.fluid.layers as layers import paddle.fluid.layers as layers
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -15,8 +15,8 @@ import numpy as np ...@@ -15,8 +15,8 @@ import numpy as np
import io import io
import copy import copy
import random import random
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class TrainReader(Reader): class TrainReader(Reader):
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 3 epochs: 3
workspace: "fleetrec.models.recall.gru4rec" workspace: "paddlerec.models.recall.gru4rec"
reader: reader:
batch_size: 5 batch_size: 5
......
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
import math import math
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class TrainReader(Reader): class TrainReader(Reader):
......
...@@ -35,7 +35,7 @@ sh data_process.sh ...@@ -35,7 +35,7 @@ sh data_process.sh
``` ```
### 训练 ### 训练
```shell ```shell
python -m fleetrec.run -m config.yaml -d cpu -e single python -m paddlerec.run -m config.yaml -d cpu -e single
``` ```
### 预测 ### 预测
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 3 epochs: 3
workspace: "fleetrec.models.recall.ssr" workspace: "paddlerec.models.recall.ssr"
reader: reader:
batch_size: 5 batch_size: 5
......
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
import math import math
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
import paddle.fluid.layers.tensor as tensor import paddle.fluid.layers.tensor as tensor
import paddle.fluid.layers.io as io import paddle.fluid.layers.io as io
import paddle.fluid.layers.control_flow as cf import paddle.fluid.layers.control_flow as cf
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
import random import random
......
...@@ -18,7 +18,7 @@ train: ...@@ -18,7 +18,7 @@ train:
strategy: "async" strategy: "async"
epochs: 2 epochs: 2
workspace: "fleetrec.models.recall.tdm" workspace: "paddlerec.models.recall.tdm"
reader: reader:
batch_size: 32 batch_size: 32
...@@ -73,7 +73,7 @@ train: ...@@ -73,7 +73,7 @@ train:
save_last: True save_last: True
evaluate: evaluate:
workspace: "fleetrec.models.recall.tdm" workspace: "paddlerec.models.recall.tdm"
reader: reader:
batch_size: 1 batch_size: 1
class: "{workspace}/tdm_evaluate_reader.py" class: "{workspace}/tdm_evaluate_reader.py"
......
...@@ -17,8 +17,8 @@ ...@@ -17,8 +17,8 @@
import paddle.fluid as fluid import paddle.fluid as fluid
import math import math
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
class EvaluateReader(Reader): class EvaluateReader(Reader):
......
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
from __future__ import print_function from __future__ import print_function
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
class TrainReader(Reader): class TrainReader(Reader):
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
evaluate: evaluate:
workspace: "fleetrec.models.recall.word2vec" workspace: "paddlerec.models.recall.word2vec"
reader: reader:
batch_size: 50 batch_size: 50
class: "{workspace}/w2v_evaluate_reader.py" class: "{workspace}/w2v_evaluate_reader.py"
...@@ -25,7 +25,7 @@ train: ...@@ -25,7 +25,7 @@ train:
strategy: "async" strategy: "async"
epochs: 2 epochs: 2
workspace: "fleetrec.models.recall.word2vec" workspace: "paddlerec.models.recall.word2vec"
reader: reader:
batch_size: 100 batch_size: 100
......
...@@ -16,8 +16,8 @@ import math ...@@ -16,8 +16,8 @@ import math
import numpy as np import numpy as np
import paddle.fluid as fluid import paddle.fluid as fluid
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.model import Model as ModelBase from paddlerec.core.model import Model as ModelBase
class Model(ModelBase): class Model(ModelBase):
......
...@@ -14,8 +14,8 @@ ...@@ -14,8 +14,8 @@
import numpy as np import numpy as np
import io import io
import six import six
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class EvaluateReader(Reader): class EvaluateReader(Reader):
......
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
import numpy as np import numpy as np
import io import io
from fleetrec.core.reader import Reader from paddlerec.core.reader import Reader
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
class NumpyRandomInt(object): class NumpyRandomInt(object):
......
...@@ -74,7 +74,7 @@ PadlleRec以预置模型为核心,具备以下特点: ...@@ -74,7 +74,7 @@ PadlleRec以预置模型为核心,具备以下特点:
- 安装方法一<PIP源直接安装> - 安装方法一<PIP源直接安装>
```bash ```bash
python -m pip install fleet-rec python -m pip install paddle-rec
``` ```
- 安装方法二 - 安装方法二
...@@ -88,8 +88,8 @@ PadlleRec以预置模型为核心,具备以下特点: ...@@ -88,8 +88,8 @@ PadlleRec以预置模型为核心,具备以下特点:
* 源码安装Fleet-Rec * 源码安装Fleet-Rec
``` ```
git clone https://github.com/seiriosPlus/FleetRec/ git clone https://github.com/PaddlePaddle/PaddleRec/
cd FleetRec cd PaddleRec
python setup.py install python setup.py install
``` ```
...@@ -99,16 +99,16 @@ PadlleRec以预置模型为核心,具备以下特点: ...@@ -99,16 +99,16 @@ PadlleRec以预置模型为核心,具备以下特点:
##### 单机训练 ##### 单机训练
```bash ```bash
cd FleetRec cd PaddleRec
python -m fleetrec.run \ python -m paddlerec.run \
-m fleetrec.models.rank.dnn \ -m paddlerec.models.rank.dnn \
-d cpu \ -d cpu \
-e single -e single
# 使用GPU资源进行训练 # 使用GPU资源进行训练
python -m fleetrec.run \ python -m paddlerec.run \
-m fleetrec.models.rank.dnn \ -m paddlerec.models.rank.dnn \
-d gpu \ -d gpu \
-e single -e single
``` ```
...@@ -116,10 +116,10 @@ python -m fleetrec.run \ ...@@ -116,10 +116,10 @@ python -m fleetrec.run \
##### 本地模拟分布式训练 ##### 本地模拟分布式训练
```bash ```bash
cd FleetRec cd PaddleRec
# 使用CPU资源进行训练 # 使用CPU资源进行训练
python -m fleetrec.run \ python -m paddlerec.run \
-m fleetrec.models.rank.dnn \ -m paddlerec.models.rank.dnn \
-d cpu \ -d cpu \
-e local_cluster -e local_cluster
``` ```
...@@ -127,10 +127,10 @@ python -m fleetrec.run \ ...@@ -127,10 +127,10 @@ python -m fleetrec.run \
##### 集群提交分布式训练<需要用户预先配置好集群环境,本提交命令不包含提交客户端> ##### 集群提交分布式训练<需要用户预先配置好集群环境,本提交命令不包含提交客户端>
```bash ```bash
cd FleetRec cd PaddleRec
python -m fleetrec.run \ python -m paddlerec.run \
-m fleetrec.models.rank.dnn \ -m paddlerec.models.rank.dnn \
-d cpu \ -d cpu \
-e cluster -e cluster
``` ```
......
...@@ -5,9 +5,9 @@ import tempfile ...@@ -5,9 +5,9 @@ import tempfile
import yaml import yaml
from fleetrec.core.factory import TrainerFactory from paddlerec.core.factory import TrainerFactory
from fleetrec.core.utils import envs from paddlerec.core.utils import envs
from fleetrec.core.utils import util from paddlerec.core.utils import util
engines = {} engines = {}
device = ["CPU", "GPU"] device = ["CPU", "GPU"]
...@@ -117,9 +117,9 @@ def cluster_engine(args): ...@@ -117,9 +117,9 @@ def cluster_engine(args):
return return
# is fleet inner models # is fleet inner models
if workspace.startswith("fleetrec."): if workspace.startswith("paddlerec."):
fleet_package = envs.get_runtime_environ("PACKAGE_BASE") fleet_package = envs.get_runtime_environ("PACKAGE_BASE")
workspace_dir = workspace.split("fleetrec.")[1].replace(".", "/") workspace_dir = workspace.split("paddlerec.")[1].replace(".", "/")
path = os.path.join(fleet_package, workspace_dir) path = os.path.join(fleet_package, workspace_dir)
else: else:
path = workspace path = workspace
...@@ -130,7 +130,7 @@ def cluster_engine(args): ...@@ -130,7 +130,7 @@ def cluster_engine(args):
cluster_envs[name] = value cluster_envs[name] = value
def master(): def master():
from fleetrec.core.engine.cluster.cluster import ClusterEngine from paddlerec.core.engine.cluster.cluster import ClusterEngine
with open(args.backend, 'r') as rb: with open(args.backend, 'r') as rb:
_envs = yaml.load(rb.read(), Loader=yaml.FullLoader) _envs = yaml.load(rb.read(), Loader=yaml.FullLoader)
...@@ -179,7 +179,7 @@ def cluster_mpi_engine(args): ...@@ -179,7 +179,7 @@ def cluster_mpi_engine(args):
def local_cluster_engine(args): def local_cluster_engine(args):
from fleetrec.core.engine.local_cluster import LocalClusterEngine from paddlerec.core.engine.local_cluster import LocalClusterEngine
trainer = get_trainer_prefix(args) + "ClusterTrainer" trainer = get_trainer_prefix(args) + "ClusterTrainer"
cluster_envs = {} cluster_envs = {}
...@@ -205,7 +205,7 @@ def local_cluster_engine(args): ...@@ -205,7 +205,7 @@ def local_cluster_engine(args):
def local_mpi_engine(args): def local_mpi_engine(args):
print("launch cluster engine with cluster to run model: {}".format(args.model)) print("launch cluster engine with cluster to run model: {}".format(args.model))
from fleetrec.core.engine.local_mpi import LocalMPIEngine from paddlerec.core.engine.local_mpi import LocalMPIEngine
print("use 1X1 MPI ClusterTraining at localhost to run model: {}".format(args.model)) print("use 1X1 MPI ClusterTraining at localhost to run model: {}".format(args.model))
...@@ -227,9 +227,9 @@ def local_mpi_engine(args): ...@@ -227,9 +227,9 @@ def local_mpi_engine(args):
def get_abs_model(model): def get_abs_model(model):
if model.startswith("fleetrec."): if model.startswith("paddlerec."):
fleet_base = envs.get_runtime_environ("PACKAGE_BASE") fleet_base = envs.get_runtime_environ("PACKAGE_BASE")
workspace_dir = model.split("fleetrec.")[1].replace(".", "/") workspace_dir = model.split("paddlerec.")[1].replace(".", "/")
path = os.path.join(fleet_base, workspace_dir, "config.yaml") path = os.path.join(fleet_base, workspace_dir, "config.yaml")
else: else:
if not os.path.isfile(model): if not os.path.isfile(model):
...@@ -239,7 +239,7 @@ def get_abs_model(model): ...@@ -239,7 +239,7 @@ def get_abs_model(model):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description='fleet-rec run') parser = argparse.ArgumentParser(description='paddle-rec run')
parser.add_argument("-m", "--model", type=str) parser.add_argument("-m", "--model", type=str)
parser.add_argument("-e", "--engine", type=str, parser.add_argument("-e", "--engine", type=str,
choices=["single", "local_cluster", "cluster", choices=["single", "local_cluster", "cluster",
......
""" """
setup for fleet-rec. setup for paddle-rec.
""" """
import os import os
from setuptools import setup, find_packages from setuptools import setup, find_packages
...@@ -12,12 +12,12 @@ requires = [ ...@@ -12,12 +12,12 @@ requires = [
] ]
about = {} about = {}
about["__title__"] = "fleet-rec" about["__title__"] = "paddle-rec"
about["__version__"] = "0.0.2" about["__version__"] = "0.0.2"
about["__description__"] = "fleet-rec" about["__description__"] = "paddle-rec"
about["__author__"] = "seiriosPlus" about["__author__"] = "paddle-dev"
about["__author_email__"] = "tangwei12@baidu.com" about["__author_email__"] = "paddle-dev@baidu.com"
about["__url__"] = "https://github.com/seiriosPlus/FleetRec" about["__url__"] = "https://github.com/PaddlePaddle/PaddleRec"
readme = "..." readme = "..."
...@@ -30,22 +30,24 @@ def run_cmd(command): ...@@ -30,22 +30,24 @@ def run_cmd(command):
def build(dirname): def build(dirname):
package_dir = os.path.dirname(os.path.abspath(__file__)) package_dir = os.path.dirname(os.path.abspath(__file__))
run_cmd("cp -r {}/* {}".format(package_dir, dirname)) run_cmd("cp -r {}/* {}".format(package_dir, dirname))
run_cmd("mkdir {}".format(os.path.join(dirname, "fleetrec"))) run_cmd("mkdir {}".format(os.path.join(dirname, "paddlerec")))
run_cmd("mv {}/* {}".format(os.path.join(dirname, "fleet_rec"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "core"), os.path.join(dirname, "paddlerec")))
run_cmd("mv {} {}".format(os.path.join(dirname, "doc"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "doc"), os.path.join(dirname, "paddlerec")))
run_cmd("mv {} {}".format(os.path.join(dirname, "models"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "models"), os.path.join(dirname, "paddlerec")))
run_cmd("mv {} {}".format(os.path.join(dirname, "tools"), os.path.join(dirname, "fleetrec"))) run_cmd("mv {} {}".format(os.path.join(dirname, "tests"), os.path.join(dirname, "paddlerec")))
run_cmd("mv {} {}".format(os.path.join(dirname, "tools"), os.path.join(dirname, "paddlerec")))
run_cmd("mv {} {}".format(os.path.join(dirname, "*.py"), os.path.join(dirname, "paddlerec")))
packages = find_packages(dirname, include=('fleetrec.*')) packages = find_packages(dirname, include=('paddlerec.*'))
package_dir = {'': dirname} package_dir = {'': dirname}
package_data = {} package_data = {}
models_copy = ['data/*.txt', 'data/*/*.txt', '*.yaml', '*.sh', 'tree/*.npy', 'tree/*.txt'] models_copy = ['data/*.txt', 'data/*/*.txt', '*.yaml', '*.sh', 'tree/*.npy', 'tree/*.txt']
engine_copy = ['*/*.sh'] engine_copy = ['*/*.sh']
for package in packages: for package in packages:
if package.startswith("fleetrec.models."): if package.startswith("paddlerec.models."):
package_data[package] = models_copy package_data[package] = models_copy
if package.startswith("fleetrec.core.engine"): if package.startswith("paddlerec.core.engine"):
package_data[package] = engine_copy package_data[package] = engine_copy
setup( setup(
...@@ -73,11 +75,11 @@ print(''' ...@@ -73,11 +75,11 @@ print('''
\033[32m \033[32m
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/ \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \
( F | L | E | E | T | - | R | E | C ) ( P | A | D | D | L | E | - | R | E | C )
\_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/ \_/
\033[0m \033[0m
\033[34m \033[34m
Installation Complete. Congratulations! Installation Complete. Congratulations!
How to use it ? Please visit our webside: https://github.com/seiriosPlus/FleetRec How to use it ? Please visit our webside: https://github.com/PaddlePaddle/PaddleRec
\033[0m \033[0m
''') ''')
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册