Created by: liuliu4348

• 版本、环境信息：    1）PaddlePaddle版本：0.14    2）CPU：Intel(R) Xeon(R) CPU E5-2620 0 @ 2.00GHz    4）系统环境：k8s(v1.10.7)，镜像信息(hub.baidubce.com/paddlepaddle/paddle: 0.14.0)
• 训练信息    1）多记多卡    2）Operator信息
• 复现信息：在k8s中先启动4个service，暴露端口6174，然后再启动与之对应的4个pod，其中两个为PSERVER，两个为TRAINER，通过service的clusterIP和端口可以访问相应的pod提供的服务。 在pod中写入相应的环境变量,以第一个PSERVER的pod为例: PADDLE_TRAINING_ROLE=PSERVER PADDLE_PSERVER_IPS=10.104.213.246,10.97.10.70 PADDLE_PSERVER_PORT=6174 PADDLE_TRAINERS=2 PADDLE_CURRENT_IP=10.104.213.246

代码为官网demo示例： import os import sys, getopt import paddle import paddle.fluid as fluid

from visualdl import LogWriter

EPOCH_NUM = 30 BATCH_SIZE = 8

train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.uci_housing.train(), buf_size=500), batch_size=BATCH_SIZE)

def train(save_dir, log_dir): y = fluid.layers.data(name='y', shape=[1], dtype='float32') x = fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None)

logwriter = LogWriter(log_dir, sync_cycle=10)
with logwriter.mode("train") as writer:
    loss_scalar = writer.scalar("loss")

loss = fluid.layers.square_error_cost(input=y_predict, label=y)
avg_loss = fluid.layers.mean(loss)
opt = fluid.optimizer.SGD(learning_rate=0.001)
opt.minimize(avg_loss)

# use cpu
place = fluid.CPUPlace()
#use gpu
#place = fluid.CUDAPlace(0)
feeder = fluid.DataFeeder(place=place, feed_list=[x, y])
exe = fluid.Executor(place)

# fetch distributed training environment setting
training_role = os.getenv("PADDLE_TRAINING_ROLE", None)
port = os.getenv("PADDLE_PSERVER_PORT", "6174")
pserver_ips = os.getenv("PADDLE_PSERVER_IPS", "")
trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0"))
eplist = []
for ip in pserver_ips.split(","):
    eplist.append(':'.join([ip, port]))
pserver_endpoints = ",".join(eplist)
trainers = int(os.getenv("PADDLE_TRAINERS"))
current_endpoint = os.getenv("PADDLE_CURRENT_IP", "") + ":" + port

t = fluid.DistributeTranspiler()
t.transpile(
    trainer_id = trainer_id,
    pservers = pserver_endpoints,
    trainers = trainers)

if training_role == "PSERVER":
    pserver_prog = t.get_pserver_program(current_endpoint)
    startup_prog = t.get_startup_program(current_endpoint, pserver_prog)
    exe.run(startup_prog)
    exe.run(pserver_prog)
elif training_role == "TRAINER":
    trainer_prog = t.get_trainer_program()
    exe.run(fluid.default_startup_program())

    for epoch in range(EPOCH_NUM):
        for batch_id, batch_data in enumerate(train_reader()):
            avg_loss_value, = exe.run(trainer_prog,
                                  feed=feeder.feed(batch_data),
                                  fetch_list=[avg_loss])
            loss_scalar.add_record(batch_id + epoch * 50, avg_loss_value[0])
            if (batch_id + 1) % 10 == 0:
                print("Epoch: {0}, Batch: {1}, loss: {2}".format(
                    epoch, batch_id, avg_loss_value[0]))
    # destory the resource of current trainer node in pserver server node
    if trainer_id == 0:
        fluid.io.save_params(executor=exe, dirname=save_dir,
             main_program=trainer_prog)
    exe.close()
else:
    raise AssertionError("PADDLE_TRAINING_ROLE should be one of [TRAINER, PSERVER]")

opts, args = getopt.getopt(sys.argv[1:], "hs:l:", ["save_dir=", "log_dir="]) save_dir="./output" log_dir="./log" for op, value in opts: if op == "-h": print 'test.py -s <save_dir>' sys.exit() elif op in ("-s", "--save_dir"): save_dir=value elif op in ("-l", "--log_dir"): log_dir=value
train(save_dir, log_dir)

• 问题描述：运行第一个PSERVER时报错： block map: {'fc_0.b_0': [(0L, 1L)], 'fc_0.w_0': [(0L, 13L)]} block map: {'fc_0.w_0@GRAD': [(0L, 13L)], 'fc_0.b_0@GRAD': [(0L, 1L)]} E0315 02:21:11.156191433 77 server_chttp2.cc:38] {"created":"@1552616471.156142167","description":"No address added out of total 1 resolved","file":"src/core/ext/transport/chttp2/server/chttp2_server.cc","file_line":305,"referenced_errors":[{"created":"@1552616471.156129936","description":"Unable to configure socket","fd":10,"file":"src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":202,"referenced_errors":[{"created":"@1552616471.156123980","description":"OS Error","errno":99,"file":"src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":175,"os_error":"Cannot assign requested address","syscall":"bind"}]}]} *** Aborted at 1552616471 (unix time) try "date -d @1552616471" if you are using GNU date *** PC: @ 0x0 (unknown) *** SIGSEGV (@0x50) received by PID 25 (TID 0x7fb41cb84700) from PID 80; stack trace: *** @ 0x7fb5816ca390 (unknown) @ 0x7fb53259bf2e grpc::ServerInterface::RegisteredAsyncRequest::IssueRequest() @ 0x7fb53255060f paddle::operators::distributed::AsyncGRPCServer::TryToRegisterNewOne() @ 0x7fb532550f4e paddle::operators::distributed::AsyncGRPCServer::StartServer() @ 0x7fb53247c364 paddle::operators::RunServer() @ 0x7fb532481587 std::thread::_Impl<>::_M_run() @ 0x7fb54053fc80 (unknown) @ 0x7fb5816c06ba start_thread @ 0x7fb5813f641d clone @ 0x0 (unknown) Segmentation fault (core dumped) 另外：通过paddle_trainer 的方式在k8s上用此方式启动分布式任务是可以成功的。