dce_aux.c 15.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 * Copyright 2012-15 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: AMD
 *
 */

26
#include <linux/delay.h>
27
#include <linux/slab.h>
28

29
#include "dm_services.h"
30
#include "core_types.h"
31 32
#include "dce_aux.h"
#include "dce/dce_11_0_sh_mask.h"
33
#include "dm_event_log.h"
34 35

#define CTX \
36
	aux110->base.ctx
37 38 39 40
#define REG(reg_name)\
	(aux110->regs->reg_name)

#define DC_LOGGER \
41
	engine->ctx->logger
42 43 44

#include "reg_helper.h"

45 46 47 48
#undef FN
#define FN(reg_name, field_name) \
	aux110->shift->field_name, aux110->mask->field_name

49 50 51 52
#define FROM_AUX_ENGINE(ptr) \
	container_of((ptr), struct aux_engine_dce110, base)

#define FROM_ENGINE(ptr) \
53
	FROM_AUX_ENGINE(container_of((ptr), struct dce_aux, base))
54 55

#define FROM_AUX_ENGINE_ENGINE(ptr) \
56
	container_of((ptr), struct dce_aux, base)
57 58 59 60 61 62
enum {
	AUX_INVALID_REPLY_RETRY_COUNTER = 1,
	AUX_TIMED_OUT_RETRY_COUNTER = 2,
	AUX_DEFER_RETRY_COUNTER = 6
};
static void release_engine(
63
	struct dce_aux *engine)
64
{
65
	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
66 67 68 69 70 71 72 73 74 75 76 77

	dal_ddc_close(engine->ddc);

	engine->ddc = NULL;

	REG_UPDATE(AUX_ARB_CONTROL, AUX_SW_DONE_USING_AUX_REG, 1);
}

#define SW_CAN_ACCESS_AUX 1
#define DMCU_CAN_ACCESS_AUX 2

static bool is_engine_available(
78
	struct dce_aux *engine)
79 80 81 82 83 84 85 86 87 88 89 90
{
	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);

	uint32_t value = REG_READ(AUX_ARB_CONTROL);
	uint32_t field = get_reg_field_value(
			value,
			AUX_ARB_CONTROL,
			AUX_REG_RW_CNTL_STATUS);

	return (field != DMCU_CAN_ACCESS_AUX);
}
static bool acquire_engine(
91
	struct dce_aux *engine)
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
{
	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);

	uint32_t value = REG_READ(AUX_ARB_CONTROL);
	uint32_t field = get_reg_field_value(
			value,
			AUX_ARB_CONTROL,
			AUX_REG_RW_CNTL_STATUS);
	if (field == DMCU_CAN_ACCESS_AUX)
		return false;
	/* enable AUX before request SW to access AUX */
	value = REG_READ(AUX_CONTROL);
	field = get_reg_field_value(value,
				AUX_CONTROL,
				AUX_EN);

	if (field == 0) {
		set_reg_field_value(
				value,
				1,
				AUX_CONTROL,
				AUX_EN);

		if (REG(AUX_RESET_MASK)) {
			/*DP_AUX block as part of the enable sequence*/
			set_reg_field_value(
				value,
				1,
				AUX_CONTROL,
				AUX_RESET);
		}

		REG_WRITE(AUX_CONTROL, value);

		if (REG(AUX_RESET_MASK)) {
			/*poll HW to make sure reset it done*/

			REG_WAIT(AUX_CONTROL, AUX_RESET_DONE, 1,
					1, 11);

			set_reg_field_value(
				value,
				0,
				AUX_CONTROL,
				AUX_RESET);

			REG_WRITE(AUX_CONTROL, value);

			REG_WAIT(AUX_CONTROL, AUX_RESET_DONE, 0,
					1, 11);
		}
	} /*if (field)*/

	/* request SW to access AUX */
	REG_UPDATE(AUX_ARB_CONTROL, AUX_SW_USE_AUX_REG_REQ, 1);

	value = REG_READ(AUX_ARB_CONTROL);
	field = get_reg_field_value(
			value,
			AUX_ARB_CONTROL,
			AUX_REG_RW_CNTL_STATUS);

	return (field == SW_CAN_ACCESS_AUX);
}

#define COMPOSE_AUX_SW_DATA_16_20(command, address) \
	((command) | ((0xF0000 & (address)) >> 16))

#define COMPOSE_AUX_SW_DATA_8_15(address) \
	((0xFF00 & (address)) >> 8)

#define COMPOSE_AUX_SW_DATA_0_7(address) \
	(0xFF & (address))

static void submit_channel_request(
167
	struct dce_aux *engine,
168 169 170 171 172 173 174 175 176 177 178 179 180 181
	struct aux_request_transaction_data *request)
{
	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
	uint32_t value;
	uint32_t length;

	bool is_write =
		((request->type == AUX_TRANSACTION_TYPE_DP) &&
		 (request->action == I2CAUX_TRANSACTION_ACTION_DP_WRITE)) ||
		((request->type == AUX_TRANSACTION_TYPE_I2C) &&
		((request->action == I2CAUX_TRANSACTION_ACTION_I2C_WRITE) ||
		 (request->action == I2CAUX_TRANSACTION_ACTION_I2C_WRITE_MOT)));
	if (REG(AUXN_IMPCAL)) {
		/* clear_aux_error */
182 183 184
		REG_UPDATE_SEQ_2(AUXN_IMPCAL,
				AUXN_CALOUT_ERROR_AK, 1,
				AUXN_CALOUT_ERROR_AK, 0);
185

186 187 188
		REG_UPDATE_SEQ_2(AUXP_IMPCAL,
				AUXP_CALOUT_ERROR_AK, 1,
				AUXP_CALOUT_ERROR_AK, 0);
189 190

		/* force_default_calibrate */
191
		REG_UPDATE_SEQ_2(AUXN_IMPCAL,
192 193 194 195 196
				AUXN_IMPCAL_ENABLE, 1,
				AUXN_IMPCAL_OVERRIDE_ENABLE, 0);

		/* bug? why AUXN update EN and OVERRIDE_EN 1 by 1 while AUX P toggles OVERRIDE? */

197 198 199
		REG_UPDATE_SEQ_2(AUXP_IMPCAL,
				AUXP_IMPCAL_OVERRIDE_ENABLE, 1,
				AUXP_IMPCAL_OVERRIDE_ENABLE, 0);
200
	}
201 202 203 204 205 206

	REG_UPDATE(AUX_INTERRUPT_CONTROL, AUX_SW_DONE_ACK, 1);

	REG_WAIT(AUX_SW_STATUS, AUX_SW_DONE, 0,
				10, aux110->timeout_period/10);

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
	/* set the delay and the number of bytes to write */

	/* The length include
	 * the 4 bit header and the 20 bit address
	 * (that is 3 byte).
	 * If the requested length is non zero this means
	 * an addition byte specifying the length is required.
	 */

	length = request->length ? 4 : 3;
	if (is_write)
		length += request->length;

	REG_UPDATE_2(AUX_SW_CONTROL,
			AUX_SW_START_DELAY, request->delay,
			AUX_SW_WR_BYTES, length);

	/* program action and address and payload data (if 'is_write') */
	value = REG_UPDATE_4(AUX_SW_DATA,
			AUX_SW_INDEX, 0,
			AUX_SW_DATA_RW, 0,
			AUX_SW_AUTOINCREMENT_DISABLE, 1,
			AUX_SW_DATA, COMPOSE_AUX_SW_DATA_16_20(request->action, request->address));

	value = REG_SET_2(AUX_SW_DATA, value,
			AUX_SW_AUTOINCREMENT_DISABLE, 0,
			AUX_SW_DATA, COMPOSE_AUX_SW_DATA_8_15(request->address));

	value = REG_SET(AUX_SW_DATA, value,
			AUX_SW_DATA, COMPOSE_AUX_SW_DATA_0_7(request->address));

	if (request->length) {
		value = REG_SET(AUX_SW_DATA, value,
				AUX_SW_DATA, request->length - 1);
	}

	if (is_write) {
		/* Load the HW buffer with the Data to be sent.
		 * This is relevant for write operation.
		 * For read, the data recived data will be
		 * processed in process_channel_reply().
		 */
		uint32_t i = 0;

		while (i < request->length) {
			value = REG_SET(AUX_SW_DATA, value,
					AUX_SW_DATA, request->data[i]);

			++i;
		}
	}

	REG_UPDATE(AUX_SW_CONTROL, AUX_SW_GO, 1);
260 261
	EVENT_LOG_AUX_REQ(engine->ddc->pin_data->en, EVENT_LOG_AUX_ORIGIN_NATIVE,
					request->action, request->address, request->length, request->data);
262 263
}

264
static int read_channel_reply(struct dce_aux *engine, uint32_t size,
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
			      uint8_t *buffer, uint8_t *reply_result,
			      uint32_t *sw_status)
{
	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);
	uint32_t bytes_replied;
	uint32_t reply_result_32;

	*sw_status = REG_GET(AUX_SW_STATUS, AUX_SW_REPLY_BYTE_COUNT,
			     &bytes_replied);

	/* In case HPD is LOW, exit AUX transaction */
	if ((*sw_status & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK))
		return -1;

	/* Need at least the status byte */
	if (!bytes_replied)
		return -1;

283
	REG_UPDATE_SEQ_3(AUX_SW_DATA,
284 285 286 287 288 289
			  AUX_SW_INDEX, 0,
			  AUX_SW_AUTOINCREMENT_DISABLE, 1,
			  AUX_SW_DATA_RW, 1);

	REG_GET(AUX_SW_DATA, AUX_SW_DATA, &reply_result_32);
	reply_result_32 = reply_result_32 >> 4;
290 291
	if (reply_result != NULL)
		*reply_result = (uint8_t)reply_result_32;
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317

	if (reply_result_32 == 0) { /* ACK */
		uint32_t i = 0;

		/* First byte was already used to get the command status */
		--bytes_replied;

		/* Do not overflow buffer */
		if (bytes_replied > size)
			return -1;

		while (i < bytes_replied) {
			uint32_t aux_sw_data_val;

			REG_GET(AUX_SW_DATA, AUX_SW_DATA, &aux_sw_data_val);
			buffer[i] = aux_sw_data_val;
			++i;
		}

		return i;
	}

	return 0;
}

static enum aux_channel_operation_result get_channel_status(
318
	struct dce_aux *engine,
319 320 321 322 323 324 325 326 327 328 329 330 331 332
	uint8_t *returned_bytes)
{
	struct aux_engine_dce110 *aux110 = FROM_AUX_ENGINE(engine);

	uint32_t value;

	if (returned_bytes == NULL) {
		/*caller pass NULL pointer*/
		ASSERT_CRITICAL(false);
		return AUX_CHANNEL_OPERATION_FAILED_REASON_UNKNOWN;
	}
	*returned_bytes = 0;

	/* poll to make sure that SW_DONE is asserted */
333
	REG_WAIT(AUX_SW_STATUS, AUX_SW_DONE, 1,
334 335
				10, aux110->timeout_period/10);

336
	value = REG_READ(AUX_SW_STATUS);
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
	/* in case HPD is LOW, exit AUX transaction */
	if ((value & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK))
		return AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON;

	/* Note that the following bits are set in 'status.bits'
	 * during CTS 4.2.1.2 (FW 3.3.1):
	 * AUX_SW_RX_MIN_COUNT_VIOL, AUX_SW_RX_INVALID_STOP,
	 * AUX_SW_RX_RECV_NO_DET, AUX_SW_RX_RECV_INVALID_H.
	 *
	 * AUX_SW_RX_MIN_COUNT_VIOL is an internal,
	 * HW debugging bit and should be ignored.
	 */
	if (value & AUX_SW_STATUS__AUX_SW_DONE_MASK) {
		if ((value & AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK) ||
			(value & AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK))
			return AUX_CHANNEL_OPERATION_FAILED_TIMEOUT;

		else if ((value & AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK) ||
			(value & AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK) ||
			(value &
				AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK) ||
			(value & AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK))
			return AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY;

		*returned_bytes = get_reg_field_value(value,
				AUX_SW_STATUS,
				AUX_SW_REPLY_BYTE_COUNT);

		if (*returned_bytes == 0)
			return
			AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY;
		else {
			*returned_bytes -= 1;
			return AUX_CHANNEL_OPERATION_SUCCEEDED;
		}
	} else {
		/*time_elapsed >= aux_engine->timeout_period
		 *  AUX_SW_STATUS__AUX_SW_HPD_DISCON = at this point
		 */
		ASSERT_CRITICAL(false);
		return AUX_CHANNEL_OPERATION_FAILED_TIMEOUT;
	}
}

enum i2caux_engine_type get_engine_type(
382
		const struct dce_aux *engine)
383 384 385 386
{
	return I2CAUX_ENGINE_TYPE_AUX;
}

387
static bool acquire(
388
	struct dce_aux *engine,
389 390 391 392
	struct ddc *ddc)
{
	enum gpio_result result;

393 394
	if (!is_engine_available(engine))
		return false;
395 396 397 398 399

	result = dal_ddc_open(ddc, GPIO_MODE_HARDWARE,
		GPIO_DDC_CONFIG_TYPE_MODE_AUX);

	if (result != GPIO_RESULT_OK)
400
		return false;
401

402
	if (!acquire_engine(engine)) {
403
		dal_ddc_close(ddc);
404
		return false;
405 406 407 408
	}

	engine->ddc = ddc;

409
	return true;
410 411
}

412
void dce110_engine_destroy(struct dce_aux **engine)
413 414
{

415
	struct aux_engine_dce110 *engine110 = FROM_AUX_ENGINE(*engine);
416 417 418 419 420

	kfree(engine110);
	*engine = NULL;

}
421

422
struct dce_aux *dce110_aux_engine_construct(struct aux_engine_dce110 *aux_engine110,
423 424 425
		struct dc_context *ctx,
		uint32_t inst,
		uint32_t timeout_period,
426 427 428
		const struct dce110_aux_registers *regs,
		const struct dce110_aux_registers_mask *mask,
		const struct dce110_aux_registers_shift *shift)
429
{
430 431
	aux_engine110->base.ddc = NULL;
	aux_engine110->base.ctx = ctx;
432 433
	aux_engine110->base.delay = 0;
	aux_engine110->base.max_defer_write_retry = 0;
434
	aux_engine110->base.inst = inst;
435 436 437
	aux_engine110->timeout_period = timeout_period;
	aux_engine110->regs = regs;

438 439
	aux_engine110->mask = mask;
	aux_engine110->shift = shift;
440 441 442
	return &aux_engine110->base;
}

443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
static enum i2caux_transaction_action i2caux_action_from_payload(struct aux_payload *payload)
{
	if (payload->i2c_over_aux) {
		if (payload->write) {
			if (payload->mot)
				return I2CAUX_TRANSACTION_ACTION_I2C_WRITE_MOT;
			return I2CAUX_TRANSACTION_ACTION_I2C_WRITE;
		}
		if (payload->mot)
			return I2CAUX_TRANSACTION_ACTION_I2C_READ_MOT;
		return I2CAUX_TRANSACTION_ACTION_I2C_READ;
	}
	if (payload->write)
		return I2CAUX_TRANSACTION_ACTION_DP_WRITE;
	return I2CAUX_TRANSACTION_ACTION_DP_READ;
}

460 461 462
int dce_aux_transfer_raw(struct ddc_service *ddc,
		struct aux_payload *payload,
		enum aux_channel_operation_result *operation_result)
463 464
{
	struct ddc *ddc_pin = ddc->ddc_pin;
465
	struct dce_aux *aux_engine;
466 467 468 469 470 471 472 473 474 475
	struct aux_request_transaction_data aux_req;
	struct aux_reply_transaction_data aux_rep;
	uint8_t returned_bytes = 0;
	int res = -1;
	uint32_t status;

	memset(&aux_req, 0, sizeof(aux_req));
	memset(&aux_rep, 0, sizeof(aux_rep));

	aux_engine = ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en];
476 477
	if (!acquire(aux_engine, ddc_pin))
		return -1;
478 479 480 481 482 483 484 485 486

	if (payload->i2c_over_aux)
		aux_req.type = AUX_TRANSACTION_TYPE_I2C;
	else
		aux_req.type = AUX_TRANSACTION_TYPE_DP;

	aux_req.action = i2caux_action_from_payload(payload);

	aux_req.address = payload->address;
487
	aux_req.delay = 0;
488 489 490
	aux_req.length = payload->length;
	aux_req.data = payload->data;

491
	submit_channel_request(aux_engine, &aux_req);
492 493 494
	*operation_result = get_channel_status(aux_engine, &returned_bytes);

	if (*operation_result == AUX_CHANNEL_OPERATION_SUCCEEDED) {
495 496
		int bytes_replied = 0;
		bytes_replied = read_channel_reply(aux_engine, payload->length,
497 498
					 payload->data, payload->reply,
					 &status);
499 500 501
		EVENT_LOG_AUX_REP(aux_engine->ddc->pin_data->en,
					EVENT_LOG_AUX_ORIGIN_NATIVE, *payload->reply,
					bytes_replied, payload->data);
502 503
		res = returned_bytes;
	} else {
504 505
		res = -1;
	}
506

507
	release_engine(aux_engine);
508 509 510
	return res;
}

511 512 513 514 515
#define AUX_MAX_RETRIES 7
#define AUX_MAX_DEFER_RETRIES 7
#define AUX_MAX_I2C_DEFER_RETRIES 7
#define AUX_MAX_INVALID_REPLY_RETRIES 2
#define AUX_MAX_TIMEOUT_RETRIES 3
516 517 518 519 520 521 522

bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
		struct aux_payload *payload)
{
	int i, ret = 0;
	uint8_t reply;
	bool payload_reply = true;
523 524 525 526 527 528
	enum aux_channel_operation_result operation_result;
	int aux_ack_retries = 0,
		aux_defer_retries = 0,
		aux_i2c_defer_retries = 0,
		aux_timeout_retries = 0,
		aux_invalid_reply_retries = 0;
529 530 531 532 533 534

	if (!payload->reply) {
		payload_reply = false;
		payload->reply = &reply;
	}

535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553
	for (i = 0; i < AUX_MAX_RETRIES; i++) {
		ret = dce_aux_transfer_raw(ddc, payload, &operation_result);
		switch (operation_result) {
		case AUX_CHANNEL_OPERATION_SUCCEEDED:
			aux_timeout_retries = 0;
			aux_invalid_reply_retries = 0;

			switch (*payload->reply) {
			case AUX_TRANSACTION_REPLY_AUX_ACK:
				if (!payload->write && payload->length != ret) {
					if (++aux_ack_retries >= AUX_MAX_RETRIES)
						goto fail;
					else
						udelay(300);
				} else
					return true;
			break;

			case AUX_TRANSACTION_REPLY_AUX_DEFER:
554 555
			case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK:
			case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER:
556
				if (++aux_defer_retries >= AUX_MAX_DEFER_RETRIES) {
557
					goto fail;
558 559 560 561 562 563 564
				} else {
					if ((*payload->reply == AUX_TRANSACTION_REPLY_AUX_DEFER) ||
						(*payload->reply == AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER)) {
						if (payload->defer_delay > 0)
							msleep(payload->defer_delay);
					}
				}
565 566 567 568 569 570 571 572 573 574 575 576
				break;

			case AUX_TRANSACTION_REPLY_I2C_DEFER:
				aux_defer_retries = 0;
				if (++aux_i2c_defer_retries >= AUX_MAX_I2C_DEFER_RETRIES)
					goto fail;
				break;

			case AUX_TRANSACTION_REPLY_AUX_NACK:
			case AUX_TRANSACTION_REPLY_HPD_DISCON:
			default:
				goto fail;
577
			}
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598
			break;

		case AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY:
			if (++aux_invalid_reply_retries >= AUX_MAX_INVALID_REPLY_RETRIES)
				goto fail;
			else
				udelay(400);
			break;

		case AUX_CHANNEL_OPERATION_FAILED_TIMEOUT:
			if (++aux_timeout_retries >= AUX_MAX_TIMEOUT_RETRIES)
				goto fail;
			else {
				/*
				 * DP 1.4, 2.8.2:  AUX Transaction Response/Reply Timeouts
				 * According to the DP spec there should be 3 retries total
				 * with a 400us wait inbetween each. Hardware already waits
				 * for 550us therefore no wait is required here.
				 */
			}
			break;
599

600 601 602 603 604
		case AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON:
		case AUX_CHANNEL_OPERATION_FAILED_REASON_UNKNOWN:
		default:
			goto fail;
		}
605
	}
606 607 608 609

fail:
	if (!payload_reply)
		payload->reply = NULL;
610 611
	return false;
}