scsi: lpfc: Streamline NVME Targe6t WQE setup

To reduce latency when initializing WQE content, created templates for the
most common wqes. This reduces the number of operations taken to set the
content. It's not a lot of speed up, but every bit helps.

This patch updates the NVME target path.

[mkp: fixed typo]
Signed-off-by: NDick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: NJames Smart <james.smart@broadcom.com>
Signed-off-by: NMartin K. Petersen <martin.petersen@oracle.com>

drivers/scsi/lpfc/lpfc_crtn.h

@@ -566,6 +566,7 @@ void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba,
 				struct lpfc_iocbq *cmdiocb,
 				struct lpfc_wcqe_complete *abts_cmpl);
 void lpfc_nvme_cmd_template(void);
+void lpfc_nvmet_cmd_template(void);
 extern int lpfc_enable_nvmet_cnt;
 extern unsigned long long lpfc_enable_nvmet[];
 extern int lpfc_no_hba_reset_cnt;

drivers/scsi/lpfc/lpfc_init.c

@@ -12584,6 +12584,7 @@ lpfc_init(void)
 		return -ENOMEM;
 	}
 	lpfc_nvme_cmd_template();
+	lpfc_nvmet_cmd_template();
 
 	/* Initialize in case vector mapping is needed */
 	lpfc_used_cpu = NULL;

drivers/scsi/lpfc/lpfc_nvmet.c

@@ -74,6 +74,149 @@ static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *,
 static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *,
 				    struct lpfc_nvmet_rcv_ctx *);
 
+union lpfc_wqe128 lpfc_tsend_cmd_template;
+union lpfc_wqe128 lpfc_treceive_cmd_template;
+union lpfc_wqe128 lpfc_trsp_cmd_template;
+
+/* Setup WQE templates for NVME IOs */
+void
+lpfc_nvmet_cmd_template()
+{
+	union lpfc_wqe128 *wqe;
+
+	/* TSEND template */
+	wqe = &lpfc_tsend_cmd_template;
+	memset(wqe, 0, sizeof(union lpfc_wqe128));
+
+	/* Word 0, 1, 2 - BDE is variable */
+
+	/* Word 3 - payload_offset_len is zero */
+
+	/* Word 4 - relative_offset is variable */
+
+	/* Word 5 - is zero */
+
+	/* Word 6 - ctxt_tag, xri_tag is variable */
+
+	/* Word 7 - wqe_ar is variable */
+	bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
+	bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF);
+	bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3);
+	bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI);
+	bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
+
+	/* Word 8 - abort_tag is variable */
+
+	/* Word 9  - reqtag, rcvoxid is variable */
+
+	/* Word 10 - wqes, xc is variable */
+	bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
+	bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
+	bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
+	bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
+	bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
+	bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12);
+
+	/* Word 11 - sup, irsp, irsplen is variable */
+	bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND);
+	bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+	bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
+	bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
+	bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
+	bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0);
+
+	/* Word 12 - fcp_data_len is variable */
+
+	/* Word 13, 14, 15 - PBDE is zero */
+
+	/* TRECEIVE template */
+	wqe = &lpfc_treceive_cmd_template;
+	memset(wqe, 0, sizeof(union lpfc_wqe128));
+
+	/* Word 0, 1, 2 - BDE is variable */
+
+	/* Word 3 */
+	wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
+
+	/* Word 4 - relative_offset is variable */
+
+	/* Word 5 - is zero */
+
+	/* Word 6 - ctxt_tag, xri_tag is variable */
+
+	/* Word 7 */
+	bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE);
+	bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF);
+	bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3);
+	bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI);
+	bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
+
+	/* Word 8 - abort_tag is variable */
+
+	/* Word 9  - reqtag, rcvoxid is variable */
+
+	/* Word 10 - xc is variable */
+	bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
+	bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
+	bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
+	bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
+	bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12);
+	bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
+
+	/* Word 11 - pbde is variable */
+	bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE);
+	bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+	bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0);
+	bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
+	bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
+	bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1);
+
+	/* Word 12 - fcp_data_len is variable */
+
+	/* Word 13, 14, 15 - PBDE is variable */
+
+	/* TRSP template */
+	wqe = &lpfc_trsp_cmd_template;
+	memset(wqe, 0, sizeof(union lpfc_wqe128));
+
+	/* Word 0, 1, 2 - BDE is variable */
+
+	/* Word 3 - response_len is variable */
+
+	/* Word 4, 5 - is zero */
+
+	/* Word 6 - ctxt_tag, xri_tag is variable */
+
+	/* Word 7 */
+	bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
+	bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED);
+	bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3);
+	bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI);
+	bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */
+
+	/* Word 8 - abort_tag is variable */
+
+	/* Word 9  - reqtag is variable */
+
+	/* Word 10 wqes, xc is variable */
+	bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1);
+	bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
+	bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
+	bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0);
+	bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE);
+	bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3);
+
+	/* Word 11 irsp, irsplen is variable */
+	bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP);
+	bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
+	bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0);
+	bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
+	bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
+	bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0);
+
+	/* Word 12, 13, 14, 15 - is zero */
+}
+
 void
 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp)
 {
@@ -1152,15 +1295,9 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
 		ctx_buf->iocbq->iocb_flag = LPFC_IO_NVMET;
 		nvmewqe = ctx_buf->iocbq;
 		wqe = &nvmewqe->wqe;
+
 		/* Initialize WQE */
 		memset(wqe, 0, sizeof(union lpfc_wqe));
-		/* Word 7 */
-		bf_set(wqe_ct, &wqe->generic.wqe_com, SLI4_CT_RPI);
-		bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3);
-		/* Word 10 */
-		bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
-		bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0);
-		bf_set(wqe_qosd, &wqe->generic.wqe_com, 0);
 
 		ctx_buf->iocbq->context1 = NULL;
 		spin_lock(&phba->sli4_hba.sgl_list_lock);
@@ -2220,6 +2357,11 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 	switch (rsp->op) {
 	case NVMET_FCOP_READDATA:
 	case NVMET_FCOP_READDATA_RSP:
+		/* From the tsend template, initialize words 7 - 11 */
+		memcpy(&wqe->words[7],
+		       &lpfc_tsend_cmd_template.words[7],
+		       sizeof(uint32_t) * 5);
+
 		/* Words 0 - 2 : The first sg segment */
 		sgel = &rsp->sg[0];
 		physaddr = sg_dma_address(sgel);
@@ -2236,6 +2378,7 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		wqe->fcp_tsend.relative_offset = ctxp->offset;
 
 		/* Word 5 */
+		wqe->fcp_tsend.reserved = 0;
 
 		/* Word 6 */
 		bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
@@ -2243,10 +2386,7 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
 		       nvmewqe->sli4_xritag);
 
-		/* Word 7 */
-		bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, 1);
-		bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
-		do_pbde = 0;
+		/* Word 7 - set ar later */
 
 		/* Word 8 */
 		wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
@@ -2255,23 +2395,12 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
 		bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
 
-		/* Word 10 */
-		bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
-		bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
-		bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
-		bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com,
-		       LPFC_WQE_LENLOC_WORD12);
-		bf_set(wqe_ebde_cnt, &wqe->fcp_tsend.wqe_com, 0);
-		bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, xc);
-		bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
-		if (phba->cfg_nvme_oas)
-			bf_set(wqe_oas, &wqe->fcp_tsend.wqe_com, 1);
+		/* Word 10 - set wqes later, in template xc=1 */
+		if (!xc)
+			bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0);
 
-		/* Word 11 */
-		bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com,
-		       LPFC_WQE_CQ_ID_DEFAULT);
-		bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com,
-		       FCP_COMMAND_TSEND);
+		/* Word 11 - set sup, irsp, irsplen later */
+		do_pbde = 0;
 
 		/* Word 12 */
 		wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
@@ -2293,16 +2422,14 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		sgl++;
 		if (rsp->op == NVMET_FCOP_READDATA_RSP) {
 			atomic_inc(&tgtp->xmt_fcp_read_rsp);
-			bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
+
+			/* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
+
 			if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
 				if (ndlp->nlp_flag & NLP_SUPPRESS_RSP)
 					bf_set(wqe_sup,
 					       &wqe->fcp_tsend.wqe_com, 1);
-				bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
-				bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
-				bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
 			} else {
-				bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
 				bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
 				bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
 				bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
@@ -2313,15 +2440,17 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		} else {
 			atomic_inc(&tgtp->xmt_fcp_read);
 
-			bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
-			bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
-			bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
+			/* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
 			bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
-			bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
 		}
 		break;
 
 	case NVMET_FCOP_WRITEDATA:
+		/* From the treceive template, initialize words 3 - 11 */
+		memcpy(&wqe->words[3],
+		       &lpfc_treceive_cmd_template.words[3],
+		       sizeof(uint32_t) * 9);
+
 		/* Words 0 - 2 : The first sg segment */
 		txrdy = dma_pool_alloc(phba->txrdy_payload_pool,
 				       GFP_KERNEL, &physaddr);
@@ -2340,14 +2469,9 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		wqe->fcp_treceive.bde.addrHigh =
 			cpu_to_le32(putPaddrHigh(physaddr));
 
-		/* Word 3 */
-		wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
-
 		/* Word 4 */
 		wqe->fcp_treceive.relative_offset = ctxp->offset;
 
-		/* Word 5 */
-
 		/* Word 6 */
 		bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
 		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
@@ -2355,14 +2479,6 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		       nvmewqe->sli4_xritag);
 
 		/* Word 7 */
-		bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, 1);
-		bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
-		bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com,
-		       CMD_FCP_TRECEIVE64_WQE);
-		if (phba->nvme_embed_pbde)
-			do_pbde = 1;
-		else
-			do_pbde = 0;
 
 		/* Word 8 */
 		wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
@@ -2371,26 +2487,17 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
 		bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
 
-		/* Word 10 */
-		bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
-		bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
-		bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
-		bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com,
-		       LPFC_WQE_LENLOC_WORD12);
-		bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, xc);
-		bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
-		bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
-		bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
-		bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
-		if (phba->cfg_nvme_oas)
-			bf_set(wqe_oas, &wqe->fcp_treceive.wqe_com, 1);
+		/* Word 10 - in template xc=1 */
+		if (!xc)
+			bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0);
 
-		/* Word 11 */
-		bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com,
-		       LPFC_WQE_CQ_ID_DEFAULT);
-		bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com,
-		       FCP_COMMAND_TRECEIVE);
-		bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
+		/* Word 11 - set pbde later */
+		if (phba->nvme_embed_pbde) {
+			do_pbde = 1;
+		} else {
+			bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0);
+			do_pbde = 0;
+		}
 
 		/* Word 12 */
 		wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
@@ -2418,6 +2525,11 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		break;
 
 	case NVMET_FCOP_RSP:
+		/* From the treceive template, initialize words 4 - 11 */
+		memcpy(&wqe->words[4],
+		       &lpfc_trsp_cmd_template.words[4],
+		       sizeof(uint32_t) * 8);
+
 		/* Words 0 - 2 */
 		physaddr = rsp->rspdma;
 		wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
@@ -2430,12 +2542,6 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		/* Word 3 */
 		wqe->fcp_trsp.response_len = rsp->rsplen;
 
-		/* Word 4 */
-		wqe->fcp_trsp.rsvd_4_5[0] = 0;
-
-
-		/* Word 5 */
-
 		/* Word 6 */
 		bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
 		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
@@ -2443,10 +2549,6 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		       nvmewqe->sli4_xritag);
 
 		/* Word 7 */
-		bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, 0);
-		bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1);
-		bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
-		do_pbde = 0;
 
 		/* Word 8 */
 		wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
@@ -2456,35 +2558,23 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
 
 		/* Word 10 */
-		bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
-		bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 0);
-		bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_WRITE);
-		bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com,
-		       LPFC_WQE_LENLOC_WORD3);
-		bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, xc);
-		bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
-		if (phba->cfg_nvme_oas)
-			bf_set(wqe_oas, &wqe->fcp_trsp.wqe_com, 1);
+		if (xc)
+			bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1);
 
 		/* Word 11 */
-		bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com,
-		       LPFC_WQE_CQ_ID_DEFAULT);
-		bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com,
-		       FCP_COMMAND_TRSP);
-		bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
-
-		if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
-			/* Good response - all zero's on wire */
-			bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
-			bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
-			bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
-		} else {
+		/* In template wqes=0 irsp=0 irsplen=0 - good response */
+		if (rsp->rsplen != LPFC_NVMET_SUCCESS_LEN) {
+			/* Bad response - embed it */
 			bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
 			bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
 			bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
 			       ((rsp->rsplen >> 2) - 1));
 			memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
 		}
+		do_pbde = 0;
+
+		/* Word 12 */
+		wqe->fcp_trsp.rsvd_12_15[0] = 0;
 
 		/* Use rspbuf, NOT sg list */
 		rsp->sg_cnt = 0;
@@ -2531,11 +2621,6 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		sgl++;
 		ctxp->offset += cnt;
 	}
-
-	if (do_pbde)
-		bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
-	else
-		bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
 	ctxp->state = LPFC_NVMET_STE_DATA;
 	ctxp->entry_cnt++;
 	return nvmewqe;