From: Roy Novich <royno@nvidia.com>
Subject: [PATCH] BACKPORT: net/sunrpc/xprtrdma/svc_rdma_rw.c

Change-Id: I2e08d1c1010bef81e0a295134ca3f82601617287
---
 net/sunrpc/xprtrdma/svc_rdma_rw.c | 446 +++++++++++++++++++++++++++++-
 1 file changed, 440 insertions(+), 6 deletions(-)

--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -12,7 +12,9 @@
 #include <linux/sunrpc/svc_rdma.h>
 
 #include "xprt_rdma.h"
+#ifdef HAVE_TRACE_RPCRDMA_H
 #include <trace/events/rpcrdma.h>
+#endif
 
 static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc);
 static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc);
@@ -71,16 +73,26 @@ svc_rdma_get_rw_ctxt(struct svcxprt_rdma
 	}
 
 	ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl;
+#ifdef HAVE_SG_ALLOC_TABLE_CHAINED_NENTS_FIRST_CHUNK_PARAM
 	if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges,
 				   ctxt->rw_sg_table.sgl,
 				   SG_CHUNK_SIZE))
+#else
+	if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges,
+#ifdef HAVE_SG_ALLOC_TABLE_CHAINED_GFP_MASK
+				   GFP_ATOMIC,
+#endif
+				   ctxt->rw_sg_table.sgl))
+#endif
 		goto out_free;
 	return ctxt;
 
 out_free:
 	kfree(ctxt);
 out_noctx:
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_no_rwctx_err(rdma, sges);
+#endif
 	return NULL;
 }
 
@@ -88,7 +100,11 @@ static void __svc_rdma_put_rw_ctxt(struc
 				   struct svc_rdma_rw_ctxt *ctxt,
 				   struct llist_head *list)
 {
+#ifdef HAVE_SG_ALLOC_TABLE_CHAINED_NENTS_FIRST_CHUNK_PARAM
 	sg_free_table_chained(&ctxt->rw_sg_table, SG_CHUNK_SIZE);
+#else
+	sg_free_table_chained(&ctxt->rw_sg_table, true);
+#endif
 	llist_add(&ctxt->rw_node, list);
 }
 
@@ -137,7 +153,9 @@ static int svc_rdma_rw_ctx_init(struct s
 			       0, offset, handle, direction);
 	if (unlikely(ret < 0)) {
 		svc_rdma_put_rw_ctxt(rdma, ctxt);
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_dma_map_rw_err(rdma, ctxt->rw_nents, ret);
+#endif
 	}
 	return ret;
 }
@@ -157,8 +175,10 @@ struct svc_rdma_chunk_ctxt {
 	struct list_head	cc_rwctxts;
 	ktime_t			cc_posttime;
 	int			cc_sqecount;
+#ifdef HAVE_SVC_RDMA_PCL
 	enum ib_wc_status	cc_status;
 	struct completion	cc_done;
+#endif
 };
 
 static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma,
@@ -214,7 +234,12 @@ static void svc_rdma_cc_release(struct s
  *  - Stores arguments for the SGL constructor functions
  */
 struct svc_rdma_write_info {
+#ifdef HAVE_SVC_RDMA_PCL
 	const struct svc_rdma_chunk	*wi_chunk;
+#else
+	unsigned int		wi_nsegs;
+	__be32			*wi_segs;
+#endif
 
 	/* write state of this chunk */
 	unsigned int		wi_seg_off;
@@ -229,8 +254,12 @@ struct svc_rdma_write_info {
 };
 
 static struct svc_rdma_write_info *
+#ifdef HAVE_SVC_RDMA_PCL
 svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma,
 			  const struct svc_rdma_chunk *chunk)
+#else
+svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, __be32 *chunk)
+#endif
 {
 	struct svc_rdma_write_info *info;
 
@@ -238,7 +267,12 @@ svc_rdma_write_info_alloc(struct svcxprt
 	if (!info)
 		return info;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	info->wi_chunk = chunk;
+#else
+	info->wi_nsegs = be32_to_cpup(++chunk);
+	info->wi_segs = ++chunk;
+#endif
 	info->wi_seg_off = 0;
 	info->wi_seg_no = 0;
 	svc_rdma_cc_init(rdma, &info->wi_cc);
@@ -268,6 +302,7 @@ static void svc_rdma_write_done(struct i
 	struct svc_rdma_write_info *info =
 			container_of(cc, struct svc_rdma_write_info, wi_cc);
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	switch (wc->status) {
 	case IB_WC_SUCCESS:
 		trace_svcrdma_wc_write(wc, &cc->cc_cid);
@@ -278,11 +313,16 @@ static void svc_rdma_write_done(struct i
 	default:
 		trace_svcrdma_wc_write_err(wc, &cc->cc_cid);
 	}
+#endif
 
 	svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount);
 
 	if (unlikely(wc->status != IB_WC_SUCCESS))
+#ifdef HAVE_SVC_XPRT_DEFERRED_CLOSE
 		svc_xprt_deferred_close(&rdma->sc_xprt);
+#else
+		set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+#endif
 
 	svc_rdma_write_info_free(info);
 }
@@ -290,11 +330,16 @@ static void svc_rdma_write_done(struct i
 /* State for pulling a Read chunk.
  */
 struct svc_rdma_read_info {
+#ifdef HAVE_SVC_RDMA_PCL
 	struct svc_rqst			*ri_rqst;
+	unsigned int			ri_totalbytes;
+#else
+	unsigned int			ri_position;
+	unsigned int			ri_chunklen;
+#endif
 	struct svc_rdma_recv_ctxt	*ri_readctxt;
 	unsigned int			ri_pageno;
 	unsigned int			ri_pageoff;
-	unsigned int			ri_totalbytes;
 
 	struct svc_rdma_chunk_ctxt	ri_cc;
 };
@@ -330,13 +375,25 @@ static void svc_rdma_wc_read_done(struct
 	struct ib_cqe *cqe = wc->wr_cqe;
 	struct svc_rdma_chunk_ctxt *cc =
 			container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
-	struct svc_rdma_read_info *info;
+#ifndef HAVE_SVC_RDMA_PCL
+	struct svcxprt_rdma *rdma = cc->cc_rdma;
+#endif
 
+#if !defined(HAVE_SVC_RDMA_PCL) || defined(HAVE_TRACE_RPCRDMA_H)
+	struct svc_rdma_read_info *info =
+			container_of(cc, struct svc_rdma_read_info, ri_cc);
+#endif
+
+#ifdef HAVE_TRACE_RPCRDMA_H
 	switch (wc->status) {
 	case IB_WC_SUCCESS:
-		info = container_of(cc, struct svc_rdma_read_info, ri_cc);
+#ifdef HAVE_SVC_RDMA_PCL
 		trace_svcrdma_wc_read(wc, &cc->cc_cid, info->ri_totalbytes,
 				      cc->cc_posttime);
+#else
+		trace_svcrdma_wc_read(wc, &cc->cc_cid, info->ri_chunklen,
+				      cc->cc_posttime);
+#endif
 		break;
 	case IB_WC_WR_FLUSH_ERR:
 		trace_svcrdma_wc_read_flush(wc, &cc->cc_cid);
@@ -344,11 +401,30 @@ static void svc_rdma_wc_read_done(struct
 	default:
 		trace_svcrdma_wc_read_err(wc, &cc->cc_cid);
 	}
+#endif
 
 	svc_rdma_wake_send_waiters(cc->cc_rdma, cc->cc_sqecount);
+#ifdef HAVE_SVC_RDMA_PCL
 	cc->cc_status = wc->status;
 	complete(&cc->cc_done);
 	return;
+#else
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+		svc_rdma_recv_ctxt_put(rdma, info->ri_readctxt);
+	} else {
+		spin_lock(&rdma->sc_rq_dto_lock);
+		list_add_tail(&info->ri_readctxt->rc_list,
+			      &rdma->sc_read_complete_q);
+		/* Note the unlock pairs with the smp_rmb in svc_xprt_ready: */
+		set_bit(XPT_DATA, &rdma->sc_xprt.xpt_flags);
+		spin_unlock(&rdma->sc_rq_dto_lock);
+
+		svc_xprt_enqueue(&rdma->sc_xprt);
+	}
+
+	svc_rdma_read_info_free(info);
+#endif
 }
 
 /* This function sleeps when the transport's Send Queue is congested.
@@ -361,6 +437,9 @@ static void svc_rdma_wc_read_done(struct
 static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc)
 {
 	struct svcxprt_rdma *rdma = cc->cc_rdma;
+#ifndef HAVE_SVC_XPRT_DEFERRED_CLOSE
+	struct svc_xprt *xprt = &rdma->sc_xprt;
+#endif
 	struct ib_send_wr *first_wr;
 	const struct ib_send_wr *bad_wr;
 	struct list_head *tmp;
@@ -392,15 +471,25 @@ static int svc_rdma_post_chunk_ctxt(stru
 		}
 
 		percpu_counter_inc(&svcrdma_stat_sq_starve);
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_sq_full(rdma);
+#endif
 		atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
 		wait_event(rdma->sc_send_wait,
 			   atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount);
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_sq_retry(rdma);
+#endif
 	} while (1);
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_sq_post_err(rdma, ret);
+#endif
+#ifdef HAVE_SVC_XPRT_DEFERRED_CLOSE
 	svc_xprt_deferred_close(&rdma->sc_xprt);
+#else
+	set_bit(XPT_CLOSE, &xprt->xpt_flags);
+#endif
 
 	/* If even one was posted, there will be a completion. */
 	if (bad_wr != first_wr)
@@ -470,10 +559,15 @@ svc_rdma_build_writes(struct svc_rdma_wr
 {
 	struct svc_rdma_chunk_ctxt *cc = &info->wi_cc;
 	struct svcxprt_rdma *rdma = cc->cc_rdma;
+#ifdef HAVE_SVC_RDMA_PCL
 	const struct svc_rdma_segment *seg;
+#else
+	__be32 *seg;
+#endif
 	struct svc_rdma_rw_ctxt *ctxt;
 	int ret;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	do {
 		unsigned int write_len;
 		u64 offset;
@@ -483,6 +577,21 @@ svc_rdma_build_writes(struct svc_rdma_wr
 
 		seg = &info->wi_chunk->ch_segments[info->wi_seg_no];
 		write_len = min(remaining, seg->rs_length - info->wi_seg_off);
+#else
+	seg = info->wi_segs + info->wi_seg_no * rpcrdma_segment_maxsz;
+	do {
+		unsigned int write_len;
+		u32 handle, length;
+		u64 offset;
+
+		if (info->wi_seg_no >= info->wi_nsegs)
+			goto out_overflow;
+
+		xdr_decode_rdma_segment(seg, &handle, &length, &offset);
+		offset += info->wi_seg_off;
+
+		write_len = min(remaining, length - info->wi_seg_off);
+#endif
 		if (!write_len)
 			goto out_overflow;
 		ctxt = svc_rdma_get_rw_ctxt(rdma,
@@ -491,8 +600,12 @@ svc_rdma_build_writes(struct svc_rdma_wr
 			return -ENOMEM;
 
 		constructor(info, write_len, ctxt);
+#ifdef HAVE_SVC_RDMA_PCL
 		offset = seg->rs_offset + info->wi_seg_off;
 		ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, seg->rs_handle,
+#else
+		ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, handle,
+#endif
 					   DMA_TO_DEVICE);
 		if (ret < 0)
 			return -EIO;
@@ -500,7 +613,12 @@ svc_rdma_build_writes(struct svc_rdma_wr
 
 		list_add(&ctxt->rw_list, &cc->cc_rwctxts);
 		cc->cc_sqecount += ret;
+#ifdef HAVE_SVC_RDMA_PCL
 		if (write_len == seg->rs_length - info->wi_seg_off) {
+#else
+		if (write_len == length - info->wi_seg_off) {
+			seg += 4;
+#endif
 			info->wi_seg_no++;
 			info->wi_seg_off = 0;
 		} else {
@@ -512,8 +630,14 @@ svc_rdma_build_writes(struct svc_rdma_wr
 	return 0;
 
 out_overflow:
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_small_wrch_err(rdma, remaining, info->wi_seg_no,
+#ifdef HAVE_SVC_RDMA_PCL
 				     info->wi_chunk->ch_segcount);
+#else
+				     info->wi_nsegs);
+#endif
+#endif
 	return -E2BIG;
 }
 
@@ -560,6 +684,7 @@ static int svc_rdma_pages_write(struct s
 				     length);
 }
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_xb_write - Construct RDMA Writes to write an xdr_buf
  * @xdr: xdr_buf to write
@@ -597,6 +722,7 @@ static int svc_rdma_xb_write(const struc
 
 	return xdr->len;
 }
+#endif
 
 /**
  * svc_rdma_send_write_chunk - Write all segments in a Write chunk
@@ -611,28 +737,54 @@ static int svc_rdma_xb_write(const struc
  *	%-ENOTCONN if posting failed (connection is lost),
  *	%-EIO if rdma_rw initialization failed (DMA mapping, etc).
  */
+#ifdef HAVE_SVC_RDMA_PCL
 int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
 			      const struct svc_rdma_chunk *chunk,
 			      const struct xdr_buf *xdr)
+#else
+int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, __be32 *wr_ch,
+			      struct xdr_buf *xdr,
+			      unsigned int offset, unsigned long length)
+#endif
 {
 	struct svc_rdma_write_info *info;
 	struct svc_rdma_chunk_ctxt *cc;
 	int ret;
 
+#ifndef HAVE_SVC_RDMA_PCL
+	if (!length)
+		return 0;
+#endif
+
+#ifdef HAVE_SVC_RDMA_PCL
 	info = svc_rdma_write_info_alloc(rdma, chunk);
+#else
+	info = svc_rdma_write_info_alloc(rdma, wr_ch);
+#endif
 	if (!info)
 		return -ENOMEM;
 	cc = &info->wi_cc;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	ret = svc_rdma_xb_write(xdr, info);
 	if (ret != xdr->len)
+#else
+	ret = svc_rdma_pages_write(info, xdr, offset, length);
+	if (ret < 0)
+#endif
 		goto out_err;
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
 	ret = svc_rdma_post_chunk_ctxt(cc);
 	if (ret < 0)
 		goto out_err;
+#ifdef HAVE_SVC_RDMA_PCL
 	return xdr->len;
+#else
+	return length;
+#endif
 
 out_err:
 	svc_rdma_write_info_free(info);
@@ -654,33 +806,77 @@ out_err:
  */
 int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
 			      const struct svc_rdma_recv_ctxt *rctxt,
+#ifdef HAVE_SVC_RDMA_PCL
 			      const struct xdr_buf *xdr)
+#else
+			      struct xdr_buf *xdr)
+#endif
 {
 	struct svc_rdma_write_info *info;
 	struct svc_rdma_chunk_ctxt *cc;
+#ifdef HAVE_SVC_RDMA_PCL
 	struct svc_rdma_chunk *chunk;
 	int ret;
+#else
+	int consumed, ret;
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 	if (pcl_is_empty(&rctxt->rc_reply_pcl))
 		return 0;
 
 	chunk = pcl_first_chunk(&rctxt->rc_reply_pcl);
 	info = svc_rdma_write_info_alloc(rdma, chunk);
+#else
+	info = svc_rdma_write_info_alloc(rdma, rctxt->rc_reply_chunk);
+#endif
 	if (!info)
 		return -ENOMEM;
 	cc = &info->wi_cc;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	ret = pcl_process_nonpayloads(&rctxt->rc_write_pcl, xdr,
 				      svc_rdma_xb_write, info);
+#else
+	ret = svc_rdma_iov_write(info, &xdr->head[0]);
+#endif
 	if (ret < 0)
 		goto out_err;
 
+#ifndef HAVE_SVC_RDMA_PCL
+	consumed = xdr->head[0].iov_len;
+
+	/* Send the page list in the Reply chunk only if the
+	 * client did not provide Write chunks.
+	 */
+	if (!rctxt->rc_write_list && xdr->page_len) {
+		ret = svc_rdma_pages_write(info, xdr, xdr->head[0].iov_len,
+					   xdr->page_len);
+		if (ret < 0)
+			goto out_err;
+		consumed += xdr->page_len;
+	}
+
+	if (xdr->tail[0].iov_len) {
+		ret = svc_rdma_iov_write(info, &xdr->tail[0]);
+		if (ret < 0)
+			goto out_err;
+		consumed += xdr->tail[0].iov_len;
+	}
+#endif
+
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
 	ret = svc_rdma_post_chunk_ctxt(cc);
 	if (ret < 0)
 		goto out_err;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	return xdr->len;
+#else
+	return consumed;
+#endif
 
 out_err:
 	svc_rdma_write_info_free(info);
@@ -699,17 +895,28 @@ out_err:
  *   %-EIO: a DMA mapping error occurred
  */
 static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
+#ifdef HAVE_SVC_RDMA_PCL
 				       const struct svc_rdma_segment *segment)
+#else
+				       struct svc_rqst *rqstp,
+				       u32 rkey, u32 len, u64 offset)
+#endif
 {
 	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
 	struct svc_rdma_chunk_ctxt *cc = &info->ri_cc;
+#ifdef HAVE_SVC_RDMA_PCL
 	struct svc_rqst *rqstp = info->ri_rqst;
 	unsigned int sge_no, seg_len, len;
+#else
+	unsigned int sge_no, seg_len;
+#endif
 	struct svc_rdma_rw_ctxt *ctxt;
 	struct scatterlist *sg;
 	int ret;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	len = segment->rs_length;
+#endif
 	sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT;
 	ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no);
 	if (!ctxt)
@@ -721,6 +928,10 @@ static int svc_rdma_build_read_segment(s
 		seg_len = min_t(unsigned int, len,
 				PAGE_SIZE - info->ri_pageoff);
 
+#ifndef HAVE_SVC_RDMA_PCL
+		head->rc_arg.pages[info->ri_pageno] =
+			rqstp->rq_pages[info->ri_pageno];
+#endif
 		if (!info->ri_pageoff)
 			head->rc_page_count++;
 
@@ -741,8 +952,13 @@ static int svc_rdma_build_read_segment(s
 			goto out_overrun;
 	}
 
+#ifdef HAVE_SVC_RDMA_PCL
 	ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, segment->rs_offset,
 				   segment->rs_handle, DMA_FROM_DEVICE);
+#else
+	ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, offset, rkey,
+				   DMA_FROM_DEVICE);
+#endif
 	if (ret < 0)
 		return -EIO;
 	percpu_counter_inc(&svcrdma_stat_read);
@@ -752,10 +968,13 @@ static int svc_rdma_build_read_segment(s
 	return 0;
 
 out_overrun:
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_page_overrun_err(cc->cc_rdma, rqstp, info->ri_pageno);
+#endif
 	return -EINVAL;
 }
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk
  * @info: context for ongoing I/O
@@ -782,7 +1001,36 @@ static int svc_rdma_build_read_chunk(str
 	}
 	return ret;
 }
+#else
+/* Walk the segments in the Read chunk starting at @p and construct
+ * RDMA Read operations to pull the chunk to the server.
+ */
+static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp,
+				     struct svc_rdma_read_info *info,
+				     __be32 *p)
+{
+	int ret;
 
+	ret = -EINVAL;
+	info->ri_chunklen = 0;
+	while (*p++ != xdr_zero && be32_to_cpup(p++) == info->ri_position) {
+		u32 handle, length;
+		u64 offset;
+
+		p = xdr_decode_rdma_segment(p, &handle, &length, &offset);
+		ret = svc_rdma_build_read_segment(info, rqstp, handle, length,
+						  offset);
+		if (ret < 0)
+			break;
+
+		info->ri_chunklen += length;
+	}
+	return ret;
+}
+#endif
+
+
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_copy_inline_range - Copy part of the inline content into pages
  * @info: context for RDMA Reads
@@ -803,7 +1051,11 @@ static int svc_rdma_copy_inline_range(st
 				      unsigned int remaining)
 {
 	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
+#ifdef HAVE_SVC_FILL_WRITE_VECTOR
 	unsigned char *dst, *src = head->rc_recv_buf;
+#else
+	unsigned char *dst, *src = page_address(head->rc_pages[0]);
+#endif
 	struct svc_rqst *rqstp = info->ri_rqst;
 	unsigned int page_no, numpages;
 
@@ -814,10 +1066,18 @@ static int svc_rdma_copy_inline_range(st
 		page_len = min_t(unsigned int, remaining,
 				 PAGE_SIZE - info->ri_pageoff);
 
+#ifndef HAVE_SVC_RDMA_PCL
+		head->rc_arg.pages[info->ri_pageno] =
+			rqstp->rq_pages[info->ri_pageno];
+#endif
 		if (!info->ri_pageoff)
 			head->rc_page_count++;
 
+#ifdef HAVE_SVC_RDMA_PCL
 		dst = page_address(rqstp->rq_pages[info->ri_pageno]);
+#else
+		dst = page_address(head->rc_arg.pages[info->ri_pageno]);
+#endif
 		memcpy(dst + info->ri_pageno, src + offset, page_len);
 
 		info->ri_totalbytes += page_len;
@@ -851,7 +1111,11 @@ static noinline int svc_rdma_read_multip
 {
 	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
 	const struct svc_rdma_pcl *pcl = &head->rc_read_pcl;
+#ifdef HAVE_SVC_RDMA_PCL
 	struct xdr_buf *buf = &info->ri_rqst->rq_arg;
+#else
+	struct xdr_buf *buf = &head->rc_arg;
+#endif
 	struct svc_rdma_chunk *chunk, *next;
 	unsigned int start, length;
 	int ret;
@@ -888,12 +1152,18 @@ static noinline int svc_rdma_read_multip
 	buf->len += info->ri_totalbytes;
 	buf->buflen += info->ri_totalbytes;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]);
-	buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes);
 	buf->pages = &info->ri_rqst->rq_pages[1];
+#else
+	head->rc_hdr_count = 1;
+	buf->head[0].iov_base = page_address(head->rc_pages[0]);
+#endif
+	buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes);
 	buf->page_len = info->ri_totalbytes - buf->head[0].iov_len;
 	return 0;
 }
+#endif
 
 /**
  * svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks
@@ -912,24 +1182,46 @@ static noinline int svc_rdma_read_multip
  *   %-ENOTCONN: posting failed (connection is lost),
  *   %-EIO: rdma_rw initialization failed (DMA mapping, etc).
  */
+#ifdef HAVE_SVC_RDMA_PCL
 static int svc_rdma_read_data_item(struct svc_rdma_read_info *info)
+#else
+static int svc_rdma_build_normal_read_chunk(struct svc_rqst *rqstp,
+					    struct svc_rdma_read_info *info,
+					    __be32 *p)
+#endif
 {
 	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
+#ifdef HAVE_SVC_RDMA_PCL
 	struct xdr_buf *buf = &info->ri_rqst->rq_arg;
 	struct svc_rdma_chunk *chunk;
 	unsigned int length;
+#endif
 	int ret;
 
+#ifndef HAVE_SVC_FILL_WRITE_VECTOR
+	info->ri_pageno = head->rc_hdr_count;
+	info->ri_pageoff = 0;
+#endif
+
+#ifdef HAVE_SVC_RDMA_PCL
 	chunk = pcl_first_chunk(&head->rc_read_pcl);
 	ret = svc_rdma_build_read_chunk(info, chunk);
+#else
+	ret = svc_rdma_build_read_chunk(rqstp, info, p);
+#endif
 	if (ret < 0)
 		goto out;
 
+#if !defined(HAVE_SVC_RDMA_PCL) && defined(HAVE_SVC_FILL_WRITE_VECTOR)
+	head->rc_hdr_count = 0;
+#endif
+
 	/* Split the Receive buffer between the head and tail
 	 * buffers at Read chunk's position. XDR roundup of the
 	 * chunk is not included in either the pagelist or in
 	 * the tail.
 	 */
+#ifdef HAVE_SVC_RDMA_PCL
 	buf->tail[0].iov_base = buf->head[0].iov_base + chunk->ch_position;
 	buf->tail[0].iov_len = buf->head[0].iov_len - chunk->ch_position;
 	buf->head[0].iov_len = chunk->ch_position;
@@ -948,11 +1240,34 @@ static int svc_rdma_read_data_item(struc
 	buf->page_len = length;
 	buf->len += length;
 	buf->buflen += length;
+#else
+	head->rc_arg.tail[0].iov_base =
+		head->rc_arg.head[0].iov_base + info->ri_position;
+	head->rc_arg.tail[0].iov_len =
+		head->rc_arg.head[0].iov_len - info->ri_position;
+	head->rc_arg.head[0].iov_len = info->ri_position;
+
+	/* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2).
+	 *
+	 * If the client already rounded up the chunk length, the
+	 * length does not change. Otherwise, the length of the page
+	 * list is increased to include XDR round-up.
+	 *
+	 * Currently these chunks always start at page offset 0,
+	 * thus the rounded-up length never crosses a page boundary.
+	 */
+	info->ri_chunklen = XDR_QUADLEN(info->ri_chunklen) << 2;
+
+	head->rc_arg.page_len = info->ri_chunklen;
+	head->rc_arg.len += info->ri_chunklen;
+	head->rc_arg.buflen += info->ri_chunklen;
+#endif
 
 out:
 	return ret;
 }
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_read_chunk_range - Build RDMA Read WQEs for portion of a chunk
  * @info: context for RDMA Reads
@@ -1050,6 +1365,7 @@ static int svc_rdma_read_call_chunk(stru
 	length = call_chunk->ch_length - start;
 	return svc_rdma_read_chunk_range(info, call_chunk, start, length);
 }
+#endif
 
 /**
  * svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message
@@ -1069,27 +1385,101 @@ static int svc_rdma_read_call_chunk(stru
  *   %-ENOTCONN: posting failed (connection is lost),
  *   %-EIO: rdma_rw initialization failed (DMA mapping, etc).
  */
+#ifdef HAVE_SVC_RDMA_PCL
 static noinline int svc_rdma_read_special(struct svc_rdma_read_info *info)
+#else
+static int svc_rdma_build_pz_read_chunk(struct svc_rqst *rqstp,
+					struct svc_rdma_read_info *info,
+					__be32 *p)
+#endif
 {
+#ifdef HAVE_SVC_RDMA_PCL
 	struct xdr_buf *buf = &info->ri_rqst->rq_arg;
+#endif
+#ifndef HAVE_SVC_RDMA_PCL
+	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
+#endif
 	int ret;
 
+#ifndef HAVE_SVC_FILL_WRITE_VECTOR
+	info->ri_pageno = head->rc_hdr_count - 1;
+	info->ri_pageoff = offset_in_page(head->rc_byte_len);
+#endif
+
+#ifdef HAVE_SVC_RDMA_PCL
 	ret = svc_rdma_read_call_chunk(info);
+#else
+	ret = svc_rdma_build_read_chunk(rqstp, info, p);
+#endif
 	if (ret < 0)
 		goto out;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	buf->len += info->ri_totalbytes;
 	buf->buflen += info->ri_totalbytes;
+#else
+	head->rc_arg.len += info->ri_chunklen;
+	head->rc_arg.buflen += info->ri_chunklen;
+#endif
 
+#ifdef HAVE_SVC_FILL_WRITE_VECTOR
+#ifdef HAVE_SVC_RDMA_PCL
 	buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]);
 	buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes);
 	buf->pages = &info->ri_rqst->rq_pages[1];
 	buf->page_len = info->ri_totalbytes - buf->head[0].iov_len;
-
+#else
+	head->rc_hdr_count = 1;
+	head->rc_arg.head[0].iov_base = page_address(head->rc_pages[0]);
+	head->rc_arg.head[0].iov_len = min_t(size_t, PAGE_SIZE,
+					     info->ri_chunklen);
+
+	head->rc_arg.page_len = info->ri_chunklen -
+				head->rc_arg.head[0].iov_len;
+#endif
+#else
+	if (head->rc_arg.buflen <= head->rc_sges[0].length) {
+		/* Transport header and RPC message fit entirely
+		 * in page where head iovec resides.
+		 */
+		head->rc_arg.head[0].iov_len = info->ri_chunklen;
+	} else {
+		/* Transport header and part of RPC message reside
+		 * in the head iovec's page.
+		 */
+		head->rc_arg.head[0].iov_len =
+			head->rc_sges[0].length - head->rc_byte_len;
+		head->rc_arg.page_len =
+			info->ri_chunklen - head->rc_arg.head[0].iov_len;
+	}
+#endif
+ 
 out:
 	return ret;
 }
 
+#ifndef HAVE_SVC_RDMA_PCL
+/* Pages under I/O have been copied to head->rc_pages. Ensure they
+ * are not released by svc_xprt_release() until the I/O is complete.
+ *
+ * This has to be done after all Read WRs are constructed to properly
+ * handle a page that is part of I/O on behalf of two different RDMA
+ * segments.
+ *
+ * Do this only if I/O has been posted. Otherwise, we do indeed want
+ * svc_xprt_release() to clean things up properly.
+ */
+static void svc_rdma_save_io_pages(struct svc_rqst *rqstp,
+				   const unsigned int start,
+				   const unsigned int num_pages)
+{
+	unsigned int i;
+
+	for (i = start; i < num_pages + start; i++)
+		rqstp->rq_pages[i] = NULL;
+}
+#endif
+
 /**
  * svc_rdma_process_read_list - Pull list of Read chunks from the client
  * @rdma: controlling RDMA transport
@@ -1113,24 +1503,52 @@ out:
  *   %-ENOTCONN: posting failed (connection is lost),
  *   %-EIO: rdma_rw initialization failed (DMA mapping, etc).
  */
+#ifdef HAVE_SVC_RDMA_PCL
 int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
 			       struct svc_rqst *rqstp,
 			       struct svc_rdma_recv_ctxt *head)
+#else
+int svc_rdma_recv_read_chunk(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp,
+			     struct svc_rdma_recv_ctxt *head, __be32 *p)
+#endif
 {
 	struct svc_rdma_read_info *info;
 	struct svc_rdma_chunk_ctxt *cc;
 	int ret;
 
+#ifndef HAVE_SVC_FILL_WRITE_VECTOR
+	head->rc_page_count = head->rc_hdr_count;
+#endif
+
+#ifndef HAVE_SVC_RDMA_PCL
+	/* The request (with page list) is constructed in
+	 * head->rc_arg. Pages involved with RDMA Read I/O are
+	 * transferred there.
+	 */
+	head->rc_arg.head[0] = rqstp->rq_arg.head[0];
+	head->rc_arg.tail[0] = rqstp->rq_arg.tail[0];
+	head->rc_arg.pages = head->rc_pages;
+	head->rc_arg.page_base = 0;
+	head->rc_arg.page_len = 0;
+	head->rc_arg.len = rqstp->rq_arg.len;
+	head->rc_arg.buflen = rqstp->rq_arg.buflen;
+#endif
+
 	info = svc_rdma_read_info_alloc(rdma);
 	if (!info)
 		return -ENOMEM;
 	cc = &info->ri_cc;
+#ifdef HAVE_SVC_RDMA_PCL
 	info->ri_rqst = rqstp;
+	info->ri_totalbytes = 0;
+#endif
 	info->ri_readctxt = head;
+#ifdef HAVE_SVC_FILL_WRITE_VECTOR
 	info->ri_pageno = 0;
 	info->ri_pageoff = 0;
-	info->ri_totalbytes = 0;
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 	if (pcl_is_empty(&head->rc_call_pcl)) {
 		if (head->rc_read_pcl.cl_count == 1)
 			ret = svc_rdma_read_data_item(info);
@@ -1138,15 +1556,27 @@ int svc_rdma_process_read_list(struct sv
 			ret = svc_rdma_read_multiple_chunks(info);
 	} else
 		ret = svc_rdma_read_special(info);
+#else
+	info->ri_position = be32_to_cpup(p + 1);
+	if (info->ri_position)
+		ret = svc_rdma_build_normal_read_chunk(rqstp, info, p);
+	else
+		ret = svc_rdma_build_pz_read_chunk(rqstp, info, p);
+#endif
 	if (ret < 0)
 		goto out_err;
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
+#ifdef HAVE_SVC_RDMA_PCL
 	init_completion(&cc->cc_done);
+#endif
 	ret = svc_rdma_post_chunk_ctxt(cc);
 	if (ret < 0)
 		goto out_err;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	ret = 1;
 	wait_for_completion(&cc->cc_done);
 	if (cc->cc_status != IB_WC_SUCCESS)
@@ -1158,6 +1588,10 @@ int svc_rdma_process_read_list(struct sv
 
 	/* Ensure svc_rdma_recv_ctxt_put() does not try to release pages */
 	head->rc_page_count = 0;
+#else
+	svc_rdma_save_io_pages(rqstp, 0, head->rc_page_count);
+	return 1;
+#endif
 
 out_err:
 	svc_rdma_read_info_free(info);
