From f46595d8a5a66bb0867702c9783fd522f913845c Mon Sep 17 00:00:00 2001
From: Chaitanya Pratapa <cpratapa@codeaurora.org>
Date: Thu, 18 Mar 2021 18:03:19 -0700
Subject: [PATCH 1/9] msm: ipa: page pool recycling enhancements

Added the following enhancements to page pool recycling logic.
1) Updated page pool implementation to use list instead of array.
2) Added provision to check for configurable number of list
elements when checking for free pages.

Change-Id: I01c43be5a169c1438fb29b8179854985e199d055
Signed-off-by: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa.c         |   9 +-
 drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c |  58 +++++++-
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c      | 140 ++++++++++--------
 drivers/platform/msm/ipa/ipa_v3/ipa_i.h       |  20 ++-
 4 files changed, 160 insertions(+), 67 deletions(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c
index aa2cc02cb268..ee60841a1798 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2023-2024 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/clk.h>
@@ -7359,6 +7359,8 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
 	ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc = 0;
 	ipa3_ctx->stats.page_recycle_stats[1].total_replenished = 0;
 	ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0;
+	memset(ipa3_ctx->stats.page_recycle_cnt, 0,
+		sizeof(ipa3_ctx->stats.page_recycle_cnt));
 	ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset;
 	ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control;
 	ipa3_ctx->ee = resource_p->ee;
@@ -7549,6 +7551,9 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
 	/* Enable ipa3_ctx->enable_napi_chain */
 	ipa3_ctx->enable_napi_chain = 1;
 
+	/* Initialize Page poll threshold. */
+	ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD;
+
 	/* assume clock is on in virtual/emulation mode */
 	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
 	    ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION)
@@ -8187,7 +8192,7 @@ static int get_ipa_dts_configuration(struct platform_device *pdev,
 	ipa_drv_res->ipa_mhi_dynamic_config = false;
 	ipa_drv_res->use_64_bit_dma_mask = false;
 	ipa_drv_res->use_bw_vote = false;
-	ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
+	ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ_WAN;
 	ipa_drv_res->lan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
 	ipa_drv_res->apply_rg10_wa = false;
 	ipa_drv_res->gsi_ch20_wa = false;
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
index a83c50716724..c8cda1cd4ddb 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2012-2020, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifdef CONFIG_DEBUG_FS
@@ -1421,24 +1421,38 @@ static ssize_t ipa3_read_odlstats(struct file *file, char __user *ubuf,
 	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
 }
 
+
 static ssize_t ipa3_read_page_recycle_stats(struct file *file,
 		char __user *ubuf, size_t count, loff_t *ppos)
 {
 	int nbytes;
-	int cnt = 0;
+	int cnt = 0, i = 0, k = 0;
 
 	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
 			"COAL : Total number of packets replenished =%llu\n"
+			"COAL : Number of page recycled packets  =%llu\n"
 			"COAL : Number of tmp alloc packets  =%llu\n"
 			"DEF  : Total number of packets replenished =%llu\n"
+			"DEF  : Number of page recycled packets =%llu\n"
 			"DEF  : Number of tmp alloc packets  =%llu\n",
 			ipa3_ctx->stats.page_recycle_stats[0].total_replenished,
+			ipa3_ctx->stats.page_recycle_stats[0].page_recycled,
 			ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc,
 			ipa3_ctx->stats.page_recycle_stats[1].total_replenished,
+			ipa3_ctx->stats.page_recycle_stats[1].page_recycled,
 			ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc);
 
 	cnt += nbytes;
 
+	for (k = 0; k < 2; k++) {
+		for (i = 0; i < ipa3_ctx->page_poll_threshold; i++) {
+			nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN,
+				"COMMON  : Page replenish efficiency[%d][%d]  =%llu\n",
+				k, i, ipa3_ctx->stats.page_recycle_cnt[k][i]);
+			cnt += nbytes;
+		}
+	}
+
 	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
 }
 static ssize_t ipa3_read_wstats(struct file *file, char __user *ubuf,
@@ -2804,6 +2818,41 @@ static ssize_t ipa3_write_gsi_wdi3_db_polling(struct file *file,
 	return count;
 }
 
+static ssize_t ipa3_read_page_poll_threshold(struct file *file,
+		char __user *buf, size_t count, loff_t *ppos) {
+
+	int nbytes;
+	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
+				"Page Poll Threshold = %d\n",
+				ipa3_ctx->page_poll_threshold);
+	return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
+
+}
+
+static ssize_t ipa3_write_page_poll_threshold(struct file *file,
+	const char __user *buf, size_t count, loff_t *ppos) {
+
+	int ret;
+	u8 page_poll_threshold =0;
+
+	if (count >= sizeof(dbg_buff))
+		return -EFAULT;
+
+	ret = kstrtou8_from_user(buf, count, 0, &page_poll_threshold);
+	if(ret)
+		return ret;
+
+	if(page_poll_threshold != 0 &&
+		page_poll_threshold <= IPA_PAGE_POLL_THRESHOLD_MAX)
+		ipa3_ctx->page_poll_threshold = page_poll_threshold;
+	else
+		IPAERR("Invalid value \n");
+
+	IPADBG("Updated page poll threshold = %d", ipa3_ctx->page_poll_threshold);
+
+	return count;
+}
+
 static const struct ipa3_debugfs_file debugfs_files[] = {
 	{
 		"gen_reg", IPA_READ_ONLY_MODE, NULL, {
@@ -2986,6 +3035,11 @@ static const struct ipa3_debugfs_file debugfs_files[] = {
 			.read = ipa3_read_gsi_wdi3_db_polling,
 			.write = ipa3_write_gsi_wdi3_db_polling,
 		}
+	}, {
+		"page_poll_threshold", IPA_READ_WRITE_MODE, NULL, {
+			.read = ipa3_read_page_poll_threshold,
+			.write = ipa3_write_page_poll_threshold,
+		}
 	},
 };
 
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index f5783895557e..4484a268a7a6 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/delay.h>
@@ -1130,7 +1130,8 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 		snprintf(buff, IPA_RESOURCE_NAME_MAX, "iparepwq%d",
 				sys_in->client);
 		ep->sys->repl_wq = alloc_workqueue(buff,
-				WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1);
+				WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | WQ_HIGHPRI,
+				1);
 		if (!ep->sys->repl_wq) {
 			IPAERR("failed to create rep wq for client %d\n",
 					sys_in->client);
@@ -1291,12 +1292,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 		atomic_set(&ep->sys->page_recycle_repl->pending, 0);
 		ep->sys->page_recycle_repl->capacity =
 				(ep->sys->rx_pool_sz + 1) * 2;
-
-		ep->sys->page_recycle_repl->cache =
-				kcalloc(ep->sys->page_recycle_repl->capacity,
-				sizeof(void *), GFP_KERNEL);
-		atomic_set(&ep->sys->page_recycle_repl->head_idx, 0);
-		atomic_set(&ep->sys->page_recycle_repl->tail_idx, 0);
+		INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
 		ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
 		if (!ep->sys->repl) {
 			IPAERR("failed to alloc repl for client %d\n",
@@ -2126,8 +2122,10 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys)
 			ipa_assert();
 			break;
 		}
+		INIT_LIST_HEAD(&rx_pkt->link);
 		rx_pkt->sys = sys;
-		sys->page_recycle_repl->cache[curr] = rx_pkt;
+		list_add_tail(&rx_pkt->link,
+			&sys->page_recycle_repl->page_repl_head);
 	}
 
 	return;
@@ -2195,6 +2193,34 @@ static inline void __trigger_repl_work(struct ipa3_sys_context *sys)
 	}
 }
 
+static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page
+(
+	struct ipa3_sys_context *sys,
+	u32 stats_i
+)
+{
+	struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
+	struct ipa3_rx_pkt_wrapper *tmp = NULL;
+	struct page *cur_page;
+	int i = 0;
+	u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold;
+
+	list_for_each_entry_safe(rx_pkt, tmp,
+		&sys->page_recycle_repl->page_repl_head, link) {
+		if (i == LOOP_THRESHOLD)
+			break;
+		cur_page = rx_pkt->page_data.page;
+		if (page_ref_count(cur_page) == 1) {
+			/* Found a free page. */
+			page_ref_inc(cur_page);
+			list_del_init(&rx_pkt->link);
+			++ipa3_ctx->stats.page_recycle_cnt[stats_i][i];
+			return rx_pkt;
+		}
+		i++;
+	}
+	return NULL;
+}
 
 static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 {
@@ -2202,10 +2228,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 	int ret;
 	int rx_len_cached = 0;
 	struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX];
-	u32 curr;
 	u32 curr_wq;
 	int idx = 0;
-	struct page *cur_page;
 	u32 stats_i = 0;
 
 	/* start replenish only when buffers go lower than the threshold */
@@ -2215,17 +2239,13 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 
 	spin_lock_bh(&sys->spinlock);
 	rx_len_cached = sys->len;
-	curr = atomic_read(&sys->page_recycle_repl->head_idx);
 	curr_wq = atomic_read(&sys->repl->head_idx);
 
 	while (rx_len_cached < sys->rx_pool_sz) {
-		cur_page = sys->page_recycle_repl->cache[curr]->page_data.page;
-		/* Found an idle page that can be used */
-		if (page_ref_count(cur_page) == 1) {
-			page_ref_inc(cur_page);
-			rx_pkt = sys->page_recycle_repl->cache[curr];
-			curr = (++curr == sys->page_recycle_repl->capacity) ?
-								0 : curr;
+		/* check for an idle page that can be used */
+		if ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL) {
+			ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++;
+
 		} else {
 			/*
 			 * Could not find idle page at curr index.
@@ -2275,7 +2295,6 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 		/* ensure write is done before setting head index */
 		mb();
 		atomic_set(&sys->repl->head_idx, curr_wq);
-		atomic_set(&sys->page_recycle_repl->head_idx, curr);
 		sys->len = rx_len_cached;
 	} else {
 		/* we don't expect this will happen */
@@ -2790,15 +2809,10 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data)
 {
 	struct ipa3_rx_pkt_wrapper *rx_pkt = (struct ipa3_rx_pkt_wrapper *)
 		xfer_user_data;
-	struct ipa3_sys_context *sys = rx_pkt->sys;
-	int i;
 
-	for (i = 0; i < sys->page_recycle_repl->capacity; i++)
-		if (sys->page_recycle_repl->cache[i] == rx_pkt)
-			break;
-	if (i < sys->page_recycle_repl->capacity) {
+	if (!rx_pkt->page_data.is_tmp_alloc) {
+		list_del_init(&rx_pkt->link);
 		page_ref_dec(rx_pkt->page_data.page);
-		sys->page_recycle_repl->cache[i] = NULL;
 	}
 	dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr,
 		rx_pkt->len, DMA_FROM_DEVICE);
@@ -2816,7 +2830,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
 	struct ipa3_rx_pkt_wrapper *r;
 	u32 head;
 	u32 tail;
-	int i;
 
 	/*
 	 * buffers not consumed by gsi are cleaned up using cleanup callback
@@ -2865,23 +2878,20 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
 		sys->repl = NULL;
 	}
 	if (sys->page_recycle_repl) {
-		for (i = 0; i < sys->page_recycle_repl->capacity; i++) {
-			rx_pkt = sys->page_recycle_repl->cache[i];
-			if (rx_pkt) {
-				dma_unmap_page(ipa3_ctx->pdev,
-					rx_pkt->page_data.dma_addr,
-					rx_pkt->len,
-					DMA_FROM_DEVICE);
-				__free_pages(rx_pkt->page_data.page,
-					rx_pkt->page_data.page_order);
-				kmem_cache_free(
-					ipa3_ctx->rx_pkt_wrapper_cache,
-					rx_pkt);
-			}
+		list_for_each_entry_safe(rx_pkt, r,
+		&sys->page_recycle_repl->page_repl_head, link) {
+			list_del(&rx_pkt->link);
+			dma_unmap_page(ipa3_ctx->pdev,
+				rx_pkt->page_data.dma_addr,
+				rx_pkt->len,
+				DMA_FROM_DEVICE);
+			__free_pages(rx_pkt->page_data.page,
+				rx_pkt->page_data.page_order);
+			kmem_cache_free(
+				ipa3_ctx->rx_pkt_wrapper_cache,
+				rx_pkt);
 		}
-		kfree(sys->page_recycle_repl->cache);
 		kfree(sys->page_recycle_repl);
-		sys->page_recycle_repl = NULL;
 	}
 }
 
@@ -3670,6 +3680,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 		IPAERR("notify->veid > GSI_VEID_MAX\n");
 		if (!rx_page.is_tmp_alloc) {
 			init_page_count(rx_page.page);
+			spin_lock_bh(&rx_pkt->sys->spinlock);
+			/* Add the element to head. */
+			list_add(&rx_pkt->link,
+				&rx_pkt->sys->page_recycle_repl->page_repl_head);
+			spin_unlock_bh(&rx_pkt->sys->spinlock);
 		} else {
 			dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
@@ -3690,20 +3705,19 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 		sys->ep->client == IPA_CLIENT_APPS_LAN_CONS) {
 		rx_skb = alloc_skb(0, GFP_ATOMIC);
 		if (unlikely(!rx_skb)) {
-			IPAERR("skb alloc failure, free all pending pages\n");
-			list_for_each_entry_safe(rx_pkt, tmp, head, link) {
-				rx_page = rx_pkt->page_data;
-				size = rx_pkt->data_len;
-				list_del_init(&rx_pkt->link);
-				if (!rx_page.is_tmp_alloc) {
-					init_page_count(rx_page.page);
-				} else {
-					dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
-						rx_pkt->len, DMA_FROM_DEVICE);
-					__free_pages(rx_pkt->page_data.page,
-								rx_pkt->page_data.page_order);
-				}
-				rx_pkt->sys->free_rx_wrapper(rx_pkt);
+			IPAERR("skb alloc failure\n");
+			list_del_init(&rx_pkt->link);
+			if (!rx_page.is_tmp_alloc) {
+				init_page_count(rx_page.page);
+				spin_lock_bh(&rx_pkt->sys->spinlock);
+				/* Add the element to head. */
+				list_add(&rx_pkt->link,
+					&rx_pkt->sys->page_recycle_repl->page_repl_head);
+				spin_unlock_bh(&rx_pkt->sys->spinlock);
+			} else {
+				dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
+					rx_pkt->len, DMA_FROM_DEVICE);
+				__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
 			}
 			IPA_STATS_INC_CNT(ipa3_ctx->stats.rx_page_drop_cnt);
 			return NULL;
@@ -3712,14 +3726,20 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 			rx_page = rx_pkt->page_data;
 			size = rx_pkt->data_len;
 
-			list_del(&rx_pkt->link);
-			if (rx_page.is_tmp_alloc)
+			list_del_init(&rx_pkt->link);
+			if (rx_page.is_tmp_alloc) {
 				dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
-			else
+			} else {
+				spin_lock_bh(&rx_pkt->sys->spinlock);
+				/* Add the element back to tail. */
+				list_add_tail(&rx_pkt->link,
+					&rx_pkt->sys->page_recycle_repl->page_repl_head);
+				spin_unlock_bh(&rx_pkt->sys->spinlock);
 				dma_sync_single_for_cpu(ipa3_ctx->pdev,
 					rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
+			}
 			rx_pkt->sys->free_rx_wrapper(rx_pkt);
 
 			skb_add_rx_frag(rx_skb,
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
index 82902d0e1c80..c2d09a4e91f3 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef _IPA3_I_H_
@@ -59,6 +59,7 @@
 #define IPA_QMAP_HEADER_LENGTH (4)
 #define IPA_DL_CHECKSUM_LENGTH (8)
 #define IPA_NUM_DESC_PER_SW_TX (3)
+#define IPA_GENERIC_RX_POOL_SZ_WAN 224
 #define IPA_GENERIC_RX_POOL_SZ 192
 #define IPA_UC_FINISH_MAX 6
 #define IPA_UC_WAIT_MIN_SLEEP 1000
@@ -84,6 +85,10 @@
 
 #define IPA_WAN_AGGR_PKT_CNT 1
 
+#define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15
+#define IPA_PAGE_POLL_THRESHOLD_MAX 30
+
+
 #define IPADBG(fmt, args...) \
 	do { \
 		pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
@@ -1062,6 +1067,12 @@ struct ipa3_repl_ctx {
 	atomic_t pending;
 };
 
+struct ipa3_page_repl_ctx {
+	struct list_head page_repl_head;
+	u32 capacity;
+	atomic_t pending;
+};
+
 /**
  * struct ipa3_sys_context - IPA GPI pipes context
  * @head_desc_list: header descriptors list
@@ -1100,7 +1111,7 @@ struct ipa3_sys_context {
 	struct work_struct repl_work;
 	void (*repl_hdlr)(struct ipa3_sys_context *sys);
 	struct ipa3_repl_ctx *repl;
-	struct ipa3_repl_ctx *page_recycle_repl;
+	struct ipa3_page_repl_ctx *page_recycle_repl;
 	u32 pkt_sent;
 	struct napi_struct *napi_obj;
 	struct list_head pending_pkts[GSI_VEID_MAX];
@@ -1431,8 +1442,10 @@ enum ipa3_config_this_ep {
 
 struct ipa3_page_recycle_stats {
 	u64 total_replenished;
+	u64 page_recycled;
 	u64 tmp_alloc;
 };
+
 struct ipa3_stats {
 	u32 tx_sw_pkts;
 	u32 tx_hw_pkts;
@@ -1459,6 +1472,7 @@ struct ipa3_stats {
 	struct ipa3_page_recycle_stats page_recycle_stats[2];
 	u64 lower_order;
 	u32 pipe_setup_fail_cnt;
+	u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX];
 };
 
 /* offset for each stats */
@@ -2174,7 +2188,7 @@ struct ipa3_context {
 	bool fnr_stats_not_supported;
 	bool is_device_crashed;
 	int ipa_pil_load;
-
+	u8 page_poll_threshold;
 };
 
 struct ipa3_plat_drv_res {

From d851d46728bb0ef89837cb701966bb03fbc5d936 Mon Sep 17 00:00:00 2001
From: Chaitanya Pratapa <cpratapa@codeaurora.org>
Date: Wed, 28 Apr 2021 13:33:00 -0700
Subject: [PATCH 2/9] msm: ipa: page pool recycling enhancements

Added the following enhancements to page pool recycling logic.

1) Common page pool for coalescing/default pipe.
2) Common page pool will hold 5 times the rx_pool_sz (224) for page
   recycling pool and 3 times for temp pool. This is done to sustain
   50 stream scenarios.

Change-Id: I55adfffa101df33c46e95b0214fb7338644e8d7c
Signed-off-by: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa.c         |   4 +
 drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c |   2 +
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c      | 131 ++++++++++--------
 drivers/platform/msm/ipa/ipa_v3/ipa_i.h       |   7 +
 4 files changed, 88 insertions(+), 56 deletions(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c
index ee60841a1798..c4141001bb30 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c
@@ -7554,6 +7554,10 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
 	/* Initialize Page poll threshold. */
 	ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD;
 
+	/* Use common page pool for Def/Coal pipe. */
+	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_9)
+		ipa3_ctx->wan_common_page_pool = true;
+
 	/* assume clock is on in virtual/emulation mode */
 	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
 	    ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION)
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
index c8cda1cd4ddb..aad988894dbc 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
@@ -1353,6 +1353,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf,
 		"act_clnt=%u\n"
 		"con_clnt_bmap=0x%x\n"
 		"wan_rx_empty=%u\n"
+		"wan_rx_empty_coal=%u\n"
 		"wan_repl_rx_empty=%u\n"
 		"lan_rx_empty=%u\n"
 		"lan_repl_rx_empty=%u\n"
@@ -1372,6 +1373,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf,
 		atomic_read(&ipa3_ctx->ipa3_active_clients.cnt),
 		connect,
 		ipa3_ctx->stats.wan_rx_empty,
+		ipa3_ctx->stats.wan_rx_empty_coal,
 		ipa3_ctx->stats.wan_repl_rx_empty,
 		ipa3_ctx->stats.lan_rx_empty,
 		ipa3_ctx->stats.lan_repl_rx_empty,
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 4484a268a7a6..42eaf09dbed7 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -1280,36 +1280,66 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 		}
 	}
 
+	/* Use common page pool for Coal and defalt pipe if applicable. */
 	if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) {
-		ep->sys->page_recycle_repl = kzalloc(
-			sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL);
-		if (!ep->sys->page_recycle_repl) {
-			IPAERR("failed to alloc repl for client %d\n",
-					sys_in->client);
-			result = -ENOMEM;
-			goto fail_napi;
-		}
-		atomic_set(&ep->sys->page_recycle_repl->pending, 0);
-		ep->sys->page_recycle_repl->capacity =
-				(ep->sys->rx_pool_sz + 1) * 2;
-		INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
-		ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
-		if (!ep->sys->repl) {
-			IPAERR("failed to alloc repl for client %d\n",
-				   sys_in->client);
-			result = -ENOMEM;
-			goto fail_page_recycle_repl;
-		}
-		ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1);
+		if (!(ipa3_ctx->wan_common_page_pool &&
+			sys_in->client == IPA_CLIENT_APPS_WAN_CONS &&
+			coal_ep_id != IPA_EP_NOT_ALLOCATED &&
+			ipa3_ctx->ep[coal_ep_id].valid == 1)) {
+			ep->sys->page_recycle_repl = kzalloc(
+				sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL);
+			if (!ep->sys->page_recycle_repl) {
+				IPAERR("failed to alloc repl for client %d\n",
+						sys_in->client);
+				result = -ENOMEM;
+				goto fail_napi;
+			}
+			atomic_set(&ep->sys->page_recycle_repl->pending, 0);
+			/* For common page pool double the pool size. */
+			if (ipa3_ctx->wan_common_page_pool &&
+				sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
+				ep->sys->page_recycle_repl->capacity =
+						(ep->sys->rx_pool_sz + 1) *
+						IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR;
+			else
+				ep->sys->page_recycle_repl->capacity =
+						(ep->sys->rx_pool_sz + 1) *
+						IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR;
+			IPADBG("Page repl capacity for client:%d, value:%d\n",
+					   sys_in->client, ep->sys->page_recycle_repl->capacity);
+			INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
+			ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
+			if (!ep->sys->repl) {
+				IPAERR("failed to alloc repl for client %d\n",
+					   sys_in->client);
+				result = -ENOMEM;
+				goto fail_page_recycle_repl;
+			}
+			/* For common page pool triple the pool size. */
+			if (ipa3_ctx->wan_common_page_pool &&
+				sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
+				ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1) *
+				IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR;
+			else
+				ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1);
+			IPADBG("Repl capacity for client:%d, value:%d\n",
+					   sys_in->client, ep->sys->repl->capacity);
+			atomic_set(&ep->sys->repl->pending, 0);
+			ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity,
+					sizeof(void *), GFP_KERNEL);
+			atomic_set(&ep->sys->repl->head_idx, 0);
+			atomic_set(&ep->sys->repl->tail_idx, 0);
 
-		atomic_set(&ep->sys->repl->pending, 0);
-		ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity,
-				sizeof(void *), GFP_KERNEL);
-		atomic_set(&ep->sys->repl->head_idx, 0);
-		atomic_set(&ep->sys->repl->tail_idx, 0);
-
-		ipa3_replenish_rx_page_cache(ep->sys);
-		ipa3_wq_page_repl(&ep->sys->repl_work);
+			ipa3_replenish_rx_page_cache(ep->sys);
+			ipa3_wq_page_repl(&ep->sys->repl_work);
+		} else {
+			/* Use pool same as coal pipe when common page pool is used */
+			ep->sys->common_buff_pool = true;
+			ep->sys->common_sys = ipa3_ctx->ep[coal_ep_id].sys;
+			ep->sys->repl = ipa3_ctx->ep[coal_ep_id].sys->repl;
+			ep->sys->page_recycle_repl =
+				ipa3_ctx->ep[coal_ep_id].sys->page_recycle_repl;
+		}
 	}
 
 	if (IPA_CLIENT_IS_CONS(sys_in->client)) {
@@ -1379,16 +1409,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 fail_gen3:
 	ipa3_disable_data_path(ipa_ep_idx);
 fail_repl:
-	if (IPA_CLIENT_IS_CONS(ep->client))
+	if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
 		ipa3_cleanup_rx(ep->sys);
 
-	if(ep->sys->repl) {
+	ep->sys->repl_hdlr = ipa3_replenish_rx_cache;
+	if (ep->sys->repl && !ep->sys->common_buff_pool) {
 		ep->sys->repl->capacity = 0;
 		kfree(ep->sys->repl);
-		ep->sys->repl = NULL;
 	}
 fail_page_recycle_repl:
-	if (ep->sys->page_recycle_repl) {
+	if (ep->sys->page_recycle_repl && !ep->sys->common_buff_pool) {
 		ep->sys->page_recycle_repl->capacity = 0;
 		kfree(ep->sys->page_recycle_repl);
 		ep->sys->page_recycle_repl = NULL;
@@ -1567,7 +1597,7 @@ int ipa3_teardown_sys_pipe(u32 clnt_hdl)
 	}
 	if (ep->sys->repl_wq)
 		flush_workqueue(ep->sys->repl_wq);
-	if (IPA_CLIENT_IS_CONS(ep->client))
+	if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
 		ipa3_cleanup_rx(ep->sys);
 
 	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(ep->client)) {
@@ -1653,7 +1683,7 @@ static int ipa3_teardown_coal_def_pipe(u32 clnt_hdl)
 
 	if (ep->sys->repl_wq)
 		flush_workqueue(ep->sys->repl_wq);
-	if (IPA_CLIENT_IS_CONS(ep->client))
+	if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
 		ipa3_cleanup_rx(ep->sys);
 
 	ep->valid = 0;
@@ -2187,7 +2217,7 @@ static inline void __trigger_repl_work(struct ipa3_sys_context *sys)
 	head = atomic_read(&sys->repl->head_idx);
 	avail = (tail - head) % sys->repl->capacity;
 
-	if (avail < sys->repl->capacity / 4) {
+	if (avail < sys->repl->capacity / 2) {
 		atomic_set(&sys->repl->pending, 1);
 		queue_work(sys->repl_wq, &sys->repl_work);
 	}
@@ -2237,7 +2267,6 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 		return;
 	stats_i = (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) ? 0 : 1;
 
-	spin_lock_bh(&sys->spinlock);
 	rx_len_cached = sys->len;
 	curr_wq = atomic_read(&sys->repl->head_idx);
 
@@ -2258,6 +2287,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 			curr_wq = (++curr_wq == sys->repl->capacity) ?
 								 0 : curr_wq;
 		}
+		rx_pkt->sys = sys;
 
 		dma_sync_single_for_device(ipa3_ctx->pdev,
 			rx_pkt->page_data.dma_addr,
@@ -2301,19 +2331,21 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 		IPAERR("failed to provide buffer: %d\n", ret);
 		ipa_assert();
 	}
-	spin_unlock_bh(&sys->spinlock);
-	__trigger_repl_work(sys);
+
+	 if (sys->common_buff_pool)
+	 	__trigger_repl_work(sys->common_sys);
+	 else
+		__trigger_repl_work(sys);
 
 	if (rx_len_cached <= IPA_DEFAULT_SYS_YELLOW_WM) {
-		if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS ||
-			sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
+		if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS)
 			IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty);
+		else if (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
+			IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty_coal);
 		else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS)
 			IPA_STATS_INC_CNT(ipa3_ctx->stats.lan_rx_empty);
 		else
 			WARN_ON(1);
-		queue_delayed_work(sys->wq, &sys->replenish_rx_work,
-				msecs_to_jiffies(1));
 	}
 
 	return;
@@ -3866,20 +3898,6 @@ static void ipa3_rx_napi_chain(struct ipa3_sys_context *sys,
 			if (prev_skb) {
 				skb_shinfo(prev_skb)->frag_list = NULL;
 				sys->pyld_hdlr(first_skb, sys);
-				/*
-				 * For coalescing, we have 2 transfer
-				 * rings to replenish
-				 */
-				ipa_ep_idx = ipa3_get_ep_mapping(
-						IPA_CLIENT_APPS_WAN_CONS);
-				if (ipa_ep_idx ==
-					IPA_EP_NOT_ALLOCATED) {
-					IPAERR("Invalid client.\n");
-					return;
-				}
-				wan_def_sys =
-					ipa3_ctx->ep[ipa_ep_idx].sys;
-				wan_def_sys->repl_hdlr(wan_def_sys);
 			}
 		}
 	}
@@ -5461,6 +5479,7 @@ start_poll:
 	cnt += weight - remain_aggr_weight * ipa3_ctx->ipa_wan_aggr_pkt_cnt;
 	/* call repl_hdlr before napi_reschedule / napi_complete */
 	ep->sys->repl_hdlr(ep->sys);
+	wan_def_sys->repl_hdlr(wan_def_sys);
 	/* When not able to replenish enough descriptors, keep in polling
 	 * mode, wait for napi-poll and replenish again.
 	 */
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
index c2d09a4e91f3..3e5138cc6c80 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
@@ -61,6 +61,9 @@
 #define IPA_NUM_DESC_PER_SW_TX (3)
 #define IPA_GENERIC_RX_POOL_SZ_WAN 224
 #define IPA_GENERIC_RX_POOL_SZ 192
+#define IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR 2
+#define IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR 5
+#define IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR 3
 #define IPA_UC_FINISH_MAX 6
 #define IPA_UC_WAIT_MIN_SLEEP 1000
 #define IPA_UC_WAII_MAX_SLEEP 1200
@@ -1121,6 +1124,8 @@ struct ipa3_sys_context {
 	u32 eob_drop_cnt;
 	struct napi_struct napi_tx;
 	atomic_t in_napi_context;
+	bool common_buff_pool;
+	struct ipa3_sys_context *common_sys;
 
 	/* ordering is important - mutable fields go above */
 	struct ipa3_ep_context *ep;
@@ -1460,6 +1465,7 @@ struct ipa3_stats {
 	u32 aggr_close;
 	u32 wan_aggr_close;
 	u32 wan_rx_empty;
+	u32 wan_rx_empty_coal;
 	u32 wan_repl_rx_empty;
 	u32 lan_rx_empty;
 	u32 lan_repl_rx_empty;
@@ -2189,6 +2195,7 @@ struct ipa3_context {
 	bool is_device_crashed;
 	int ipa_pil_load;
 	u8 page_poll_threshold;
+	bool wan_common_page_pool;
 };
 
 struct ipa3_plat_drv_res {

From d32afa862e23f4dcb4df56ff1fd0479c0971ed40 Mon Sep 17 00:00:00 2001
From: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
Date: Fri, 24 Sep 2021 00:35:29 +0530
Subject: [PATCH 3/9] msm: ipa3: Changes to enhance find free pages from list

Adding changes to enhance find free pages from list,
 1. When free page not available instead of keep trying
    using temporary pages.
 2. In tasklet search for free pages adding to head.

Change-Id: I991336167519322081faf8f82feec4e3f0d7b161
Signed-off-by: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa.c         |   9 ++
 drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c |  84 ++++++++++-
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c      | 133 +++++++++++++++---
 drivers/platform/msm/ipa/ipa_v3/ipa_i.h       |  13 ++
 4 files changed, 219 insertions(+), 20 deletions(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c
index c4141001bb30..09f5b1cc2fa5 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c
@@ -7361,6 +7361,11 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
 	ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0;
 	memset(ipa3_ctx->stats.page_recycle_cnt, 0,
 		sizeof(ipa3_ctx->stats.page_recycle_cnt));
+	ipa3_ctx->stats.num_sort_tasklet_sched[0] = 0;
+	ipa3_ctx->stats.num_sort_tasklet_sched[1] = 0;
+	ipa3_ctx->stats.num_sort_tasklet_sched[2] = 0;
+	ipa3_ctx->stats.num_of_times_wq_reschd = 0;
+	ipa3_ctx->stats.page_recycle_cnt_in_tasklet = 0;
 	ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset;
 	ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control;
 	ipa3_ctx->ee = resource_p->ee;
@@ -7554,6 +7559,10 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
 	/* Initialize Page poll threshold. */
 	ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD;
 
+	/*Initialize number napi without prealloc buff*/
+	ipa3_ctx->ipa_max_napi_sort_page_thrshld = IPA_MAX_NAPI_SORT_PAGE_THRSHLD;
+	ipa3_ctx->page_wq_reschd_time = IPA_MAX_PAGE_WQ_RESCHED_TIME;
+
 	/* Use common page pool for Def/Coal pipe. */
 	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_9)
 		ipa3_ctx->wan_common_page_pool = true;
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
index aad988894dbc..426cbbea09fa 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
@@ -1434,15 +1434,23 @@ static ssize_t ipa3_read_page_recycle_stats(struct file *file,
 			"COAL : Total number of packets replenished =%llu\n"
 			"COAL : Number of page recycled packets  =%llu\n"
 			"COAL : Number of tmp alloc packets  =%llu\n"
+			"COAL  : Number of times tasklet scheduled  =%llu\n"
 			"DEF  : Total number of packets replenished =%llu\n"
 			"DEF  : Number of page recycled packets =%llu\n"
-			"DEF  : Number of tmp alloc packets  =%llu\n",
+			"DEF  : Number of tmp alloc packets  =%llu\n"
+			"DEF  : Number of times tasklet scheduled  =%llu\n"
+			"COMMON  : Number of page recycled in tasklet  =%llu\n"
+			"COMMON  : Number of times free pages not found in tasklet =%llu\n",
 			ipa3_ctx->stats.page_recycle_stats[0].total_replenished,
 			ipa3_ctx->stats.page_recycle_stats[0].page_recycled,
 			ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc,
+			ipa3_ctx->stats.num_sort_tasklet_sched[0],
 			ipa3_ctx->stats.page_recycle_stats[1].total_replenished,
 			ipa3_ctx->stats.page_recycle_stats[1].page_recycled,
-			ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc);
+			ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc,
+			ipa3_ctx->stats.num_sort_tasklet_sched[1],
+			ipa3_ctx->stats.page_recycle_cnt_in_tasklet,
+			ipa3_ctx->stats.num_of_times_wq_reschd);
 
 	cnt += nbytes;
 
@@ -2788,6 +2796,68 @@ static ssize_t ipa3_enable_ipc_low(struct file *file,
 	return count;
 }
 
+static ssize_t ipa3_read_ipa_max_napi_sort_page_thrshld(struct file *file,
+	char __user *buf, size_t count, loff_t *ppos) {
+
+	int nbytes;
+	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
+				"page max napi without free page = %d\n",
+				ipa3_ctx->ipa_max_napi_sort_page_thrshld);
+	return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
+
+}
+
+static ssize_t ipa3_write_ipa_max_napi_sort_page_thrshld(struct file *file,
+	const char __user *buf, size_t count, loff_t *ppos) {
+
+	int ret;
+	u8 ipa_max_napi_sort_page_thrshld = 0;
+
+	if (count >= sizeof(dbg_buff))
+		return -EFAULT;
+
+	ret = kstrtou8_from_user(buf, count, 0, &ipa_max_napi_sort_page_thrshld);
+	if(ret)
+		return ret;
+
+	ipa3_ctx->ipa_max_napi_sort_page_thrshld = ipa_max_napi_sort_page_thrshld;
+
+	IPADBG("napi cnt without prealloc pages = %d", ipa3_ctx->ipa_max_napi_sort_page_thrshld);
+
+	return count;
+}
+
+static ssize_t ipa3_read_page_wq_reschd_time(struct file *file,
+	char __user *buf, size_t count, loff_t *ppos) {
+
+	int nbytes;
+	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
+				"Page WQ reschduule time = %d\n",
+				ipa3_ctx->page_wq_reschd_time);
+	return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
+
+}
+
+static ssize_t ipa3_write_page_wq_reschd_time(struct file *file,
+	const char __user *buf, size_t count, loff_t *ppos) {
+
+	int ret;
+	u8 page_wq_reschd_time = 0;
+
+	if (count >= sizeof(dbg_buff))
+		return -EFAULT;
+
+	ret = kstrtou8_from_user(buf, count, 0, &page_wq_reschd_time);
+	if(ret)
+		return ret;
+
+	ipa3_ctx->page_wq_reschd_time = page_wq_reschd_time;
+
+	IPADBG("Updated page WQ reschedule time = %d", ipa3_ctx->page_wq_reschd_time);
+
+	return count;
+}
+
 static ssize_t ipa3_read_gsi_wdi3_db_polling(struct file *file,
 	char __user *buf, size_t count, loff_t *ppos) {
 
@@ -3042,6 +3112,16 @@ static const struct ipa3_debugfs_file debugfs_files[] = {
 			.read = ipa3_read_page_poll_threshold,
 			.write = ipa3_write_page_poll_threshold,
 		}
+	}, {
+		"page_wq_reschd_time", IPA_READ_WRITE_MODE, NULL, {
+			.read = ipa3_read_page_wq_reschd_time,
+			.write = ipa3_write_page_wq_reschd_time,
+		}
+	}, {
+		"ipa_max_napi_sort_page_thrshld", IPA_READ_WRITE_MODE, NULL, {
+			.read = ipa3_read_ipa_max_napi_sort_page_thrshld,
+			.write = ipa3_write_ipa_max_napi_sort_page_thrshld,
+		}
 	},
 };
 
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 42eaf09dbed7..2714e1920a47 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -137,6 +137,7 @@ static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys,
 static unsigned long tag_to_pointer_wa(uint64_t tag);
 static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt);
 static void ipa3_tasklet_rx_notify(unsigned long data);
+static void ipa3_tasklet_find_freepage(unsigned long data);
 
 static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit);
 
@@ -1048,6 +1049,60 @@ static void ipa_pm_sys_pipe_cb(void *p, enum ipa_pm_cb_event event)
 	}
 }
 
+static void ipa3_schd_freepage_work(struct work_struct *work)
+{
+	struct delayed_work *dwork;
+	struct ipa3_sys_context *sys;
+
+	dwork = container_of(work, struct delayed_work, work);
+	sys = container_of(dwork, struct ipa3_sys_context, freepage_work);
+
+	IPADBG_LOW("WQ scheduled, reschedule sort tasklet\n");
+
+	tasklet_schedule(&sys->tasklet_find_freepage);
+}
+
+static void ipa3_tasklet_find_freepage(unsigned long data)
+{
+	struct ipa3_sys_context *sys;
+	struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
+	struct ipa3_rx_pkt_wrapper *tmp = NULL;
+	struct page *cur_page;
+	int found_free_page = 0;
+	struct list_head temp_head;
+
+	sys = (struct ipa3_sys_context *)data;
+
+	INIT_LIST_HEAD(&temp_head);
+	spin_lock_bh(&sys->common_sys->spinlock);
+	list_for_each_entry_safe(rx_pkt, tmp,
+		&sys->page_recycle_repl->page_repl_head, link) {
+		cur_page = rx_pkt->page_data.page;
+		if (page_ref_count(cur_page) == 1) {
+			/* Found a free page. */
+			list_del_init(&rx_pkt->link);
+			list_add(&rx_pkt->link, &temp_head);
+			found_free_page++;
+		}
+	}
+	if (!found_free_page) {
+		/*Not found free page rescheduling tasklet after 2msec*/
+		IPADBG_LOW("Scheduling WQ not found free pages\n");
+		++ipa3_ctx->stats.num_of_times_wq_reschd;
+		queue_delayed_work(sys->freepage_wq,
+				&sys->freepage_work,
+				msecs_to_jiffies(ipa3_ctx->page_wq_reschd_time));
+	} else {
+		/*Allow to use pre-allocated buffers*/
+		list_splice(&temp_head, &sys->page_recycle_repl->page_repl_head);
+		ipa3_ctx->stats.page_recycle_cnt_in_tasklet += found_free_page;
+		IPADBG_LOW("found free pages count = %d\n", found_free_page);
+		atomic_set(&sys->common_sys->page_avilable, 1);
+	}
+	spin_unlock_bh(&sys->common_sys->spinlock);
+
+}
+
 /**
  * ipa3_setup_sys_pipe() - Setup an IPA GPI pipe and perform
  * IPA EP configuration
@@ -1139,6 +1194,9 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 			goto fail_wq2;
 		}
 
+		snprintf(buff, IPA_RESOURCE_NAME_MAX, "ipafreepagewq%d",
+				sys_in->client);
+
 		INIT_LIST_HEAD(&ep->sys->head_desc_list);
 		INIT_LIST_HEAD(&ep->sys->rcycl_list);
 		INIT_LIST_HEAD(&ep->sys->avail_tx_wrapper_list);
@@ -1155,6 +1213,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 			/* Use coalescing pipe PM handle for default pipe also*/
 			ep->sys->pm_hdl = ipa3_ctx->ep[coal_ep_id].sys->pm_hdl;
 		} else if (IPA_CLIENT_IS_CONS(sys_in->client)) {
+			ep->sys->freepage_wq = alloc_workqueue(buff,
+					WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS |
+					WQ_HIGHPRI, 1);
+			if (!ep->sys->freepage_wq) {
+				IPAERR("failed to create freepage wq for client %d\n",
+						sys_in->client);
+				result = -EFAULT;
+				goto fail_wq3;
+			}
+
 			pm_reg.name = ipa_clients_strings[sys_in->client];
 			pm_reg.callback = ipa_pm_sys_pipe_cb;
 			pm_reg.user_data = ep->sys;
@@ -1255,6 +1323,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 	}
 
 	*clnt_hdl = ipa_ep_idx;
+	ep->sys->common_sys = ipa3_ctx->ep[ipa_ep_idx].sys;
 
 	if (ep->sys->repl_hdlr == ipa3_fast_replenish_rx_cache) {
 		ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
@@ -1330,6 +1399,10 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 			atomic_set(&ep->sys->repl->head_idx, 0);
 			atomic_set(&ep->sys->repl->tail_idx, 0);
 
+			tasklet_init(&ep->sys->tasklet_find_freepage,
+					ipa3_tasklet_find_freepage, (unsigned long) ep->sys);
+			INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work);
+			ep->sys->napi_sort_page_thrshld_cnt = 0;
 			ipa3_replenish_rx_page_cache(ep->sys);
 			ipa3_wq_page_repl(&ep->sys->repl_work);
 		} else {
@@ -1431,6 +1504,8 @@ fail_napi:
 fail_gen2:
 	ipa_pm_deregister(ep->sys->pm_hdl);
 fail_pm:
+	destroy_workqueue(ep->sys->freepage_wq);
+fail_wq3:
 	destroy_workqueue(ep->sys->repl_wq);
 fail_wq2:
 	destroy_workqueue(ep->sys->wq);
@@ -2157,6 +2232,7 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys)
 		list_add_tail(&rx_pkt->link,
 			&sys->page_recycle_repl->page_repl_head);
 	}
+	atomic_set(&sys->common_sys->page_avilable, 1);
 
 	return;
 
@@ -2235,6 +2311,7 @@ static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page
 	int i = 0;
 	u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold;
 
+	spin_lock_bh(&sys->common_sys->spinlock);
 	list_for_each_entry_safe(rx_pkt, tmp,
 		&sys->page_recycle_repl->page_repl_head, link) {
 		if (i == LOOP_THRESHOLD)
@@ -2245,10 +2322,23 @@ static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page
 			page_ref_inc(cur_page);
 			list_del_init(&rx_pkt->link);
 			++ipa3_ctx->stats.page_recycle_cnt[stats_i][i];
+			sys->common_sys->napi_sort_page_thrshld_cnt = 0;
+			spin_unlock_bh(&sys->common_sys->spinlock);
 			return rx_pkt;
 		}
 		i++;
 	}
+	spin_unlock_bh(&sys->common_sys->spinlock);
+	IPADBG_LOW("napi_sort_page_thrshld_cnt = %d ipa_max_napi_sort_page_thrshld = %d\n",
+			sys->common_sys->napi_sort_page_thrshld_cnt,
+			ipa3_ctx->ipa_max_napi_sort_page_thrshld);
+	/*Scheduling tasklet to find the free page*/
+	if (sys->common_sys->napi_sort_page_thrshld_cnt >=
+			ipa3_ctx->ipa_max_napi_sort_page_thrshld) {
+		atomic_set(&sys->common_sys->page_avilable, 0);
+		tasklet_schedule(&sys->common_sys->tasklet_find_freepage);
+		++ipa3_ctx->stats.num_sort_tasklet_sched[stats_i];
+	}
 	return NULL;
 }
 
@@ -2272,7 +2362,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 
 	while (rx_len_cached < sys->rx_pool_sz) {
 		/* check for an idle page that can be used */
-		if ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL) {
+		if (atomic_read(&sys->common_sys->page_avilable) &&
+			((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL)) {
 			ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++;
 
 		} else {
@@ -3712,11 +3803,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 		IPAERR("notify->veid > GSI_VEID_MAX\n");
 		if (!rx_page.is_tmp_alloc) {
 			init_page_count(rx_page.page);
-			spin_lock_bh(&rx_pkt->sys->spinlock);
+			spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
 			/* Add the element to head. */
 			list_add(&rx_pkt->link,
 				&rx_pkt->sys->page_recycle_repl->page_repl_head);
-			spin_unlock_bh(&rx_pkt->sys->spinlock);
+			spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
 		} else {
 			dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
@@ -3737,19 +3828,24 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 		sys->ep->client == IPA_CLIENT_APPS_LAN_CONS) {
 		rx_skb = alloc_skb(0, GFP_ATOMIC);
 		if (unlikely(!rx_skb)) {
-			IPAERR("skb alloc failure\n");
-			list_del_init(&rx_pkt->link);
-			if (!rx_page.is_tmp_alloc) {
-				init_page_count(rx_page.page);
-				spin_lock_bh(&rx_pkt->sys->spinlock);
-				/* Add the element to head. */
-				list_add(&rx_pkt->link,
-					&rx_pkt->sys->page_recycle_repl->page_repl_head);
-				spin_unlock_bh(&rx_pkt->sys->spinlock);
-			} else {
-				dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
-					rx_pkt->len, DMA_FROM_DEVICE);
-				__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
+			IPAERR("skb alloc failure, free all pending pages\n");
+			list_for_each_entry_safe(rx_pkt, tmp, head, link) {
+				rx_page = rx_pkt->page_data;
+				size = rx_pkt->data_len;
+				list_del_init(&rx_pkt->link);
+				if (!rx_page.is_tmp_alloc) {
+					init_page_count(rx_page.page);
+					spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
+					/* Add the element to head. */
+					list_add(&rx_pkt->link,
+						&rx_pkt->sys->page_recycle_repl->page_repl_head);
+					spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
+				} else {
+					dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
+						rx_pkt->len, DMA_FROM_DEVICE);
+					__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
+				}
+				rx_pkt->sys->free_rx_wrapper(rx_pkt);
 			}
 			IPA_STATS_INC_CNT(ipa3_ctx->stats.rx_page_drop_cnt);
 			return NULL;
@@ -3763,11 +3859,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 				dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
 			} else {
-				spin_lock_bh(&rx_pkt->sys->spinlock);
+				spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
 				/* Add the element back to tail. */
 				list_add_tail(&rx_pkt->link,
 					&rx_pkt->sys->page_recycle_repl->page_repl_head);
-				spin_unlock_bh(&rx_pkt->sys->spinlock);
+				spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
 				dma_sync_single_for_cpu(ipa3_ctx->pdev,
 					rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
@@ -5446,6 +5542,7 @@ int ipa3_rx_poll(u32 clnt_hdl, int weight)
 		return -EINVAL;
 	}
 
+	ep->sys->common_sys->napi_sort_page_thrshld_cnt++;
 start_poll:
 	/*
 	 * it is guaranteed we already have clock here.
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
index 3e5138cc6c80..9c10e4233043 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
@@ -91,6 +91,9 @@
 #define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15
 #define IPA_PAGE_POLL_THRESHOLD_MAX 30
 
+#define IPA_MAX_NAPI_SORT_PAGE_THRSHLD 3
+#define IPA_MAX_PAGE_WQ_RESCHED_TIME 2
+
 
 #define IPADBG(fmt, args...) \
 	do { \
@@ -1126,6 +1129,10 @@ struct ipa3_sys_context {
 	atomic_t in_napi_context;
 	bool common_buff_pool;
 	struct ipa3_sys_context *common_sys;
+	struct tasklet_struct tasklet_find_freepage;
+	atomic_t page_avilable;
+	struct delayed_work freepage_work;
+	u32 napi_sort_page_thrshld_cnt;
 
 	/* ordering is important - mutable fields go above */
 	struct ipa3_ep_context *ep;
@@ -1141,6 +1148,7 @@ struct ipa3_sys_context {
 	u32 pm_hdl;
 	unsigned int napi_sch_cnt;
 	unsigned int napi_comp_cnt;
+	struct workqueue_struct *freepage_wq;
 	/* ordering is important - other immutable fields go below */
 };
 
@@ -1479,6 +1487,9 @@ struct ipa3_stats {
 	u64 lower_order;
 	u32 pipe_setup_fail_cnt;
 	u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX];
+	u64 num_sort_tasklet_sched[3];
+	u64 num_of_times_wq_reschd;
+	u64 page_recycle_cnt_in_tasklet;
 };
 
 /* offset for each stats */
@@ -2196,6 +2207,8 @@ struct ipa3_context {
 	int ipa_pil_load;
 	u8 page_poll_threshold;
 	bool wan_common_page_pool;
+	u32 ipa_max_napi_sort_page_thrshld;
+	u32 page_wq_reschd_time;
 };
 
 struct ipa3_plat_drv_res {

From ddeae0e52f4a2ff7ca4fe280f4bd117ca2588e6d Mon Sep 17 00:00:00 2001
From: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
Date: Thu, 24 Mar 2022 13:34:16 +0530
Subject: [PATCH 4/9] msm: ipa3: Fix to destory workqueue only if created

Freepage workqueue created only for few pipes, while handling error
scenarios freeing it for all pipes. So adding check to free work queue
only if created.

Change-Id: I8bf097c02f6acd0a4a22b3d4bc5b66f46884fc1e
Signed-off-by: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 2714e1920a47..3da7642625c0 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -1504,7 +1504,8 @@ fail_napi:
 fail_gen2:
 	ipa_pm_deregister(ep->sys->pm_hdl);
 fail_pm:
-	destroy_workqueue(ep->sys->freepage_wq);
+	if (ep->sys->freepage_wq)
+		destroy_workqueue(ep->sys->freepage_wq);
 fail_wq3:
 	destroy_workqueue(ep->sys->repl_wq);
 fail_wq2:

From ceaf68b200980fbac38d44b7324a4e55a407bc3e Mon Sep 17 00:00:00 2001
From: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
Date: Thu, 3 Mar 2022 17:13:03 +0530
Subject: [PATCH 5/9] msm: ipa3: Fix to flush to workqueue during teardown pipe

Workqueue was getting scheduled after teardwon the pipe.
Adding changes to flush to workqueue during teardown pipe.

Change-Id: I5a6b3464d76a9835bfbf7668e8dbb1cdf00c73fe
Signed-off-by: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c | 6 ++++++
 drivers/platform/msm/ipa/ipa_v3/ipa_i.h  | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 3da7642625c0..779e1a61dcff 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -1673,6 +1673,12 @@ int ipa3_teardown_sys_pipe(u32 clnt_hdl)
 	}
 	if (ep->sys->repl_wq)
 		flush_workqueue(ep->sys->repl_wq);
+
+	if(ep->sys->common_sys) {
+		cancel_delayed_work_sync(&ep->sys->common_sys->freepage_work);
+		tasklet_kill(&ep->sys->common_sys->tasklet_find_freepage);
+	}
+
 	if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
 		ipa3_cleanup_rx(ep->sys);
 
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
index 9c10e4233043..1c8ffddaa1ec 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
@@ -1131,7 +1131,6 @@ struct ipa3_sys_context {
 	struct ipa3_sys_context *common_sys;
 	struct tasklet_struct tasklet_find_freepage;
 	atomic_t page_avilable;
-	struct delayed_work freepage_work;
 	u32 napi_sort_page_thrshld_cnt;
 
 	/* ordering is important - mutable fields go above */
@@ -1149,6 +1148,7 @@ struct ipa3_sys_context {
 	unsigned int napi_sch_cnt;
 	unsigned int napi_comp_cnt;
 	struct workqueue_struct *freepage_wq;
+	struct delayed_work freepage_work;
 	/* ordering is important - other immutable fields go below */
 };
 

From 28b3e2400af6ab12aa3e67165257a9ea87b4f38b Mon Sep 17 00:00:00 2001
From: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
Date: Tue, 15 Mar 2022 21:27:48 +0530
Subject: [PATCH 6/9] msm: ipa3: Flush free page WQ only for page recyle
 replenish handler

Free page work init only for page recycle replenish handler.

Change-Id: Ieb5e7b2deb109fa46d3ab38be44eaf34fffd4ad2
Signed-off-by: Ashok Vuyyuru <quic_avuyyuru@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 779e1a61dcff..6b78bed73980 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -1674,7 +1674,7 @@ int ipa3_teardown_sys_pipe(u32 clnt_hdl)
 	if (ep->sys->repl_wq)
 		flush_workqueue(ep->sys->repl_wq);
 
-	if(ep->sys->common_sys) {
+	if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) {
 		cancel_delayed_work_sync(&ep->sys->common_sys->freepage_work);
 		tasklet_kill(&ep->sys->common_sys->tasklet_find_freepage);
 	}

From 8a2df635024f8c5f0e5eaa2c101d6c1eaa6cc87b Mon Sep 17 00:00:00 2001
From: Chaitanya Pratapa <quic_cpratapa@quicinc.com>
Date: Wed, 16 Feb 2022 23:03:51 -0800
Subject: [PATCH 7/9] msm: ipa: avoid OOM Killer for temp allocations

Make changes to use __GFP_RETRY_MAYFAIL to avoid triggering
OOM Killer when allocating pages for temp allocations.

Change-Id: I2f5843c9a1c0a9f41e215c8241713f4670e4b35c
Signed-off-by: Chaitanya Pratapa <quic_cpratapa@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 6b78bed73980..0d4650181bc0 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -2189,6 +2189,9 @@ static struct ipa3_rx_pkt_wrapper *ipa3_alloc_rx_pkt_page(
 		return NULL;
 
 	rx_pkt->page_data.page_order = IPA_WAN_PAGE_ORDER;
+	/* For temporary allocations, avoid triggering OOM Killer. */
+	if (is_tmp_alloc)
+		flag |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
 	/* Try a lower order page for order 3 pages in case allocation fails. */
 	rx_pkt->page_data.page = ipa3_alloc_page(flag,
 				&rx_pkt->page_data.page_order,

From a7823d280e9a1444767a39fda9dc6420bd20abfd Mon Sep 17 00:00:00 2001
From: Chaitanya Pratapa <quic_cpratapa@quicinc.com>
Date: Thu, 7 Apr 2022 17:58:01 -0700
Subject: [PATCH 8/9] msm: ipa: allocate page recycling buffers only once

Order-3 allocations are costly when system is in full use.
Make changes to allocate page recycling buffers only once on boot up.

Change-Id: I445e27ecc03aaa00483e7a46a468ef0a6f4bfcfb
Signed-off-by: Chaitanya Pratapa <quic_cpratapa@quicinc.com>
---
 drivers/platform/msm/ipa/ipa_v3/ipa_dp.c | 87 ++++++++++++------------
 drivers/platform/msm/ipa/ipa_v3/ipa_i.h  |  4 +-
 2 files changed, 44 insertions(+), 47 deletions(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
index 0d4650181bc0..bea0e82699b7 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
@@ -1355,28 +1355,40 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 			sys_in->client == IPA_CLIENT_APPS_WAN_CONS &&
 			coal_ep_id != IPA_EP_NOT_ALLOCATED &&
 			ipa3_ctx->ep[coal_ep_id].valid == 1)) {
-			ep->sys->page_recycle_repl = kzalloc(
-				sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL);
+			/* Allocate page recycling pool only once. */
 			if (!ep->sys->page_recycle_repl) {
-				IPAERR("failed to alloc repl for client %d\n",
-						sys_in->client);
-				result = -ENOMEM;
-				goto fail_napi;
+				ep->sys->page_recycle_repl = kzalloc(
+					sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL);
+				if (!ep->sys->page_recycle_repl) {
+					IPAERR("failed to alloc repl for client %d\n",
+							sys_in->client);
+					result = -ENOMEM;
+					goto fail_napi;
+				}
+				atomic_set(&ep->sys->page_recycle_repl->pending, 0);
+				/* For common page pool double the pool size. */
+				if (ipa3_ctx->wan_common_page_pool &&
+					sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
+					ep->sys->page_recycle_repl->capacity =
+							(ep->sys->rx_pool_sz + 1) *
+							IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR;
+				else
+					ep->sys->page_recycle_repl->capacity =
+							(ep->sys->rx_pool_sz + 1) *
+							IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR;
+				IPADBG("Page repl capacity for client:%d, value:%d\n",
+						   sys_in->client, ep->sys->page_recycle_repl->capacity);
+				INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
+				INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work);
+				tasklet_init(&ep->sys->tasklet_find_freepage,
+					ipa3_tasklet_find_freepage, (unsigned long) ep->sys);
+				ipa3_replenish_rx_page_cache(ep->sys);
+			} else {
+ 				ep->sys->napi_sort_page_thrshld_cnt = 0;
+				/* Sort the pages once. */
+				ipa3_tasklet_find_freepage((unsigned long) ep->sys);
 			}
-			atomic_set(&ep->sys->page_recycle_repl->pending, 0);
-			/* For common page pool double the pool size. */
-			if (ipa3_ctx->wan_common_page_pool &&
-				sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
-				ep->sys->page_recycle_repl->capacity =
-						(ep->sys->rx_pool_sz + 1) *
-						IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR;
-			else
-				ep->sys->page_recycle_repl->capacity =
-						(ep->sys->rx_pool_sz + 1) *
-						IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR;
-			IPADBG("Page repl capacity for client:%d, value:%d\n",
-					   sys_in->client, ep->sys->page_recycle_repl->capacity);
-			INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
+
 			ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
 			if (!ep->sys->repl) {
 				IPAERR("failed to alloc repl for client %d\n",
@@ -1399,11 +1411,6 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 			atomic_set(&ep->sys->repl->head_idx, 0);
 			atomic_set(&ep->sys->repl->tail_idx, 0);
 
-			tasklet_init(&ep->sys->tasklet_find_freepage,
-					ipa3_tasklet_find_freepage, (unsigned long) ep->sys);
-			INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work);
-			ep->sys->napi_sort_page_thrshld_cnt = 0;
-			ipa3_replenish_rx_page_cache(ep->sys);
 			ipa3_wq_page_repl(&ep->sys->repl_work);
 		} else {
 			/* Use pool same as coal pipe when common page pool is used */
@@ -2946,11 +2953,17 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data)
 	if (!rx_pkt->page_data.is_tmp_alloc) {
 		list_del_init(&rx_pkt->link);
 		page_ref_dec(rx_pkt->page_data.page);
+		spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
+		/* Add the element to head. */
+		list_add(&rx_pkt->link,
+			&rx_pkt->sys->page_recycle_repl->page_repl_head);
+		spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
+	} else {
+		dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr,
+			rx_pkt->len, DMA_FROM_DEVICE);
+		__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
+		kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
 	}
-	dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr,
-		rx_pkt->len, DMA_FROM_DEVICE);
-	__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
-	kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
 }
 
 /**
@@ -3010,22 +3023,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
 		kfree(sys->repl);
 		sys->repl = NULL;
 	}
-	if (sys->page_recycle_repl) {
-		list_for_each_entry_safe(rx_pkt, r,
-		&sys->page_recycle_repl->page_repl_head, link) {
-			list_del(&rx_pkt->link);
-			dma_unmap_page(ipa3_ctx->pdev,
-				rx_pkt->page_data.dma_addr,
-				rx_pkt->len,
-				DMA_FROM_DEVICE);
-			__free_pages(rx_pkt->page_data.page,
-				rx_pkt->page_data.page_order);
-			kmem_cache_free(
-				ipa3_ctx->rx_pkt_wrapper_cache,
-				rx_pkt);
-		}
-		kfree(sys->page_recycle_repl);
-	}
 }
 
 static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len)
diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
index 1c8ffddaa1ec..ababce0a4cd6 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
@@ -1117,7 +1117,6 @@ struct ipa3_sys_context {
 	struct work_struct repl_work;
 	void (*repl_hdlr)(struct ipa3_sys_context *sys);
 	struct ipa3_repl_ctx *repl;
-	struct ipa3_page_repl_ctx *page_recycle_repl;
 	u32 pkt_sent;
 	struct napi_struct *napi_obj;
 	struct list_head pending_pkts[GSI_VEID_MAX];
@@ -1129,7 +1128,6 @@ struct ipa3_sys_context {
 	atomic_t in_napi_context;
 	bool common_buff_pool;
 	struct ipa3_sys_context *common_sys;
-	struct tasklet_struct tasklet_find_freepage;
 	atomic_t page_avilable;
 	u32 napi_sort_page_thrshld_cnt;
 
@@ -1147,8 +1145,10 @@ struct ipa3_sys_context {
 	u32 pm_hdl;
 	unsigned int napi_sch_cnt;
 	unsigned int napi_comp_cnt;
+	struct ipa3_page_repl_ctx *page_recycle_repl;
 	struct workqueue_struct *freepage_wq;
 	struct delayed_work freepage_work;
+	struct tasklet_struct tasklet_find_freepage;
 	/* ordering is important - other immutable fields go below */
 };
 

From 4579b40320793031a200cdd70e4aa2c5657e573a Mon Sep 17 00:00:00 2001
From: Jagadeesh Ponduru <quic_jponduru@quicinc.com>
Date: Sun, 26 Nov 2023 08:51:28 +0530
Subject: [PATCH 9/9] msm: ipa3: reduce rx-wan pool and cmn page, temp pool
 sizes

Changes to reduce the rx wan pool, cmn page and temp pool sizes

Change-Id: Ibc86c4f2cd0f440d8eafd247600f944175e08803
---
 drivers/platform/msm/ipa/ipa_v3/ipa_i.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
index ababce0a4cd6..ceb439f729b2 100644
--- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
+++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
@@ -59,11 +59,11 @@
 #define IPA_QMAP_HEADER_LENGTH (4)
 #define IPA_DL_CHECKSUM_LENGTH (8)
 #define IPA_NUM_DESC_PER_SW_TX (3)
-#define IPA_GENERIC_RX_POOL_SZ_WAN 224
+#define IPA_GENERIC_RX_POOL_SZ_WAN 192
 #define IPA_GENERIC_RX_POOL_SZ 192
 #define IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR 2
-#define IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR 5
-#define IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR 3
+#define IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR 4
+#define IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR 2
 #define IPA_UC_FINISH_MAX 6
 #define IPA_UC_WAIT_MIN_SLEEP 1000
 #define IPA_UC_WAII_MAX_SLEEP 1200