Merge tag 'LA.UM.9.14.r1-24200-LAHAINA.QSSI13.0' of https://git.codelinaro.org/clo/la/platform/vendor/opensource/dataipa into android13-5.4-lahaina
"LA.UM.9.14.r1-24200-LAHAINA.QSSI13.0" * tag 'LA.UM.9.14.r1-24200-LAHAINA.QSSI13.0' of https://git.codelinaro.org/clo/la/platform/vendor/opensource/dataipa: msm: ipa3: reduce rx-wan pool and cmn page, temp pool sizes msm: ipa: allocate page recycling buffers only once msm: ipa: avoid OOM Killer for temp allocations msm: ipa3: Flush free page WQ only for page recyle replenish handler msm: ipa3: Fix to flush to workqueue during teardown pipe msm: ipa3: Fix to destory workqueue only if created msm: ipa3: Changes to enhance find free pages from list msm: ipa: page pool recycling enhancements msm: ipa: page pool recycling enhancements Change-Id: I21c0f5974f08fc032605d2710858eb28e592d1f0
This commit is contained in:
Michael Bestas
commit
edd5792efd
@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2023-2024 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/clk.h>
|
||||
@ -7359,6 +7359,13 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
|
||||
ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc = 0;
|
||||
ipa3_ctx->stats.page_recycle_stats[1].total_replenished = 0;
|
||||
ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0;
|
||||
memset(ipa3_ctx->stats.page_recycle_cnt, 0,
|
||||
sizeof(ipa3_ctx->stats.page_recycle_cnt));
|
||||
ipa3_ctx->stats.num_sort_tasklet_sched[0] = 0;
|
||||
ipa3_ctx->stats.num_sort_tasklet_sched[1] = 0;
|
||||
ipa3_ctx->stats.num_sort_tasklet_sched[2] = 0;
|
||||
ipa3_ctx->stats.num_of_times_wq_reschd = 0;
|
||||
ipa3_ctx->stats.page_recycle_cnt_in_tasklet = 0;
|
||||
ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset;
|
||||
ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control;
|
||||
ipa3_ctx->ee = resource_p->ee;
|
||||
@ -7549,6 +7556,17 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
|
||||
/* Enable ipa3_ctx->enable_napi_chain */
|
||||
ipa3_ctx->enable_napi_chain = 1;
|
||||
|
||||
/* Initialize Page poll threshold. */
|
||||
ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD;
|
||||
|
||||
/*Initialize number napi without prealloc buff*/
|
||||
ipa3_ctx->ipa_max_napi_sort_page_thrshld = IPA_MAX_NAPI_SORT_PAGE_THRSHLD;
|
||||
ipa3_ctx->page_wq_reschd_time = IPA_MAX_PAGE_WQ_RESCHED_TIME;
|
||||
|
||||
/* Use common page pool for Def/Coal pipe. */
|
||||
if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_9)
|
||||
ipa3_ctx->wan_common_page_pool = true;
|
||||
|
||||
/* assume clock is on in virtual/emulation mode */
|
||||
if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
|
||||
ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION)
|
||||
@ -8187,7 +8205,7 @@ static int get_ipa_dts_configuration(struct platform_device *pdev,
|
||||
ipa_drv_res->ipa_mhi_dynamic_config = false;
|
||||
ipa_drv_res->use_64_bit_dma_mask = false;
|
||||
ipa_drv_res->use_bw_vote = false;
|
||||
ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
|
||||
ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ_WAN;
|
||||
ipa_drv_res->lan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
|
||||
ipa_drv_res->apply_rg10_wa = false;
|
||||
ipa_drv_res->gsi_ch20_wa = false;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2012-2020, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifdef CONFIG_DEBUG_FS
|
||||
@ -1353,6 +1353,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf,
|
||||
"act_clnt=%u\n"
|
||||
"con_clnt_bmap=0x%x\n"
|
||||
"wan_rx_empty=%u\n"
|
||||
"wan_rx_empty_coal=%u\n"
|
||||
"wan_repl_rx_empty=%u\n"
|
||||
"lan_rx_empty=%u\n"
|
||||
"lan_repl_rx_empty=%u\n"
|
||||
@ -1372,6 +1373,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf,
|
||||
atomic_read(&ipa3_ctx->ipa3_active_clients.cnt),
|
||||
connect,
|
||||
ipa3_ctx->stats.wan_rx_empty,
|
||||
ipa3_ctx->stats.wan_rx_empty_coal,
|
||||
ipa3_ctx->stats.wan_repl_rx_empty,
|
||||
ipa3_ctx->stats.lan_rx_empty,
|
||||
ipa3_ctx->stats.lan_repl_rx_empty,
|
||||
@ -1421,24 +1423,46 @@ static ssize_t ipa3_read_odlstats(struct file *file, char __user *ubuf,
|
||||
return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
|
||||
}
|
||||
|
||||
|
||||
static ssize_t ipa3_read_page_recycle_stats(struct file *file,
|
||||
char __user *ubuf, size_t count, loff_t *ppos)
|
||||
{
|
||||
int nbytes;
|
||||
int cnt = 0;
|
||||
int cnt = 0, i = 0, k = 0;
|
||||
|
||||
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
|
||||
"COAL : Total number of packets replenished =%llu\n"
|
||||
"COAL : Number of page recycled packets =%llu\n"
|
||||
"COAL : Number of tmp alloc packets =%llu\n"
|
||||
"COAL : Number of times tasklet scheduled =%llu\n"
|
||||
"DEF : Total number of packets replenished =%llu\n"
|
||||
"DEF : Number of tmp alloc packets =%llu\n",
|
||||
"DEF : Number of page recycled packets =%llu\n"
|
||||
"DEF : Number of tmp alloc packets =%llu\n"
|
||||
"DEF : Number of times tasklet scheduled =%llu\n"
|
||||
"COMMON : Number of page recycled in tasklet =%llu\n"
|
||||
"COMMON : Number of times free pages not found in tasklet =%llu\n",
|
||||
ipa3_ctx->stats.page_recycle_stats[0].total_replenished,
|
||||
ipa3_ctx->stats.page_recycle_stats[0].page_recycled,
|
||||
ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc,
|
||||
ipa3_ctx->stats.num_sort_tasklet_sched[0],
|
||||
ipa3_ctx->stats.page_recycle_stats[1].total_replenished,
|
||||
ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc);
|
||||
ipa3_ctx->stats.page_recycle_stats[1].page_recycled,
|
||||
ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc,
|
||||
ipa3_ctx->stats.num_sort_tasklet_sched[1],
|
||||
ipa3_ctx->stats.page_recycle_cnt_in_tasklet,
|
||||
ipa3_ctx->stats.num_of_times_wq_reschd);
|
||||
|
||||
cnt += nbytes;
|
||||
|
||||
for (k = 0; k < 2; k++) {
|
||||
for (i = 0; i < ipa3_ctx->page_poll_threshold; i++) {
|
||||
nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN,
|
||||
"COMMON : Page replenish efficiency[%d][%d] =%llu\n",
|
||||
k, i, ipa3_ctx->stats.page_recycle_cnt[k][i]);
|
||||
cnt += nbytes;
|
||||
}
|
||||
}
|
||||
|
||||
return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
|
||||
}
|
||||
static ssize_t ipa3_read_wstats(struct file *file, char __user *ubuf,
|
||||
@ -2772,6 +2796,68 @@ static ssize_t ipa3_enable_ipc_low(struct file *file,
|
||||
return count;
|
||||
}
|
||||
|
||||
static ssize_t ipa3_read_ipa_max_napi_sort_page_thrshld(struct file *file,
|
||||
char __user *buf, size_t count, loff_t *ppos) {
|
||||
|
||||
int nbytes;
|
||||
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
|
||||
"page max napi without free page = %d\n",
|
||||
ipa3_ctx->ipa_max_napi_sort_page_thrshld);
|
||||
return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
|
||||
|
||||
}
|
||||
|
||||
static ssize_t ipa3_write_ipa_max_napi_sort_page_thrshld(struct file *file,
|
||||
const char __user *buf, size_t count, loff_t *ppos) {
|
||||
|
||||
int ret;
|
||||
u8 ipa_max_napi_sort_page_thrshld = 0;
|
||||
|
||||
if (count >= sizeof(dbg_buff))
|
||||
return -EFAULT;
|
||||
|
||||
ret = kstrtou8_from_user(buf, count, 0, &ipa_max_napi_sort_page_thrshld);
|
||||
if(ret)
|
||||
return ret;
|
||||
|
||||
ipa3_ctx->ipa_max_napi_sort_page_thrshld = ipa_max_napi_sort_page_thrshld;
|
||||
|
||||
IPADBG("napi cnt without prealloc pages = %d", ipa3_ctx->ipa_max_napi_sort_page_thrshld);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
static ssize_t ipa3_read_page_wq_reschd_time(struct file *file,
|
||||
char __user *buf, size_t count, loff_t *ppos) {
|
||||
|
||||
int nbytes;
|
||||
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
|
||||
"Page WQ reschduule time = %d\n",
|
||||
ipa3_ctx->page_wq_reschd_time);
|
||||
return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
|
||||
|
||||
}
|
||||
|
||||
static ssize_t ipa3_write_page_wq_reschd_time(struct file *file,
|
||||
const char __user *buf, size_t count, loff_t *ppos) {
|
||||
|
||||
int ret;
|
||||
u8 page_wq_reschd_time = 0;
|
||||
|
||||
if (count >= sizeof(dbg_buff))
|
||||
return -EFAULT;
|
||||
|
||||
ret = kstrtou8_from_user(buf, count, 0, &page_wq_reschd_time);
|
||||
if(ret)
|
||||
return ret;
|
||||
|
||||
ipa3_ctx->page_wq_reschd_time = page_wq_reschd_time;
|
||||
|
||||
IPADBG("Updated page WQ reschedule time = %d", ipa3_ctx->page_wq_reschd_time);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
static ssize_t ipa3_read_gsi_wdi3_db_polling(struct file *file,
|
||||
char __user *buf, size_t count, loff_t *ppos) {
|
||||
|
||||
@ -2804,6 +2890,41 @@ static ssize_t ipa3_write_gsi_wdi3_db_polling(struct file *file,
|
||||
return count;
|
||||
}
|
||||
|
||||
static ssize_t ipa3_read_page_poll_threshold(struct file *file,
|
||||
char __user *buf, size_t count, loff_t *ppos) {
|
||||
|
||||
int nbytes;
|
||||
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
|
||||
"Page Poll Threshold = %d\n",
|
||||
ipa3_ctx->page_poll_threshold);
|
||||
return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
|
||||
|
||||
}
|
||||
|
||||
static ssize_t ipa3_write_page_poll_threshold(struct file *file,
|
||||
const char __user *buf, size_t count, loff_t *ppos) {
|
||||
|
||||
int ret;
|
||||
u8 page_poll_threshold =0;
|
||||
|
||||
if (count >= sizeof(dbg_buff))
|
||||
return -EFAULT;
|
||||
|
||||
ret = kstrtou8_from_user(buf, count, 0, &page_poll_threshold);
|
||||
if(ret)
|
||||
return ret;
|
||||
|
||||
if(page_poll_threshold != 0 &&
|
||||
page_poll_threshold <= IPA_PAGE_POLL_THRESHOLD_MAX)
|
||||
ipa3_ctx->page_poll_threshold = page_poll_threshold;
|
||||
else
|
||||
IPAERR("Invalid value \n");
|
||||
|
||||
IPADBG("Updated page poll threshold = %d", ipa3_ctx->page_poll_threshold);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
static const struct ipa3_debugfs_file debugfs_files[] = {
|
||||
{
|
||||
"gen_reg", IPA_READ_ONLY_MODE, NULL, {
|
||||
@ -2986,6 +3107,21 @@ static const struct ipa3_debugfs_file debugfs_files[] = {
|
||||
.read = ipa3_read_gsi_wdi3_db_polling,
|
||||
.write = ipa3_write_gsi_wdi3_db_polling,
|
||||
}
|
||||
}, {
|
||||
"page_poll_threshold", IPA_READ_WRITE_MODE, NULL, {
|
||||
.read = ipa3_read_page_poll_threshold,
|
||||
.write = ipa3_write_page_poll_threshold,
|
||||
}
|
||||
}, {
|
||||
"page_wq_reschd_time", IPA_READ_WRITE_MODE, NULL, {
|
||||
.read = ipa3_read_page_wq_reschd_time,
|
||||
.write = ipa3_write_page_wq_reschd_time,
|
||||
}
|
||||
}, {
|
||||
"ipa_max_napi_sort_page_thrshld", IPA_READ_WRITE_MODE, NULL, {
|
||||
.read = ipa3_read_ipa_max_napi_sort_page_thrshld,
|
||||
.write = ipa3_write_ipa_max_napi_sort_page_thrshld,
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/delay.h>
|
||||
@ -137,6 +137,7 @@ static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys,
|
||||
static unsigned long tag_to_pointer_wa(uint64_t tag);
|
||||
static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt);
|
||||
static void ipa3_tasklet_rx_notify(unsigned long data);
|
||||
static void ipa3_tasklet_find_freepage(unsigned long data);
|
||||
|
||||
static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit);
|
||||
|
||||
@ -1048,6 +1049,60 @@ static void ipa_pm_sys_pipe_cb(void *p, enum ipa_pm_cb_event event)
|
||||
}
|
||||
}
|
||||
|
||||
static void ipa3_schd_freepage_work(struct work_struct *work)
|
||||
{
|
||||
struct delayed_work *dwork;
|
||||
struct ipa3_sys_context *sys;
|
||||
|
||||
dwork = container_of(work, struct delayed_work, work);
|
||||
sys = container_of(dwork, struct ipa3_sys_context, freepage_work);
|
||||
|
||||
IPADBG_LOW("WQ scheduled, reschedule sort tasklet\n");
|
||||
|
||||
tasklet_schedule(&sys->tasklet_find_freepage);
|
||||
}
|
||||
|
||||
static void ipa3_tasklet_find_freepage(unsigned long data)
|
||||
{
|
||||
struct ipa3_sys_context *sys;
|
||||
struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
|
||||
struct ipa3_rx_pkt_wrapper *tmp = NULL;
|
||||
struct page *cur_page;
|
||||
int found_free_page = 0;
|
||||
struct list_head temp_head;
|
||||
|
||||
sys = (struct ipa3_sys_context *)data;
|
||||
|
||||
INIT_LIST_HEAD(&temp_head);
|
||||
spin_lock_bh(&sys->common_sys->spinlock);
|
||||
list_for_each_entry_safe(rx_pkt, tmp,
|
||||
&sys->page_recycle_repl->page_repl_head, link) {
|
||||
cur_page = rx_pkt->page_data.page;
|
||||
if (page_ref_count(cur_page) == 1) {
|
||||
/* Found a free page. */
|
||||
list_del_init(&rx_pkt->link);
|
||||
list_add(&rx_pkt->link, &temp_head);
|
||||
found_free_page++;
|
||||
}
|
||||
}
|
||||
if (!found_free_page) {
|
||||
/*Not found free page rescheduling tasklet after 2msec*/
|
||||
IPADBG_LOW("Scheduling WQ not found free pages\n");
|
||||
++ipa3_ctx->stats.num_of_times_wq_reschd;
|
||||
queue_delayed_work(sys->freepage_wq,
|
||||
&sys->freepage_work,
|
||||
msecs_to_jiffies(ipa3_ctx->page_wq_reschd_time));
|
||||
} else {
|
||||
/*Allow to use pre-allocated buffers*/
|
||||
list_splice(&temp_head, &sys->page_recycle_repl->page_repl_head);
|
||||
ipa3_ctx->stats.page_recycle_cnt_in_tasklet += found_free_page;
|
||||
IPADBG_LOW("found free pages count = %d\n", found_free_page);
|
||||
atomic_set(&sys->common_sys->page_avilable, 1);
|
||||
}
|
||||
spin_unlock_bh(&sys->common_sys->spinlock);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* ipa3_setup_sys_pipe() - Setup an IPA GPI pipe and perform
|
||||
* IPA EP configuration
|
||||
@ -1130,7 +1185,8 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
snprintf(buff, IPA_RESOURCE_NAME_MAX, "iparepwq%d",
|
||||
sys_in->client);
|
||||
ep->sys->repl_wq = alloc_workqueue(buff,
|
||||
WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1);
|
||||
WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | WQ_HIGHPRI,
|
||||
1);
|
||||
if (!ep->sys->repl_wq) {
|
||||
IPAERR("failed to create rep wq for client %d\n",
|
||||
sys_in->client);
|
||||
@ -1138,6 +1194,9 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
goto fail_wq2;
|
||||
}
|
||||
|
||||
snprintf(buff, IPA_RESOURCE_NAME_MAX, "ipafreepagewq%d",
|
||||
sys_in->client);
|
||||
|
||||
INIT_LIST_HEAD(&ep->sys->head_desc_list);
|
||||
INIT_LIST_HEAD(&ep->sys->rcycl_list);
|
||||
INIT_LIST_HEAD(&ep->sys->avail_tx_wrapper_list);
|
||||
@ -1154,6 +1213,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
/* Use coalescing pipe PM handle for default pipe also*/
|
||||
ep->sys->pm_hdl = ipa3_ctx->ep[coal_ep_id].sys->pm_hdl;
|
||||
} else if (IPA_CLIENT_IS_CONS(sys_in->client)) {
|
||||
ep->sys->freepage_wq = alloc_workqueue(buff,
|
||||
WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS |
|
||||
WQ_HIGHPRI, 1);
|
||||
if (!ep->sys->freepage_wq) {
|
||||
IPAERR("failed to create freepage wq for client %d\n",
|
||||
sys_in->client);
|
||||
result = -EFAULT;
|
||||
goto fail_wq3;
|
||||
}
|
||||
|
||||
pm_reg.name = ipa_clients_strings[sys_in->client];
|
||||
pm_reg.callback = ipa_pm_sys_pipe_cb;
|
||||
pm_reg.user_data = ep->sys;
|
||||
@ -1254,6 +1323,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
}
|
||||
|
||||
*clnt_hdl = ipa_ep_idx;
|
||||
ep->sys->common_sys = ipa3_ctx->ep[ipa_ep_idx].sys;
|
||||
|
||||
if (ep->sys->repl_hdlr == ipa3_fast_replenish_rx_cache) {
|
||||
ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
|
||||
@ -1279,7 +1349,14 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
}
|
||||
}
|
||||
|
||||
/* Use common page pool for Coal and defalt pipe if applicable. */
|
||||
if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) {
|
||||
if (!(ipa3_ctx->wan_common_page_pool &&
|
||||
sys_in->client == IPA_CLIENT_APPS_WAN_CONS &&
|
||||
coal_ep_id != IPA_EP_NOT_ALLOCATED &&
|
||||
ipa3_ctx->ep[coal_ep_id].valid == 1)) {
|
||||
/* Allocate page recycling pool only once. */
|
||||
if (!ep->sys->page_recycle_repl) {
|
||||
ep->sys->page_recycle_repl = kzalloc(
|
||||
sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL);
|
||||
if (!ep->sys->page_recycle_repl) {
|
||||
@ -1289,14 +1366,29 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
goto fail_napi;
|
||||
}
|
||||
atomic_set(&ep->sys->page_recycle_repl->pending, 0);
|
||||
/* For common page pool double the pool size. */
|
||||
if (ipa3_ctx->wan_common_page_pool &&
|
||||
sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
|
||||
ep->sys->page_recycle_repl->capacity =
|
||||
(ep->sys->rx_pool_sz + 1) * 2;
|
||||
(ep->sys->rx_pool_sz + 1) *
|
||||
IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR;
|
||||
else
|
||||
ep->sys->page_recycle_repl->capacity =
|
||||
(ep->sys->rx_pool_sz + 1) *
|
||||
IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR;
|
||||
IPADBG("Page repl capacity for client:%d, value:%d\n",
|
||||
sys_in->client, ep->sys->page_recycle_repl->capacity);
|
||||
INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
|
||||
INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work);
|
||||
tasklet_init(&ep->sys->tasklet_find_freepage,
|
||||
ipa3_tasklet_find_freepage, (unsigned long) ep->sys);
|
||||
ipa3_replenish_rx_page_cache(ep->sys);
|
||||
} else {
|
||||
ep->sys->napi_sort_page_thrshld_cnt = 0;
|
||||
/* Sort the pages once. */
|
||||
ipa3_tasklet_find_freepage((unsigned long) ep->sys);
|
||||
}
|
||||
|
||||
ep->sys->page_recycle_repl->cache =
|
||||
kcalloc(ep->sys->page_recycle_repl->capacity,
|
||||
sizeof(void *), GFP_KERNEL);
|
||||
atomic_set(&ep->sys->page_recycle_repl->head_idx, 0);
|
||||
atomic_set(&ep->sys->page_recycle_repl->tail_idx, 0);
|
||||
ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
|
||||
if (!ep->sys->repl) {
|
||||
IPAERR("failed to alloc repl for client %d\n",
|
||||
@ -1304,16 +1396,30 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
result = -ENOMEM;
|
||||
goto fail_page_recycle_repl;
|
||||
}
|
||||
/* For common page pool triple the pool size. */
|
||||
if (ipa3_ctx->wan_common_page_pool &&
|
||||
sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
|
||||
ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1) *
|
||||
IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR;
|
||||
else
|
||||
ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1);
|
||||
|
||||
IPADBG("Repl capacity for client:%d, value:%d\n",
|
||||
sys_in->client, ep->sys->repl->capacity);
|
||||
atomic_set(&ep->sys->repl->pending, 0);
|
||||
ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity,
|
||||
sizeof(void *), GFP_KERNEL);
|
||||
atomic_set(&ep->sys->repl->head_idx, 0);
|
||||
atomic_set(&ep->sys->repl->tail_idx, 0);
|
||||
|
||||
ipa3_replenish_rx_page_cache(ep->sys);
|
||||
ipa3_wq_page_repl(&ep->sys->repl_work);
|
||||
} else {
|
||||
/* Use pool same as coal pipe when common page pool is used */
|
||||
ep->sys->common_buff_pool = true;
|
||||
ep->sys->common_sys = ipa3_ctx->ep[coal_ep_id].sys;
|
||||
ep->sys->repl = ipa3_ctx->ep[coal_ep_id].sys->repl;
|
||||
ep->sys->page_recycle_repl =
|
||||
ipa3_ctx->ep[coal_ep_id].sys->page_recycle_repl;
|
||||
}
|
||||
}
|
||||
|
||||
if (IPA_CLIENT_IS_CONS(sys_in->client)) {
|
||||
@ -1383,16 +1489,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
|
||||
fail_gen3:
|
||||
ipa3_disable_data_path(ipa_ep_idx);
|
||||
fail_repl:
|
||||
if (IPA_CLIENT_IS_CONS(ep->client))
|
||||
if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
|
||||
ipa3_cleanup_rx(ep->sys);
|
||||
|
||||
if(ep->sys->repl) {
|
||||
ep->sys->repl_hdlr = ipa3_replenish_rx_cache;
|
||||
if (ep->sys->repl && !ep->sys->common_buff_pool) {
|
||||
ep->sys->repl->capacity = 0;
|
||||
kfree(ep->sys->repl);
|
||||
ep->sys->repl = NULL;
|
||||
}
|
||||
fail_page_recycle_repl:
|
||||
if (ep->sys->page_recycle_repl) {
|
||||
if (ep->sys->page_recycle_repl && !ep->sys->common_buff_pool) {
|
||||
ep->sys->page_recycle_repl->capacity = 0;
|
||||
kfree(ep->sys->page_recycle_repl);
|
||||
ep->sys->page_recycle_repl = NULL;
|
||||
@ -1405,6 +1511,9 @@ fail_napi:
|
||||
fail_gen2:
|
||||
ipa_pm_deregister(ep->sys->pm_hdl);
|
||||
fail_pm:
|
||||
if (ep->sys->freepage_wq)
|
||||
destroy_workqueue(ep->sys->freepage_wq);
|
||||
fail_wq3:
|
||||
destroy_workqueue(ep->sys->repl_wq);
|
||||
fail_wq2:
|
||||
destroy_workqueue(ep->sys->wq);
|
||||
@ -1571,7 +1680,13 @@ int ipa3_teardown_sys_pipe(u32 clnt_hdl)
|
||||
}
|
||||
if (ep->sys->repl_wq)
|
||||
flush_workqueue(ep->sys->repl_wq);
|
||||
if (IPA_CLIENT_IS_CONS(ep->client))
|
||||
|
||||
if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) {
|
||||
cancel_delayed_work_sync(&ep->sys->common_sys->freepage_work);
|
||||
tasklet_kill(&ep->sys->common_sys->tasklet_find_freepage);
|
||||
}
|
||||
|
||||
if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
|
||||
ipa3_cleanup_rx(ep->sys);
|
||||
|
||||
if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(ep->client)) {
|
||||
@ -1657,7 +1772,7 @@ static int ipa3_teardown_coal_def_pipe(u32 clnt_hdl)
|
||||
|
||||
if (ep->sys->repl_wq)
|
||||
flush_workqueue(ep->sys->repl_wq);
|
||||
if (IPA_CLIENT_IS_CONS(ep->client))
|
||||
if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
|
||||
ipa3_cleanup_rx(ep->sys);
|
||||
|
||||
ep->valid = 0;
|
||||
@ -2081,6 +2196,9 @@ static struct ipa3_rx_pkt_wrapper *ipa3_alloc_rx_pkt_page(
|
||||
return NULL;
|
||||
|
||||
rx_pkt->page_data.page_order = IPA_WAN_PAGE_ORDER;
|
||||
/* For temporary allocations, avoid triggering OOM Killer. */
|
||||
if (is_tmp_alloc)
|
||||
flag |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
|
||||
/* Try a lower order page for order 3 pages in case allocation fails. */
|
||||
rx_pkt->page_data.page = ipa3_alloc_page(flag,
|
||||
&rx_pkt->page_data.page_order,
|
||||
@ -2126,9 +2244,12 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys)
|
||||
ipa_assert();
|
||||
break;
|
||||
}
|
||||
INIT_LIST_HEAD(&rx_pkt->link);
|
||||
rx_pkt->sys = sys;
|
||||
sys->page_recycle_repl->cache[curr] = rx_pkt;
|
||||
list_add_tail(&rx_pkt->link,
|
||||
&sys->page_recycle_repl->page_repl_head);
|
||||
}
|
||||
atomic_set(&sys->common_sys->page_avilable, 1);
|
||||
|
||||
return;
|
||||
|
||||
@ -2189,12 +2310,54 @@ static inline void __trigger_repl_work(struct ipa3_sys_context *sys)
|
||||
head = atomic_read(&sys->repl->head_idx);
|
||||
avail = (tail - head) % sys->repl->capacity;
|
||||
|
||||
if (avail < sys->repl->capacity / 4) {
|
||||
if (avail < sys->repl->capacity / 2) {
|
||||
atomic_set(&sys->repl->pending, 1);
|
||||
queue_work(sys->repl_wq, &sys->repl_work);
|
||||
}
|
||||
}
|
||||
|
||||
static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page
|
||||
(
|
||||
struct ipa3_sys_context *sys,
|
||||
u32 stats_i
|
||||
)
|
||||
{
|
||||
struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
|
||||
struct ipa3_rx_pkt_wrapper *tmp = NULL;
|
||||
struct page *cur_page;
|
||||
int i = 0;
|
||||
u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold;
|
||||
|
||||
spin_lock_bh(&sys->common_sys->spinlock);
|
||||
list_for_each_entry_safe(rx_pkt, tmp,
|
||||
&sys->page_recycle_repl->page_repl_head, link) {
|
||||
if (i == LOOP_THRESHOLD)
|
||||
break;
|
||||
cur_page = rx_pkt->page_data.page;
|
||||
if (page_ref_count(cur_page) == 1) {
|
||||
/* Found a free page. */
|
||||
page_ref_inc(cur_page);
|
||||
list_del_init(&rx_pkt->link);
|
||||
++ipa3_ctx->stats.page_recycle_cnt[stats_i][i];
|
||||
sys->common_sys->napi_sort_page_thrshld_cnt = 0;
|
||||
spin_unlock_bh(&sys->common_sys->spinlock);
|
||||
return rx_pkt;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
spin_unlock_bh(&sys->common_sys->spinlock);
|
||||
IPADBG_LOW("napi_sort_page_thrshld_cnt = %d ipa_max_napi_sort_page_thrshld = %d\n",
|
||||
sys->common_sys->napi_sort_page_thrshld_cnt,
|
||||
ipa3_ctx->ipa_max_napi_sort_page_thrshld);
|
||||
/*Scheduling tasklet to find the free page*/
|
||||
if (sys->common_sys->napi_sort_page_thrshld_cnt >=
|
||||
ipa3_ctx->ipa_max_napi_sort_page_thrshld) {
|
||||
atomic_set(&sys->common_sys->page_avilable, 0);
|
||||
tasklet_schedule(&sys->common_sys->tasklet_find_freepage);
|
||||
++ipa3_ctx->stats.num_sort_tasklet_sched[stats_i];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
|
||||
{
|
||||
@ -2202,10 +2365,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
|
||||
int ret;
|
||||
int rx_len_cached = 0;
|
||||
struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX];
|
||||
u32 curr;
|
||||
u32 curr_wq;
|
||||
int idx = 0;
|
||||
struct page *cur_page;
|
||||
u32 stats_i = 0;
|
||||
|
||||
/* start replenish only when buffers go lower than the threshold */
|
||||
@ -2213,19 +2374,15 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
|
||||
return;
|
||||
stats_i = (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) ? 0 : 1;
|
||||
|
||||
spin_lock_bh(&sys->spinlock);
|
||||
rx_len_cached = sys->len;
|
||||
curr = atomic_read(&sys->page_recycle_repl->head_idx);
|
||||
curr_wq = atomic_read(&sys->repl->head_idx);
|
||||
|
||||
while (rx_len_cached < sys->rx_pool_sz) {
|
||||
cur_page = sys->page_recycle_repl->cache[curr]->page_data.page;
|
||||
/* Found an idle page that can be used */
|
||||
if (page_ref_count(cur_page) == 1) {
|
||||
page_ref_inc(cur_page);
|
||||
rx_pkt = sys->page_recycle_repl->cache[curr];
|
||||
curr = (++curr == sys->page_recycle_repl->capacity) ?
|
||||
0 : curr;
|
||||
/* check for an idle page that can be used */
|
||||
if (atomic_read(&sys->common_sys->page_avilable) &&
|
||||
((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL)) {
|
||||
ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++;
|
||||
|
||||
} else {
|
||||
/*
|
||||
* Could not find idle page at curr index.
|
||||
@ -2238,6 +2395,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
|
||||
curr_wq = (++curr_wq == sys->repl->capacity) ?
|
||||
0 : curr_wq;
|
||||
}
|
||||
rx_pkt->sys = sys;
|
||||
|
||||
dma_sync_single_for_device(ipa3_ctx->pdev,
|
||||
rx_pkt->page_data.dma_addr,
|
||||
@ -2275,26 +2433,27 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
|
||||
/* ensure write is done before setting head index */
|
||||
mb();
|
||||
atomic_set(&sys->repl->head_idx, curr_wq);
|
||||
atomic_set(&sys->page_recycle_repl->head_idx, curr);
|
||||
sys->len = rx_len_cached;
|
||||
} else {
|
||||
/* we don't expect this will happen */
|
||||
IPAERR("failed to provide buffer: %d\n", ret);
|
||||
ipa_assert();
|
||||
}
|
||||
spin_unlock_bh(&sys->spinlock);
|
||||
|
||||
if (sys->common_buff_pool)
|
||||
__trigger_repl_work(sys->common_sys);
|
||||
else
|
||||
__trigger_repl_work(sys);
|
||||
|
||||
if (rx_len_cached <= IPA_DEFAULT_SYS_YELLOW_WM) {
|
||||
if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS ||
|
||||
sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
|
||||
if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS)
|
||||
IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty);
|
||||
else if (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
|
||||
IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty_coal);
|
||||
else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS)
|
||||
IPA_STATS_INC_CNT(ipa3_ctx->stats.lan_rx_empty);
|
||||
else
|
||||
WARN_ON(1);
|
||||
queue_delayed_work(sys->wq, &sys->replenish_rx_work,
|
||||
msecs_to_jiffies(1));
|
||||
}
|
||||
|
||||
return;
|
||||
@ -2790,20 +2949,21 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data)
|
||||
{
|
||||
struct ipa3_rx_pkt_wrapper *rx_pkt = (struct ipa3_rx_pkt_wrapper *)
|
||||
xfer_user_data;
|
||||
struct ipa3_sys_context *sys = rx_pkt->sys;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < sys->page_recycle_repl->capacity; i++)
|
||||
if (sys->page_recycle_repl->cache[i] == rx_pkt)
|
||||
break;
|
||||
if (i < sys->page_recycle_repl->capacity) {
|
||||
if (!rx_pkt->page_data.is_tmp_alloc) {
|
||||
list_del_init(&rx_pkt->link);
|
||||
page_ref_dec(rx_pkt->page_data.page);
|
||||
sys->page_recycle_repl->cache[i] = NULL;
|
||||
}
|
||||
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
/* Add the element to head. */
|
||||
list_add(&rx_pkt->link,
|
||||
&rx_pkt->sys->page_recycle_repl->page_repl_head);
|
||||
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
} else {
|
||||
dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr,
|
||||
rx_pkt->len, DMA_FROM_DEVICE);
|
||||
__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
|
||||
kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2816,7 +2976,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
|
||||
struct ipa3_rx_pkt_wrapper *r;
|
||||
u32 head;
|
||||
u32 tail;
|
||||
int i;
|
||||
|
||||
/*
|
||||
* buffers not consumed by gsi are cleaned up using cleanup callback
|
||||
@ -2864,25 +3023,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
|
||||
kfree(sys->repl);
|
||||
sys->repl = NULL;
|
||||
}
|
||||
if (sys->page_recycle_repl) {
|
||||
for (i = 0; i < sys->page_recycle_repl->capacity; i++) {
|
||||
rx_pkt = sys->page_recycle_repl->cache[i];
|
||||
if (rx_pkt) {
|
||||
dma_unmap_page(ipa3_ctx->pdev,
|
||||
rx_pkt->page_data.dma_addr,
|
||||
rx_pkt->len,
|
||||
DMA_FROM_DEVICE);
|
||||
__free_pages(rx_pkt->page_data.page,
|
||||
rx_pkt->page_data.page_order);
|
||||
kmem_cache_free(
|
||||
ipa3_ctx->rx_pkt_wrapper_cache,
|
||||
rx_pkt);
|
||||
}
|
||||
}
|
||||
kfree(sys->page_recycle_repl->cache);
|
||||
kfree(sys->page_recycle_repl);
|
||||
sys->page_recycle_repl = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len)
|
||||
@ -3670,6 +3810,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
|
||||
IPAERR("notify->veid > GSI_VEID_MAX\n");
|
||||
if (!rx_page.is_tmp_alloc) {
|
||||
init_page_count(rx_page.page);
|
||||
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
/* Add the element to head. */
|
||||
list_add(&rx_pkt->link,
|
||||
&rx_pkt->sys->page_recycle_repl->page_repl_head);
|
||||
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
} else {
|
||||
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
|
||||
rx_pkt->len, DMA_FROM_DEVICE);
|
||||
@ -3697,11 +3842,15 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
|
||||
list_del_init(&rx_pkt->link);
|
||||
if (!rx_page.is_tmp_alloc) {
|
||||
init_page_count(rx_page.page);
|
||||
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
/* Add the element to head. */
|
||||
list_add(&rx_pkt->link,
|
||||
&rx_pkt->sys->page_recycle_repl->page_repl_head);
|
||||
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
} else {
|
||||
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
|
||||
rx_pkt->len, DMA_FROM_DEVICE);
|
||||
__free_pages(rx_pkt->page_data.page,
|
||||
rx_pkt->page_data.page_order);
|
||||
__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
|
||||
}
|
||||
rx_pkt->sys->free_rx_wrapper(rx_pkt);
|
||||
}
|
||||
@ -3712,14 +3861,20 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
|
||||
rx_page = rx_pkt->page_data;
|
||||
size = rx_pkt->data_len;
|
||||
|
||||
list_del(&rx_pkt->link);
|
||||
if (rx_page.is_tmp_alloc)
|
||||
list_del_init(&rx_pkt->link);
|
||||
if (rx_page.is_tmp_alloc) {
|
||||
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
|
||||
rx_pkt->len, DMA_FROM_DEVICE);
|
||||
else
|
||||
} else {
|
||||
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
/* Add the element back to tail. */
|
||||
list_add_tail(&rx_pkt->link,
|
||||
&rx_pkt->sys->page_recycle_repl->page_repl_head);
|
||||
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
|
||||
dma_sync_single_for_cpu(ipa3_ctx->pdev,
|
||||
rx_page.dma_addr,
|
||||
rx_pkt->len, DMA_FROM_DEVICE);
|
||||
}
|
||||
rx_pkt->sys->free_rx_wrapper(rx_pkt);
|
||||
|
||||
skb_add_rx_frag(rx_skb,
|
||||
@ -3846,20 +4001,6 @@ static void ipa3_rx_napi_chain(struct ipa3_sys_context *sys,
|
||||
if (prev_skb) {
|
||||
skb_shinfo(prev_skb)->frag_list = NULL;
|
||||
sys->pyld_hdlr(first_skb, sys);
|
||||
/*
|
||||
* For coalescing, we have 2 transfer
|
||||
* rings to replenish
|
||||
*/
|
||||
ipa_ep_idx = ipa3_get_ep_mapping(
|
||||
IPA_CLIENT_APPS_WAN_CONS);
|
||||
if (ipa_ep_idx ==
|
||||
IPA_EP_NOT_ALLOCATED) {
|
||||
IPAERR("Invalid client.\n");
|
||||
return;
|
||||
}
|
||||
wan_def_sys =
|
||||
ipa3_ctx->ep[ipa_ep_idx].sys;
|
||||
wan_def_sys->repl_hdlr(wan_def_sys);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -5408,6 +5549,7 @@ int ipa3_rx_poll(u32 clnt_hdl, int weight)
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
ep->sys->common_sys->napi_sort_page_thrshld_cnt++;
|
||||
start_poll:
|
||||
/*
|
||||
* it is guaranteed we already have clock here.
|
||||
@ -5441,6 +5583,7 @@ start_poll:
|
||||
cnt += weight - remain_aggr_weight * ipa3_ctx->ipa_wan_aggr_pkt_cnt;
|
||||
/* call repl_hdlr before napi_reschedule / napi_complete */
|
||||
ep->sys->repl_hdlr(ep->sys);
|
||||
wan_def_sys->repl_hdlr(wan_def_sys);
|
||||
/* When not able to replenish enough descriptors, keep in polling
|
||||
* mode, wait for napi-poll and replenish again.
|
||||
*/
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _IPA3_I_H_
|
||||
@ -59,7 +59,11 @@
|
||||
#define IPA_QMAP_HEADER_LENGTH (4)
|
||||
#define IPA_DL_CHECKSUM_LENGTH (8)
|
||||
#define IPA_NUM_DESC_PER_SW_TX (3)
|
||||
#define IPA_GENERIC_RX_POOL_SZ_WAN 192
|
||||
#define IPA_GENERIC_RX_POOL_SZ 192
|
||||
#define IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR 2
|
||||
#define IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR 4
|
||||
#define IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR 2
|
||||
#define IPA_UC_FINISH_MAX 6
|
||||
#define IPA_UC_WAIT_MIN_SLEEP 1000
|
||||
#define IPA_UC_WAII_MAX_SLEEP 1200
|
||||
@ -84,6 +88,13 @@
|
||||
|
||||
#define IPA_WAN_AGGR_PKT_CNT 1
|
||||
|
||||
#define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15
|
||||
#define IPA_PAGE_POLL_THRESHOLD_MAX 30
|
||||
|
||||
#define IPA_MAX_NAPI_SORT_PAGE_THRSHLD 3
|
||||
#define IPA_MAX_PAGE_WQ_RESCHED_TIME 2
|
||||
|
||||
|
||||
#define IPADBG(fmt, args...) \
|
||||
do { \
|
||||
pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
|
||||
@ -1062,6 +1073,12 @@ struct ipa3_repl_ctx {
|
||||
atomic_t pending;
|
||||
};
|
||||
|
||||
struct ipa3_page_repl_ctx {
|
||||
struct list_head page_repl_head;
|
||||
u32 capacity;
|
||||
atomic_t pending;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct ipa3_sys_context - IPA GPI pipes context
|
||||
* @head_desc_list: header descriptors list
|
||||
@ -1100,7 +1117,6 @@ struct ipa3_sys_context {
|
||||
struct work_struct repl_work;
|
||||
void (*repl_hdlr)(struct ipa3_sys_context *sys);
|
||||
struct ipa3_repl_ctx *repl;
|
||||
struct ipa3_repl_ctx *page_recycle_repl;
|
||||
u32 pkt_sent;
|
||||
struct napi_struct *napi_obj;
|
||||
struct list_head pending_pkts[GSI_VEID_MAX];
|
||||
@ -1110,6 +1126,10 @@ struct ipa3_sys_context {
|
||||
u32 eob_drop_cnt;
|
||||
struct napi_struct napi_tx;
|
||||
atomic_t in_napi_context;
|
||||
bool common_buff_pool;
|
||||
struct ipa3_sys_context *common_sys;
|
||||
atomic_t page_avilable;
|
||||
u32 napi_sort_page_thrshld_cnt;
|
||||
|
||||
/* ordering is important - mutable fields go above */
|
||||
struct ipa3_ep_context *ep;
|
||||
@ -1125,6 +1145,10 @@ struct ipa3_sys_context {
|
||||
u32 pm_hdl;
|
||||
unsigned int napi_sch_cnt;
|
||||
unsigned int napi_comp_cnt;
|
||||
struct ipa3_page_repl_ctx *page_recycle_repl;
|
||||
struct workqueue_struct *freepage_wq;
|
||||
struct delayed_work freepage_work;
|
||||
struct tasklet_struct tasklet_find_freepage;
|
||||
/* ordering is important - other immutable fields go below */
|
||||
};
|
||||
|
||||
@ -1431,8 +1455,10 @@ enum ipa3_config_this_ep {
|
||||
|
||||
struct ipa3_page_recycle_stats {
|
||||
u64 total_replenished;
|
||||
u64 page_recycled;
|
||||
u64 tmp_alloc;
|
||||
};
|
||||
|
||||
struct ipa3_stats {
|
||||
u32 tx_sw_pkts;
|
||||
u32 tx_hw_pkts;
|
||||
@ -1447,6 +1473,7 @@ struct ipa3_stats {
|
||||
u32 aggr_close;
|
||||
u32 wan_aggr_close;
|
||||
u32 wan_rx_empty;
|
||||
u32 wan_rx_empty_coal;
|
||||
u32 wan_repl_rx_empty;
|
||||
u32 lan_rx_empty;
|
||||
u32 lan_repl_rx_empty;
|
||||
@ -1459,6 +1486,10 @@ struct ipa3_stats {
|
||||
struct ipa3_page_recycle_stats page_recycle_stats[2];
|
||||
u64 lower_order;
|
||||
u32 pipe_setup_fail_cnt;
|
||||
u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX];
|
||||
u64 num_sort_tasklet_sched[3];
|
||||
u64 num_of_times_wq_reschd;
|
||||
u64 page_recycle_cnt_in_tasklet;
|
||||
};
|
||||
|
||||
/* offset for each stats */
|
||||
@ -2174,7 +2205,10 @@ struct ipa3_context {
|
||||
bool fnr_stats_not_supported;
|
||||
bool is_device_crashed;
|
||||
int ipa_pil_load;
|
||||
|
||||
u8 page_poll_threshold;
|
||||
bool wan_common_page_pool;
|
||||
u32 ipa_max_napi_sort_page_thrshld;
|
||||
u32 page_wq_reschd_time;
|
||||
};
|
||||
|
||||
struct ipa3_plat_drv_res {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user