lisa/android_kernel_xiaomi_sm8350
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge tag 'LA.UM.9.14.r1-24200-LAHAINA.QSSI13.0' of https://git.codelinaro.org/clo/la/platform/vendor/opensource/dataipa into android13-5.4-lahaina

Browse Source 
"LA.UM.9.14.r1-24200-LAHAINA.QSSI13.0"

* tag 'LA.UM.9.14.r1-24200-LAHAINA.QSSI13.0' of https://git.codelinaro.org/clo/la/platform/vendor/opensource/dataipa:
  msm: ipa3: reduce rx-wan pool and cmn page, temp pool sizes
  msm: ipa: allocate page recycling buffers only once
  msm: ipa: avoid OOM Killer for temp allocations
  msm: ipa3: Flush free page WQ only for page recyle replenish handler
  msm: ipa3: Fix to flush to workqueue during teardown pipe
  msm: ipa3: Fix to destory workqueue only if created
  msm: ipa3: Changes to enhance find free pages from list
  msm: ipa: page pool recycling enhancements
  msm: ipa: page pool recycling enhancements

Change-Id: I21c0f5974f08fc032605d2710858eb28e592d1f0
This commit is contained in:
Michael Bestas 2024-03-23 17:41:41 +02:00
commit edd5792efd
No known key found for this signature in database
GPG Key ID: CC95044519BE6669
4 changed files with 451 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa.c
View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) 2023-2024 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#include <linux/clk.h>
@ -7359,6 +7359,13 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc = 0;
ipa3_ctx->stats.page_recycle_stats[1].total_replenished = 0;
ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0;
memset(ipa3_ctx->stats.page_recycle_cnt, 0,
sizeof(ipa3_ctx->stats.page_recycle_cnt));
ipa3_ctx->stats.num_sort_tasklet_sched[0] = 0;
ipa3_ctx->stats.num_sort_tasklet_sched[1] = 0;
ipa3_ctx->stats.num_sort_tasklet_sched[2] = 0;
ipa3_ctx->stats.num_of_times_wq_reschd = 0;
ipa3_ctx->stats.page_recycle_cnt_in_tasklet = 0;
ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset;
ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control;
ipa3_ctx->ee = resource_p->ee;
@ -7549,6 +7556,17 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
/* Enable ipa3_ctx->enable_napi_chain */
ipa3_ctx->enable_napi_chain = 1;
/* Initialize Page poll threshold. */
ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD;
/*Initialize number napi without prealloc buff*/
ipa3_ctx->ipa_max_napi_sort_page_thrshld = IPA_MAX_NAPI_SORT_PAGE_THRSHLD;
ipa3_ctx->page_wq_reschd_time = IPA_MAX_PAGE_WQ_RESCHED_TIME;
/* Use common page pool for Def/Coal pipe. */
if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_9)
ipa3_ctx->wan_common_page_pool = true;
/* assume clock is on in virtual/emulation mode */
if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION)
@ -8187,7 +8205,7 @@ static int get_ipa_dts_configuration(struct platform_device *pdev,
ipa_drv_res->ipa_mhi_dynamic_config = false;
ipa_drv_res->use_64_bit_dma_mask = false;
ipa_drv_res->use_bw_vote = false;
ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ_WAN;
ipa_drv_res->lan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
ipa_drv_res->apply_rg10_wa = false;
ipa_drv_res->gsi_ch20_wa = false;

144
techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c
View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2012-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#ifdef CONFIG_DEBUG_FS
@ -1353,6 +1353,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf,
"act_clnt=%u\n"
"con_clnt_bmap=0x%x\n"
"wan_rx_empty=%u\n"
"wan_rx_empty_coal=%u\n"
"wan_repl_rx_empty=%u\n"
"lan_rx_empty=%u\n"
"lan_repl_rx_empty=%u\n"
@ -1372,6 +1373,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf,
atomic_read(&ipa3_ctx->ipa3_active_clients.cnt),
connect,
ipa3_ctx->stats.wan_rx_empty,
ipa3_ctx->stats.wan_rx_empty_coal,
ipa3_ctx->stats.wan_repl_rx_empty,
ipa3_ctx->stats.lan_rx_empty,
ipa3_ctx->stats.lan_repl_rx_empty,
@ -1421,24 +1423,46 @@ static ssize_t ipa3_read_odlstats(struct file *file, char __user *ubuf,
return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
}
static ssize_t ipa3_read_page_recycle_stats(struct file *file,
char __user *ubuf, size_t count, loff_t *ppos)
{
int nbytes;
int cnt = 0;
int cnt = 0, i = 0, k = 0;
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
"COAL : Total number of packets replenished =%llu\n"
"COAL : Number of page recycled packets =%llu\n"
"COAL : Number of tmp alloc packets =%llu\n"
"COAL : Number of times tasklet scheduled =%llu\n"
"DEF : Total number of packets replenished =%llu\n"
"DEF : Number of tmp alloc packets =%llu\n",
"DEF : Number of page recycled packets =%llu\n"
"DEF : Number of tmp alloc packets =%llu\n"
"DEF : Number of times tasklet scheduled =%llu\n"
"COMMON : Number of page recycled in tasklet =%llu\n"
"COMMON : Number of times free pages not found in tasklet =%llu\n",
ipa3_ctx->stats.page_recycle_stats[0].total_replenished,
ipa3_ctx->stats.page_recycle_stats[0].page_recycled,
ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc,
ipa3_ctx->stats.num_sort_tasklet_sched[0],
ipa3_ctx->stats.page_recycle_stats[1].total_replenished,
ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc);
ipa3_ctx->stats.page_recycle_stats[1].page_recycled,
ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc,
ipa3_ctx->stats.num_sort_tasklet_sched[1],
ipa3_ctx->stats.page_recycle_cnt_in_tasklet,
ipa3_ctx->stats.num_of_times_wq_reschd);
cnt += nbytes;
for (k = 0; k < 2; k++) {
for (i = 0; i < ipa3_ctx->page_poll_threshold; i++) {
nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN,
"COMMON : Page replenish efficiency[%d][%d] =%llu\n",
k, i, ipa3_ctx->stats.page_recycle_cnt[k][i]);
cnt += nbytes;
}
}
return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
}
static ssize_t ipa3_read_wstats(struct file *file, char __user *ubuf,
@ -2772,6 +2796,68 @@ static ssize_t ipa3_enable_ipc_low(struct file *file,
return count;
}
static ssize_t ipa3_read_ipa_max_napi_sort_page_thrshld(struct file *file,
char __user *buf, size_t count, loff_t *ppos) {
int nbytes;
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
"page max napi without free page = %d\n",
ipa3_ctx->ipa_max_napi_sort_page_thrshld);
return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
}
static ssize_t ipa3_write_ipa_max_napi_sort_page_thrshld(struct file *file,
const char __user *buf, size_t count, loff_t *ppos) {
int ret;
u8 ipa_max_napi_sort_page_thrshld = 0;
if (count >= sizeof(dbg_buff))
return -EFAULT;
ret = kstrtou8_from_user(buf, count, 0, &ipa_max_napi_sort_page_thrshld);
if(ret)
return ret;
ipa3_ctx->ipa_max_napi_sort_page_thrshld = ipa_max_napi_sort_page_thrshld;
IPADBG("napi cnt without prealloc pages = %d", ipa3_ctx->ipa_max_napi_sort_page_thrshld);
return count;
}
static ssize_t ipa3_read_page_wq_reschd_time(struct file *file,
char __user *buf, size_t count, loff_t *ppos) {
int nbytes;
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
"Page WQ reschduule time = %d\n",
ipa3_ctx->page_wq_reschd_time);
return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
}
static ssize_t ipa3_write_page_wq_reschd_time(struct file *file,
const char __user *buf, size_t count, loff_t *ppos) {
int ret;
u8 page_wq_reschd_time = 0;
if (count >= sizeof(dbg_buff))
return -EFAULT;
ret = kstrtou8_from_user(buf, count, 0, &page_wq_reschd_time);
if(ret)
return ret;
ipa3_ctx->page_wq_reschd_time = page_wq_reschd_time;
IPADBG("Updated page WQ reschedule time = %d", ipa3_ctx->page_wq_reschd_time);
return count;
}
static ssize_t ipa3_read_gsi_wdi3_db_polling(struct file *file,
char __user *buf, size_t count, loff_t *ppos) {
@ -2804,6 +2890,41 @@ static ssize_t ipa3_write_gsi_wdi3_db_polling(struct file *file,
return count;
}
static ssize_t ipa3_read_page_poll_threshold(struct file *file,
char __user *buf, size_t count, loff_t *ppos) {
int nbytes;
nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
"Page Poll Threshold = %d\n",
ipa3_ctx->page_poll_threshold);
return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
}
static ssize_t ipa3_write_page_poll_threshold(struct file *file,
const char __user *buf, size_t count, loff_t *ppos) {
int ret;
u8 page_poll_threshold =0;
if (count >= sizeof(dbg_buff))
return -EFAULT;
ret = kstrtou8_from_user(buf, count, 0, &page_poll_threshold);
if(ret)
return ret;
if(page_poll_threshold != 0 &&
page_poll_threshold <= IPA_PAGE_POLL_THRESHOLD_MAX)
ipa3_ctx->page_poll_threshold = page_poll_threshold;
else
IPAERR("Invalid value \n");
IPADBG("Updated page poll threshold = %d", ipa3_ctx->page_poll_threshold);
return count;
}
static const struct ipa3_debugfs_file debugfs_files[] = {
{
"gen_reg", IPA_READ_ONLY_MODE, NULL, {
@ -2986,6 +3107,21 @@ static const struct ipa3_debugfs_file debugfs_files[] = {
.read = ipa3_read_gsi_wdi3_db_polling,
.write = ipa3_write_gsi_wdi3_db_polling,
}
}, {
"page_poll_threshold", IPA_READ_WRITE_MODE, NULL, {
.read = ipa3_read_page_poll_threshold,
.write = ipa3_write_page_poll_threshold,
}
}, {
"page_wq_reschd_time", IPA_READ_WRITE_MODE, NULL, {
.read = ipa3_read_page_wq_reschd_time,
.write = ipa3_write_page_wq_reschd_time,
}
}, {
"ipa_max_napi_sort_page_thrshld", IPA_READ_WRITE_MODE, NULL, {
.read = ipa3_read_ipa_max_napi_sort_page_thrshld,
.write = ipa3_write_ipa_max_napi_sort_page_thrshld,
}
},
};

365
techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c
View File

@ -2,7 +2,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#include <linux/delay.h>
@ -137,6 +137,7 @@ static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys,
static unsigned long tag_to_pointer_wa(uint64_t tag);
static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt);
static void ipa3_tasklet_rx_notify(unsigned long data);
static void ipa3_tasklet_find_freepage(unsigned long data);
static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit);
@ -1048,6 +1049,60 @@ static void ipa_pm_sys_pipe_cb(void *p, enum ipa_pm_cb_event event)
}
}
static void ipa3_schd_freepage_work(struct work_struct *work)
{
struct delayed_work *dwork;
struct ipa3_sys_context *sys;
dwork = container_of(work, struct delayed_work, work);
sys = container_of(dwork, struct ipa3_sys_context, freepage_work);
IPADBG_LOW("WQ scheduled, reschedule sort tasklet\n");
tasklet_schedule(&sys->tasklet_find_freepage);
}
static void ipa3_tasklet_find_freepage(unsigned long data)
{
struct ipa3_sys_context *sys;
struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
struct ipa3_rx_pkt_wrapper *tmp = NULL;
struct page *cur_page;
int found_free_page = 0;
struct list_head temp_head;
sys = (struct ipa3_sys_context *)data;
INIT_LIST_HEAD(&temp_head);
spin_lock_bh(&sys->common_sys->spinlock);
list_for_each_entry_safe(rx_pkt, tmp,
&sys->page_recycle_repl->page_repl_head, link) {
cur_page = rx_pkt->page_data.page;
if (page_ref_count(cur_page) == 1) {
/* Found a free page. */
list_del_init(&rx_pkt->link);
list_add(&rx_pkt->link, &temp_head);
found_free_page++;
}
}
if (!found_free_page) {
/*Not found free page rescheduling tasklet after 2msec*/
IPADBG_LOW("Scheduling WQ not found free pages\n");
++ipa3_ctx->stats.num_of_times_wq_reschd;
queue_delayed_work(sys->freepage_wq,
&sys->freepage_work,
msecs_to_jiffies(ipa3_ctx->page_wq_reschd_time));
} else {
/*Allow to use pre-allocated buffers*/
list_splice(&temp_head, &sys->page_recycle_repl->page_repl_head);
ipa3_ctx->stats.page_recycle_cnt_in_tasklet += found_free_page;
IPADBG_LOW("found free pages count = %d\n", found_free_page);
atomic_set(&sys->common_sys->page_avilable, 1);
}
spin_unlock_bh(&sys->common_sys->spinlock);
}
/**
* ipa3_setup_sys_pipe() - Setup an IPA GPI pipe and perform
* IPA EP configuration
@ -1130,7 +1185,8 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
snprintf(buff, IPA_RESOURCE_NAME_MAX, "iparepwq%d",
sys_in->client);
ep->sys->repl_wq = alloc_workqueue(buff,
WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1);
WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | WQ_HIGHPRI,
1);
if (!ep->sys->repl_wq) {
IPAERR("failed to create rep wq for client %d\n",
sys_in->client);
@ -1138,6 +1194,9 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
goto fail_wq2;
}
snprintf(buff, IPA_RESOURCE_NAME_MAX, "ipafreepagewq%d",
sys_in->client);
INIT_LIST_HEAD(&ep->sys->head_desc_list);
INIT_LIST_HEAD(&ep->sys->rcycl_list);
INIT_LIST_HEAD(&ep->sys->avail_tx_wrapper_list);
@ -1154,6 +1213,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
/* Use coalescing pipe PM handle for default pipe also*/
ep->sys->pm_hdl = ipa3_ctx->ep[coal_ep_id].sys->pm_hdl;
} else if (IPA_CLIENT_IS_CONS(sys_in->client)) {
ep->sys->freepage_wq = alloc_workqueue(buff,
WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS |
WQ_HIGHPRI, 1);
if (!ep->sys->freepage_wq) {
IPAERR("failed to create freepage wq for client %d\n",
sys_in->client);
result = -EFAULT;
goto fail_wq3;
}
pm_reg.name = ipa_clients_strings[sys_in->client];
pm_reg.callback = ipa_pm_sys_pipe_cb;
pm_reg.user_data = ep->sys;
@ -1254,6 +1323,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
}
*clnt_hdl = ipa_ep_idx;
ep->sys->common_sys = ipa3_ctx->ep[ipa_ep_idx].sys;
if (ep->sys->repl_hdlr == ipa3_fast_replenish_rx_cache) {
ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
@ -1279,41 +1349,77 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
}
}
/* Use common page pool for Coal and defalt pipe if applicable. */
if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) {
ep->sys->page_recycle_repl = kzalloc(
sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL);
if (!ep->sys->page_recycle_repl) {
IPAERR("failed to alloc repl for client %d\n",
sys_in->client);
result = -ENOMEM;
goto fail_napi;
if (!(ipa3_ctx->wan_common_page_pool &&
sys_in->client == IPA_CLIENT_APPS_WAN_CONS &&
coal_ep_id != IPA_EP_NOT_ALLOCATED &&
ipa3_ctx->ep[coal_ep_id].valid == 1)) {
/* Allocate page recycling pool only once. */
if (!ep->sys->page_recycle_repl) {
ep->sys->page_recycle_repl = kzalloc(
sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL);
if (!ep->sys->page_recycle_repl) {
IPAERR("failed to alloc repl for client %d\n",
sys_in->client);
result = -ENOMEM;
goto fail_napi;
}
atomic_set(&ep->sys->page_recycle_repl->pending, 0);
/* For common page pool double the pool size. */
if (ipa3_ctx->wan_common_page_pool &&
sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
ep->sys->page_recycle_repl->capacity =
(ep->sys->rx_pool_sz + 1) *
IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR;
else
ep->sys->page_recycle_repl->capacity =
(ep->sys->rx_pool_sz + 1) *
IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR;
IPADBG("Page repl capacity for client:%d, value:%d\n",
sys_in->client, ep->sys->page_recycle_repl->capacity);
INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work);
tasklet_init(&ep->sys->tasklet_find_freepage,
ipa3_tasklet_find_freepage, (unsigned long) ep->sys);
ipa3_replenish_rx_page_cache(ep->sys);
} else {
ep->sys->napi_sort_page_thrshld_cnt = 0;
/* Sort the pages once. */
ipa3_tasklet_find_freepage((unsigned long) ep->sys);
}
ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
if (!ep->sys->repl) {
IPAERR("failed to alloc repl for client %d\n",
sys_in->client);
result = -ENOMEM;
goto fail_page_recycle_repl;
}
/* For common page pool triple the pool size. */
if (ipa3_ctx->wan_common_page_pool &&
sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1) *
IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR;
else
ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1);
IPADBG("Repl capacity for client:%d, value:%d\n",
sys_in->client, ep->sys->repl->capacity);
atomic_set(&ep->sys->repl->pending, 0);
ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity,
sizeof(void *), GFP_KERNEL);
atomic_set(&ep->sys->repl->head_idx, 0);
atomic_set(&ep->sys->repl->tail_idx, 0);
ipa3_wq_page_repl(&ep->sys->repl_work);
} else {
/* Use pool same as coal pipe when common page pool is used */
ep->sys->common_buff_pool = true;
ep->sys->common_sys = ipa3_ctx->ep[coal_ep_id].sys;
ep->sys->repl = ipa3_ctx->ep[coal_ep_id].sys->repl;
ep->sys->page_recycle_repl =
ipa3_ctx->ep[coal_ep_id].sys->page_recycle_repl;
}
atomic_set(&ep->sys->page_recycle_repl->pending, 0);
ep->sys->page_recycle_repl->capacity =
(ep->sys->rx_pool_sz + 1) * 2;
ep->sys->page_recycle_repl->cache =
kcalloc(ep->sys->page_recycle_repl->capacity,
sizeof(void *), GFP_KERNEL);
atomic_set(&ep->sys->page_recycle_repl->head_idx, 0);
atomic_set(&ep->sys->page_recycle_repl->tail_idx, 0);
ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
if (!ep->sys->repl) {
IPAERR("failed to alloc repl for client %d\n",
sys_in->client);
result = -ENOMEM;
goto fail_page_recycle_repl;
}
ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1);
atomic_set(&ep->sys->repl->pending, 0);
ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity,
sizeof(void *), GFP_KERNEL);
atomic_set(&ep->sys->repl->head_idx, 0);
atomic_set(&ep->sys->repl->tail_idx, 0);
ipa3_replenish_rx_page_cache(ep->sys);
ipa3_wq_page_repl(&ep->sys->repl_work);
}
if (IPA_CLIENT_IS_CONS(sys_in->client)) {
@ -1383,16 +1489,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
fail_gen3:
ipa3_disable_data_path(ipa_ep_idx);
fail_repl:
if (IPA_CLIENT_IS_CONS(ep->client))
if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
ipa3_cleanup_rx(ep->sys);
if(ep->sys->repl) {
ep->sys->repl_hdlr = ipa3_replenish_rx_cache;
if (ep->sys->repl && !ep->sys->common_buff_pool) {
ep->sys->repl->capacity = 0;
kfree(ep->sys->repl);
ep->sys->repl = NULL;
}
fail_page_recycle_repl:
if (ep->sys->page_recycle_repl) {
if (ep->sys->page_recycle_repl && !ep->sys->common_buff_pool) {
ep->sys->page_recycle_repl->capacity = 0;
kfree(ep->sys->page_recycle_repl);
ep->sys->page_recycle_repl = NULL;
@ -1405,6 +1511,9 @@ fail_napi:
fail_gen2:
ipa_pm_deregister(ep->sys->pm_hdl);
fail_pm:
if (ep->sys->freepage_wq)
destroy_workqueue(ep->sys->freepage_wq);
fail_wq3:
destroy_workqueue(ep->sys->repl_wq);
fail_wq2:
destroy_workqueue(ep->sys->wq);
@ -1571,7 +1680,13 @@ int ipa3_teardown_sys_pipe(u32 clnt_hdl)
}
if (ep->sys->repl_wq)
flush_workqueue(ep->sys->repl_wq);
if (IPA_CLIENT_IS_CONS(ep->client))
if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) {
cancel_delayed_work_sync(&ep->sys->common_sys->freepage_work);
tasklet_kill(&ep->sys->common_sys->tasklet_find_freepage);
}
if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
ipa3_cleanup_rx(ep->sys);
if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(ep->client)) {
@ -1657,7 +1772,7 @@ static int ipa3_teardown_coal_def_pipe(u32 clnt_hdl)
if (ep->sys->repl_wq)
flush_workqueue(ep->sys->repl_wq);
if (IPA_CLIENT_IS_CONS(ep->client))
if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool)
ipa3_cleanup_rx(ep->sys);
ep->valid = 0;
@ -2081,6 +2196,9 @@ static struct ipa3_rx_pkt_wrapper *ipa3_alloc_rx_pkt_page(
return NULL;
rx_pkt->page_data.page_order = IPA_WAN_PAGE_ORDER;
/* For temporary allocations, avoid triggering OOM Killer. */
if (is_tmp_alloc)
flag |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
/* Try a lower order page for order 3 pages in case allocation fails. */
rx_pkt->page_data.page = ipa3_alloc_page(flag,
&rx_pkt->page_data.page_order,
@ -2126,9 +2244,12 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys)
ipa_assert();
break;
}
INIT_LIST_HEAD(&rx_pkt->link);
rx_pkt->sys = sys;
sys->page_recycle_repl->cache[curr] = rx_pkt;
list_add_tail(&rx_pkt->link,
&sys->page_recycle_repl->page_repl_head);
}
atomic_set(&sys->common_sys->page_avilable, 1);
return;
@ -2189,12 +2310,54 @@ static inline void __trigger_repl_work(struct ipa3_sys_context *sys)
head = atomic_read(&sys->repl->head_idx);
avail = (tail - head) % sys->repl->capacity;
if (avail < sys->repl->capacity / 4) {
if (avail < sys->repl->capacity / 2) {
atomic_set(&sys->repl->pending, 1);
queue_work(sys->repl_wq, &sys->repl_work);
}
}
static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page
(
struct ipa3_sys_context *sys,
u32 stats_i
)
{
struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
struct ipa3_rx_pkt_wrapper *tmp = NULL;
struct page *cur_page;
int i = 0;
u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold;
spin_lock_bh(&sys->common_sys->spinlock);
list_for_each_entry_safe(rx_pkt, tmp,
&sys->page_recycle_repl->page_repl_head, link) {
if (i == LOOP_THRESHOLD)
break;
cur_page = rx_pkt->page_data.page;
if (page_ref_count(cur_page) == 1) {
/* Found a free page. */
page_ref_inc(cur_page);
list_del_init(&rx_pkt->link);
++ipa3_ctx->stats.page_recycle_cnt[stats_i][i];
sys->common_sys->napi_sort_page_thrshld_cnt = 0;
spin_unlock_bh(&sys->common_sys->spinlock);
return rx_pkt;
}
i++;
}
spin_unlock_bh(&sys->common_sys->spinlock);
IPADBG_LOW("napi_sort_page_thrshld_cnt = %d ipa_max_napi_sort_page_thrshld = %d\n",
sys->common_sys->napi_sort_page_thrshld_cnt,
ipa3_ctx->ipa_max_napi_sort_page_thrshld);
/*Scheduling tasklet to find the free page*/
if (sys->common_sys->napi_sort_page_thrshld_cnt >=
ipa3_ctx->ipa_max_napi_sort_page_thrshld) {
atomic_set(&sys->common_sys->page_avilable, 0);
tasklet_schedule(&sys->common_sys->tasklet_find_freepage);
++ipa3_ctx->stats.num_sort_tasklet_sched[stats_i];
}
return NULL;
}
static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
{
@ -2202,10 +2365,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
int ret;
int rx_len_cached = 0;
struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX];
u32 curr;
u32 curr_wq;
int idx = 0;
struct page *cur_page;
u32 stats_i = 0;
/* start replenish only when buffers go lower than the threshold */
@ -2213,19 +2374,15 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
return;
stats_i = (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) ? 0 : 1;
spin_lock_bh(&sys->spinlock);
rx_len_cached = sys->len;
curr = atomic_read(&sys->page_recycle_repl->head_idx);
curr_wq = atomic_read(&sys->repl->head_idx);
while (rx_len_cached < sys->rx_pool_sz) {
cur_page = sys->page_recycle_repl->cache[curr]->page_data.page;
/* Found an idle page that can be used */
if (page_ref_count(cur_page) == 1) {
page_ref_inc(cur_page);
rx_pkt = sys->page_recycle_repl->cache[curr];
curr = (++curr == sys->page_recycle_repl->capacity) ?
0 : curr;
/* check for an idle page that can be used */
if (atomic_read(&sys->common_sys->page_avilable) &&
((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL)) {
ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++;
} else {
/*
* Could not find idle page at curr index.
@ -2238,6 +2395,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
curr_wq = (++curr_wq == sys->repl->capacity) ?
0 : curr_wq;
}
rx_pkt->sys = sys;
dma_sync_single_for_device(ipa3_ctx->pdev,
rx_pkt->page_data.dma_addr,
@ -2275,26 +2433,27 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
/* ensure write is done before setting head index */
mb();
atomic_set(&sys->repl->head_idx, curr_wq);
atomic_set(&sys->page_recycle_repl->head_idx, curr);
sys->len = rx_len_cached;
} else {
/* we don't expect this will happen */
IPAERR("failed to provide buffer: %d\n", ret);
ipa_assert();
}
spin_unlock_bh(&sys->spinlock);
__trigger_repl_work(sys);
if (sys->common_buff_pool)
__trigger_repl_work(sys->common_sys);
else
__trigger_repl_work(sys);
if (rx_len_cached <= IPA_DEFAULT_SYS_YELLOW_WM) {
if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS ||
sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS)
IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty);
else if (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty_coal);
else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS)
IPA_STATS_INC_CNT(ipa3_ctx->stats.lan_rx_empty);
else
WARN_ON(1);
queue_delayed_work(sys->wq, &sys->replenish_rx_work,
msecs_to_jiffies(1));
}
return;
@ -2790,20 +2949,21 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data)
{
struct ipa3_rx_pkt_wrapper *rx_pkt = (struct ipa3_rx_pkt_wrapper *)
xfer_user_data;
struct ipa3_sys_context *sys = rx_pkt->sys;
int i;
for (i = 0; i < sys->page_recycle_repl->capacity; i++)
if (sys->page_recycle_repl->cache[i] == rx_pkt)
break;
if (i < sys->page_recycle_repl->capacity) {
if (!rx_pkt->page_data.is_tmp_alloc) {
list_del_init(&rx_pkt->link);
page_ref_dec(rx_pkt->page_data.page);
sys->page_recycle_repl->cache[i] = NULL;
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
/* Add the element to head. */
list_add(&rx_pkt->link,
&rx_pkt->sys->page_recycle_repl->page_repl_head);
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
} else {
dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
}
dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
}
/**
@ -2816,7 +2976,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
struct ipa3_rx_pkt_wrapper *r;
u32 head;
u32 tail;
int i;
/*
* buffers not consumed by gsi are cleaned up using cleanup callback
@ -2864,25 +3023,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
kfree(sys->repl);
sys->repl = NULL;
}
if (sys->page_recycle_repl) {
for (i = 0; i < sys->page_recycle_repl->capacity; i++) {
rx_pkt = sys->page_recycle_repl->cache[i];
if (rx_pkt) {
dma_unmap_page(ipa3_ctx->pdev,
rx_pkt->page_data.dma_addr,
rx_pkt->len,
DMA_FROM_DEVICE);
__free_pages(rx_pkt->page_data.page,
rx_pkt->page_data.page_order);
kmem_cache_free(
ipa3_ctx->rx_pkt_wrapper_cache,
rx_pkt);
}
}
kfree(sys->page_recycle_repl->cache);
kfree(sys->page_recycle_repl);
sys->page_recycle_repl = NULL;
}
}
static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len)
@ -3670,6 +3810,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
IPAERR("notify->veid > GSI_VEID_MAX\n");
if (!rx_page.is_tmp_alloc) {
init_page_count(rx_page.page);
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
/* Add the element to head. */
list_add(&rx_pkt->link,
&rx_pkt->sys->page_recycle_repl->page_repl_head);
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
} else {
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
@ -3697,11 +3842,15 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
list_del_init(&rx_pkt->link);
if (!rx_page.is_tmp_alloc) {
init_page_count(rx_page.page);
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
/* Add the element to head. */
list_add(&rx_pkt->link,
&rx_pkt->sys->page_recycle_repl->page_repl_head);
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
} else {
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
__free_pages(rx_pkt->page_data.page,
rx_pkt->page_data.page_order);
__free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order);
}
rx_pkt->sys->free_rx_wrapper(rx_pkt);
}
@ -3712,14 +3861,20 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
rx_page = rx_pkt->page_data;
size = rx_pkt->data_len;
list_del(&rx_pkt->link);
if (rx_page.is_tmp_alloc)
list_del_init(&rx_pkt->link);
if (rx_page.is_tmp_alloc) {
dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
else
} else {
spin_lock_bh(&rx_pkt->sys->common_sys->spinlock);
/* Add the element back to tail. */
list_add_tail(&rx_pkt->link,
&rx_pkt->sys->page_recycle_repl->page_repl_head);
spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock);
dma_sync_single_for_cpu(ipa3_ctx->pdev,
rx_page.dma_addr,
rx_pkt->len, DMA_FROM_DEVICE);
}
rx_pkt->sys->free_rx_wrapper(rx_pkt);
skb_add_rx_frag(rx_skb,
@ -3846,20 +4001,6 @@ static void ipa3_rx_napi_chain(struct ipa3_sys_context *sys,
if (prev_skb) {
skb_shinfo(prev_skb)->frag_list = NULL;
sys->pyld_hdlr(first_skb, sys);
/*
* For coalescing, we have 2 transfer
* rings to replenish
*/
ipa_ep_idx = ipa3_get_ep_mapping(
IPA_CLIENT_APPS_WAN_CONS);
if (ipa_ep_idx ==
IPA_EP_NOT_ALLOCATED) {
IPAERR("Invalid client.\n");
return;
}
wan_def_sys =
ipa3_ctx->ep[ipa_ep_idx].sys;
wan_def_sys->repl_hdlr(wan_def_sys);
}
}
}
@ -5408,6 +5549,7 @@ int ipa3_rx_poll(u32 clnt_hdl, int weight)
return -EINVAL;
}
ep->sys->common_sys->napi_sort_page_thrshld_cnt++;
start_poll:
/*
* it is guaranteed we already have clock here.
@ -5441,6 +5583,7 @@ start_poll:
cnt += weight - remain_aggr_weight * ipa3_ctx->ipa_wan_aggr_pkt_cnt;
/* call repl_hdlr before napi_reschedule / napi_complete */
ep->sys->repl_hdlr(ep->sys);
wan_def_sys->repl_hdlr(wan_def_sys);
/* When not able to replenish enough descriptors, keep in polling
* mode, wait for napi-poll and replenish again.
*/

40
techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_i.h
View File

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
* Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#ifndef _IPA3_I_H_
@ -59,7 +59,11 @@
#define IPA_QMAP_HEADER_LENGTH (4)
#define IPA_DL_CHECKSUM_LENGTH (8)
#define IPA_NUM_DESC_PER_SW_TX (3)
#define IPA_GENERIC_RX_POOL_SZ_WAN 192
#define IPA_GENERIC_RX_POOL_SZ 192
#define IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR 2
#define IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR 4
#define IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR 2
#define IPA_UC_FINISH_MAX 6
#define IPA_UC_WAIT_MIN_SLEEP 1000
#define IPA_UC_WAII_MAX_SLEEP 1200
@ -84,6 +88,13 @@
#define IPA_WAN_AGGR_PKT_CNT 1
#define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15
#define IPA_PAGE_POLL_THRESHOLD_MAX 30
#define IPA_MAX_NAPI_SORT_PAGE_THRSHLD 3
#define IPA_MAX_PAGE_WQ_RESCHED_TIME 2
#define IPADBG(fmt, args...) \
do { \
pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
@ -1062,6 +1073,12 @@ struct ipa3_repl_ctx {
atomic_t pending;
};
struct ipa3_page_repl_ctx {
struct list_head page_repl_head;
u32 capacity;
atomic_t pending;
};
/**
* struct ipa3_sys_context - IPA GPI pipes context
* @head_desc_list: header descriptors list
@ -1100,7 +1117,6 @@ struct ipa3_sys_context {
struct work_struct repl_work;
void (*repl_hdlr)(struct ipa3_sys_context *sys);
struct ipa3_repl_ctx *repl;
struct ipa3_repl_ctx *page_recycle_repl;
u32 pkt_sent;
struct napi_struct *napi_obj;
struct list_head pending_pkts[GSI_VEID_MAX];
@ -1110,6 +1126,10 @@ struct ipa3_sys_context {
u32 eob_drop_cnt;
struct napi_struct napi_tx;
atomic_t in_napi_context;
bool common_buff_pool;
struct ipa3_sys_context *common_sys;
atomic_t page_avilable;
u32 napi_sort_page_thrshld_cnt;
/* ordering is important - mutable fields go above */
struct ipa3_ep_context *ep;
@ -1125,6 +1145,10 @@ struct ipa3_sys_context {
u32 pm_hdl;
unsigned int napi_sch_cnt;
unsigned int napi_comp_cnt;
struct ipa3_page_repl_ctx *page_recycle_repl;
struct workqueue_struct *freepage_wq;
struct delayed_work freepage_work;
struct tasklet_struct tasklet_find_freepage;
/* ordering is important - other immutable fields go below */
};
@ -1431,8 +1455,10 @@ enum ipa3_config_this_ep {
struct ipa3_page_recycle_stats {
u64 total_replenished;
u64 page_recycled;
u64 tmp_alloc;
};
struct ipa3_stats {
u32 tx_sw_pkts;
u32 tx_hw_pkts;
@ -1447,6 +1473,7 @@ struct ipa3_stats {
u32 aggr_close;
u32 wan_aggr_close;
u32 wan_rx_empty;
u32 wan_rx_empty_coal;
u32 wan_repl_rx_empty;
u32 lan_rx_empty;
u32 lan_repl_rx_empty;
@ -1459,6 +1486,10 @@ struct ipa3_stats {
struct ipa3_page_recycle_stats page_recycle_stats[2];
u64 lower_order;
u32 pipe_setup_fail_cnt;
u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX];
u64 num_sort_tasklet_sched[3];
u64 num_of_times_wq_reschd;
u64 page_recycle_cnt_in_tasklet;
};
/* offset for each stats */
@ -2174,7 +2205,10 @@ struct ipa3_context {
bool fnr_stats_not_supported;
bool is_device_crashed;
int ipa_pil_load;
u8 page_poll_threshold;
bool wan_common_page_pool;
u32 ipa_max_napi_sort_page_thrshld;
u32 page_wq_reschd_time;
};
struct ipa3_plat_drv_res {