diff --git a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa.c b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa.c index aa2cc02cb268..09f5b1cc2fa5 100644 --- a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa.c +++ b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2023-2024 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -7359,6 +7359,13 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc = 0; ipa3_ctx->stats.page_recycle_stats[1].total_replenished = 0; ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0; + memset(ipa3_ctx->stats.page_recycle_cnt, 0, + sizeof(ipa3_ctx->stats.page_recycle_cnt)); + ipa3_ctx->stats.num_sort_tasklet_sched[0] = 0; + ipa3_ctx->stats.num_sort_tasklet_sched[1] = 0; + ipa3_ctx->stats.num_sort_tasklet_sched[2] = 0; + ipa3_ctx->stats.num_of_times_wq_reschd = 0; + ipa3_ctx->stats.page_recycle_cnt_in_tasklet = 0; ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset; ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control; ipa3_ctx->ee = resource_p->ee; @@ -7549,6 +7556,17 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, /* Enable ipa3_ctx->enable_napi_chain */ ipa3_ctx->enable_napi_chain = 1; + /* Initialize Page poll threshold. */ + ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD; + + /*Initialize number napi without prealloc buff*/ + ipa3_ctx->ipa_max_napi_sort_page_thrshld = IPA_MAX_NAPI_SORT_PAGE_THRSHLD; + ipa3_ctx->page_wq_reschd_time = IPA_MAX_PAGE_WQ_RESCHED_TIME; + + /* Use common page pool for Def/Coal pipe. */ + if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_9) + ipa3_ctx->wan_common_page_pool = true; + /* assume clock is on in virtual/emulation mode */ if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL || ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) @@ -8187,7 +8205,7 @@ static int get_ipa_dts_configuration(struct platform_device *pdev, ipa_drv_res->ipa_mhi_dynamic_config = false; ipa_drv_res->use_64_bit_dma_mask = false; ipa_drv_res->use_bw_vote = false; - ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ; + ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ_WAN; ipa_drv_res->lan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ; ipa_drv_res->apply_rg10_wa = false; ipa_drv_res->gsi_ch20_wa = false; diff --git a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c index a83c50716724..426cbbea09fa 100644 --- a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c +++ b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2012-2020, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifdef CONFIG_DEBUG_FS @@ -1353,6 +1353,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf, "act_clnt=%u\n" "con_clnt_bmap=0x%x\n" "wan_rx_empty=%u\n" + "wan_rx_empty_coal=%u\n" "wan_repl_rx_empty=%u\n" "lan_rx_empty=%u\n" "lan_repl_rx_empty=%u\n" @@ -1372,6 +1373,7 @@ static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf, atomic_read(&ipa3_ctx->ipa3_active_clients.cnt), connect, ipa3_ctx->stats.wan_rx_empty, + ipa3_ctx->stats.wan_rx_empty_coal, ipa3_ctx->stats.wan_repl_rx_empty, ipa3_ctx->stats.lan_rx_empty, ipa3_ctx->stats.lan_repl_rx_empty, @@ -1421,24 +1423,46 @@ static ssize_t ipa3_read_odlstats(struct file *file, char __user *ubuf, return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt); } + static ssize_t ipa3_read_page_recycle_stats(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { int nbytes; - int cnt = 0; + int cnt = 0, i = 0, k = 0; nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, "COAL : Total number of packets replenished =%llu\n" + "COAL : Number of page recycled packets =%llu\n" "COAL : Number of tmp alloc packets =%llu\n" + "COAL : Number of times tasklet scheduled =%llu\n" "DEF : Total number of packets replenished =%llu\n" - "DEF : Number of tmp alloc packets =%llu\n", + "DEF : Number of page recycled packets =%llu\n" + "DEF : Number of tmp alloc packets =%llu\n" + "DEF : Number of times tasklet scheduled =%llu\n" + "COMMON : Number of page recycled in tasklet =%llu\n" + "COMMON : Number of times free pages not found in tasklet =%llu\n", ipa3_ctx->stats.page_recycle_stats[0].total_replenished, + ipa3_ctx->stats.page_recycle_stats[0].page_recycled, ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc, + ipa3_ctx->stats.num_sort_tasklet_sched[0], ipa3_ctx->stats.page_recycle_stats[1].total_replenished, - ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc); + ipa3_ctx->stats.page_recycle_stats[1].page_recycled, + ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc, + ipa3_ctx->stats.num_sort_tasklet_sched[1], + ipa3_ctx->stats.page_recycle_cnt_in_tasklet, + ipa3_ctx->stats.num_of_times_wq_reschd); cnt += nbytes; + for (k = 0; k < 2; k++) { + for (i = 0; i < ipa3_ctx->page_poll_threshold; i++) { + nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN, + "COMMON : Page replenish efficiency[%d][%d] =%llu\n", + k, i, ipa3_ctx->stats.page_recycle_cnt[k][i]); + cnt += nbytes; + } + } + return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt); } static ssize_t ipa3_read_wstats(struct file *file, char __user *ubuf, @@ -2772,6 +2796,68 @@ static ssize_t ipa3_enable_ipc_low(struct file *file, return count; } +static ssize_t ipa3_read_ipa_max_napi_sort_page_thrshld(struct file *file, + char __user *buf, size_t count, loff_t *ppos) { + + int nbytes; + nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, + "page max napi without free page = %d\n", + ipa3_ctx->ipa_max_napi_sort_page_thrshld); + return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes); + +} + +static ssize_t ipa3_write_ipa_max_napi_sort_page_thrshld(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) { + + int ret; + u8 ipa_max_napi_sort_page_thrshld = 0; + + if (count >= sizeof(dbg_buff)) + return -EFAULT; + + ret = kstrtou8_from_user(buf, count, 0, &ipa_max_napi_sort_page_thrshld); + if(ret) + return ret; + + ipa3_ctx->ipa_max_napi_sort_page_thrshld = ipa_max_napi_sort_page_thrshld; + + IPADBG("napi cnt without prealloc pages = %d", ipa3_ctx->ipa_max_napi_sort_page_thrshld); + + return count; +} + +static ssize_t ipa3_read_page_wq_reschd_time(struct file *file, + char __user *buf, size_t count, loff_t *ppos) { + + int nbytes; + nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, + "Page WQ reschduule time = %d\n", + ipa3_ctx->page_wq_reschd_time); + return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes); + +} + +static ssize_t ipa3_write_page_wq_reschd_time(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) { + + int ret; + u8 page_wq_reschd_time = 0; + + if (count >= sizeof(dbg_buff)) + return -EFAULT; + + ret = kstrtou8_from_user(buf, count, 0, &page_wq_reschd_time); + if(ret) + return ret; + + ipa3_ctx->page_wq_reschd_time = page_wq_reschd_time; + + IPADBG("Updated page WQ reschedule time = %d", ipa3_ctx->page_wq_reschd_time); + + return count; +} + static ssize_t ipa3_read_gsi_wdi3_db_polling(struct file *file, char __user *buf, size_t count, loff_t *ppos) { @@ -2804,6 +2890,41 @@ static ssize_t ipa3_write_gsi_wdi3_db_polling(struct file *file, return count; } +static ssize_t ipa3_read_page_poll_threshold(struct file *file, + char __user *buf, size_t count, loff_t *ppos) { + + int nbytes; + nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, + "Page Poll Threshold = %d\n", + ipa3_ctx->page_poll_threshold); + return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes); + +} + +static ssize_t ipa3_write_page_poll_threshold(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) { + + int ret; + u8 page_poll_threshold =0; + + if (count >= sizeof(dbg_buff)) + return -EFAULT; + + ret = kstrtou8_from_user(buf, count, 0, &page_poll_threshold); + if(ret) + return ret; + + if(page_poll_threshold != 0 && + page_poll_threshold <= IPA_PAGE_POLL_THRESHOLD_MAX) + ipa3_ctx->page_poll_threshold = page_poll_threshold; + else + IPAERR("Invalid value \n"); + + IPADBG("Updated page poll threshold = %d", ipa3_ctx->page_poll_threshold); + + return count; +} + static const struct ipa3_debugfs_file debugfs_files[] = { { "gen_reg", IPA_READ_ONLY_MODE, NULL, { @@ -2986,6 +3107,21 @@ static const struct ipa3_debugfs_file debugfs_files[] = { .read = ipa3_read_gsi_wdi3_db_polling, .write = ipa3_write_gsi_wdi3_db_polling, } + }, { + "page_poll_threshold", IPA_READ_WRITE_MODE, NULL, { + .read = ipa3_read_page_poll_threshold, + .write = ipa3_write_page_poll_threshold, + } + }, { + "page_wq_reschd_time", IPA_READ_WRITE_MODE, NULL, { + .read = ipa3_read_page_wq_reschd_time, + .write = ipa3_write_page_wq_reschd_time, + } + }, { + "ipa_max_napi_sort_page_thrshld", IPA_READ_WRITE_MODE, NULL, { + .read = ipa3_read_ipa_max_napi_sort_page_thrshld, + .write = ipa3_write_ipa_max_napi_sort_page_thrshld, + } }, }; diff --git a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c index f5783895557e..bea0e82699b7 100644 --- a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c +++ b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -137,6 +137,7 @@ static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys, static unsigned long tag_to_pointer_wa(uint64_t tag); static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt); static void ipa3_tasklet_rx_notify(unsigned long data); +static void ipa3_tasklet_find_freepage(unsigned long data); static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit); @@ -1048,6 +1049,60 @@ static void ipa_pm_sys_pipe_cb(void *p, enum ipa_pm_cb_event event) } } +static void ipa3_schd_freepage_work(struct work_struct *work) +{ + struct delayed_work *dwork; + struct ipa3_sys_context *sys; + + dwork = container_of(work, struct delayed_work, work); + sys = container_of(dwork, struct ipa3_sys_context, freepage_work); + + IPADBG_LOW("WQ scheduled, reschedule sort tasklet\n"); + + tasklet_schedule(&sys->tasklet_find_freepage); +} + +static void ipa3_tasklet_find_freepage(unsigned long data) +{ + struct ipa3_sys_context *sys; + struct ipa3_rx_pkt_wrapper *rx_pkt = NULL; + struct ipa3_rx_pkt_wrapper *tmp = NULL; + struct page *cur_page; + int found_free_page = 0; + struct list_head temp_head; + + sys = (struct ipa3_sys_context *)data; + + INIT_LIST_HEAD(&temp_head); + spin_lock_bh(&sys->common_sys->spinlock); + list_for_each_entry_safe(rx_pkt, tmp, + &sys->page_recycle_repl->page_repl_head, link) { + cur_page = rx_pkt->page_data.page; + if (page_ref_count(cur_page) == 1) { + /* Found a free page. */ + list_del_init(&rx_pkt->link); + list_add(&rx_pkt->link, &temp_head); + found_free_page++; + } + } + if (!found_free_page) { + /*Not found free page rescheduling tasklet after 2msec*/ + IPADBG_LOW("Scheduling WQ not found free pages\n"); + ++ipa3_ctx->stats.num_of_times_wq_reschd; + queue_delayed_work(sys->freepage_wq, + &sys->freepage_work, + msecs_to_jiffies(ipa3_ctx->page_wq_reschd_time)); + } else { + /*Allow to use pre-allocated buffers*/ + list_splice(&temp_head, &sys->page_recycle_repl->page_repl_head); + ipa3_ctx->stats.page_recycle_cnt_in_tasklet += found_free_page; + IPADBG_LOW("found free pages count = %d\n", found_free_page); + atomic_set(&sys->common_sys->page_avilable, 1); + } + spin_unlock_bh(&sys->common_sys->spinlock); + +} + /** * ipa3_setup_sys_pipe() - Setup an IPA GPI pipe and perform * IPA EP configuration @@ -1130,7 +1185,8 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) snprintf(buff, IPA_RESOURCE_NAME_MAX, "iparepwq%d", sys_in->client); ep->sys->repl_wq = alloc_workqueue(buff, - WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1); + WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | WQ_HIGHPRI, + 1); if (!ep->sys->repl_wq) { IPAERR("failed to create rep wq for client %d\n", sys_in->client); @@ -1138,6 +1194,9 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) goto fail_wq2; } + snprintf(buff, IPA_RESOURCE_NAME_MAX, "ipafreepagewq%d", + sys_in->client); + INIT_LIST_HEAD(&ep->sys->head_desc_list); INIT_LIST_HEAD(&ep->sys->rcycl_list); INIT_LIST_HEAD(&ep->sys->avail_tx_wrapper_list); @@ -1154,6 +1213,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) /* Use coalescing pipe PM handle for default pipe also*/ ep->sys->pm_hdl = ipa3_ctx->ep[coal_ep_id].sys->pm_hdl; } else if (IPA_CLIENT_IS_CONS(sys_in->client)) { + ep->sys->freepage_wq = alloc_workqueue(buff, + WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | + WQ_HIGHPRI, 1); + if (!ep->sys->freepage_wq) { + IPAERR("failed to create freepage wq for client %d\n", + sys_in->client); + result = -EFAULT; + goto fail_wq3; + } + pm_reg.name = ipa_clients_strings[sys_in->client]; pm_reg.callback = ipa_pm_sys_pipe_cb; pm_reg.user_data = ep->sys; @@ -1254,6 +1323,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) } *clnt_hdl = ipa_ep_idx; + ep->sys->common_sys = ipa3_ctx->ep[ipa_ep_idx].sys; if (ep->sys->repl_hdlr == ipa3_fast_replenish_rx_cache) { ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL); @@ -1279,41 +1349,77 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) } } + /* Use common page pool for Coal and defalt pipe if applicable. */ if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) { - ep->sys->page_recycle_repl = kzalloc( - sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL); - if (!ep->sys->page_recycle_repl) { - IPAERR("failed to alloc repl for client %d\n", - sys_in->client); - result = -ENOMEM; - goto fail_napi; + if (!(ipa3_ctx->wan_common_page_pool && + sys_in->client == IPA_CLIENT_APPS_WAN_CONS && + coal_ep_id != IPA_EP_NOT_ALLOCATED && + ipa3_ctx->ep[coal_ep_id].valid == 1)) { + /* Allocate page recycling pool only once. */ + if (!ep->sys->page_recycle_repl) { + ep->sys->page_recycle_repl = kzalloc( + sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL); + if (!ep->sys->page_recycle_repl) { + IPAERR("failed to alloc repl for client %d\n", + sys_in->client); + result = -ENOMEM; + goto fail_napi; + } + atomic_set(&ep->sys->page_recycle_repl->pending, 0); + /* For common page pool double the pool size. */ + if (ipa3_ctx->wan_common_page_pool && + sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS) + ep->sys->page_recycle_repl->capacity = + (ep->sys->rx_pool_sz + 1) * + IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR; + else + ep->sys->page_recycle_repl->capacity = + (ep->sys->rx_pool_sz + 1) * + IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR; + IPADBG("Page repl capacity for client:%d, value:%d\n", + sys_in->client, ep->sys->page_recycle_repl->capacity); + INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head); + INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work); + tasklet_init(&ep->sys->tasklet_find_freepage, + ipa3_tasklet_find_freepage, (unsigned long) ep->sys); + ipa3_replenish_rx_page_cache(ep->sys); + } else { + ep->sys->napi_sort_page_thrshld_cnt = 0; + /* Sort the pages once. */ + ipa3_tasklet_find_freepage((unsigned long) ep->sys); + } + + ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL); + if (!ep->sys->repl) { + IPAERR("failed to alloc repl for client %d\n", + sys_in->client); + result = -ENOMEM; + goto fail_page_recycle_repl; + } + /* For common page pool triple the pool size. */ + if (ipa3_ctx->wan_common_page_pool && + sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS) + ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1) * + IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR; + else + ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1); + IPADBG("Repl capacity for client:%d, value:%d\n", + sys_in->client, ep->sys->repl->capacity); + atomic_set(&ep->sys->repl->pending, 0); + ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity, + sizeof(void *), GFP_KERNEL); + atomic_set(&ep->sys->repl->head_idx, 0); + atomic_set(&ep->sys->repl->tail_idx, 0); + + ipa3_wq_page_repl(&ep->sys->repl_work); + } else { + /* Use pool same as coal pipe when common page pool is used */ + ep->sys->common_buff_pool = true; + ep->sys->common_sys = ipa3_ctx->ep[coal_ep_id].sys; + ep->sys->repl = ipa3_ctx->ep[coal_ep_id].sys->repl; + ep->sys->page_recycle_repl = + ipa3_ctx->ep[coal_ep_id].sys->page_recycle_repl; } - atomic_set(&ep->sys->page_recycle_repl->pending, 0); - ep->sys->page_recycle_repl->capacity = - (ep->sys->rx_pool_sz + 1) * 2; - - ep->sys->page_recycle_repl->cache = - kcalloc(ep->sys->page_recycle_repl->capacity, - sizeof(void *), GFP_KERNEL); - atomic_set(&ep->sys->page_recycle_repl->head_idx, 0); - atomic_set(&ep->sys->page_recycle_repl->tail_idx, 0); - ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL); - if (!ep->sys->repl) { - IPAERR("failed to alloc repl for client %d\n", - sys_in->client); - result = -ENOMEM; - goto fail_page_recycle_repl; - } - ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1); - - atomic_set(&ep->sys->repl->pending, 0); - ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity, - sizeof(void *), GFP_KERNEL); - atomic_set(&ep->sys->repl->head_idx, 0); - atomic_set(&ep->sys->repl->tail_idx, 0); - - ipa3_replenish_rx_page_cache(ep->sys); - ipa3_wq_page_repl(&ep->sys->repl_work); } if (IPA_CLIENT_IS_CONS(sys_in->client)) { @@ -1383,16 +1489,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) fail_gen3: ipa3_disable_data_path(ipa_ep_idx); fail_repl: - if (IPA_CLIENT_IS_CONS(ep->client)) + if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool) ipa3_cleanup_rx(ep->sys); - if(ep->sys->repl) { + ep->sys->repl_hdlr = ipa3_replenish_rx_cache; + if (ep->sys->repl && !ep->sys->common_buff_pool) { ep->sys->repl->capacity = 0; kfree(ep->sys->repl); - ep->sys->repl = NULL; } fail_page_recycle_repl: - if (ep->sys->page_recycle_repl) { + if (ep->sys->page_recycle_repl && !ep->sys->common_buff_pool) { ep->sys->page_recycle_repl->capacity = 0; kfree(ep->sys->page_recycle_repl); ep->sys->page_recycle_repl = NULL; @@ -1405,6 +1511,9 @@ fail_napi: fail_gen2: ipa_pm_deregister(ep->sys->pm_hdl); fail_pm: + if (ep->sys->freepage_wq) + destroy_workqueue(ep->sys->freepage_wq); +fail_wq3: destroy_workqueue(ep->sys->repl_wq); fail_wq2: destroy_workqueue(ep->sys->wq); @@ -1571,7 +1680,13 @@ int ipa3_teardown_sys_pipe(u32 clnt_hdl) } if (ep->sys->repl_wq) flush_workqueue(ep->sys->repl_wq); - if (IPA_CLIENT_IS_CONS(ep->client)) + + if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) { + cancel_delayed_work_sync(&ep->sys->common_sys->freepage_work); + tasklet_kill(&ep->sys->common_sys->tasklet_find_freepage); + } + + if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool) ipa3_cleanup_rx(ep->sys); if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(ep->client)) { @@ -1657,7 +1772,7 @@ static int ipa3_teardown_coal_def_pipe(u32 clnt_hdl) if (ep->sys->repl_wq) flush_workqueue(ep->sys->repl_wq); - if (IPA_CLIENT_IS_CONS(ep->client)) + if (IPA_CLIENT_IS_CONS(ep->client) && !ep->sys->common_buff_pool) ipa3_cleanup_rx(ep->sys); ep->valid = 0; @@ -2081,6 +2196,9 @@ static struct ipa3_rx_pkt_wrapper *ipa3_alloc_rx_pkt_page( return NULL; rx_pkt->page_data.page_order = IPA_WAN_PAGE_ORDER; + /* For temporary allocations, avoid triggering OOM Killer. */ + if (is_tmp_alloc) + flag |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN; /* Try a lower order page for order 3 pages in case allocation fails. */ rx_pkt->page_data.page = ipa3_alloc_page(flag, &rx_pkt->page_data.page_order, @@ -2126,9 +2244,12 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys) ipa_assert(); break; } + INIT_LIST_HEAD(&rx_pkt->link); rx_pkt->sys = sys; - sys->page_recycle_repl->cache[curr] = rx_pkt; + list_add_tail(&rx_pkt->link, + &sys->page_recycle_repl->page_repl_head); } + atomic_set(&sys->common_sys->page_avilable, 1); return; @@ -2189,12 +2310,54 @@ static inline void __trigger_repl_work(struct ipa3_sys_context *sys) head = atomic_read(&sys->repl->head_idx); avail = (tail - head) % sys->repl->capacity; - if (avail < sys->repl->capacity / 4) { + if (avail < sys->repl->capacity / 2) { atomic_set(&sys->repl->pending, 1); queue_work(sys->repl_wq, &sys->repl_work); } } +static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page +( + struct ipa3_sys_context *sys, + u32 stats_i +) +{ + struct ipa3_rx_pkt_wrapper *rx_pkt = NULL; + struct ipa3_rx_pkt_wrapper *tmp = NULL; + struct page *cur_page; + int i = 0; + u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold; + + spin_lock_bh(&sys->common_sys->spinlock); + list_for_each_entry_safe(rx_pkt, tmp, + &sys->page_recycle_repl->page_repl_head, link) { + if (i == LOOP_THRESHOLD) + break; + cur_page = rx_pkt->page_data.page; + if (page_ref_count(cur_page) == 1) { + /* Found a free page. */ + page_ref_inc(cur_page); + list_del_init(&rx_pkt->link); + ++ipa3_ctx->stats.page_recycle_cnt[stats_i][i]; + sys->common_sys->napi_sort_page_thrshld_cnt = 0; + spin_unlock_bh(&sys->common_sys->spinlock); + return rx_pkt; + } + i++; + } + spin_unlock_bh(&sys->common_sys->spinlock); + IPADBG_LOW("napi_sort_page_thrshld_cnt = %d ipa_max_napi_sort_page_thrshld = %d\n", + sys->common_sys->napi_sort_page_thrshld_cnt, + ipa3_ctx->ipa_max_napi_sort_page_thrshld); + /*Scheduling tasklet to find the free page*/ + if (sys->common_sys->napi_sort_page_thrshld_cnt >= + ipa3_ctx->ipa_max_napi_sort_page_thrshld) { + atomic_set(&sys->common_sys->page_avilable, 0); + tasklet_schedule(&sys->common_sys->tasklet_find_freepage); + ++ipa3_ctx->stats.num_sort_tasklet_sched[stats_i]; + } + return NULL; +} static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) { @@ -2202,10 +2365,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) int ret; int rx_len_cached = 0; struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX]; - u32 curr; u32 curr_wq; int idx = 0; - struct page *cur_page; u32 stats_i = 0; /* start replenish only when buffers go lower than the threshold */ @@ -2213,19 +2374,15 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) return; stats_i = (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) ? 0 : 1; - spin_lock_bh(&sys->spinlock); rx_len_cached = sys->len; - curr = atomic_read(&sys->page_recycle_repl->head_idx); curr_wq = atomic_read(&sys->repl->head_idx); while (rx_len_cached < sys->rx_pool_sz) { - cur_page = sys->page_recycle_repl->cache[curr]->page_data.page; - /* Found an idle page that can be used */ - if (page_ref_count(cur_page) == 1) { - page_ref_inc(cur_page); - rx_pkt = sys->page_recycle_repl->cache[curr]; - curr = (++curr == sys->page_recycle_repl->capacity) ? - 0 : curr; + /* check for an idle page that can be used */ + if (atomic_read(&sys->common_sys->page_avilable) && + ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL)) { + ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++; + } else { /* * Could not find idle page at curr index. @@ -2238,6 +2395,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) curr_wq = (++curr_wq == sys->repl->capacity) ? 0 : curr_wq; } + rx_pkt->sys = sys; dma_sync_single_for_device(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr, @@ -2275,26 +2433,27 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) /* ensure write is done before setting head index */ mb(); atomic_set(&sys->repl->head_idx, curr_wq); - atomic_set(&sys->page_recycle_repl->head_idx, curr); sys->len = rx_len_cached; } else { /* we don't expect this will happen */ IPAERR("failed to provide buffer: %d\n", ret); ipa_assert(); } - spin_unlock_bh(&sys->spinlock); - __trigger_repl_work(sys); + + if (sys->common_buff_pool) + __trigger_repl_work(sys->common_sys); + else + __trigger_repl_work(sys); if (rx_len_cached <= IPA_DEFAULT_SYS_YELLOW_WM) { - if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS || - sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) + if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS) IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty); + else if (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) + IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty_coal); else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS) IPA_STATS_INC_CNT(ipa3_ctx->stats.lan_rx_empty); else WARN_ON(1); - queue_delayed_work(sys->wq, &sys->replenish_rx_work, - msecs_to_jiffies(1)); } return; @@ -2790,20 +2949,21 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data) { struct ipa3_rx_pkt_wrapper *rx_pkt = (struct ipa3_rx_pkt_wrapper *) xfer_user_data; - struct ipa3_sys_context *sys = rx_pkt->sys; - int i; - for (i = 0; i < sys->page_recycle_repl->capacity; i++) - if (sys->page_recycle_repl->cache[i] == rx_pkt) - break; - if (i < sys->page_recycle_repl->capacity) { + if (!rx_pkt->page_data.is_tmp_alloc) { + list_del_init(&rx_pkt->link); page_ref_dec(rx_pkt->page_data.page); - sys->page_recycle_repl->cache[i] = NULL; + spin_lock_bh(&rx_pkt->sys->common_sys->spinlock); + /* Add the element to head. */ + list_add(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock); + } else { + dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr, + rx_pkt->len, DMA_FROM_DEVICE); + __free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order); + kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt); } - dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr, - rx_pkt->len, DMA_FROM_DEVICE); - __free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order); - kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt); } /** @@ -2816,7 +2976,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys) struct ipa3_rx_pkt_wrapper *r; u32 head; u32 tail; - int i; /* * buffers not consumed by gsi are cleaned up using cleanup callback @@ -2864,25 +3023,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys) kfree(sys->repl); sys->repl = NULL; } - if (sys->page_recycle_repl) { - for (i = 0; i < sys->page_recycle_repl->capacity; i++) { - rx_pkt = sys->page_recycle_repl->cache[i]; - if (rx_pkt) { - dma_unmap_page(ipa3_ctx->pdev, - rx_pkt->page_data.dma_addr, - rx_pkt->len, - DMA_FROM_DEVICE); - __free_pages(rx_pkt->page_data.page, - rx_pkt->page_data.page_order); - kmem_cache_free( - ipa3_ctx->rx_pkt_wrapper_cache, - rx_pkt); - } - } - kfree(sys->page_recycle_repl->cache); - kfree(sys->page_recycle_repl); - sys->page_recycle_repl = NULL; - } } static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len) @@ -3670,6 +3810,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify IPAERR("notify->veid > GSI_VEID_MAX\n"); if (!rx_page.is_tmp_alloc) { init_page_count(rx_page.page); + spin_lock_bh(&rx_pkt->sys->common_sys->spinlock); + /* Add the element to head. */ + list_add(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock); } else { dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); @@ -3697,11 +3842,15 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify list_del_init(&rx_pkt->link); if (!rx_page.is_tmp_alloc) { init_page_count(rx_page.page); + spin_lock_bh(&rx_pkt->sys->common_sys->spinlock); + /* Add the element to head. */ + list_add(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock); } else { dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); - __free_pages(rx_pkt->page_data.page, - rx_pkt->page_data.page_order); + __free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order); } rx_pkt->sys->free_rx_wrapper(rx_pkt); } @@ -3712,14 +3861,20 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify rx_page = rx_pkt->page_data; size = rx_pkt->data_len; - list_del(&rx_pkt->link); - if (rx_page.is_tmp_alloc) + list_del_init(&rx_pkt->link); + if (rx_page.is_tmp_alloc) { dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); - else + } else { + spin_lock_bh(&rx_pkt->sys->common_sys->spinlock); + /* Add the element back to tail. */ + list_add_tail(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock); dma_sync_single_for_cpu(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); + } rx_pkt->sys->free_rx_wrapper(rx_pkt); skb_add_rx_frag(rx_skb, @@ -3846,20 +4001,6 @@ static void ipa3_rx_napi_chain(struct ipa3_sys_context *sys, if (prev_skb) { skb_shinfo(prev_skb)->frag_list = NULL; sys->pyld_hdlr(first_skb, sys); - /* - * For coalescing, we have 2 transfer - * rings to replenish - */ - ipa_ep_idx = ipa3_get_ep_mapping( - IPA_CLIENT_APPS_WAN_CONS); - if (ipa_ep_idx == - IPA_EP_NOT_ALLOCATED) { - IPAERR("Invalid client.\n"); - return; - } - wan_def_sys = - ipa3_ctx->ep[ipa_ep_idx].sys; - wan_def_sys->repl_hdlr(wan_def_sys); } } } @@ -5408,6 +5549,7 @@ int ipa3_rx_poll(u32 clnt_hdl, int weight) return -EINVAL; } + ep->sys->common_sys->napi_sort_page_thrshld_cnt++; start_poll: /* * it is guaranteed we already have clock here. @@ -5441,6 +5583,7 @@ start_poll: cnt += weight - remain_aggr_weight * ipa3_ctx->ipa_wan_aggr_pkt_cnt; /* call repl_hdlr before napi_reschedule / napi_complete */ ep->sys->repl_hdlr(ep->sys); + wan_def_sys->repl_hdlr(wan_def_sys); /* When not able to replenish enough descriptors, keep in polling * mode, wait for napi-poll and replenish again. */ diff --git a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_i.h index 82902d0e1c80..ceb439f729b2 100644 --- a/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_i.h +++ b/techpack/dataipa/drivers/platform/msm/ipa/ipa_v3/ipa_i.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _IPA3_I_H_ @@ -59,7 +59,11 @@ #define IPA_QMAP_HEADER_LENGTH (4) #define IPA_DL_CHECKSUM_LENGTH (8) #define IPA_NUM_DESC_PER_SW_TX (3) +#define IPA_GENERIC_RX_POOL_SZ_WAN 192 #define IPA_GENERIC_RX_POOL_SZ 192 +#define IPA_GENERIC_RX_PAGE_POOL_SZ_FACTOR 2 +#define IPA_GENERIC_RX_CMN_PAGE_POOL_SZ_FACTOR 4 +#define IPA_GENERIC_RX_CMN_TEMP_POOL_SZ_FACTOR 2 #define IPA_UC_FINISH_MAX 6 #define IPA_UC_WAIT_MIN_SLEEP 1000 #define IPA_UC_WAII_MAX_SLEEP 1200 @@ -84,6 +88,13 @@ #define IPA_WAN_AGGR_PKT_CNT 1 +#define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15 +#define IPA_PAGE_POLL_THRESHOLD_MAX 30 + +#define IPA_MAX_NAPI_SORT_PAGE_THRSHLD 3 +#define IPA_MAX_PAGE_WQ_RESCHED_TIME 2 + + #define IPADBG(fmt, args...) \ do { \ pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\ @@ -1062,6 +1073,12 @@ struct ipa3_repl_ctx { atomic_t pending; }; +struct ipa3_page_repl_ctx { + struct list_head page_repl_head; + u32 capacity; + atomic_t pending; +}; + /** * struct ipa3_sys_context - IPA GPI pipes context * @head_desc_list: header descriptors list @@ -1100,7 +1117,6 @@ struct ipa3_sys_context { struct work_struct repl_work; void (*repl_hdlr)(struct ipa3_sys_context *sys); struct ipa3_repl_ctx *repl; - struct ipa3_repl_ctx *page_recycle_repl; u32 pkt_sent; struct napi_struct *napi_obj; struct list_head pending_pkts[GSI_VEID_MAX]; @@ -1110,6 +1126,10 @@ struct ipa3_sys_context { u32 eob_drop_cnt; struct napi_struct napi_tx; atomic_t in_napi_context; + bool common_buff_pool; + struct ipa3_sys_context *common_sys; + atomic_t page_avilable; + u32 napi_sort_page_thrshld_cnt; /* ordering is important - mutable fields go above */ struct ipa3_ep_context *ep; @@ -1125,6 +1145,10 @@ struct ipa3_sys_context { u32 pm_hdl; unsigned int napi_sch_cnt; unsigned int napi_comp_cnt; + struct ipa3_page_repl_ctx *page_recycle_repl; + struct workqueue_struct *freepage_wq; + struct delayed_work freepage_work; + struct tasklet_struct tasklet_find_freepage; /* ordering is important - other immutable fields go below */ }; @@ -1431,8 +1455,10 @@ enum ipa3_config_this_ep { struct ipa3_page_recycle_stats { u64 total_replenished; + u64 page_recycled; u64 tmp_alloc; }; + struct ipa3_stats { u32 tx_sw_pkts; u32 tx_hw_pkts; @@ -1447,6 +1473,7 @@ struct ipa3_stats { u32 aggr_close; u32 wan_aggr_close; u32 wan_rx_empty; + u32 wan_rx_empty_coal; u32 wan_repl_rx_empty; u32 lan_rx_empty; u32 lan_repl_rx_empty; @@ -1459,6 +1486,10 @@ struct ipa3_stats { struct ipa3_page_recycle_stats page_recycle_stats[2]; u64 lower_order; u32 pipe_setup_fail_cnt; + u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX]; + u64 num_sort_tasklet_sched[3]; + u64 num_of_times_wq_reschd; + u64 page_recycle_cnt_in_tasklet; }; /* offset for each stats */ @@ -2174,7 +2205,10 @@ struct ipa3_context { bool fnr_stats_not_supported; bool is_device_crashed; int ipa_pil_load; - + u8 page_poll_threshold; + bool wan_common_page_pool; + u32 ipa_max_napi_sort_page_thrshld; + u32 page_wq_reschd_time; }; struct ipa3_plat_drv_res {