From a1937b42425cd22bd4b2a607f90f08930f9f3070 Mon Sep 17 00:00:00 2001 From: Hari Krishna Chowdary Yennana Date: Mon, 14 Aug 2023 15:12:43 +0530 Subject: [PATCH] dsp: add support to get Source Track data from FNN module - Add support for both audio record and voip call usecases to get STT params from dsp. Change-Id: Ifde484be0109ca25fb7cb9572abc506c1a70e4a4 --- dsp/q6adm.c | 95 +++++++++++++++++++++ dsp/q6voice.c | 167 +++++++++++++++++++++++++++++++++++++ include/dsp/apr_audio-v2.h | 36 +++++++- include/dsp/q6adm-v2.h | 3 + include/dsp/q6voice.h | 15 ++++ 5 files changed, 315 insertions(+), 1 deletion(-) diff --git a/dsp/q6adm.c b/dsp/q6adm.c index 712ba088baeb..32a5b29d3a1c 100644 --- a/dsp/q6adm.c +++ b/dsp/q6adm.c @@ -5983,6 +5983,101 @@ done: } EXPORT_SYMBOL(adm_get_source_tracking); +/** + * adm_get_fnn_source_tracking - + * Retrieve sound track info + * + * @port_id: Port ID number + * @copp_idx: copp index assigned + * @FnnSourceTrackingData: pointer for source track data to be updated with + * + * Returns 0 on success or error on failure + */ +int adm_get_fnn_source_tracking(int port_id, int copp_idx, + struct fluence_nn_source_tracking_param *FnnSourceTrackingData) +{ + int ret = 0, i; + char *params_value; + uint32_t max_param_size = 0; + struct adm_param_fluence_nn_source_tracking_t *fnn_sourcetrack_params = NULL; + struct param_hdr_v3 param_hdr; + + pr_debug("%s: Enter, port_id %d, copp_idx %d\n", + __func__, port_id, copp_idx); + + max_param_size = sizeof(struct adm_param_fluence_nn_source_tracking_t) + + sizeof(union param_hdrs); + params_value = kzalloc(max_param_size, GFP_KERNEL); + if (!params_value) + return -ENOMEM; + + memset(¶m_hdr, 0, sizeof(param_hdr)); + param_hdr.module_id = MODULE_ID_FLUENCE_NN; + param_hdr.instance_id = INSTANCE_ID_0; + param_hdr.param_id = AUDPROC_PARAM_ID_FLUENCE_NN_SOURCE_TRACKING; + param_hdr.param_size = max_param_size; + ret = adm_get_pp_params(port_id, copp_idx, + ADM_CLIENT_ID_SOURCE_TRACKING, NULL, ¶m_hdr, + params_value); + if (ret) { + pr_err("%s: get parameters failed ret:%d\n", __func__, ret); + ret = -EINVAL; + goto done; + } + + if (this_adm.sourceTrackingData.apr_cmd_status != 0) { + pr_err("%s - get params returned error [%s]\n", + __func__, adsp_err_get_err_str( + this_adm.sourceTrackingData.apr_cmd_status)); + ret = adsp_err_get_lnx_err_code( + this_adm.sourceTrackingData.apr_cmd_status); + goto done; + } + + fnn_sourcetrack_params = (struct adm_param_fluence_nn_source_tracking_t *) params_value; + if ((!FnnSourceTrackingData) || (!fnn_sourcetrack_params)) { + pr_err("%s: Caught NULL pointer \n", __func__); + ret = -EINVAL; + goto done; + } + + FnnSourceTrackingData->speech_probablity_q20 = + fnn_sourcetrack_params->speech_probablity_q20; + pr_debug("%s: speech_probablity_q20 = %d\n", + __func__, FnnSourceTrackingData->speech_probablity_q20); + + for (i = 0; i < MAX_TOP_SPEAKERS; i++) { + FnnSourceTrackingData->speakers[i] = + fnn_sourcetrack_params->speakers[i]; + pr_debug("%s: speakers[%d] = %d\n", + __func__, i, FnnSourceTrackingData->speakers[i]); + } + + for (i = 0; i < MAX_POLAR_ACTIVITY_INDICATORS; i++) { + FnnSourceTrackingData->polarActivity[i] = + fnn_sourcetrack_params->polarActivity[i]; + pr_debug("%s: polarActivity[%d] = %d\n", + __func__, i, FnnSourceTrackingData->polarActivity[i]); + } + + FnnSourceTrackingData->session_time_lsw = + fnn_sourcetrack_params->session_time_lsw; + pr_debug("%s: session_time_lsw = %x\n", + __func__, FnnSourceTrackingData->session_time_lsw); + + FnnSourceTrackingData->session_time_msw = + fnn_sourcetrack_params->session_time_msw; + pr_debug("%s: session_time_msw = %x\n", + __func__, FnnSourceTrackingData->session_time_msw); + +done: + pr_debug("%s: Exit, ret = %d\n", __func__, ret); + + kfree(params_value); + return ret; +} +EXPORT_SYMBOL(adm_get_fnn_source_tracking); + /** * adm_get_doa_tracking_mon - * Retrieve doa tracking monitor info diff --git a/dsp/q6voice.c b/dsp/q6voice.c index f0f2a05b636d..f5115d225655 100644 --- a/dsp/q6voice.c +++ b/dsp/q6voice.c @@ -147,6 +147,8 @@ static int voice_send_get_sound_focus_cmd(struct voice_data *v, struct sound_focus_param *soundFocusData); static int voice_send_get_source_tracking_cmd(struct voice_data *v, struct source_tracking_param *sourceTrackingData); +static int voice_send_get_fnn_source_track_cmd(struct voice_data *v, + struct fluence_nn_source_tracking_param *FnnSourceTrackingData); static int voice_pack_and_set_cvp_param(struct voice_data *v, struct param_hdr_v3 param_hdr, u8 *param_data); @@ -8431,6 +8433,19 @@ static int32_t qdsp_cvp_callback(struct apr_client_data *data, void *priv) v->async_err = ptr[1]; wake_up(&v->cvp_wait); break; + case VSS_IFNN_ST_CMD_GET_ACTIVITY: + if (!ptr[1]) { + /* Read data from shared memory */ + common.is_source_tracking_resp_success = true; + } else { + common.is_source_tracking_resp_success = false; + pr_err("%s: Error received for source tracking params\n", + __func__); + } + v->cvp_state = CMD_STATUS_SUCCESS; + v->async_err = ptr[1]; + wake_up(&v->cvp_wait); + break; default: pr_debug("%s: not match cmd = 0x%x\n", __func__, ptr[0]); @@ -10028,6 +10043,158 @@ int voc_get_source_tracking(struct source_tracking_param *sourceTrackingData) } EXPORT_SYMBOL(voc_get_source_tracking); +static int voice_send_get_fnn_source_track_cmd(struct voice_data *v, + struct fluence_nn_source_tracking_param *FnnSourceTrackingData) +{ + struct apr_hdr cvp_get_fnn_st_param_cmd; + int ret = 0; + void *apr_cvp; + u16 cvp_handle; + int i; + + if (!v) { + pr_err("%s: v is NULL\n", __func__); + return -EINVAL; + } + + apr_cvp = common.apr_q6_cvp; + if (!apr_cvp) { + pr_err("%s: apr_cvp is NULL.\n", __func__); + return -EINVAL; + } + + if (!FnnSourceTrackingData) { + pr_err("%s: Caught NULL pointer \n", __func__); + ret = -EINVAL; + goto done; + } + + cvp_handle = voice_get_cvp_handle(v); + + /* send APR command to retrieve Sound Track Params from fluence NN Module */ + cvp_get_fnn_st_param_cmd.hdr_field = APR_HDR_FIELD(APR_MSG_TYPE_SEQ_CMD, + APR_HDR_LEN(APR_HDR_SIZE), + APR_PKT_VER); + cvp_get_fnn_st_param_cmd.pkt_size = APR_PKT_SIZE(APR_HDR_SIZE, + sizeof(cvp_get_fnn_st_param_cmd) - APR_HDR_SIZE); + cvp_get_fnn_st_param_cmd.src_port = voice_get_idx_for_session(v->session_id); + cvp_get_fnn_st_param_cmd.dest_port = cvp_handle; + cvp_get_fnn_st_param_cmd.token = 0; + cvp_get_fnn_st_param_cmd.opcode = VSS_IFNN_ST_CMD_GET_ACTIVITY; + + v->cvp_state = CMD_STATUS_FAIL; + v->async_err = 0; + ret = apr_send_pkt(apr_cvp, + (uint32_t *) &cvp_get_fnn_st_param_cmd); + if (ret < 0) { + pr_err("%s: Error in sending APR command\n", __func__); + + ret = -EINVAL; + goto done; + } + ret = wait_event_timeout(v->cvp_wait, + (v->cvp_state == CMD_STATUS_SUCCESS), + msecs_to_jiffies(TIMEOUT_MS)); + if (!ret) { + pr_err("%s: wait_event timeout\n", __func__); + + ret = -ETIME; + goto done; + } + + if (v->async_err > 0) { + pr_err("%s: DSP returned error[%s]\n", + __func__, adsp_err_get_err_str( + v->async_err)); + ret = adsp_err_get_lnx_err_code( + v->async_err); + goto done; + } + + if (!common.is_fnn_source_tracking_resp_success) { + pr_err("%s: Error response received from CVD\n", __func__); + + ret = -EINVAL; + goto done; + } + + FnnSourceTrackingData->speech_probablity_q20 = + common.FnnSourceTrackingResponse.speech_probablity_q20; + pr_debug("%s: speech_probablity = %d\n", + __func__, FnnSourceTrackingData->speech_probablity_q20); + + for (i = 0; i < MAX_TOP_SPEAKERS; i++) { + FnnSourceTrackingData->speakers[i] = + common.FnnSourceTrackingResponse.speakers[i]; + pr_debug("%s: speakers[%d] = %d\n", + __func__, i, FnnSourceTrackingData->speakers[i]); + } + + for (i = 0; i < MAX_POLAR_ACTIVITY_INDICATORS; i++) { + FnnSourceTrackingData->polarActivity[i] = + common.FnnSourceTrackingResponse.polarActivity[i]; + pr_debug("%s: polarActivity[%d] = %d\n", + __func__, i, FnnSourceTrackingData->polarActivity[i]); + } + + FnnSourceTrackingData->session_time_lsw = + common.FnnSourceTrackingResponse.session_time_lsw; + pr_debug("%s: session_time_lsw = %d\n", + __func__, FnnSourceTrackingData->session_time_lsw); + + FnnSourceTrackingData->session_time_msw = + common.FnnSourceTrackingResponse.session_time_msw; + pr_debug("%s: session_time_msw = %d\n", + __func__, FnnSourceTrackingData->session_time_msw); + + common.is_fnn_source_tracking_resp_success = false; + +done: + pr_debug("%s: Exit, ret=%d\n", __func__, ret); + + return ret; +} + +/** + * voc_get_fnn_source_tracking - retrieves source track data. + * + * @sourceTrackingData: pointer to be updated with source track data. + * + * Returns 0 on success or error on failure + */ +int voc_get_fnn_source_tracking(struct fluence_nn_source_tracking_param *FnnSourceTrackingData) +{ + struct voice_data *v = NULL; + int ret = -EINVAL; + struct voice_session_itr itr; + + mutex_lock(&common.common_lock); + + voice_itr_init(&itr, ALL_SESSION_VSID); + while (voice_itr_get_next_session(&itr, &v)) { + if (v == NULL) { + pr_err("%s: invalid session\n", __func__); + + break; + } + + mutex_lock(&v->lock); + if (is_voc_state_active(v->voc_state) && + (v->lch_mode != VOICE_LCH_START) && + !v->disable_topology) + ret = voice_send_get_fnn_source_track_cmd(v, + FnnSourceTrackingData); + mutex_unlock(&v->lock); + + } + + mutex_unlock(&common.common_lock); + pr_debug("%s: Exit, ret=%d\n", __func__, ret); + + return ret; +} +EXPORT_SYMBOL(voc_get_fnn_source_tracking); + static int voice_set_cvp_param(struct voice_data *v, struct vss_icommon_mem_mapping_hdr *mem_hdr, u32 *param_data, u32 param_size) diff --git a/include/dsp/apr_audio-v2.h b/include/dsp/apr_audio-v2.h index ed5963474da6..ee0bfd5f2f23 100644 --- a/include/dsp/apr_audio-v2.h +++ b/include/dsp/apr_audio-v2.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved. */ @@ -13519,10 +13519,29 @@ struct adm_set_compressed_device_latency { #define VOICEPROC_MODULE_ID_FLUENCE_PRO_VC_TX 0x00010F35 #define VOICEPROC_PARAM_ID_FLUENCE_SOUNDFOCUS 0x00010E37 #define VOICEPROC_PARAM_ID_FLUENCE_SOURCETRACKING 0x00010E38 +#define AUDPROC_PARAM_ID_FLUENCE_NN_SOURCE_TRACKING 0x00010B83 +#define MODULE_ID_FLUENCE_NN 0x00010B0F #define MAX_SECTORS 8 #define MAX_NOISE_SOURCE_INDICATORS 3 #define MAX_POLAR_ACTIVITY_INDICATORS 360 #define MAX_DOA_TRACKING_ANGLES 2 +#define MAX_TOP_SPEAKERS 5 +#define MAX_FOCUS_DIRECTION 2 + +struct fluence_nn_sound_focus_param { + int16_t mode; + int16_t focus_direction[MAX_FOCUS_DIRECTION]; + int16_t focus_width; +} __packed; + +struct fluence_nn_source_tracking_param { + int32_t speech_probablity_q20; + int16_t speakers[MAX_TOP_SPEAKERS]; + int16_t reserved; + uint8_t polarActivity[MAX_POLAR_ACTIVITY_INDICATORS]; + uint32_t session_time_lsw; + uint32_t session_time_msw; +} __packed; struct sound_focus_param { uint16_t start_angle[MAX_SECTORS]; @@ -13543,6 +13562,21 @@ struct doa_tracking_mon_param { uint8_t polar_activity[MAX_POLAR_ACTIVITY_INDICATORS]; } __packed; +struct adm_param_fluence_nn_sound_focus_t { + int16_t mode; + int16_t focus_direction[2]; + int16_t focus_width; +} __packed; + +struct adm_param_fluence_nn_source_tracking_t { + int32_t speech_probablity_q20; + int16_t speakers[5]; + int16_t reserved; + uint8_t polarActivity[360]; + uint32_t session_time_lsw; + uint32_t session_time_msw; +} __packed; + struct adm_param_fluence_soundfocus_t { uint16_t start_angles[MAX_SECTORS]; uint8_t enables[MAX_SECTORS]; diff --git a/include/dsp/q6adm-v2.h b/include/dsp/q6adm-v2.h index a010b56cb5ac..d96318ee1ccc 100644 --- a/include/dsp/q6adm-v2.h +++ b/include/dsp/q6adm-v2.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef __Q6_ADM_V2_H__ #define __Q6_ADM_V2_H__ @@ -229,6 +230,8 @@ int adm_get_sound_focus(int port_id, int copp_idx, struct sound_focus_param *soundFocusData); int adm_get_source_tracking(int port_id, int copp_idx, struct source_tracking_param *sourceTrackingData); +int adm_get_fnn_source_tracking(int port_id, int copp_idx, + struct fluence_nn_source_tracking_param *FnnSourceTrackingData); int adm_get_doa_tracking_mon(int port_id, int copp_idx, struct doa_tracking_mon_param *doa_tracking_data); int adm_set_custom_chmix_cfg(int port_id, int copp_idx, diff --git a/include/dsp/q6voice.h b/include/dsp/q6voice.h index 9ada57a5ad3e..0994cc0069bb 100644 --- a/include/dsp/q6voice.h +++ b/include/dsp/q6voice.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2012-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef __QDSP6VOICE_H__ #define __QDSP6VOICE_H__ @@ -1839,6 +1840,7 @@ struct share_memory_info { #define VSS_ISOUNDFOCUS_CMD_GET_SECTORS 0x00013134 #define VSS_ISOUNDFOCUS_RSP_GET_SECTORS 0x00013135 #define VSS_ISOURCETRACK_CMD_GET_ACTIVITY 0x00013136 +#define VSS_IFNN_ST_CMD_GET_ACTIVITY 0x00013137 struct vss_isoundfocus_cmd_set_sectors_t { uint16_t start_angles[8]; @@ -1880,6 +1882,16 @@ struct vss_isourcetrack_activity_data_t { uint8_t sound_strength[360]; } __packed; +/* Payload structure for the VSS_IFNN_ST_CMD_GET_ACTIVITY command */ +struct vss_ifnn_st_cmd_get_activity_t { + int32_t speech_probablity_q20; + int16_t speakers[MAX_TOP_SPEAKERS]; + int16_t reserved; + uint8_t polarActivity[MAX_POLAR_ACTIVITY_INDICATORS]; + uint32_t session_time_lsw; + uint32_t session_time_msw; +} __packed; + struct shared_mem_info { uint32_t mem_handle; struct mem_map_table sh_mem_block; @@ -1995,9 +2007,11 @@ struct common_data { bool is_per_vocoder_cal_enabled; bool is_sound_focus_resp_success; bool is_source_tracking_resp_success; + bool is_fnn_source_tracking_resp_success; struct vss_isoundfocus_rsp_get_sectors_t soundFocusResponse; struct shared_mem_info source_tracking_sh_mem; struct vss_isourcetrack_activity_data_t sourceTrackingResponse; + struct vss_ifnn_st_cmd_get_activity_t FnnSourceTrackingResponse; bool sidetone_enable; bool mic_break_enable; struct audio_uevent_data *uevent_data; @@ -2144,6 +2158,7 @@ int voice_set_topology_specific_info(struct voice_data *v, int voc_set_sound_focus(struct sound_focus_param sound_focus_param); int voc_get_sound_focus(struct sound_focus_param *soundFocusData); int voc_get_source_tracking(struct source_tracking_param *sourceTrackingData); +int voc_get_fnn_source_tracking(struct fluence_nn_source_tracking_param *FnnSourceTrackingData); int voc_set_afe_sidetone(uint32_t session_id, bool sidetone_enable); bool voc_get_afe_sidetone(void); #endif