[feat][ai聊天][语音识别]

index.html

@@ -210,6 +210,13 @@
     </div>
   </div>
   <script type="module" src="/src/main.js"></script>
+<!--  <script src="src/utils/ai/iat/dist/index.cjs.js"></script>
+  <script src="src/utils/ai/iat/dist/index.d.ts"></script>
+  <script src="src/utils/ai/iat/dist/index.esm.js"></script>-->
+
+<!--  <script src="src/utils/ai/iat/dist/processor.worker.js"></script>-->
+<!--  <script src="src/utils/ai/iat/dist/processor.worklet.js"></script>-->
+
 </body>
 
 </html>

src/main.js

@@ -16,6 +16,7 @@ import directive from './directive' // directive
 // 注册指令
 import plugins from './plugins' // plugins
 import { download } from '@/utils/request'
+// import * as RecorderManager from '@/utils/ai/iat/dist/index.d.ts'
 
 // svg图标
 import 'virtual:svg-icons-register'

src/utils/ai/iat/dist/index.d.ts

@@ -1,30 +0,0 @@
-declare class RecorderManager {
-    /**
-     * 构造函数
-     * @param processorPath processor的文件路径，如果processor.worker.js的访问地址为`/a/b/processor.worker.js`,则processorPath 为`/a/b`
-     *
-     */
-    constructor(processorPath: string);
-    private audioBuffers;
-    private processorPath;
-    private audioContext?;
-    private audioTracks?;
-    private audioWorklet?;
-    onStop?: (audioBuffers: ArrayBuffer[]) => void;
-    onFrameRecorded?: (params: {
-        isLastFrame: boolean;
-        frameBuffer: ArrayBuffer;
-    }) => void;
-    /**
-     * 监听录音开始事件
-     */
-    onStart?: () => void;
-    start({ sampleRate, frameSize, arrayBufferType, }: {
-        sampleRate?: number;
-        frameSize?: number;
-        arrayBufferType?: "short16" | "float32";
-    }): Promise<void>;
-    stop(): void;
-}
-
-export { RecorderManager as default };

src/views/xjc-integratedmachine/common/ai/chat.vue

@@ -59,7 +59,7 @@
                     <el-button style="width:140px">退出</el-button>
                 </div>
                 <div class="control-button">
-                    <el-button style="width:140px">静音</el-button>
+                    <el-button style="width:140px" @click="startVoice">静音</el-button>
                 </div>
                 <div class="control-button">
                     <el-button style="width:140px">新会话</el-button>
@@ -73,9 +73,14 @@
 </template>
 
 <script setup>
+    import {nextTick, onMounted} from 'vue'
     import {aiChatRecordList} from '@/api/xjc-integratedmachine/common/aiChat.js'
     import { getToken } from '@/utils/auth'
+    // md转换为html
     import { marked } from 'marked'
+    // 语音识别
+    import CryptoJS from 'crypto-js';
+    import * as RecorderManager from "/public/ai/iat/dist/index.umd.js"
 
     // 聊天记录
     let chatRecordList = ref([])
@@ -199,8 +204,269 @@
         }
     }
 
+    /* 语音识别 ↓*/
+    // 控制录音弹窗
+    let voiceOpen = false
+    // 是否开始录音
+    let startVoiceStatus = false
+    // 识别中状态
+    let identifyStatus = false
+    let recorder = null
+    let transcription = ''
+    let btnStatus = ''
+    let resultText = ''
+    let resultTextTemp = ''
+    let countdownInterval = null
+    let iatWS = null
+    let recognition = null
+    let buttonDisabled = true
+    let loading = false
+    let xfIatKeys = {
+        APPID: '5a2643f4',
+        APISecret: 'MTg4MWIzY2VmYTg2YTEwMjliMDY1N2Iz',
+        APIKey: '8b1a53486bec887eb817b4410aa743ed',
+    }
+    // 初始化语音识别
+    function initRecognize(){
+        recorder = new window.RecorderManager('/ai/iat/dist');
+        recorder.onStart = () => {
+            changeBtnStatus('OPEN');
+        };
+        recorder.onFrameRecorded = ({ isLastFrame, frameBuffer }) => {
+            if (iatWS.readyState === iatWS.OPEN) {
+                iatWS.send(
+                    JSON.stringify({
+                        data: {
+                            status: isLastFrame ? 2 : 1,
+                            format: 'audio/L16;rate=16000',
+                            encoding: 'raw',
+                            audio: toBase64(frameBuffer),
+                        },
+                    }),
+                );
+                if (isLastFrame) {
+                    changeBtnStatus('CLOSING');
+                }
+            }
+        };
+        recorder.onStop = () => {
+            console.log('录音结束，停止定时器');
+            clearInterval(countdownInterval);
+            startVoiceStatus = false;
+        };
+
+    }
+
+    async function startVoice() {
+        if (loading) {
+            return;
+        }
+        voiceOpen = true;
+        await playIatVoice();
+    }
+
+    async function playIatVoice() {
+        if (loading) {
+            return;
+        }
+        startVoiceStatus = !startVoiceStatus;
+        // 浏览器自带的识别
+        if (recognition) {
+            if (startVoiceStatus) {
+                recognition.start();
+            } else {
+                recognition.stop();
+            }
+            return;
+        }
+        if (startVoiceStatus) {
+            connectWebSocket();
+        } else {
+            recorder.stop();
+        }
+    }
+
+    /**
+     * 关闭录音弹窗
+     */
+    function closeVoiceOpen() {
+        voiceOpen = false;
+        startVoiceStatus = false;
+        if (recorder) {
+            recorder.stop();
+        }
+        if (recognition) {
+            recognition.stop();
+        }
+        transcription = '';
+    }
+
+    function renderResult(resultData) {
+        // 识别结束
+        const jsonData = JSON.parse(resultData);
+        if (jsonData.data && jsonData.data.result) {
+            const data = jsonData.data.result;
+            let str = '';
+            const { ws } = data;
+            for (let i = 0; i < ws.length; i += 1) {
+                str += ws[i].cw[0].w;
+            }
+            // 开启wpgs会有此字段(前提：在控制台开通动态修正功能)
+            // 取值为 "apd"时表示该片结果是追加到前面的最终结果；取值为"rpl" 时表示替换前面的部分结果，替换范围为rg字段
+            if (data.pgs) {
+                if (data.pgs === 'apd') {
+                    // 将resultTextTemp同步给resultText
+                    resultText = resultTextTemp;
+                }
+                // 将结果存储在resultTextTemp中
+                resultTextTemp = resultText + str;
+            } else {
+                resultText += str;
+            }
+            transcription = resultTextTemp || resultText || '';
+            console.log("识别："+transcription);
+        }
+        if (jsonData.code === 0 && jsonData.data.status === 2) {
+            iatWS.close();
+        }
+        if (jsonData.code !== 0) {
+            iatWS.close();
+            console.error(jsonData);
+        }
+    }
+
+    function connectWebSocket() {
+        const websocketUrl = getWebSocketUrl();
+        if ('WebSocket' in window) {
+            iatWS = new window.WebSocket(websocketUrl);
+        } else if ('MozWebSocket' in window) {
+            iatWS = new window.MozWebSocket(websocketUrl);
+        } else {
+            message.error('浏览器不支持WebSocket');
+            return;
+        }
+        changeBtnStatus('CONNECTING');
+        iatWS.onopen = e => {
+            console.log('iatWS.onopen', e);
+            // 开始录音
+            recorder.start({
+                sampleRate: 16000,
+                frameSize: 1280,
+            });
+            const params = {
+                common: {
+                    app_id: xfIatKeys.APPID,
+                },
+                business: {
+                    language: 'zh_cn',
+                    domain: 'iat',
+                    accent: 'mandarin',
+                    vad_eos: 5000,
+                    dwa: 'wpgs',
+                    nbest: 1,
+                    wbest: 1,
+                },
+                data: {
+                    status: 0,
+                    format: 'audio/L16;rate=16000',
+                    encoding: 'raw',
+                },
+            };
+            iatWS.send(JSON.stringify(params));
+        };
+        iatWS.onmessage = e => {
+            renderResult(e.data);
+        };
+        iatWS.onerror = e => {
+            console.error(e);
+            recorder.stop();
+            changeBtnStatus('CLOSED');
+        };
+        iatWS.onclose = e => {
+            console.log(e);
+            recorder.stop();
+            changeBtnStatus('CLOSED');
+        };
+    }
+
+    function getWebSocketUrl() {
+        const { APIKey, APISecret } = xfIatKeys;
+        if (!APIKey) {
+            message.error('语音识别配置未生效');
+            return null;
+        }
+        // 请求地址根据语种不同变化
+        let url = 'wss://iat-api.xfyun.cn/v2/iat';
+        const host = 'iat-api.xfyun.cn';
+        const apiKey = APIKey;
+        const apiSecret = APISecret;
+        const date = new Date().toGMTString();
+        const algorithm = 'hmac-sha256';
+        const headers = 'host date request-line';
+        const signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v2/iat HTTP/1.1`;
+        const signatureSha = CryptoJS.HmacSHA256(signatureOrigin, apiSecret);
+        const signature = CryptoJS.enc.Base64.stringify(signatureSha);
+        const authorizationOrigin = `api_key="${apiKey}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;
+        const authorization = btoa(authorizationOrigin);
+        url = `${url}?authorization=${authorization}&date=${date}&host=${host}`;
+        return url;
+    }
+
+    function countdown() {
+        let seconds = 10;
+        console.log(`录音中（${seconds}s）`);
+        countdownInterval = setInterval(() => {
+            seconds -= 1;
+            if (seconds <= 0) {
+                clearInterval(countdownInterval);
+                recorder.stop();
+            } else {
+                console.log(`录音中（${seconds}s）`);
+            }
+        }, 1000);
+    }
+
+    function changeBtnStatus(status) {
+        btnStatus = status;
+        if (status === 'CONNECTING') {
+            console.log('建立连接中');
+            resultText = '';
+            resultTextTemp = '';
+        } else if (status === 'OPEN') {
+            if (recorder) {
+                countdown();
+            }
+        } else if (status === 'CLOSING') {
+            console.log('关闭连接中');
+        } else if (status === 'CLOSED') {
+            console.log('开始录音');
+        }
+    }
+
+    function toBase64(buffer) {
+        let binary = '';
+        const bytes = new Uint8Array(buffer);
+        const len = bytes.byteLength;
+        for (let i = 0; i < len; i += 1) {
+            binary += String.fromCharCode(bytes[i]);
+        }
+        return window.btoa(binary);
+    }
+
+    // 发送消息
+    function sendMsg(text) {
+        // $emit('sendMsg', text);
+        // transcription = '';
+    }
+    /* 语音识别 ↑*/
+
+
     onMounted(()=>{
-        sendMessage()
+        nextTick(()=>{
+            initRecognize();
+            sendMessage()
+        })
+
     })
 
 </script>