diff --git a/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json b/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json deleted file mode 100644 index f860ad4c..00000000 --- a/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "enabled": false, - "mode": "TEXT", - "serverBaseUrl": "http://192.168.2.28:24605", - - "recordSeconds": 5, - "useDumpStateAudio": true, - - "useAsrServiceStt": true, - "asrServiceHost": "127.0.0.1", - "asrServicePort": 8088, - "asrAudioSourceId": "alsa1", - "asrTimeoutMs": 15000, - "asrServiceDebugWs": false, - "asrAutoStart": true, - - "wakeupChitchatPhrases": [ - "hello", - "howdy", - "hi", - "hey", - "look what i found", - "nice to see you", - "good morning", - "good afternoon", - "good evening" - ], - - "followupEnabled": true, - "followupDelayMs": 250 -} diff --git a/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js b/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js deleted file mode 100644 index 12b9cf26..00000000 --- a/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge.js +++ /dev/null @@ -1,2716 +0,0 @@ -"use strict"; - -// Modular AI bridge for Jibo BE. -// - TEXT mode: uses jetstream turn results (ASR text) and forwards to a server. -// - AUDIO mode: records a short WAV clip (arecord) and forwards to a server. -// Server is expected to return JSON: { reply: "..." }. - -var http = require("http"); -var https = require("https"); -var urlLib = require("url"); -var fs = require("fs"); -var net = require("net"); -var crypto = null; -try { - crypto = require("crypto"); -} catch (e) { - crypto = null; -} -var childProcess = require("child_process"); -var spawn = childProcess.spawn; - -var rlog = null; -try { - rlog = require("./robot-logger"); -} catch (e) { - // ignore -} - -var tunable = null; -try { - // Optional: drives the :3333 panel - tunable = require("jibo-tunable"); -} catch (e) { - // ignore -} - -var beFramework = null; -try { - beFramework = require("@be/be-framework"); -} catch (e) { - // ignore -} - -// Many builds already have Tunables under the "Jibo Embodied Dialog" window. -// Keeping AI bridge controls there makes them easier to find. -var WIN = "Jibo Embodied Dialog"; -var LABEL = { - enabled: "AI Bridge: Enabled", - input: "AI Bridge: Input", - audioCheckbox: "AI Bridge: Audio input (off = text)", - serverUrl: "AI Bridge: Server URL", - recordSeconds: "AI Bridge: Record seconds", - alsaDevice: "AI Bridge: ALSA device (arecord -D)", - audioSampleRate: "AI Bridge: Audio sample rate", - audioChannels: "AI Bridge: Audio channels", - audioFormat: "AI Bridge: Audio format (arecord -f)", - debugAudio: "AI Bridge: Debug audio capture", - useDumpStateAudio: "AI Bridge: Use audio-service dump-state (fix mic busy)", - useAsrServiceStt: "AI Bridge: Use ASR-service STT (8088)", - asrServiceDebugWs: "AI Bridge: Debug ASR-service WS", -}; - -var InputMode = { - TEXT: "TEXT", - AUDIO: "AUDIO", -}; - -var WsLib = null; -try { - WsLib = require("ws"); -} catch (e) { - WsLib = null; -} - -function escapeRegExp(s) { - return String(s || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -function stripKnownEdgePhrases(text, phrases) { - // Remove known "Jibo wake chitchat" phrases from the start/end. - // This helps avoid TTS bleed-through becoming the user's prompt. - try { - var t = String(text || ""); - if (!t) return ""; - phrases = phrases && phrases.length ? phrases : []; - for (var pass = 0; pass < 2; pass += 1) { - for (var i = 0; i < phrases.length; i += 1) { - var p = String(phrases[i] || "").trim(); - if (!p) continue; - var startRe = new RegExp("^\\s*" + escapeRegExp(p) + "[\\s,!.?;:]*", "i"); - var endRe = new RegExp("[\\s,!.?;:]*" + escapeRegExp(p) + "\\s*$", "i"); - t = t.replace(startRe, ""); - t = t.replace(endRe, ""); - } - t = t.replace(/^\s+|\s+$/g, ""); - } - return t; - } catch (e) { - return String(text || ""); - } -} - -function isChitchatOnly(text, phrases) { - try { - var t = String(text || "").trim().toLowerCase(); - if (!t) return true; - phrases = phrases && phrases.length ? phrases : []; - for (var i = 0; i < phrases.length; i += 1) { - var p = String(phrases[i] || "").trim().toLowerCase(); - if (!p) continue; - if (t === p) return true; - } - return false; - } catch (e) { - return false; - } -} - -function endsWithQuestion(text) { - try { - var t = String(text || "").replace(/\s+/g, " ").trim(); - if (!t) return false; - // Consider "?" at end (optionally followed by quotes). - return /\?\s*["']?$/.test(t); - } catch (e) { - return false; - } -} - -function httpJsonPost(urlString, payload, timeoutMs) { - timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 15000; - - var parsed = urlLib.parse(urlString); - var isHttps = parsed.protocol === "https:"; - var bodyStr = JSON.stringify(payload || {}); - // Buffer.from is not available on some older Node builds - var body = new Buffer(bodyStr, "utf8"); - - var requestOptions = { - protocol: parsed.protocol, - hostname: parsed.hostname, - port: parsed.port || (isHttps ? 443 : 80), - path: parsed.path || "/", - method: "POST", - headers: { - "Content-Type": "application/json", - "Content-Length": body.length, - }, - timeout: timeoutMs, - }; - - return new Promise(function (resolve, reject) { - var req = (isHttps ? https : http).request(requestOptions, function (res) { - var chunks = []; - res.on("data", function (d) { - chunks.push(d); - }); - res.on("end", function () { - var raw = Buffer.concat(chunks).toString("utf8"); - if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { - try { - resolve(JSON.parse(raw)); - } catch (e) { - reject(new Error("AI Bridge: bad JSON response (" + res.statusCode + "): " + raw)); - } - return; - } - reject(new Error("AI Bridge: HTTP " + res.statusCode + ": " + raw)); - }); - }); - - req.on("error", reject); - req.on("timeout", function () { - req.destroy(new Error("AI Bridge: request timeout")); - }); - - req.write(body); - req.end(); - }); -} - -// Streaming NDJSON POST — reads the server response line-by-line and calls -// onSentence(sentenceString) for each complete sentence as it arrives. -// This replicates the original Jibo hub pattern: the robot starts speaking -// while the server is still generating the rest of the response. -// Returns a Promise that resolves to the full reply string. -function httpStreamingPost(urlString, payload, onSentence, timeoutMs) { - timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 120000; - - var parsed = urlLib.parse(urlString); - var isHttps = parsed.protocol === "https:"; - var bodyStr = JSON.stringify(payload || {}); - var body = new Buffer(bodyStr, "utf8"); - - var requestOptions = { - protocol: parsed.protocol, - hostname: parsed.hostname, - port: parsed.port || (isHttps ? 443 : 80), - path: parsed.path || "/", - method: "POST", - headers: { - "Content-Type": "application/json", - "Content-Length": body.length, - }, - timeout: timeoutMs, - }; - - return new Promise(function (resolve, reject) { - var req = (isHttps ? https : http).request(requestOptions, function (res) { - if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) { - var errChunks = []; - res.on("data", function (d) { errChunks.push(d); }); - res.on("end", function () { - reject(new Error("AI Bridge streaming: HTTP " + res.statusCode + ": " + Buffer.concat(errChunks).toString("utf8"))); - }); - return; - } - - var fullReply = ""; - var lineBuf = ""; - - res.on("data", function (chunk) { - lineBuf += chunk.toString("utf8"); - - // Process complete lines (NDJSON = one JSON object per line) - var lines = lineBuf.split("\n"); - // Keep the last (possibly incomplete) line in the buffer - lineBuf = lines.pop() || ""; - - for (var i = 0; i < lines.length; i++) { - var line = lines[i].replace(/^\s+|\s+$/g, ""); - if (!line) continue; - try { - var obj = JSON.parse(line); - } catch (e) { - continue; // skip malformed lines - } - - if (obj.done && obj.reply) { - fullReply = obj.reply; - } - if (obj.sentence && !obj.done) { - try { - onSentence(obj.sentence); - } catch (e) { - // don't let TTS errors kill the stream - } - } - // Handle error/fallback in the done+sentence case (ollama down) - if (obj.done && obj.sentence) { - try { - onSentence(obj.sentence); - } catch (e) { - // ignore - } - if (!fullReply) fullReply = obj.sentence; - } - } - }); - - res.on("end", function () { - // Process any remaining data in the buffer - var remaining = (lineBuf || "").replace(/^\s+|\s+$/g, ""); - if (remaining) { - try { - var obj = JSON.parse(remaining); - if (obj.reply) fullReply = obj.reply; - if (obj.sentence && !obj.done) { - try { onSentence(obj.sentence); } catch (e) { /* ignore */ } - } - if (obj.done && obj.sentence) { - try { onSentence(obj.sentence); } catch (e) { /* ignore */ } - if (!fullReply) fullReply = obj.sentence; - } - } catch (e) { - // ignore - } - } - resolve(fullReply || ""); - }); - }); - - req.on("error", reject); - req.on("timeout", function () { - req.destroy(new Error("AI Bridge: streaming request timeout")); - }); - - req.write(body); - req.end(); - }); -} - -function httpJsonPostRaw(urlString, payload, timeoutMs) { - timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 6000; - - var parsed = urlLib.parse(urlString); - var isHttps = parsed.protocol === "https:"; - var bodyStr = JSON.stringify(payload || {}); - var body = new Buffer(bodyStr, "utf8"); - var requestOptions = { - protocol: parsed.protocol, - hostname: parsed.hostname, - port: parsed.port || (isHttps ? 443 : 80), - path: parsed.path || "/", - method: "POST", - headers: { - "Content-Type": "application/json", - "Content-Length": body.length, - }, - timeout: timeoutMs, - }; - - return new Promise(function (resolve, reject) { - var req = (isHttps ? https : http).request(requestOptions, function (res) { - var chunks = []; - res.on("data", function (d) { - chunks.push(d); - }); - res.on("end", function () { - var raw = Buffer.concat(chunks).toString("utf8"); - resolve({ statusCode: res.statusCode, headers: res.headers || {}, body: raw }); - }); - }); - req.on("error", reject); - req.on("timeout", function () { - req.destroy(new Error("POST timeout")); - }); - req.write(body); - req.end(); - }); -} - -function parseWsUrl(wsUrl) { - // Very small ws:// URL parser for embedded Node. - // Returns { host, port, path }. - if (typeof wsUrl !== "string") return null; - var s = wsUrl.trim(); - var m = s.match(/^wss?:\/\/([^\/]+)(\/.*)?$/i); - if (!m) return null; - var hostPort = m[1]; - var path = m[2] || "/"; - var hp = hostPort.split(":"); - var host = hp[0]; - var port = hp.length > 1 ? Number(hp[1]) : s.toLowerCase().indexOf("wss://") === 0 ? 443 : 80; - if (!host) return null; - if (!port || Number.isNaN(port)) port = 80; - return { host: host, port: port, path: path }; -} - -function connectWebSocketText(wsUrl, timeoutMs) { - // Returns a Promise resolving to { onMessage(fn), close() }. - // Uses `ws` module if available; otherwise uses a minimal RFC6455 client. - timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 4000; - - if (WsLib) { - return new Promise(function (resolve, reject) { - var ws; - try { - ws = new WsLib(wsUrl); - } catch (e) { - reject(e); - return; - } - var timer = setTimeout(function () { - try { - ws.terminate && ws.terminate(); - } catch (e) { - // ignore - } - reject(new Error("ws connect timeout")); - }, timeoutMs); - ws.on("open", function () { - clearTimeout(timer); - resolve({ - onMessage: function (fn) { - ws.on("message", function (msg) { - fn(typeof msg === "string" ? msg : msg && msg.toString ? msg.toString("utf8") : String(msg)); - }); - }, - close: function () { - try { - ws.close(); - } catch (e) { - // ignore - } - }, - }); - }); - ws.on("error", function (e) { - clearTimeout(timer); - reject(e); - }); - }); - } - - // Minimal client without external deps. - return new Promise(function (resolve, reject) { - var findHeaderEnd = function (buf) { - // Find \r\n\r\n in a Buffer. - if (!buf || !buf.length) return -1; - for (var i = 0; i + 3 < buf.length; i += 1) { - if (buf[i] === 13 && buf[i + 1] === 10 && buf[i + 2] === 13 && buf[i + 3] === 10) return i; - } - return -1; - }; - - var info = parseWsUrl(wsUrl); - if (!info) { - reject(new Error("bad ws url")); - return; - } - var key = null; - try { - key = crypto && crypto.randomBytes ? crypto.randomBytes(16).toString("base64") : new Buffer(String(Date.now()) + ":" + String(Math.random())).toString("base64"); - } catch (e) { - key = new Buffer(String(Date.now()) + ":" + String(Math.random())).toString("base64"); - } - - var socket = net.connect({ host: info.host, port: info.port }); - var timer = setTimeout(function () { - try { - socket.destroy(); - } catch (e) { - // ignore - } - reject(new Error("ws connect timeout")); - }, timeoutMs); - - var state = { handshook: false, buf: new Buffer(0) }; - var msgHandlers = []; - var closed = false; - var fragType = null; // 0x1 text, 0x2 binary - var fragChunks = []; - - var emitMessage = function (s) { - for (var i = 0; i < msgHandlers.length; i += 1) { - try { - msgHandlers[i](s); - } catch (e) { - // ignore - } - } - }; - - var sendPong = function (payload) { - try { - // server->client frames are unmasked; client->server must be masked. - var mask = crypto && crypto.randomBytes ? crypto.randomBytes(4) : new Buffer([1, 2, 3, 4]); - var len = payload ? payload.length : 0; - var header = null; - if (len < 126) { - header = new Buffer(2); - header[0] = 0x8a; // FIN + pong - header[1] = 0x80 | len; - } else { - header = new Buffer(4); - header[0] = 0x8a; - header[1] = 0x80 | 126; - header.writeUInt16BE(len, 2); - } - var out = new Buffer(header.length + 4 + len); - header.copy(out, 0); - mask.copy(out, header.length); - for (var i = 0; i < len; i += 1) { - out[header.length + 4 + i] = payload[i] ^ mask[i % 4]; - } - socket.write(out); - } catch (e) { - // ignore - } - }; - - var parseFrames = function () { - // parse server->client frames (unmasked) - while (state.buf.length >= 2) { - var b0 = state.buf[0]; - var b1 = state.buf[1]; - var fin = (b0 & 0x80) !== 0; - var opcode = b0 & 0x0f; - var masked = (b1 & 0x80) !== 0; - var len = b1 & 0x7f; - var off = 2; - if (len === 126) { - if (state.buf.length < off + 2) return; - len = state.buf.readUInt16BE(off); - off += 2; - } else if (len === 127) { - // Not expected; avoid overflow. - if (state.buf.length < off + 8) return; - // Only support up to 2^31-1 - var hi = state.buf.readUInt32BE(off); - var lo = state.buf.readUInt32BE(off + 4); - off += 8; - if (hi !== 0) { - // too large - try { - socket.destroy(); - } catch (e) { - // ignore - } - return; - } - len = lo; - } - var maskKey = null; - if (masked) { - if (state.buf.length < off + 4) return; - maskKey = state.buf.slice(off, off + 4); - off += 4; - } - if (state.buf.length < off + len) return; - var payload = state.buf.slice(off, off + len); - state.buf = state.buf.slice(off + len); - - if (masked && maskKey) { - for (var i = 0; i < payload.length; i += 1) { - payload[i] = payload[i] ^ maskKey[i % 4]; - } - } - - if (opcode === 0x8) { - try { - socket.end(); - } catch (e) { - // ignore - } - return; - } - if (opcode === 0x9) { - // ping - sendPong(payload); - continue; - } - // Continuation frames for fragmented messages. - if (opcode === 0x0) { - if (!fragType) { - continue; - } - fragChunks.push(payload); - if (fin) { - var whole = Buffer.concat(fragChunks); - fragType = null; - fragChunks = []; - try { - emitMessage(whole.toString("utf8")); - } catch (e) { - // ignore - } - } - continue; - } - - // Text or binary payloads (ASR may send either). - if (opcode === 0x1 || opcode === 0x2) { - if (!fin) { - fragType = opcode; - fragChunks = [payload]; - continue; - } - try { - emitMessage(payload.toString("utf8")); - } catch (e) { - // ignore - } - continue; - } - } - }; - - socket.on("connect", function () { - var req = ""; - req += "GET " + info.path + " HTTP/1.1\r\n"; - req += "Host: " + info.host + ":" + String(info.port) + "\r\n"; - req += "Upgrade: websocket\r\n"; - req += "Connection: Upgrade\r\n"; - req += "Sec-WebSocket-Key: " + key + "\r\n"; - req += "Sec-WebSocket-Version: 13\r\n"; - req += "\r\n"; - socket.write(req); - }); - - socket.on("data", function (d) { - if (closed) return; - state.buf = Buffer.concat([state.buf, d]); - if (!state.handshook) { - var endIdx = findHeaderEnd(state.buf); - if (endIdx < 0) return; - var headerText = state.buf.slice(0, endIdx).toString("utf8"); - state.buf = state.buf.slice(endIdx + 4); - if (!/^HTTP\/1\.1 101/i.test(headerText)) { - closed = true; - clearTimeout(timer); - reject(new Error("ws handshake failed: " + headerText.split("\r\n")[0])); - try { - socket.destroy(); - } catch (e) { - // ignore - } - return; - } - state.handshook = true; - clearTimeout(timer); - resolve({ - onMessage: function (fn) { - msgHandlers.push(fn); - }, - close: function () { - if (closed) return; - closed = true; - try { - socket.end(); - } catch (e) { - // ignore - } - }, - }); - } - // Parse any remaining frames. - if (state.handshook) parseFrames(); - }); - socket.on("error", function (e) { - if (closed) return; - closed = true; - clearTimeout(timer); - reject(e); - }); - socket.on("close", function () { - if (closed) return; - closed = true; - clearTimeout(timer); - }); - }); -} - -function pickBestAsrUtterance(utterances) { - // utterances: [{utterance, score}] possibly. - try { - if (!utterances || !utterances.length) return ""; - var bestText = ""; - var bestScore = -1e99; - for (var i = 0; i < utterances.length; i += 1) { - var u = utterances[i]; - var text = u && (u.utterance || u.Utterance || u.text) ? String(u.utterance || u.Utterance || u.text) : ""; - text = text.replace(/^\s+|\s+$/g, ""); - if (!text) continue; - // Light cleanup: sometimes a leading duplicated char occurs. - if (text.length >= 2 && text[0].toLowerCase && text[1].toLowerCase && text[0].toLowerCase() === text[1].toLowerCase()) { - text = text.slice(1); - } - var score = typeof u.score === "number" ? u.score : typeof u.Score === "number" ? u.Score : 0; - if (!bestText) { - bestText = text; - bestScore = score; - continue; - } - if (score > bestScore) { - bestText = text; - bestScore = score; - } - } - // Fallback: choose the shortest distinct utterance. - if (!bestText) { - var uniq = {}; - var arr = []; - for (var j = 0; j < utterances.length; j += 1) { - var u2 = utterances[j]; - var t2 = u2 && (u2.utterance || u2.Utterance || u2.text) ? String(u2.utterance || u2.Utterance || u2.text) : ""; - t2 = t2.replace(/^\s+|\s+$/g, ""); - if (!t2) continue; - var k = t2.toLowerCase ? t2.toLowerCase() : t2; - if (uniq[k]) continue; - uniq[k] = true; - arr.push(t2); - } - arr.sort(function (a, b) { - return a.length - b.length; - }); - bestText = arr[0] || ""; - } - return bestText; - } catch (e) { - return ""; - } -} - -function recordWavWithArecord(durationSec, sampleRate, channels, device, format, debug) { - durationSec = typeof durationSec === "number" ? durationSec : 5; - sampleRate = typeof sampleRate === "number" ? sampleRate : 16000; - channels = typeof channels === "number" ? channels : 1; - device = typeof device === "string" ? device : ""; - format = typeof format === "string" ? format : "S16_LE"; - debug = !!debug; - if (["S16_LE", "S32_LE"].indexOf(format) < 0) { - format = "S16_LE"; - } - // Uses `arecord` to capture a short wav to stdout. - // Args chosen to be broadly compatible with embedded ALSA. - var args = [ - // Quiet unless debugging; arecord prints useful device/rate info on stderr. - // (When not quiet, we still only log stderr when debug=true.) - // "-q", - "-f", - format, - "-r", - String(sampleRate), - "-c", - String(channels), - "-d", - String(durationSec), - "-t", - "wav", - ]; - - if (!debug) { - args.unshift("-q"); - } - if (device && device.trim()) { - args.unshift(device.trim()); - args.unshift("-D"); - } - - return new Promise(function (resolve, reject) { - var child = spawn("arecord", args, { stdio: ["ignore", "pipe", "pipe"] }); - var stdout = []; - var stderr = []; - - child.stdout.on("data", function (d) { - stdout.push(d); - }); - child.stderr.on("data", function (d) { - stderr.push(d); - }); - child.on("error", function (err) { - reject(err); - }); - child.on("close", function (code) { - if (code !== 0) { - var errText = Buffer.concat(stderr).toString("utf8"); - reject(new Error("arecord failed (" + code + "): " + errText)); - return; - } - if (debug && rlog) { - try { - var s = Buffer.concat(stderr).toString("utf8"); - if (s && s.trim()) { - rlog.debug("ai-bridge", "arecord stderr", { text: s.slice(0, 500) }); - } - } catch (e) { - // ignore - } - } - resolve(Buffer.concat(stdout)); - }); - }); -} - -function wavSignalStats16(wavBuffer, maxSamples) { - // Best-effort parser for PCM 16-bit WAV emitted by arecord. - // Returns { ok, samples, zeroFrac, rms, min, max } or { ok:false, err }. - maxSamples = typeof maxSamples === "number" ? maxSamples : 16000; - try { - if (!wavBuffer || !Buffer.isBuffer(wavBuffer) || wavBuffer.length < 44) { - return { ok: false, err: "no buffer" }; - } - if (wavBuffer.toString("ascii", 0, 4) !== "RIFF" || wavBuffer.toString("ascii", 8, 12) !== "WAVE") { - return { ok: false, err: "not wav" }; - } - var offset = 12; - var dataOffset = -1; - var dataSize = 0; - var bitsPerSample = null; - while (offset + 8 <= wavBuffer.length) { - var chunkId = wavBuffer.toString("ascii", offset, offset + 4); - var chunkSize = wavBuffer.readUInt32LE(offset + 4); - offset += 8; - if (chunkId === "fmt ") { - if (offset + 16 <= wavBuffer.length) { - bitsPerSample = wavBuffer.readUInt16LE(offset + 14); - } - } else if (chunkId === "data") { - dataOffset = offset; - dataSize = chunkSize; - break; - } - offset += chunkSize; - if (chunkSize % 2 === 1) offset += 1; // padding - } - if (dataOffset < 0) return { ok: false, err: "no data chunk" }; - if (bitsPerSample !== 16) return { ok: false, err: "bitsPerSample=" + String(bitsPerSample) }; - - var available = Math.min(dataSize, wavBuffer.length - dataOffset); - var sampleCount = Math.floor(available / 2); - sampleCount = Math.max(0, Math.min(sampleCount, maxSamples)); - if (sampleCount <= 0) return { ok: false, err: "no samples" }; - - var zeros = 0; - var min = 32767; - var max = -32768; - var sumSq = 0; - for (var i = 0; i < sampleCount; i++) { - var s = wavBuffer.readInt16LE(dataOffset + i * 2); - if (s === 0) zeros += 1; - if (s < min) min = s; - if (s > max) max = s; - sumSq += s * s; - } - var rms = Math.sqrt(sumSq / sampleCount); - return { - ok: true, - samples: sampleCount, - zeroFrac: zeros / sampleCount, - rms: rms, - min: min, - max: max, - }; - } catch (e) { - return { ok: false, err: String(e && (e.stack || e.message || e)) }; - } -} - -function parseWavHeader(wavBuffer) { - // Minimal WAV header parser (RIFF/WAVE + fmt/data). Used for logging/metadata only. - try { - if (!wavBuffer || !Buffer.isBuffer(wavBuffer) || wavBuffer.length < 44) return { ok: false, err: "short" }; - if (wavBuffer.toString("ascii", 0, 4) !== "RIFF" || wavBuffer.toString("ascii", 8, 12) !== "WAVE") return { ok: false, err: "not wav" }; - - var offset = 12; - var channels = null; - var sampleRate = null; - var bitsPerSample = null; - var dataBytes = null; - while (offset + 8 <= wavBuffer.length) { - var chunkId = wavBuffer.toString("ascii", offset, offset + 4); - var chunkSize = wavBuffer.readUInt32LE(offset + 4); - offset += 8; - if (chunkId === "fmt ") { - if (offset + 16 <= wavBuffer.length) { - channels = wavBuffer.readUInt16LE(offset + 2); - sampleRate = wavBuffer.readUInt32LE(offset + 4); - bitsPerSample = wavBuffer.readUInt16LE(offset + 14); - } - } else if (chunkId === "data") { - dataBytes = chunkSize; - break; - } - offset += chunkSize; - if (chunkSize % 2 === 1) offset += 1; - } - - return { ok: true, channels: channels, sampleRate: sampleRate, bitsPerSample: bitsPerSample, dataBytes: dataBytes }; - } catch (e) { - return { ok: false, err: String(e && (e.stack || e.message || e)) }; - } -} - -function runAudioDumpState(cwd, timeoutMs) { - // Runs /usr/local/bin/jibo-audio-dump-state (on-robot) which emits sin.wav/sout.wav/etc. - cwd = typeof cwd === "string" && cwd ? cwd : "/tmp"; - timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 7000; - - var trimDir = function (d) { - return String(d || "/tmp").replace(/\/+$/, "") || "/tmp"; - }; - var fileExistsNonEmpty = function (p) { - try { - var st = fs.statSync(p); - return !!(st && st.isFile && st.isFile() && st.size > 0); - } catch (e) { - return false; - } - }; - // Audio service appears to write its state log as /audiolog.bin - // (see strings in /usr/local/bin/jibo-audio-service). - var dir = trimDir(cwd); - // Some builds appear to write the log at an absolute path: /audiolog.bin. - var candidates = [dir + "/audiolog.bin", "/tmp/audiolog.bin", "/opt/tmp/audiolog.bin", "/audiolog.bin"]; - var infile = null; - for (var i = 0; i < candidates.length; i += 1) { - if (fileExistsNonEmpty(candidates[i])) { - infile = candidates[i]; - break; - } - } - var args = []; - if (infile) { - args = ["--infile", infile]; - } - - return new Promise(function (resolve, reject) { - var child; - try { - child = spawn("/usr/local/bin/jibo-audio-dump-state", args, { stdio: ["ignore", "pipe", "pipe"], cwd: cwd }); - } catch (e) { - reject(e); - return; - } - var out = []; - var err = []; - var killed = false; - var timer = setTimeout(function () { - killed = true; - try { - child.kill("SIGKILL"); - } catch (e) { - // ignore - } - }, timeoutMs); - - child.stdout.on("data", function (d) { - out.push(d); - }); - child.stderr.on("data", function (d) { - err.push(d); - }); - child.on("error", function (e) { - clearTimeout(timer); - reject(e); - }); - child.on("close", function (code) { - clearTimeout(timer); - resolve({ - code: code, - timedOut: killed, - infile: infile, - stdout: Buffer.concat(out).toString("utf8"), - stderr: Buffer.concat(err).toString("utf8"), - }); - }); - }); -} - -function getAudioLogInfo(dir) { - dir = typeof dir === "string" && dir ? dir : "/tmp"; - var base = dir.replace(/\/+$/, ""); - var candidates = [base + "/audiolog.bin", "/tmp/audiolog.bin", "/opt/tmp/audiolog.bin", "/audiolog.bin"]; - for (var i = 0; i < candidates.length; i += 1) { - var p = candidates[i]; - try { - var st = fs.statSync(p); - if (st && st.isFile && st.isFile() && st.size > 0) { - return { path: p, exists: true, bytes: st.size, mtimeMs: st.mtime ? st.mtime.getTime() : null }; - } - } catch (e) { - // ignore - } - } - return { path: candidates[0], exists: false, candidates: candidates }; -} - -function httpTextGet(urlString, timeoutMs) { - timeoutMs = typeof timeoutMs === "number" ? timeoutMs : 3000; - var parsed = urlLib.parse(urlString); - var isHttps = parsed.protocol === "https:"; - var requestOptions = { - protocol: parsed.protocol, - hostname: parsed.hostname, - port: parsed.port || (isHttps ? 443 : 80), - path: parsed.path || "/", - method: "GET", - headers: { Accept: "application/json,text/plain,*/*" }, - timeout: timeoutMs, - }; - return new Promise(function (resolve, reject) { - var req = (isHttps ? https : http).request(requestOptions, function (res) { - var chunks = []; - res.on("data", function (d) { - chunks.push(d); - }); - res.on("end", function () { - var raw = Buffer.concat(chunks).toString("utf8"); - resolve({ statusCode: res.statusCode, headers: res.headers || {}, body: raw }); - }); - }); - req.on("error", reject); - req.on("timeout", function () { - req.destroy(new Error("GET timeout")); - }); - req.end(); - }); -} - -function listTmpAudioFiles(dir) { - dir = typeof dir === "string" && dir ? dir : "/tmp"; - try { - var names = fs.readdirSync(dir); - var out = []; - names.forEach(function (n) { - if (!/\.(raw|wav)$/i.test(n)) return; - try { - var p = dir.replace(/\/+$/, "") + "/" + n; - var st = fs.statSync(p); - if (!st || !st.isFile || !st.isFile()) return; - out.push({ name: n, bytes: st.size, mtimeMs: st.mtime ? st.mtime.getTime() : null }); - } catch (e) { - // ignore - } - }); - out.sort(function (a, b) { - return (b.mtimeMs || 0) - (a.mtimeMs || 0); - }); - return out; - } catch (e) { - return []; - } -} - -function pickBestDumpWav(dir) { - // Prefer multi-mic capture if present. - dir = typeof dir === "string" && dir ? dir : "/tmp"; - var candidates = ["sin.wav", "sout.wav", "ref.wav", "rin.wav", "rout.wav"]; - for (var i = 0; i < candidates.length; i += 1) { - var p = dir.replace(/\/+$/, "") + "/" + candidates[i]; - try { - var st = fs.statSync(p); - if (st && st.isFile && st.isFile() && st.size > 64) { - return { path: p, wav: fs.readFileSync(p) }; - } - } catch (e) { - // ignore - } - } - return null; -} - -function extractAsrText(turnResult) { - // Different builds/components surface listen results differently. - // Prefer the "real" ASR transcript, but accept multiple shapes. - try { - if (!turnResult) return null; - // Already a string - if (typeof turnResult === "string") return turnResult; - // jetstream global/local turn result (common) - if (turnResult.result && turnResult.result.asr && typeof turnResult.result.asr.text === "string") { - return turnResult.result.asr.text; - } - // Sometimes the payload is already the listen result - if (turnResult.asr && typeof turnResult.asr.text === "string") { - return turnResult.asr.text; - } - // Some command-library style payloads use Speech - if (typeof turnResult.Speech === "string") { - return turnResult.Speech; - } - // Some internal events embed options - if (turnResult.options && turnResult.options.asr && typeof turnResult.options.asr.text === "string") { - return turnResult.options.asr.text; - } - // Some events wrap details under `message` - if (turnResult.message) { - var msg = turnResult.message; - // JSON string? - if (typeof msg === "string") { - var trimmed = msg.trim(); - if (trimmed && (trimmed[0] === "{" || trimmed[0] === "[")) { - try { - var parsed = JSON.parse(trimmed); - var nested = extractAsrText(parsed); - if (nested) return nested; - } catch (e) { - // ignore parse errors - } - } - // Sometimes the message is literally the transcript - if (trimmed) return trimmed; - } - if (typeof msg === "object") { - var nested2 = extractAsrText(msg); - if (nested2) return nested2; - } - } - // Defensive last resort: asr.text - if (turnResult.asr && typeof turnResult.asr === "object" && typeof turnResult.asr.text === "string") { - return turnResult.asr.text; - } - return null; - } catch (e) { - return null; - } -} - -function safePreview(obj, limit) { - limit = typeof limit === "number" ? limit : 400; - try { - var s = JSON.stringify(obj); - if (typeof s !== "string") return null; - if (s.length > limit) return s.slice(0, limit) + "..."; - return s; - } catch (e) { - return null; - } -} - -function normalizeServerBaseUrl(input) { - if (typeof input !== "string") return null; - var s = input.replace(/\s+/g, "").trim(); - if (!s) return null; - - // Common paste mistakes from the tunables UI (fires on each keystroke) - // - http://http://... - // - http://host/:8020 - s = s.replace(/^https?:\/\/https?:\/\//i, function (m) { - // keep the first scheme only - var scheme = m.toLowerCase().indexOf("https://") === 0 ? "https://" : "http://"; - return scheme; - }); - if (!/^https?:\/\//i.test(s)) { - s = "http://" + s; - } - // Fix "/:8020" after host - s = s.replace(/\/:(\d{2,5})(\/|$)/, ":$1$2"); - // Remove any path; we only want scheme://host:port - var m = s.match(/^(https?:\/\/[^\/]+)(\/.*)?$/i); - if (m && m[1]) { - s = m[1]; - } - // Strip trailing slash - s = s.replace(/\/+$/, ""); - - // Validate very loosely: scheme://host[:port] - if (!/^https?:\/\/[^\/:]+(:\d{2,5})?$/i.test(s)) { - return null; - } - - return s; -} - -function isCloudAsrFailureText(text) { - if (typeof text !== "string") return false; - var s = text.trim(); - if (!s) return false; - // Heuristics: avoid sending cloud connectivity errors as "ASR text". - // Observed examples: - // CloudConnection::open (poco exception): Host not found: neo-hub.jibo.com - if (/CloudConnection::open/i.test(s)) return true; - if (/poco exception/i.test(s) && /Host not found/i.test(s)) return true; - if (/neo-hub\.jibo\.com/i.test(s)) return true; - return false; -} - -function isMicBusyError(err) { - var s = String(err || ""); - return /Device or resource busy/i.test(s) || /audio device is busy/i.test(s); -} - -function looksLikeHeyJiboIntent(results) { - try { - if (!results || typeof results !== "object") return false; - var nlu = results.nlu; - if (!nlu || typeof nlu !== "object") return false; - var intent = nlu.intent || nlu.intentName || nlu.topIntent; - if (typeof intent !== "string") return false; - return /heyjibo/i.test(intent); - } catch (e) { - return false; - } -} - - -function AIBridge(skill, jibo) { - this.skill = skill; - this.jibo = jibo; - - this.configPath = "/opt/jibo/Jibo/Skills/@be/be/be/ai-bridge-config.json"; - - this.enabled = true; - // Default to TEXT because the built-in ASR pipeline already owns the mic - // on many builds; AUDIO/arecord often fails with "Device or resource busy". - this.mode = InputMode.TEXT; - this.serverBaseUrl = "http://127.0.0.1:8020"; - this.recordSeconds = 5; - this.arecordDevice = ""; - this.audioSampleRate = 16000; - this.audioChannels = 1; - this.audioFormat = "S16_LE"; - this.debugAudioCapture = false; - // Prefer grabbing audio from the system audio service (avoids ALSA mic lock). - this.useDumpStateAudio = true; - this.dumpStateDir = "/tmp"; - // Offline STT via jibo-asr-service (HTTP + WebSocket) when available. - this.useAsrServiceStt = false; - this.asrServiceHost = "127.0.0.1"; - this.asrServicePort = 8088; - this.asrAudioSourceId = "alsa1"; - this.asrTimeoutMs = 15000; - this.asrServiceDebugWs = false; - this._asrServiceInFlight = false; - this.asrExecutable = "/usr/local/bin/jibo-asr-service"; - this.asrLocalConfigPath = "/opt/jibo/Jibo/Skills/@be/be/be/jibo-asr-service.local.json"; - this.asrSystemConfigPath = "/usr/local/etc/jibo-asr-service.json"; - this.asrAutoStart = true; - this._asrSpawnAttempted = false; - // Filter for common wake-up phrases Jibo says (to avoid self-prompting). - this.wakeupChitchatPhrases = ["hello", "howdy", "hi", "hey", "look what i found", "nice to see you", "good morning", "good afternoon", "good evening"]; - // Follow-up mode: if our AI reply ends with '?', listen once more automatically. - this.followupEnabled = true; - this.followupDelayMs = 250; // after TTS completes - this._followupTurnsLeft = 0; - - this._inFlight = false; - this._lastHJAt = 0; - this._hjWindowMs = 15000; - this._pendingTextFallback = false; - this._allowAnyTextNoHJ = false; - this._lastSentText = ""; - this._lastSentAt = 0; - this._openHookInstalled = false; - this._offlineAudioAttempts = 0; - this._audioFallbackTimer = null; - - var self = this; - this._onHJHeard = function () { - return self._handleHJHeard(); - }; - this._onGlobalTurnResult = function (data) { - return self._handleTurnResult(data, "globalTurnResult"); - }; - this._onLocalTurnResult = function (data) { - return self._handleTurnResult(data, "localTurnResult"); - }; -} - -AIBridge.prototype._loadConfig = function () { - var self = this; - try { - var p = String(self.configPath || "").trim(); - if (!p) return; - var raw = fs.readFileSync(p, "utf8"); - if (!raw) return; - var cfg = JSON.parse(raw); - if (!cfg || typeof cfg !== "object") return; - - if (typeof cfg.enabled === "boolean") self.enabled = cfg.enabled; - if (typeof cfg.mode === "string" && (cfg.mode === InputMode.TEXT || cfg.mode === InputMode.AUDIO)) self.mode = cfg.mode; - if (typeof cfg.serverBaseUrl === "string") { - var normalized = normalizeServerBaseUrl(cfg.serverBaseUrl); - if (normalized) self.serverBaseUrl = normalized; - } - if (typeof cfg.recordSeconds === "number") self.recordSeconds = Math.max(1, Math.min(15, Math.floor(cfg.recordSeconds))); - if (typeof cfg.useDumpStateAudio === "boolean") self.useDumpStateAudio = cfg.useDumpStateAudio; - if (typeof cfg.useAsrServiceStt === "boolean") self.useAsrServiceStt = cfg.useAsrServiceStt; - if (typeof cfg.asrServiceHost === "string") self.asrServiceHost = cfg.asrServiceHost; - if (typeof cfg.asrServicePort === "number") self.asrServicePort = cfg.asrServicePort; - if (typeof cfg.asrAudioSourceId === "string") self.asrAudioSourceId = cfg.asrAudioSourceId; - if (typeof cfg.asrTimeoutMs === "number") self.asrTimeoutMs = Math.max(2000, Math.min(60000, Math.floor(cfg.asrTimeoutMs))); - if (typeof cfg.asrServiceDebugWs === "boolean") self.asrServiceDebugWs = cfg.asrServiceDebugWs; - if (typeof cfg.asrAutoStart === "boolean") self.asrAutoStart = cfg.asrAutoStart; - if (typeof cfg.followupEnabled === "boolean") self.followupEnabled = cfg.followupEnabled; - if (typeof cfg.followupDelayMs === "number") self.followupDelayMs = Math.max(0, Math.min(5000, Math.floor(cfg.followupDelayMs))); - if (cfg.wakeupChitchatPhrases && cfg.wakeupChitchatPhrases.length) self.wakeupChitchatPhrases = cfg.wakeupChitchatPhrases; - - if (rlog) { - rlog.info("ai-bridge", "loaded config", { - path: p, - enabled: self.enabled, - mode: self.mode, - url: self.serverBaseUrl, - useAsrServiceStt: self.useAsrServiceStt, - }); - } - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "config load failed", { err: String(e && (e.stack || e.message || e)), path: String(self.configPath || "") }); - } - } -}; - -AIBridge.prototype._ensureAsrServiceRunning = function () { - var self = this; - if (!self.asrAutoStart) return Promise.resolve(); - var host = String(self.asrServiceHost || "127.0.0.1"); - var port = typeof self.asrServicePort === "number" ? self.asrServicePort : Number(self.asrServicePort) || 8088; - - var tryConnect = function (ms) { - ms = typeof ms === "number" ? ms : 350; - return new Promise(function (resolve, reject) { - var sock = net.connect({ host: host, port: port }); - var done = false; - var timer = setTimeout(function () { - if (done) return; - done = true; - try { - sock.destroy(); - } catch (e) { - // ignore - } - reject(new Error("asr port timeout")); - }, ms); - sock.on("connect", function () { - if (done) return; - done = true; - clearTimeout(timer); - try { - sock.end(); - } catch (e) { - // ignore - } - resolve(); - }); - sock.on("error", function (e) { - if (done) return; - done = true; - clearTimeout(timer); - reject(e); - }); - }); - }; - - return tryConnect(250) - .catch(function () { - if (self._asrSpawnAttempted) return; - self._asrSpawnAttempted = true; - var exe = String(self.asrExecutable || ""); - if (!exe) return; - var cfgPath = self.asrSystemConfigPath; - try { - var st = fs.statSync(self.asrLocalConfigPath); - if (st && st.isFile && st.isFile()) cfgPath = self.asrLocalConfigPath; - } catch (e) { - // ignore - } - try { - var child = spawn(exe, ["-c", String(cfgPath)], { stdio: ["ignore", "ignore", "ignore"], detached: true }); - child.unref && child.unref(); - if (rlog) { - rlog.warn("ai-bridge", "attempted to start asr-service", { exe: exe, cfg: String(cfgPath) }); - } - } catch (e2) { - if (rlog) { - rlog.warn("ai-bridge", "failed to spawn asr-service", { err: String(e2 && (e2.stack || e2.message || e2)) }); - } - } - return new Promise(function (resolve) { - setTimeout(resolve, 450); - }); - }) - .then(function () { - return tryConnect(350); - }) - .catch(function () { - // ignore; normal if system-manager manages the service. - }); -}; - -AIBridge.prototype._captureTextViaAsrService = function (source) { - var self = this; - if (!self.enabled) return Promise.resolve(null); - if (self._inFlight || self._asrServiceInFlight) return Promise.resolve(null); - self._asrServiceInFlight = true; - - var host = String(self.asrServiceHost || "127.0.0.1"); - var port = typeof self.asrServicePort === "number" ? self.asrServicePort : Number(self.asrServicePort) || 8088; - var baseHttp = "http://" + host + ":" + String(port); - var wsUrl = "ws://" + host + ":" + String(port) + "/simple_port"; - var taskId = "DEBUG:ai-bridge-" + String(Date.now()) + "-" + String(Math.floor(Math.random() * 1e9)); - var requestId = "stt_start_" + String(Date.now()) + "_" + String(Math.floor(Math.random() * 1e9)); - var timeoutMs = typeof self.asrTimeoutMs === "number" ? self.asrTimeoutMs : 15000; - var debugWs = !!self.asrServiceDebugWs; - - var wsClient = null; - var stopAlways = function () { - var stopPayload = { command: "stop", task_id: taskId, request_id: "stt_stop_" + String(Date.now()) + "_" + String(Math.floor(Math.random() * 1e9)) }; - return httpJsonPostRaw(baseHttp + "/asr_simple_interface", stopPayload, 5000) - .then(function () { - // ignore - }) - .catch(function () { - // ignore - }); - }; - - var done = function () { - self._asrServiceInFlight = false; - }; - - var fail = function (e) { - if (rlog) { - rlog.warn("ai-bridge", "asr-service stt failed", { err: String(e && (e.stack || e.message || e)), source: source || "asr-service" }); - } - self._asrServiceInFlight = false; - }; - - var startPayload = { - command: "start", - task_id: taskId, - audio_source_id: String(self.asrAudioSourceId || "alsa1"), - hotphrase: "none", - speech_to_text: true, - request_id: requestId, - }; - - var waitForFinal = function () { - return new Promise(function (resolve, reject) { - var timer = setTimeout(function () { - reject(new Error("asr-service stt timeout")); - }, timeoutMs); - var resolved = false; - var seen = 0; - wsClient.onMessage(function (msg) { - if (resolved) return; - seen += 1; - if (debugWs && rlog && seen <= 6) { - rlog.debug("ai-bridge", "asr-service ws msg", { idx: seen, raw: String(msg || "").slice(0, 500) }); - } - var evt = null; - try { - evt = JSON.parse(String(msg || "")); - } catch (e) { - return; - } - if (!evt || typeof evt !== "object") return; - var eventType = evt.event_type || evt.eventType || evt.event || evt.type; - if (eventType !== "speech_to_text_final") return; - - // Correlate to our request when possible, but don't hard-fail on schema drift. - var evTaskId = evt.task_id || evt.taskId || (evt.payload && (evt.payload.task_id || evt.payload.taskId)); - var evRequestId = evt.request_id || evt.requestId || (evt.payload && (evt.payload.request_id || evt.payload.requestId)); - var idMatches = false; - if (evTaskId && String(evTaskId) === String(taskId)) idMatches = true; - if (evRequestId && String(evRequestId) === String(requestId)) idMatches = true; - // If the event carries neither id, accept it (single in-flight capture). - if (!evTaskId && !evRequestId) idMatches = true; - if (!idMatches) return; - - var utterances = evt.utterances || evt.Utterances || (evt.payload && (evt.payload.utterances || evt.payload.Utterances)); - var best = pickBestAsrUtterance(utterances); - if (!best || !String(best).trim()) return; - var rawText = String(best); - var cleaned = stripKnownEdgePhrases(rawText, self.wakeupChitchatPhrases); - if (cleaned !== rawText && rlog) { - rlog.info("ai-bridge", "asr-service sanitized text", { - raw: rawText.slice(0, 160), - clean: cleaned.slice(0, 160), - source: source || "asr-service", - }); - } - cleaned = String(cleaned || "").trim(); - if (!cleaned) return; - if (isChitchatOnly(cleaned, self.wakeupChitchatPhrases)) { - // Keep listening; this is probably Jibo's own wake chatter. - if (rlog) { - rlog.warn("ai-bridge", "ignored wake chitchat transcript", { text: cleaned.slice(0, 80), source: source || "asr-service" }); - } - return; - } - resolved = true; - clearTimeout(timer); - resolve(String(cleaned)); - }); - }); - }; - - var t0 = Date.now(); - return self._ensureAsrServiceRunning().then(function () { - return connectWebSocketText(wsUrl, 4000); - }) - .then(function (ws) { - wsClient = ws; - return httpJsonPostRaw(baseHttp + "/asr_simple_interface", startPayload, 6000); - }) - .then(function (resp) { - if (rlog) { - rlog.info("ai-bridge", "asr-service stt start", { - status: resp && resp.statusCode, - ms: Date.now() - t0, - ws: wsUrl, - http: baseHttp, - source: source || "asr-service", - body: resp && resp.body ? String(resp.body).slice(0, 500) : "", - }); - } - if (!resp || !resp.statusCode || resp.statusCode < 200 || resp.statusCode >= 300) { - throw new Error("asr-service start failed: " + String(resp && resp.statusCode)); - } - return waitForFinal(); - }) - .then(function (text) { - return stopAlways().then(function () { - return text; - }); - }) - .then(function (text) { - try { - wsClient && wsClient.close && wsClient.close(); - } catch (e) { - // ignore - } - if (rlog) { - rlog.info("ai-bridge", "asr-service text", { chars: String(text).length, text: String(text).slice(0, 160), ms: Date.now() - t0, source: source || "asr-service" }); - } - return self._sendText(text, "asr-service:" + String(source || "")); - }) - .then(function () { - done(); - }) - .catch(function (e) { - try { - wsClient && wsClient.close && wsClient.close(); - } catch (e2) { - // ignore - } - return stopAlways() - .then(function () { - throw e; - }) - .catch(function () { - throw e; - }); - }) - .catch(function (e) { - fail(e); - }); -}; - -AIBridge.prototype._scheduleOfflineAudioFallback = function (reason, delayMs) { - var self = this; - if (!self.enabled) return; - if (self._offlineAudioAttempts >= 2) return; - if (!self._allowAnyTextNoHJ && Date.now() - self._lastHJAt > self._hjWindowMs) return; - if (self._inFlight) return; - - if (self._audioFallbackTimer) { - clearTimeout(self._audioFallbackTimer); - self._audioFallbackTimer = null; - } - - delayMs = typeof delayMs === "number" ? delayMs : 1200; - if (rlog) { - rlog.warn("ai-bridge", "scheduling offline audio fallback", { reason: String(reason || ""), delayMs: delayMs }); - } - - self._audioFallbackTimer = setTimeout(function () { - self._audioFallbackTimer = null; - self._offlineAudioAttempts += 1; - self - ._captureAndSendAudio("offlineFallback:" + String(reason || "")) - .catch(function (e) { - if (rlog) { - rlog.warn("ai-bridge", "offline audio fallback failed", { err: String(e && (e.stack || e.message || e)) }); - } - // Common case: mic is still owned by the built-in pipeline. Retry once. - if (isMicBusyError(String(e && (e.stack || e.message || e)))) { - self._scheduleOfflineAudioFallback("retryMicBusy:" + String(reason || ""), 2400); - } - }); - }, delayMs); -}; - -AIBridge.prototype._captureAndSendAudio = function (source) { - var self = this; - if (!self.enabled) { - return Promise.resolve(); - } - if (self._inFlight) { - return Promise.resolve(); - } - self._inFlight = true; - - var t0 = Date.now(); - var rate = typeof self.audioSampleRate === "number" ? self.audioSampleRate : 16000; - var ch = typeof self.audioChannels === "number" ? self.audioChannels : 1; - var dev = typeof self.arecordDevice === "string" ? self.arecordDevice : ""; - var fmt = typeof self.audioFormat === "string" ? self.audioFormat : "S16_LE"; - if (["S16_LE", "S32_LE"].indexOf(fmt) < 0) fmt = "S16_LE"; - var dbg = !!self.debugAudioCapture; - - var dumpDir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; - - var captureViaDumpState = function () { - var tDump0 = Date.now(); - return runAudioDumpState(dumpDir, 7000).then(function (res) { - if (rlog) { - rlog.info("ai-bridge", "dump-state complete", { - ms: Date.now() - tDump0, - code: res.code, - timedOut: res.timedOut, - infile: res.infile || null, - stdout: String(res.stdout || "").slice(0, 700), - stderr: String(res.stderr || "").slice(0, 700), - }); - } - var picked = pickBestDumpWav(dumpDir); - if (!picked || !picked.wav) { - throw new Error( - "dump-state produced no wavs; files=" + - JSON.stringify(listTmpAudioFiles(dumpDir).slice(0, 12)) + - " audiolog=" + - JSON.stringify(getAudioLogInfo(dumpDir)) - ); - } - var hdr = parseWavHeader(picked.wav); - return { wav: picked.wav, meta: { method: "dump-state", path: picked.path, header: hdr } }; - }); - }; - - var captureViaArecord = function () { - return recordWavWithArecord(self.recordSeconds, rate, ch, dev, fmt, dbg).then(function (wav) { - return { wav: wav, meta: { method: "arecord", device: dev || "(default)", rate: rate, ch: ch, format: fmt } }; - }); - }; - - var capturePromise; - if (self.useDumpStateAudio) { - capturePromise = captureViaDumpState().catch(function (e) { - if (rlog) { - rlog.warn("ai-bridge", "dump-state capture failed; falling back to arecord", { err: String(e && (e.stack || e.message || e)) }); - } - return captureViaArecord(); - }); - } else { - capturePromise = captureViaArecord().catch(function (e) { - if (isMicBusyError(String(e && (e.stack || e.message || e)))) { - return captureViaDumpState(); - } - throw e; - }); - } - - return capturePromise - .then(function (cap) { - var wav = cap && cap.wav ? cap.wav : null; - var meta = cap && cap.meta ? cap.meta : null; - if (!wav) throw new Error("no audio captured"); - var url = self.serverBaseUrl.replace(/\/+$/, "") + "/v1/chat/audio"; - if (rlog) { - rlog.info("ai-bridge", "audio captured", { - bytes: wav.length, - ms: Date.now() - t0, - url: url, - source: source || "audio", - capture: meta || { method: "unknown" }, - }); - } - var t1 = Date.now(); - return httpJsonPost(url, { - wav_base64: wav.toString("base64"), - source: source || "audio", - capture: meta || null, - }).then(function (resp) { - if (rlog) { - rlog.info("ai-bridge", "audio request complete", { ms: Date.now() - t1, ok: !!(resp && resp.reply), source: source || "audio" }); - } - if (rlog && resp && typeof resp.text === "string") { - rlog.info("ai-bridge", "server transcript", { - chars: resp.text.length, - text: resp.text.slice(0, 160), - }); - } - if (resp && resp.reply) { - return self._speak(resp.reply); - } - }); - }) - .catch(function (e) { - self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge audio capture error", e && (e.stack || e.message || e)); - if (rlog) { - rlog.warn("ai-bridge", "audio capture error", { err: String(e && (e.stack || e.message || e)), source: source || "audio" }); - } - throw e; - }) - .then( - function (v) { - self._inFlight = false; - return v; - }, - function (e) { - self._inFlight = false; - throw e; - } - ); -}; - -AIBridge.prototype._installOpenHook = function () { - var self = this; - if (self._openHookInstalled) return; - self._openHookInstalled = true; - - if (!beFramework || !beFramework.BeSkill || typeof beFramework.BeSkill.registerOpenHook !== "function") { - if (rlog) { - rlog.warn("ai-bridge", "be-framework open hook unavailable; cannot intercept skill launches", {}); - } - return; - } - - try { - beFramework.BeSkill.registerOpenHook(function (lastSkill, nextSkill, results) { - return function (resolve) { - try { - if (!self.enabled) return resolve(); - if (self.mode !== InputMode.TEXT && !self._pendingTextFallback) return resolve(); - if (!self._allowAnyTextNoHJ && Date.now() - self._lastHJAt > self._hjWindowMs) return resolve(); - - var text = extractAsrText(results); - if (!text) { - if (rlog) { - rlog.debug("ai-bridge", "open hook saw no asr text", { - nextSkill: String(nextSkill || ""), - keys: results && typeof results === "object" ? Object.keys(results).slice(0, 25) : [], - preview: safePreview(results, 320), - }); - } - // If cloud ASR is down, Hey-Jibo intents often have empty asr.text. - // Try a one-shot offline audio fallback after the pipeline releases the mic. - if (looksLikeHeyJiboIntent(results) && !self.useAsrServiceStt) { - self._scheduleOfflineAudioFallback("openHookNoText", 1400); - } - return resolve(); - } - text = String(text); - if (!text.trim()) return resolve(); - if (isCloudAsrFailureText(text)) { - if (rlog) { - rlog.warn("ai-bridge", "ignoring cloud ASR error text (open hook)", { text: text.slice(0, 200) }); - } - if (!self.useAsrServiceStt) { - self._scheduleOfflineAudioFallback("cloudAsrErrorText", 1400); - } - return resolve(); - } - - if (rlog) { - rlog.info("ai-bridge", "open hook asr", { - nextSkill: String(nextSkill || ""), - text: text.slice(0, 160), - }); - } - self._sendText(text, "openHook:" + String(nextSkill || "")); - return resolve(); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "open hook error", { err: String(e && (e.stack || e.message || e)) }); - } - return resolve(); - } - }; - }); - - if (rlog) { - rlog.info("ai-bridge", "installed BeSkill open hook", {}); - } - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "failed to install open hook", { err: String(e && (e.stack || e.message || e)) }); - } - } -}; - -AIBridge.prototype.setupTunables = function () { - if (!tunable || !tunable.Tunable) { - if (rlog && typeof rlog.raw === "function") { - rlog.raw("[AI-BRIDGE] jibo-tunable not available; UI controls disabled"); - } - return; - } - - var self = this; - var Tunable = tunable.Tunable; - var safe = function (fn) { - try { - return fn(); - } catch (e) { - if (rlog && typeof rlog.raw === "function") { - rlog.raw("[AI-BRIDGE] Tunable registration failed: " + String(e && (e.stack || e.message || e))); - } - return null; - } - }; - - safe(function () { - return Tunable.getCheckboxField(LABEL.enabled, self.enabled, WIN).events.change.on(function (v) { - self.enabled = !!v; - if (rlog) { - rlog.info("ai-bridge", "enabled changed", { enabled: self.enabled }); - } - }); - }); - - // Prefer dropdown if available; otherwise fallback to a checkbox. - if (typeof Tunable.getDropdownField === "function") { - safe(function () { - return Tunable.getDropdownField(LABEL.input, [InputMode.AUDIO, InputMode.TEXT], 0, WIN).events.change.on(function (idxOrVal) { - var options = [InputMode.AUDIO, InputMode.TEXT]; - self.mode = options[idxOrVal] || idxOrVal || InputMode.AUDIO; - if (rlog) { - rlog.info("ai-bridge", "mode changed", { mode: self.mode }); - } - }); - }); - } else { - safe(function () { - return Tunable.getCheckboxField(LABEL.audioCheckbox, self.mode === InputMode.AUDIO, WIN).events.change.on(function (v) { - self.mode = v ? InputMode.AUDIO : InputMode.TEXT; - if (rlog) { - rlog.info("ai-bridge", "mode changed", { mode: self.mode }); - } - }); - }); - } - - safe(function () { - return Tunable.getStringField(LABEL.serverUrl, self.serverBaseUrl, WIN).events.change.on(function (v) { - var normalized = normalizeServerBaseUrl(v); - if (normalized) { - self.serverBaseUrl = normalized; - if (rlog) { - rlog.info("ai-bridge", "server url changed", { url: self.serverBaseUrl }); - } - } else { - if (rlog) { - rlog.warn("ai-bridge", "ignored invalid server url", { url: String(v).slice(0, 120) }); - } - } - }); - }); - - // Audio capture tuning for offline fallback and AUDIO mode. - safe(function () { - return Tunable.getStringField(LABEL.alsaDevice, self.arecordDevice, WIN).events.change.on(function (v) { - self.arecordDevice = String(v || "").trim(); - if (rlog) { - rlog.info("ai-bridge", "alsa device changed", { device: self.arecordDevice || "(default)" }); - } - }); - }); - - var setRate = function (v) { - var n = Number(v); - if (Number.isNaN(n)) return; - self.audioSampleRate = Math.max(8000, Math.min(48000, Math.floor(n))); - if (rlog) { - rlog.info("ai-bridge", "audio sample rate changed", { hz: self.audioSampleRate }); - } - }; - if (typeof Tunable.getNumberField === "function") { - safe(function () { - return Tunable.getNumberField(LABEL.audioSampleRate, self.audioSampleRate, 8000, 48000, 1000, WIN).events.change.on(setRate); - }); - } else { - safe(function () { - return Tunable.getStringField(LABEL.audioSampleRate, String(self.audioSampleRate), WIN).events.change.on(setRate); - }); - } - - var setCh = function (v) { - var n = Number(v); - if (Number.isNaN(n)) return; - self.audioChannels = Math.max(1, Math.min(8, Math.floor(n))); - if (rlog) { - rlog.info("ai-bridge", "audio channels changed", { ch: self.audioChannels }); - } - }; - if (typeof Tunable.getNumberField === "function") { - safe(function () { - return Tunable.getNumberField(LABEL.audioChannels, self.audioChannels, 1, 8, 1, WIN).events.change.on(setCh); - }); - } else { - safe(function () { - return Tunable.getStringField(LABEL.audioChannels, String(self.audioChannels), WIN).events.change.on(setCh); - }); - } - - // Sample format (default S16_LE). Keep it whitelisted. - if (typeof Tunable.getDropdownField === "function") { - safe(function () { - var opts = ["S16_LE", "S32_LE"]; - var idx = Math.max(0, opts.indexOf(self.audioFormat)); - return Tunable.getDropdownField(LABEL.audioFormat, opts, idx, WIN).events.change.on(function (idxOrVal) { - var v = opts[idxOrVal] || idxOrVal; - v = String(v || "").trim(); - if (opts.indexOf(v) >= 0) { - self.audioFormat = v; - if (rlog) { - rlog.info("ai-bridge", "audio format changed", { format: self.audioFormat }); - } - } - }); - }); - } else { - safe(function () { - return Tunable.getStringField(LABEL.audioFormat, String(self.audioFormat), WIN).events.change.on(function (v) { - v = String(v || "").trim(); - if (["S16_LE", "S32_LE"].indexOf(v) >= 0) { - self.audioFormat = v; - if (rlog) { - rlog.info("ai-bridge", "audio format changed", { format: self.audioFormat }); - } - } else if (rlog) { - rlog.warn("ai-bridge", "ignored invalid audio format", { format: String(v).slice(0, 32) }); - } - }); - }); - } - - safe(function () { - return Tunable.getCheckboxField(LABEL.debugAudio, self.debugAudioCapture, WIN).events.change.on(function (v) { - self.debugAudioCapture = !!v; - if (rlog) { - rlog.info("ai-bridge", "debug audio capture changed", { enabled: self.debugAudioCapture }); - } - }); - }); - - // Prefer the system audio service's dump-state output over arecord (avoids mic busy). - safe(function () { - return Tunable.getCheckboxField(LABEL.useDumpStateAudio, self.useDumpStateAudio, WIN).events.change.on(function (v) { - self.useDumpStateAudio = !!v; - if (rlog) { - rlog.info("ai-bridge", "use dump-state audio changed", { enabled: self.useDumpStateAudio }); - } - }); - }); - - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Dump /tmp audio files", WIN).events.change.on(function () { - try { - var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; - var files = listTmpAudioFiles(dir); - if (rlog) { - rlog.info("ai-bridge", "tmp audio files", { dir: dir, files: files.slice(0, 60), audiolog: getAudioLogInfo(dir) }); - } - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "dump tmp audio files failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Run audio dump-state now", WIN).events.change.on(function () { - try { - var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; - var t0 = Date.now(); - runAudioDumpState(dir, 8000) - .then(function (res) { - var picked = pickBestDumpWav(dir); - if (rlog) { - rlog.info("ai-bridge", "manual dump-state", { - ms: Date.now() - t0, - code: res.code, - timedOut: res.timedOut, - infile: res.infile || null, - picked: picked ? picked.path : null, - files: listTmpAudioFiles(dir).slice(0, 20), - audiolog: getAudioLogInfo(dir), - stderr: String(res.stderr || "").slice(0, 500), - }); - } - }) - .catch(function (e) { - if (rlog) { - rlog.warn("ai-bridge", "manual dump-state failed", { err: String(e && (e.stack || e.message || e)) }); - } - }); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "manual dump-state crashed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - // Dump hardware params for the currently-selected settings. - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Dump hw params", WIN).events.change.on(function () { - try { - var dev = (self.arecordDevice || "").trim(); - var rate = typeof self.audioSampleRate === "number" ? self.audioSampleRate : 16000; - var ch = typeof self.audioChannels === "number" ? self.audioChannels : 1; - var fmt = (self.audioFormat || "S16_LE").trim(); - if (["S16_LE", "S32_LE"].indexOf(fmt) < 0) fmt = "S16_LE"; - var args = ["--dump-hw-params", "-d", "1", "-t", "wav", "-f", fmt, "-r", String(rate), "-c", String(ch)]; - if (dev) { - args.unshift(dev); - args.unshift("-D"); - } - args.push("/dev/null"); - var child = spawn("arecord", args, { stdio: ["ignore", "pipe", "pipe"] }); - var out = []; - var err = []; - child.stdout.on("data", function (d) { - out.push(d); - }); - child.stderr.on("data", function (d) { - err.push(d); - }); - child.on("close", function (code) { - var o = Buffer.concat(out).toString("utf8"); - var e = Buffer.concat(err).toString("utf8"); - if (rlog) { - rlog.info("ai-bridge", "arecord hw params", { - code: code, - device: dev || "(default)", - rate: rate, - ch: ch, - format: fmt, - stdout: (o || "").slice(0, 1400), - stderr: (e || "").slice(0, 1400), - }); - } - }); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "dump hw params failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - // Quick probes for the built-in audio service HTTP API. - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: AudioSvc stream_state", WIN).events.change.on(function () { - try { - var u = "http://127.0.0.1:8383/stream_state"; - var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; - httpTextGet(u, 2500) - .then(function (res) { - if (rlog) { - rlog.info("ai-bridge", "audiosvc stream_state", { - url: u, - status: res.statusCode, - body: String(res.body || "").slice(0, 1200), - audiolog: getAudioLogInfo(dir), - }); - } - }) - .catch(function (e) { - if (rlog) { - rlog.warn("ai-bridge", "audiosvc stream_state failed", { err: String(e && (e.stack || e.message || e)) }); - } - }); - } catch (e) { - if (rlog) rlog.warn("ai-bridge", "audiosvc stream_state crashed", { err: String(e && (e.stack || e.message || e)) }); - } - }); - }); - } - - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: AudioSvc log", WIN).events.change.on(function () { - try { - var u = "http://127.0.0.1:8383/log"; - var dir = typeof self.dumpStateDir === "string" && self.dumpStateDir ? self.dumpStateDir : "/tmp"; - httpTextGet(u, 2500) - .then(function (res) { - if (rlog) { - rlog.info("ai-bridge", "audiosvc log", { - url: u, - status: res.statusCode, - body: String(res.body || "").slice(0, 1200), - audiolog: getAudioLogInfo(dir), - }); - } - }) - .catch(function (e) { - if (rlog) { - rlog.warn("ai-bridge", "audiosvc log failed", { err: String(e && (e.stack || e.message || e)) }); - } - }); - } catch (e) { - if (rlog) rlog.warn("ai-bridge", "audiosvc log crashed", { err: String(e && (e.stack || e.message || e)) }); - } - }); - }); - } - - // Probe a few candidate devices to find a usable mic tap when the real ADC is busy. - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Probe audio devices", WIN).events.change.on(function () { - try { - if (self._inFlight) { - if (rlog) rlog.warn("ai-bridge", "probe skipped; in flight", {}); - return; - } - var rate = typeof self.audioSampleRate === "number" ? self.audioSampleRate : 16000; - var ch = typeof self.audioChannels === "number" ? self.audioChannels : 1; - var fmt = typeof self.audioFormat === "string" ? self.audioFormat : "S16_LE"; - if (["S16_LE", "S32_LE"].indexOf(fmt) < 0) fmt = "S16_LE"; - // Signal stats helper only understands 16-bit PCM, so probe with S16_LE even if - // the main audio capture is set to S32_LE. - var probeFmt = "S16_LE"; - var devices = []; - // 1) configured - devices.push((self.arecordDevice || "").trim()); - // 2) default - devices.push(""); - // 3) Try shareable plugin PCMs (may work even if hw is busy) - devices.push("dsnoop"); - devices.push("dsnoop:CARD=ADC,DEV=0"); - devices.push("dsnoop:CARD=1,DEV=0"); - devices.push("plughw:1,0"); - devices.push("hw:1,0"); - // 4) ALSA Loopback common capture endpoints - devices.push("plughw:2,0"); - devices.push("plughw:2,1"); - // 5) Dummy (should be silence) - devices.push("plughw:3,0"); - // De-dupe - var seen = {}; - devices = devices.filter(function (d) { - d = String(d || ""); - if (seen[d]) return false; - seen[d] = true; - return true; - }); - - if (rlog) { - rlog.info("ai-bridge", "probe starting", { rate: rate, ch: ch, format: fmt, probeFormat: probeFmt, devices: devices }); - } - - var p = Promise.resolve(); - devices.forEach(function (dev) { - p = p.then(function () { - return recordWavWithArecord(1, rate, ch, dev, probeFmt, true) - .then(function (wav) { - var stats = wavSignalStats16(wav, Math.max(8000, rate)); - if (rlog) { - rlog.info("ai-bridge", "probe result", { - device: dev || "(default)", - bytes: wav.length, - format: probeFmt, - stats: stats, - }); - } - }); - }).catch(function (e) { - if (rlog) { - rlog.warn("ai-bridge", "probe error", { device: dev || "(default)", err: String(e && (e.stack || e.message || e)) }); - } - }); - }); - - p.then(function () { - if (rlog) rlog.info("ai-bridge", "probe done", {}); - }); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "probe failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - // Helper for finding the right -D device string. - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Dump ALSA (arecord -l)", WIN).events.change.on(function () { - try { - var child = spawn("arecord", ["-l"], { stdio: ["ignore", "pipe", "pipe"] }); - var out = []; - var err = []; - child.stdout.on("data", function (d) { - out.push(d); - }); - child.stderr.on("data", function (d) { - err.push(d); - }); - child.on("close", function (code) { - var o = Buffer.concat(out).toString("utf8"); - var e = Buffer.concat(err).toString("utf8"); - if (rlog) { - rlog.info("ai-bridge", "arecord -l", { code: code, stdout: (o || "").slice(0, 2000), stderr: (e || "").slice(0, 500) }); - } - }); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "arecord -l failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - // Helper for listing all ALSA PCMs (useful for discovering dsnoop/plug routes). - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Dump ALSA PCMs (arecord -L)", WIN).events.change.on(function () { - try { - var child = spawn("arecord", ["-L"], { stdio: ["ignore", "pipe", "pipe"] }); - var out = []; - var err = []; - child.stdout.on("data", function (d) { - out.push(d); - }); - child.stderr.on("data", function (d) { - err.push(d); - }); - child.on("close", function (code) { - var o = Buffer.concat(out).toString("utf8"); - var e = Buffer.concat(err).toString("utf8"); - if (rlog) { - rlog.info("ai-bridge", "arecord -L", { code: code, stdout: (o || "").slice(0, 4000), stderr: (e || "").slice(0, 500) }); - } - }); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "arecord -L failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - // Dump who owns the Tobor ADC capture device, via /proc/asound and (if available) fuser. - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Dump ADC owner (/proc/asound)", WIN).events.change.on(function () { - try { - var fs = require("fs"); - function readText(path) { - try { - return fs.readFileSync(path, "utf8"); - } catch (e) { - return ""; - } - } - - var statusPath = "/proc/asound/card1/pcm0c/sub0/status"; - var hwParamsPath = "/proc/asound/card1/pcm0c/sub0/hw_params"; - var infoPath = "/proc/asound/card1/pcm0c/sub0/info"; - var pcmPath = "/proc/asound/pcm"; - var cardsPath = "/proc/asound/cards"; - var idPath = "/proc/asound/card1/id"; - - var status = readText(statusPath); - var hwParams = readText(hwParamsPath); - var info = readText(infoPath); - var pcm = readText(pcmPath); - var cards = readText(cardsPath); - var cardId = readText(idPath); - - // Extract owning PID if present. - var pid = null; - var m = (status || "").match(/owner_pid\s*:\s*(\d+)/); - if (m && m[1]) pid = Number(m[1]); - var cmdline = ""; - if (pid && pid > 0) { - cmdline = readText("/proc/" + pid + "/cmdline").replace(/\u0000/g, " ").trim(); - } - - if (rlog) { - rlog.info("ai-bridge", "adc owner", { - cardId: (cardId || "").trim(), - status: (status || "").trim().slice(0, 2000), - hw_params: (hwParams || "").trim().slice(0, 2000), - info: (info || "").trim().slice(0, 2000), - pcm: (pcm || "").trim().slice(0, 2000), - cards: (cards || "").trim().slice(0, 2000), - owner_pid: pid || null, - owner_cmdline: cmdline || "", - }); - } - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "dump adc owner failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Dump ADC owner (fuser)", WIN).events.change.on(function () { - try { - var devs = ["/dev/snd/pcmC1D0c", "/dev/snd/controlC1", "/dev/snd/timer"]; // best-effort - var child = spawn("fuser", ["-v"].concat(devs), { stdio: ["ignore", "pipe", "pipe"] }); - var out = []; - var err = []; - child.stdout.on("data", function (d) { - out.push(d); - }); - child.stderr.on("data", function (d) { - err.push(d); - }); - child.on("close", function (code) { - var o = Buffer.concat(out).toString("utf8"); - var e = Buffer.concat(err).toString("utf8"); - if (rlog) { - rlog.info("ai-bridge", "fuser adc", { code: code, stdout: (o || "").slice(0, 2000), stderr: (e || "").slice(0, 2000) }); - } - }); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "fuser failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - // Simple one-shot connectivity test. - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Test server", WIN).events.change.on(function () { - try { - self._sendText("(test) hello from Jibo", "test"); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "test send failed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - // Offline STT via jibo-asr-service (requires service running). - safe(function () { - return Tunable.getCheckboxField(LABEL.useAsrServiceStt, self.useAsrServiceStt, WIN).events.change.on(function (v) { - self.useAsrServiceStt = !!v; - if (rlog) { - rlog.info("ai-bridge", "use asr-service stt changed", { enabled: self.useAsrServiceStt }); - } - }); - }); - - safe(function () { - return Tunable.getCheckboxField(LABEL.asrServiceDebugWs, self.asrServiceDebugWs, WIN).events.change.on(function (v) { - self.asrServiceDebugWs = !!v; - if (rlog) { - rlog.info("ai-bridge", "asr-service debug ws changed", { enabled: self.asrServiceDebugWs }); - } - }); - }); - if (typeof Tunable.getButtonField === "function") { - safe(function () { - return Tunable.getButtonField("AI Bridge: Run ASR-service STT now", WIN).events.change.on(function () { - try { - self._captureTextViaAsrService("manual"); - } catch (e) { - if (rlog) { - rlog.warn("ai-bridge", "manual asr-service stt crashed", { err: String(e && (e.stack || e.message || e)) }); - } - } - }); - }); - } - - if (typeof Tunable.getNumberField === "function") { - safe(function () { - return Tunable.getNumberField(LABEL.recordSeconds, self.recordSeconds, 1, 15, 1, WIN).events.change.on(function (v) { - var n = Number(v); - if (!Number.isNaN(n)) { - self.recordSeconds = Math.max(1, Math.min(15, Math.floor(n))); - if (rlog) { - rlog.info("ai-bridge", "record seconds changed", { seconds: self.recordSeconds }); - } - } - }); - }); - } else { - safe(function () { - return Tunable.getStringField(LABEL.recordSeconds, String(self.recordSeconds), WIN).events.change.on(function (v) { - var n = Number(v); - if (!Number.isNaN(n)) { - self.recordSeconds = Math.max(1, Math.min(15, Math.floor(n))); - if (rlog) { - rlog.info("ai-bridge", "record seconds changed", { seconds: self.recordSeconds }); - } - } - }); - }); - } - }; - -AIBridge.prototype.start = function () { - try { - this._loadConfig(); - } catch (e0) { - // ignore - } - try { - this.setupTunables(); - } catch (e) { - if (rlog && typeof rlog.raw === "function") { - rlog.raw("[AI-BRIDGE] setupTunables crashed: " + String(e && (e.stack || e.message || e))); - } - } - try { - var ev = this.jibo && this.jibo.jetstream && this.jibo.jetstream.events; - if (ev && ev.hjHeard && typeof ev.hjHeard.on === "function") { - ev.hjHeard.on(this._onHJHeard); - } - if (ev && ev.globalTurnResult && typeof ev.globalTurnResult.on === "function") { - ev.globalTurnResult.on(this._onGlobalTurnResult); - } - if (ev && ev.localTurnResult && typeof ev.localTurnResult.on === "function") { - ev.localTurnResult.on(this._onLocalTurnResult); - } - if (rlog) { - rlog.info("ai-bridge", "started", { mode: this.mode, serverUrl: this.serverBaseUrl, recordSeconds: this.recordSeconds }); - } - if (rlog && typeof rlog.raw === "function") { - rlog.raw("[AI-BRIDGE] started mode=" + this.mode + " url=" + this.serverBaseUrl); - } - this._installOpenHook(); - } catch (e) { - this.skill.log && this.skill.log.warn && this.skill.log.warn("AI Bridge failed to attach listeners", e); - if (rlog) { - rlog.error("ai-bridge", "failed to attach listeners", { err: String(e && (e.stack || e.message || e)) }); - } - if (rlog && typeof rlog.raw === "function") { - rlog.raw("[AI-BRIDGE] failed to attach listeners: " + String(e && (e.stack || e.message || e))); - } - } - }; - -AIBridge.prototype._speak = function (text) { - var self = this; - if (!text || !String(text).trim()) { - return Promise.resolve(); - } - - // Keep TTS inputs conservative: some builds have fragile service-client JSON parsing - // on long or multi-line payloads. - try { - text = String(text) - .replace(/[\r\n\t]+/g, " ") - .replace(/\s{2,}/g, " ") - .trim(); - if (text.length > 280) { - text = text.slice(0, 280); - } - } catch (e) { - // ignore - } - - try { - if (rlog) { - rlog.info("ai-bridge", "speaking", { chars: String(text).length }); - } - } catch (e) { - // ignore - } - - var ret; - try { - // Use SSML mode to support ESML tags (animations, pitch, breaks, etc.) - var ttsMode = self.jibo.tts.TTSMode ? self.jibo.tts.TTSMode.SSML : undefined; - ret = self.jibo.tts.speak(String(text), { mode: ttsMode }); - } catch (e) { - self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge TTS error", e); - if (rlog) { - rlog.warn("ai-bridge", "tts error", { err: String(e && (e.stack || e.message || e)) }); - } - return Promise.resolve(); - } - - return Promise.resolve(ret).catch(function (e) { - self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge TTS error", e); - if (rlog) { - rlog.warn("ai-bridge", "tts error", { err: String(e && (e.stack || e.message || e)) }); - } - }); -}; - -AIBridge.prototype._handleHJHeard = function () { - var self = this; - self._lastHJAt = Date.now(); - // Clear any prior fallback request window. - self._pendingTextFallback = false; - self._offlineAudioAttempts = 0; - if (self._audioFallbackTimer) { - clearTimeout(self._audioFallbackTimer); - self._audioFallbackTimer = null; - } - if (rlog) { - rlog.debug("ai-bridge", "hjHeard", { mode: self.mode, enabled: self.enabled }); - } - if (!self.enabled) { - return; - } - if (self.useAsrServiceStt) { - if (self._inFlight) return; - return self._captureTextViaAsrService("hjHeard"); - } - if (self.mode !== InputMode.AUDIO) { - return; - } - if (self._inFlight) { - return; - } - - return self._captureAndSendAudio("hjHeard") - .then(function () { - // done - }) - .catch(function (e) { - // _captureAndSendAudio already logs; keep compatibility with existing fallback behavior - var msg = String(e && (e.stack || e.message || e)); - if (isMicBusyError(msg)) { - self._scheduleOfflineAudioFallback("micBusyAfterHJ", 1600); - } - self._pendingTextFallback = true; - }); -}; - -AIBridge.prototype._handleTurnResult = function (data, eventName) { - var self = this; - if (!self.enabled) { - return; - } - // If we're using the local ASR service STT pipeline, ignore cloud turn-results - // to avoid duplicate sends. - if (self.useAsrServiceStt) { - return; - } - if (rlog) { - try { - rlog.debug("ai-bridge", "turn result", { event: eventName }); - } catch (e) { - // ignore - } - } - // Normal behavior: only send ASR in TEXT mode. - // Fallback behavior: if AUDIO failed during this HJ window, allow a one-shot - // ASR send even though the mode is AUDIO. - if (self.mode !== InputMode.TEXT && !self._pendingTextFallback) { - return; - } - if (!self._allowAnyTextNoHJ && Date.now() - self._lastHJAt > self._hjWindowMs) { - return; - } - - var text = extractAsrText(data); - if (!text) { - if (rlog) { - try { - var t = typeof data; - var keys = data && typeof data === "object" ? Object.keys(data).slice(0, 25) : []; - rlog.debug("ai-bridge", "turn result had no text", { event: eventName, type: t, keys: keys }); - } catch (e) { - // ignore - } - } - return; - } - text = String(text); - if (!text.trim()) { - return; - } - if (isCloudAsrFailureText(text)) { - if (rlog) { - rlog.warn("ai-bridge", "ignoring cloud ASR error text", { event: eventName, text: text.slice(0, 220) }); - } - if (!self.useAsrServiceStt) { - self._scheduleOfflineAudioFallback("cloudAsrErrorText:" + String(eventName || "turnResult"), 1400); - } - return; - } - - // Deduplicate rapid double-emits (some builds fire both local & global). - var now = Date.now(); - if (self._lastSentText === text && now - self._lastSentAt < 2000) { - if (rlog) { - rlog.debug("ai-bridge", "deduped transcript", { event: eventName }); - } - return; - } - self._lastSentText = text; - self._lastSentAt = now; - - if (rlog) { - rlog.info("ai-bridge", "asr text", { event: eventName, text: String(text).slice(0, 160) }); - } - if (self._inFlight) { - return; - } - - return self._sendText(text, "asr:" + String(eventName || "turnResult")); -}; - -AIBridge.prototype._sendText = function (text, source) { - var self = this; - if (!text) { - return Promise.resolve(); - } - if (self._inFlight) { - return Promise.resolve(); - } - - self._inFlight = true; - self._pendingTextFallback = false; - - var t0 = Date.now(); - var url = self.serverBaseUrl.replace(/\/+$/, "") + "/v1/chat/text"; - if (rlog) { - rlog.info("ai-bridge", "sending text (streaming)", { - source: source || "text", - chars: String(text).length, - url: url, - text: String(text).slice(0, 160), - }); - } - - // TTS sentence queue: speak sentences in order, overlapping network + TTS. - // This is how the original Jibo system achieved sub-5-second perceived latency: - // start speaking the first sentence while the LLM is still generating the rest. - var ttsQueue = []; - var ttsRunning = false; - var firstSentenceAt = 0; - - function drainTtsQueue() { - if (ttsRunning) return; - if (ttsQueue.length === 0) return; - ttsRunning = true; - var sentence = ttsQueue.shift(); - if (!firstSentenceAt) { - firstSentenceAt = Date.now(); - if (rlog) { - rlog.info("ai-bridge", "first sentence TTS start", { - ms: firstSentenceAt - t0, - chars: String(sentence).length, - sentence: String(sentence).slice(0, 120), - }); - } - } - self._speak(sentence).then(function () { - ttsRunning = false; - drainTtsQueue(); - }).catch(function () { - ttsRunning = false; - drainTtsQueue(); - }); - } - - function onSentence(sentence) { - if (!sentence || !String(sentence).trim()) return; - if (rlog) { - rlog.info("ai-bridge", "stream sentence received", { - ms: Date.now() - t0, - chars: String(sentence).length, - sentence: String(sentence).slice(0, 120), - queueLen: ttsQueue.length, - }); - } - ttsQueue.push(String(sentence)); - drainTtsQueue(); - } - - return httpStreamingPost(url, { text: text }, onSentence) - .then(function (fullReply) { - if (rlog) { - rlog.info("ai-bridge", "streaming request complete", { - ms: Date.now() - t0, - firstSentenceMs: firstSentenceAt ? firstSentenceAt - t0 : null, - replyChars: String(fullReply).length, - }); - } - - // Wait for remaining TTS queue to finish before releasing in-flight. - return new Promise(function (resolve) { - function waitForTts() { - if (ttsQueue.length === 0 && !ttsRunning) { - resolve({ reply: fullReply || "" }); - } else { - setTimeout(waitForTts, 100); - } - } - waitForTts(); - }); - }) - .catch(function (e) { - self.skill.log && self.skill.log.warn && self.skill.log.warn("AI Bridge text mode error", e && (e.stack || e.message || e)); - if (rlog) { - rlog.warn("ai-bridge", "text mode error", { err: String(e && (e.stack || e.message || e)) }); - } - return { reply: "" }; - }) - .then(function (res) { - // Release in-flight before any follow-up listening. - self._inFlight = false; - var reply = res && res.reply ? String(res.reply) : ""; - if (!reply) return; - if (!self.followupEnabled) return; - if (!self.useAsrServiceStt) return; - if (!endsWithQuestion(reply)) return; - if (self._followupTurnsLeft > 0) return; - self._followupTurnsLeft = 1; - var delay = typeof self.followupDelayMs === "number" ? self.followupDelayMs : 250; - setTimeout(function () { - // Start one follow-up listen. If it fails, just drop it. - self - ._captureTextViaAsrService("followup") - .then(function () { - self._followupTurnsLeft = 0; - }) - .catch(function () { - self._followupTurnsLeft = 0; - }); - }, Math.max(0, delay)); - }); -}; - -exports.InputMode = InputMode; -exports.AIBridge = AIBridge; - -exports.initAIBridge = function initAIBridge(skill, jibo) { - try { - var bridge = new AIBridge(skill, jibo); - bridge.start(); - return bridge; - } catch (e) { - skill && skill.log && skill.log.warn && skill.log.warn("AI Bridge init failed", e); - return null; - } -}; diff --git a/opt/jibo/Jibo/Skills/@be/be/be/postinit.js b/opt/jibo/Jibo/Skills/@be/be/be/postinit.js index 82316e1d..97108b45 100644 --- a/opt/jibo/Jibo/Skills/@be/be/be/postinit.js +++ b/opt/jibo/Jibo/Skills/@be/be/be/postinit.js @@ -74,17 +74,17 @@ exports.postInit = function (err) { this.log.warn('Dynamic skill loader failed (non-fatal):', e.message || e); } - // Optional: AI Bridge (modular; can run models off-robot for now) - try { - if (rlog && typeof rlog.raw === 'function') { - rlog.raw('[BE] initializing AI bridge'); - } - require('./ai-bridge').initAIBridge(this, jibo); - this.log.info('AI bridge initialized'); - if (rlog && typeof rlog.raw === 'function') { - rlog.raw('[BE] AI bridge initialized'); - } - } +// Optional: AI Bridge (DEPRICALED) +// try { +// if (rlog && typeof rlog.raw === 'function') { +// rlog.raw('[BE] initializing AI bridge'); +// } +// require('./ai-bridge').initAIBridge(this, jibo); +// this.log.info('AI bridge initialized'); +// if (rlog && typeof rlog.raw === 'function') { +// rlog.raw('[BE] AI bridge initialized'); +// } +// } catch (e) { this.log.warn('AI bridge init failed (non-fatal):', e.message || e); try { @@ -99,23 +99,23 @@ exports.postInit = function (err) { } } // Optional: rosbridge connector (connects to external rosbridge websocket) - try { - try { - var rosbridge = require('./rosbridge'); - if (rosbridge && typeof rosbridge.init === 'function') { - rosbridge.init(this, jibo); - if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE] rosbridge connector initialized'); - if (rlog && typeof rlog.info === 'function') rlog.info('be', 'rosbridge connector initialized'); - else this.log.info('rosbridge connector initialized'); - } - } catch (e) { - if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE] rosbridge module not present or failed to init (ok)'); - else this.log.info('rosbridge module not present or failed to init (ok)'); - } - } catch (e) { - if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE] rosbridge init failed: ' + String(e && (e.stack || e.message || e))); - this.log.warn('rosbridge init failed (non-fatal):', e.message || e); - } + //try { +// try { +// var rosbridge = require('./rosbridge'); +// if (rosbridge && typeof rosbridge.init === 'function') { +// rosbridge.init(this, jibo); +// if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE] rosbridge connector initialized'); +// if (rlog && typeof rlog.info === 'function') rlog.info('be', 'rosbridge connector initialized'); +// else this.log.info('rosbridge connector initialized'); +// } +// } catch (e) { +// if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE] rosbridge module not present or failed to init (ok)'); +// else this.log.info('rosbridge module not present or failed to init (ok)'); +// } +// } catch (e) { +// if (rlog && typeof rlog.raw === 'function') rlog.raw('[BE] rosbridge init failed: ' + String(e && (e.stack || e.message || e))); +// this.log.warn('rosbridge init failed (non-fatal):', e.message || e); +// } jibo.face.views.changeView({ removeAll: true, leaveEmpty: true }, () => { this.selectFirstSkill(this.launchFirstSkill.bind(this)); diff --git a/opt/jibo/Jibo/Skills/@be/be/be/rosbridge.js b/opt/jibo/Jibo/Skills/@be/be/be/rosbridge.js deleted file mode 100644 index 6863fd31..00000000 --- a/opt/jibo/Jibo/Skills/@be/be/be/rosbridge.js +++ /dev/null @@ -1,463 +0,0 @@ -"use strict"; -// Simple ROS bridge client for Jibo BE -// - Connects to a rosbridge websocket and subscribes to /jibo_remote -// - Handles do_enter_rosbridge_skill and do_exit_rosbridge_skill commands - -var WebSocket = null; -try { WebSocket = require('ws'); } catch (e) { WebSocket = null; } -var urlLib = require('url'); - -var DEFAULT_WS = process.env.ROSBRIDGE_WS || 'ws://192.168.1.5:9090'; - -var state = { - ws: null, - subId: null, - reconnectTimer: null, - lastProcessed: {}, -}; - -// Robot logger (available on BE runtime) -var rlog = null; -try { - if (typeof global !== 'undefined' && global.__rlog) rlog = global.__rlog; - if (!rlog) rlog = require('./robot-logger'); -} catch (e) { rlog = null; } - -function rlogRaw(s) { - try { if (rlog && typeof rlog.raw === 'function') return rlog.raw(String(s || '')); } catch (e) {} - try { console.log(String(s || '')); } catch (e) {} -} -function rlogInfo(tag, text, data) { - try { - if (rlog && typeof rlog.info === 'function') return rlog.info(String(tag || 'rosbridge'), String(text || ''), data || {}); - } catch (e) {} - try { console.log('[INFO]', String(tag || 'rosbridge'), String(text || ''), data || ''); } catch (e) {} -} -function rlogWarn(tag, text, data) { - try { - if (rlog && typeof rlog.warn === 'function') return rlog.warn(String(tag || 'rosbridge'), String(text || ''), data || {}); - } catch (e) {} - try { console.warn('[WARN]', String(tag || 'rosbridge'), String(text || ''), data || ''); } catch (e) {} -} - -function parseWsUrl(s) { - try { return String(s || '').trim(); } catch (e) { return DEFAULT_WS; } -} - -function getCandidateWsUrls(beRuntime) { - var list = []; - try { - var envUrl = process.env.ROSBRIDGE_WS; - if (envUrl) list.push(parseWsUrl(envUrl)); - } catch (e) {} - try { - var cfg = beRuntime && beRuntime.config && beRuntime.config.rosbridge && beRuntime.config.rosbridge.ws; - if (cfg) list.push(parseWsUrl(cfg)); - } catch (e) {} - - // Common fallbacks - list.push('ws://127.0.0.1:9090'); - list.push(DEFAULT_WS); - - // Attempt gateway-derived host if available - try { - var os = require('os'); - var ifaces = os.networkInterfaces(); - Object.keys(ifaces || {}).forEach(function (k) { - (ifaces[k] || []).forEach(function (info) { - if (!info || info.internal || info.family !== 'IPv4') return; - var parts = String(info.address).split('.'); - if (parts.length === 4) { - // guess the gateway as .1 - parts[3] = '1'; - list.push('ws://' + parts.join('.') + ':9090'); - } - }); - }); - } catch (e) {} - - // Deduplicate while keeping order - var seen = {}; - var out = []; - for (var i = 0; i < list.length; i++) { - try { - var v = String(list[i] || '').trim(); - if (!v) continue; - if (!seen[v]) { seen[v] = true; out.push(v); } - } catch (e) {} - } - return out; -} - -function sendWs(obj) { - try { - if (!state.ws || state.ws.readyState !== 1) { - rlogWarn('rosbridge', 'ws not open, drop send', { obj: obj }); - return; - } - var payload = JSON.stringify(obj); - rlogInfo('rosbridge', 'ws.send', { payload: obj }); - state.ws.send(payload); - } catch (e) { /* ignore */ } -} - -function subscribe(topic, type) { - // rosbridge subscribe message - var id = 'sub_' + Date.now() + '_' + Math.floor(Math.random() * 1000); - state.subId = id; - rlogInfo('rosbridge', 'subscribe', { id: id, topic: topic, type: type }); - sendWs({ op: 'subscribe', id: id, type: type || '', topic: topic }); -} - -function unsubscribe() { - if (!state.subId) return; - rlogInfo('rosbridge', 'unsubscribe', { id: state.subId }); - sendWs({ op: 'unsubscribe', id: state.subId }); - state.subId = null; -} - -function connect(wsUrl, onMessage) { - var url = parseWsUrl(wsUrl || DEFAULT_WS); - rlogInfo('rosbridge', 'connect attempt', { url: url }); - try { - if (WebSocket) { - state.ws = new WebSocket(url); - } else { - // try built-in if ws not present (not ideal) - var Ws = require('websocket').w3cwebsocket; - state.ws = new Ws(url); - } - } catch (e) { - rlogWarn('rosbridge', 'connect failed', { err: String(e) }); - scheduleReconnect(wsUrl, onMessage); - return; - } - - state.ws.onopen = function () { - rlogInfo('rosbridge', 'ws open'); - try { - subscribe('/jibo_remote', '/jibo_msgs/JiboRemote'); - } catch (e) { rlogWarn('rosbridge', 'subscribe failed on open /jibo_remote', { err: String(e) }); } - try { - subscribe('/jibo', '/jibo_msgs/JiboAction'); - } catch (e) { rlogWarn('rosbridge', 'subscribe failed on open /jibo', { err: String(e) }); } - }; - - state.ws.onmessage = function (evt) { - rlogRaw('[rosbridge] raw message: ' + (evt && evt.data ? String(evt.data) : '')); - var data = null; - try { data = JSON.parse(evt.data); } catch (e) { rlogWarn('rosbridge', 'json parse failed', { err: String(e), raw: String(evt && evt.data) }); return; } - // rosbridge wraps messages with { op: 'publish', topic: '...', msg: {...} } - if (data && data.op === 'publish') { - try { - var topic = data.topic || 'unknown'; - // Throttle frequent messages per-topic to avoid blocking the BE event loop. - var minMs = parseInt(process.env.ROSBRIDGE_MIN_INTERVAL_MS || '200', 10) || 200; - var now = Date.now(); - var last = state.lastProcessed[topic] || 0; - if (now - last < minMs) { - rlogInfo('rosbridge', 'throttled publish', { topic: topic, droppedMs: now - last, minMs: minMs }); - return; - } - state.lastProcessed[topic] = now; - - // Defer handling so heavy work doesn't block the socket message parser. - var handler = function () { - try { - rlogInfo('rosbridge', 'publish received', { topic: data.topic, msg: data.msg }); - if (data.msg) onMessage && onMessage(data.msg, data.topic); - } catch (e) { rlogWarn('rosbridge', 'publish handler error', { err: String(e) }); } - }; - if (typeof setImmediate === 'function') setImmediate(handler); else setTimeout(handler, 0); - } catch (e) { - rlogWarn('rosbridge', 'error handling publish', { err: String(e) }); - } - return; - } - rlogInfo('rosbridge', 'ws message', { op: data && data.op, data: data }); - }; - - state.ws.onclose = function (ev) { rlogWarn('rosbridge', 'ws closed', { code: ev && ev.code, reason: ev && ev.reason }); scheduleReconnect(wsUrl, onMessage); }; - state.ws.onerror = function (err) { rlogWarn('rosbridge', 'ws error', { err: String(err) }); }; -} - -function scheduleReconnect(wsUrl, onMessage) { - if (state.reconnectTimer) return; - rlogInfo('rosbridge', 'scheduling reconnect', { delayMs: 5000 }); - state.reconnectTimer = setTimeout(function () { - state.reconnectTimer = null; - rlogInfo('rosbridge', 'reconnecting now'); - connect(wsUrl, onMessage); - }, 5000); -} - -// Try a list of candidate URLs sequentially until one connects. -function connectToCandidates(beRuntime, onMessage) { - var candidates = getCandidateWsUrls(beRuntime); - var idx = 0; - - function tryNext() { - if (state.ws && state.ws.readyState === 1) return; // already connected - if (idx >= candidates.length) { - rlogWarn('rosbridge', 'no candidates left, will schedule reconnect'); - scheduleReconnect(candidates[0], onMessage); - return; - } - var url = candidates[idx++]; - rlogInfo('rosbridge', 'trying candidate', { url: url }); - - // attempt connect and use a short timeout to move to next candidate - var tried = false; - var timeout = setTimeout(function () { - if (tried) return; - tried = true; - try { if (state.ws) state.ws.close(); } catch (e) {} - rlogWarn('rosbridge', 'candidate timeout, trying next', { url: url }); - // small delay before next - setTimeout(tryNext, 250); - }, 3500); - - try { - // reuse existing connect path but attach temporary handlers - var prevOnOpen = state.ws && state.ws.onopen; - connect(url, function (msg, topic) { - clearTimeout(timeout); - onMessage && onMessage(msg, topic); - }); - // when open, cancel other attempts - (function (u) { - var wsInst = state.ws; - if (!wsInst) return; - var origOnOpen = wsInst.onopen; - wsInst.onopen = function (ev) { - clearTimeout(timeout); - rlogInfo('rosbridge', 'connected candidate', { url: u }); - try { if (typeof origOnOpen === 'function') origOnOpen.call(wsInst, ev); } catch (e) {} - }; - // if it closes or errors before open, try next - var origOnClose = wsInst.onclose; - wsInst.onclose = function (ev) { - clearTimeout(timeout); - if (!tried) { - tried = true; - rlogWarn('rosbridge', 'candidate closed before ready, next', { url: u }); - setTimeout(tryNext, 250); - } - try { if (typeof origOnClose === 'function') origOnClose.call(wsInst, ev); } catch (e) {} - }; - var origOnError = wsInst.onerror; - wsInst.onerror = function (err) { - clearTimeout(timeout); - if (!tried) { - tried = true; - rlogWarn('rosbridge', 'candidate error, next', { url: u, err: String(err) }); - try { if (wsInst) wsInst.close(); } catch (e) {} - setTimeout(tryNext, 250); - } - try { if (typeof origOnError === 'function') origOnError.call(wsInst, err); } catch (e) {} - }; - })(url); - } catch (e) { - clearTimeout(timeout); - rlogWarn('rosbridge', 'connect threw, trying next', { url: url, err: String(e) }); - setTimeout(tryNext, 250); - } - } - - tryNext(); -} - -function close() { - rlogInfo('rosbridge', 'close requested'); - try { unsubscribe(); } catch (e) { rlogWarn('rosbridge', 'unsubscribe failed', { err: String(e) }); } - try { if (state.ws) state.ws.close(); } catch (e) { rlogWarn('rosbridge', 'ws close failed', { err: String(e) }); } - state.ws = null; - if (state.reconnectTimer) { clearTimeout(state.reconnectTimer); state.reconnectTimer = null; } -} - -exports.init = function (beRuntime, jibo) { - // Prefer robot-logger when available - var rlog = null; - try { - if (beRuntime && beRuntime.rlog) rlog = beRuntime.rlog; - if (!rlog && typeof global !== 'undefined' && global.__rlog) rlog = global.__rlog; - if (!rlog) rlog = require('./robot-logger'); - } catch (e) { - rlog = null; - } - var log = rlog || (beRuntime && beRuntime.log) || console; - var wsUrl = (beRuntime && beRuntime.config && beRuntime.config.rosbridge && beRuntime.config.rosbridge.ws) || DEFAULT_WS; - - function logInfo(text, data) { - try { - if (rlog && typeof rlog.info === 'function') return rlog.info('rosbridge', String(text || ''), data || {}); - if (log && typeof log.info === 'function') return log.info(String(text || '')); - console.log(String(text || '')); - } catch (e) { /* ignore */ } - } - function logWarn(text, data) { - try { - if (rlog && typeof rlog.warn === 'function') return rlog.warn('rosbridge', String(text || ''), data || {}); - if (log && typeof log.warn === 'function') return log.warn(String(text || '')); - console.warn(String(text || '')); - } catch (e) { /* ignore */ } - } - - function handleMsg(msg, topic) { - try { - if (msg.do_enter_rosbridge_skill) { - logInfo('enter request', msg); - // Launch a named skill if provided - var skillName = msg.launch_skill || msg.skill || '@be/main-menu'; - try { - // Attempt lifecycle-based redirect for a proper skill switch - var path = require('path'); - var SkillSwitchData = null; - try { - if (typeof global !== 'undefined' && global && global.be && global.be.constructor) { - SkillSwitchData = global.be.constructor.SkillSwitchData; - } - } catch (e) { SkillSwitchData = null; } - function _interop(m) { return (m && (m.__esModule || m.default)) ? (m.default || m) : m; } - var SkillSwitchDataCtor = null; - if (SkillSwitchData) SkillSwitchDataCtor = _interop(SkillSwitchData); - if (!SkillSwitchDataCtor) { - try { SkillSwitchDataCtor = require(path.join(jibo.utils.PathUtils.findRoot(), 'SkillSwitchData')); } catch (e) { try { const Root = require(path.join(jibo.utils.PathUtils.findRoot(), 'index.js')); SkillSwitchDataCtor = (Root && (Root.SkillSwitchData || (Root.default && Root.default.SkillSwitchData))) || undefined; } catch (e2) { SkillSwitchDataCtor = null; } } - } - var skillObj = beRuntime && beRuntime.skills ? beRuntime.skills[skillName] : null; - if (skillObj && SkillSwitchDataCtor) { - var ssd = new (SkillSwitchDataCtor)(skillObj, {}); - try { require('./lifecycle').redirect.call(beRuntime, ssd); logInfo('requested skill redirect', { skill: skillName }); } catch (e) { logWarn('skill redirect failed', { err: String(e), skill: skillName }); } - } else { - logWarn('skill not found or SkillSwitchDataCtor missing', { skill: skillName }); - } - } catch (e) { - logWarn('enter handling failed', { err: String(e) }); - } - } - if (msg.do_exit_rosbridge_skill) { - logInfo('exit request', msg); - try { - // Redirect to idle via lifecycle - var path2 = require('path'); - var SkillSwitchData2 = null; - try { - if (typeof global !== 'undefined' && global && global.be && global.be.constructor) { - SkillSwitchData2 = global.be.constructor.SkillSwitchData; - } - } catch (e) { SkillSwitchData2 = null; } - var SkillSwitchDataCtor2 = SkillSwitchData2 ? _interop(SkillSwitchData2) : null; - if (!SkillSwitchDataCtor2) { - try { SkillSwitchDataCtor2 = require(path2.join(jibo.utils.PathUtils.findRoot(), 'SkillSwitchData')); } catch (e) { try { const Root = require(path2.join(jibo.utils.PathUtils.findRoot(), 'index.js')); SkillSwitchDataCtor2 = (Root && (Root.SkillSwitchData || (Root.default && Root.default.SkillSwitchData))) || undefined; } catch (e2) { SkillSwitchDataCtor2 = null; } } - } - var idleSkill = beRuntime && beRuntime.idle ? beRuntime.idle : null; - if (idleSkill && SkillSwitchDataCtor2) { - var ssd2 = new (SkillSwitchDataCtor2)(idleSkill, {}); - try { require('./lifecycle').redirect.call(beRuntime, ssd2); logInfo('requested redirect to idle'); } catch (e) { logWarn('idle redirect failed', { err: String(e) }); } - } else { - logWarn('idle redirect failed - missing idleSkill or ctor'); - } - } catch (e) { logWarn('exit handling failed', { err: String(e) }); } - } - // Handle /jibo actions (e.g., TTS) - if (topic === '/jibo' || topic === 'jibo') { - try { - if (msg.do_tts || msg.do_tts === true || msg.tts_text) { - var t = msg.tts_text || msg.tts || msg.text || ''; - if (t && t.length) { - logInfo('jibo action TTS', { text: t }); - try { - // Normalize: if payload is JSON object string like '{"text":"..."}', extract. - try { - if (typeof t === 'string' && t.trim().charAt(0) === '{' && t.indexOf('"text"') !== -1) { - var parsed = JSON.parse(t); - if (parsed && parsed.text) t = parsed.text; - } - } catch (e) { /* ignore parse error */ } - - // Detect ESML/SSML-like input and request SSML mode when present - function _isEsml(s) { - try { - if (!s || typeof s !== 'string') return false; - var ls = s.toLowerCase(); - return ls.indexOf('' + s + ''; - } catch (e) { return s; } - } - - var useEsml = _isEsml(t); - if (useEsml && jibo && jibo.tts && typeof jibo.tts.speak === 'function') { - var payload = _ensureSpeakWrapper(String(t)); - jibo.tts.speak(payload, { mode: jibo.tts.TTSMode ? jibo.tts.TTSMode.SSML : undefined }); - } else if (jibo && jibo.tts && typeof jibo.tts.speak === 'function') { - jibo.tts.speak(String(t), { mode: jibo.tts.TTSMode ? jibo.tts.TTSMode.TEXT : undefined }); - } else if (beRuntime && beRuntime.api && typeof beRuntime.api.speak === 'function') { beRuntime.api.speak({ text: String(t), mode: useEsml ? 'ssml' : 'text' }); - } else if (jibo && jibo.api && typeof jibo.api.speak === 'function') { - jibo.api.speak({ text: String(t), mode: useEsml ? 'ssml' : 'text' }); - } else { - logWarn('no speak API available', { t: t }); - } - } catch (e) { logWarn('tts speak failed', { err: String(e) }); } - } - } - } catch (e) { logWarn('failed handling /jibo action', { err: String(e), topic: topic, msg: msg }); } - } - if (msg.tts_text) { - try { - // Normalize and speak similar to /jibo handling - var txt = msg.tts_text; - try { - if (typeof txt === 'string' && txt.trim().charAt(0) === '{' && txt.indexOf('"text"') !== -1) { - var p2 = JSON.parse(txt); - if (p2 && p2.text) txt = p2.text; - } - } catch (e) {} - // Reuse ESML detection logic - function _isEsml2(s) { - try { - if (!s || typeof s !== 'string') return false; - var ls = s.toLowerCase(); - return ls.indexOf('' + s + ''; - } catch (e) { return s; } - } - var useEsml2 = _isEsml2(txt); - if (useEsml2 && jibo && jibo.tts && typeof jibo.tts.speak === 'function') { - var payload2 = _ensureSpeakWrapper2(String(txt)); - jibo.tts.speak(payload2, { mode: jibo.tts.TTSMode ? jibo.tts.TTSMode.SSML : undefined }); - } else if (jibo && jibo.tts && typeof jibo.tts.speak === 'function') { - jibo.tts.speak(String(txt), { mode: jibo.tts.TTSMode ? jibo.tts.TTSMode.TEXT : undefined }); - } else if (beRuntime && beRuntime.api && typeof beRuntime.api.speak === 'function') { - beRuntime.api.speak({ text: String(txt), mode: useEsml2 ? 'ssml' : 'text' }); - } else if (jibo && jibo.api && typeof jibo.api.speak === 'function') { - jibo.api.speak({ text: String(txt), mode: useEsml2 ? 'ssml' : 'text' }); - } else { - logWarn('no speak API available for tts_text', { tts_text: txt }); - } - } catch (e) { logWarn('tts speak failed', { err: String(e) }); } - } - } catch (e) { logWarn('rosbridge handleMsg error', { err: String(e) }); } - } - - connect(wsUrl, handleMsg); - - return { - close: close, - }; -}; - -exports.shutdown = function () { close(); }; diff --git a/opt/jibo/Jibo/Skills/@be/menu-entries.d/.10-test.json.kate-swp b/opt/jibo/Jibo/Skills/@be/menu-entries.d/.10-test.json.kate-swp deleted file mode 100644 index 7590021f..00000000 Binary files a/opt/jibo/Jibo/Skills/@be/menu-entries.d/.10-test.json.kate-swp and /dev/null differ diff --git a/opt/jibo/Jibo/Skills/FunStuffTest/ClockOne/menuEntry.json b/opt/jibo/Jibo/Skills/FunStuffTest/ClockOne/menuEntry.json deleted file mode 100644 index 398d2446..00000000 --- a/opt/jibo/Jibo/Skills/FunStuffTest/ClockOne/menuEntry.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "skill", - "title": "Clock One (launch jibo-tbd)", - "icon": "resources/icons/clock.png", - "color": "blue", - "order": 20, - "skillId": "jibo-tbd" -} diff --git a/opt/jibo/Jibo/Skills/FunStuffTest/FunOne/menuEntry.json b/opt/jibo/Jibo/Skills/FunStuffTest/FunOne/menuEntry.json deleted file mode 100644 index 342db2fd..00000000 --- a/opt/jibo/Jibo/Skills/FunStuffTest/FunOne/menuEntry.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "skill", - "title": "Fun One (launch jibo-tbd)", - "icon": "resources/icons/fun-stuff.png", - "color": "orange", - "order": 10, - "skillId": "jibo-tbd" -} diff --git a/opt/jibo/Jibo/Skills/FunStuffTest/menuEntry.json b/opt/jibo/Jibo/Skills/FunStuffTest/menuEntry.json deleted file mode 100644 index 51647f55..00000000 --- a/opt/jibo/Jibo/Skills/FunStuffTest/menuEntry.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "hidden": true, - "title": "FunStuffTest Root" -}