2026-04-11 07:12:57 -05:00
|
|
|
using System.Text.Json;
|
|
|
|
|
using Jibo.Cloud.Application.Abstractions;
|
|
|
|
|
using Jibo.Cloud.Domain.Models;
|
|
|
|
|
|
|
|
|
|
namespace Jibo.Cloud.Application.Services;
|
|
|
|
|
|
|
|
|
|
public sealed class JiboWebSocketService(
|
|
|
|
|
ICloudStateStore stateStore,
|
2026-04-12 09:00:17 -05:00
|
|
|
IWebSocketTelemetrySink telemetrySink,
|
2026-04-11 22:11:08 -05:00
|
|
|
WebSocketTurnFinalizationService turnFinalizationService)
|
2026-04-11 07:12:57 -05:00
|
|
|
{
|
2026-04-12 09:00:17 -05:00
|
|
|
public CloudSession GetOrCreateSession(WebSocketMessageEnvelope envelope)
|
|
|
|
|
{
|
|
|
|
|
return stateStore.FindSessionByToken(envelope.Token ?? string.Empty) ??
|
|
|
|
|
stateStore.OpenSession(envelope.Kind, null, envelope.Token, envelope.HostName, envelope.Path);
|
|
|
|
|
}
|
|
|
|
|
|
2026-05-17 08:08:11 -05:00
|
|
|
public async Task<IReadOnlyList<WebSocketReply>> HandleMessageAsync(WebSocketMessageEnvelope envelope,
|
|
|
|
|
CancellationToken cancellationToken = default)
|
2026-04-11 07:12:57 -05:00
|
|
|
{
|
2026-04-12 09:00:17 -05:00
|
|
|
var session = GetOrCreateSession(envelope);
|
2026-04-11 21:50:26 -05:00
|
|
|
session.LastSeenUtc = DateTimeOffset.UtcNow;
|
2026-04-11 07:12:57 -05:00
|
|
|
|
|
|
|
|
if (envelope.IsBinary)
|
|
|
|
|
{
|
2026-04-15 14:33:43 -05:00
|
|
|
var replies = await turnFinalizationService.HandleBinaryAudioAsync(session, envelope, cancellationToken);
|
2026-05-17 08:08:11 -05:00
|
|
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "binary_audio_received",
|
|
|
|
|
new Dictionary<string, object?>
|
|
|
|
|
{
|
|
|
|
|
["bytes"] = envelope.Binary?.Length ?? 0,
|
|
|
|
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
|
|
|
|
}, cancellationToken);
|
2026-04-12 09:00:17 -05:00
|
|
|
return replies;
|
2026-04-11 07:12:57 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var parsedType = ReadMessageType(envelope.Text);
|
2026-04-11 21:50:26 -05:00
|
|
|
session.LastMessageType = parsedType;
|
2026-04-29 09:00:04 -05:00
|
|
|
var containsInlineTurnPayload = parsedType == "LISTEN" && ContainsInlineTurnPayload(envelope.Text);
|
2026-05-07 06:24:30 -05:00
|
|
|
var staleListenRecovered = false;
|
|
|
|
|
var staleListenAgeMs = 0;
|
2026-04-29 09:00:04 -05:00
|
|
|
if (parsedType == "LISTEN" &&
|
|
|
|
|
!containsInlineTurnPayload &&
|
|
|
|
|
WebSocketTurnFinalizationService.ShouldIgnoreLateListenSetup(session, envelope.Text))
|
|
|
|
|
{
|
2026-05-06 20:10:31 -05:00
|
|
|
var (lateTransId, lateRules) = ResolveLateListenNoInputPayload(session, envelope.Text);
|
|
|
|
|
var replies = ResponsePlanToSocketMessagesMapper
|
|
|
|
|
.MapNoInputAndRedirectToSkill(lateTransId, lateRules, "@be/idle")
|
|
|
|
|
.Select(map => new WebSocketReply
|
|
|
|
|
{
|
|
|
|
|
Text = map.Text,
|
|
|
|
|
DelayMs = map.DelayMs
|
|
|
|
|
})
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
2026-05-17 08:08:11 -05:00
|
|
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "late_listen_ignored",
|
|
|
|
|
new Dictionary<string, object?>
|
|
|
|
|
{
|
|
|
|
|
["messageType"] = parsedType,
|
|
|
|
|
["activeTransID"] = session.TurnState.TransId,
|
|
|
|
|
["ignoredTransID"] = lateTransId,
|
|
|
|
|
["replyCount"] = replies.Length
|
|
|
|
|
}, cancellationToken);
|
2026-05-06 20:10:31 -05:00
|
|
|
return replies;
|
2026-04-29 09:00:04 -05:00
|
|
|
}
|
|
|
|
|
|
2026-05-07 06:24:30 -05:00
|
|
|
if (parsedType == "LISTEN" &&
|
|
|
|
|
!containsInlineTurnPayload &&
|
|
|
|
|
WebSocketTurnFinalizationService.TryRecoverStalePendingListen(session, out staleListenAgeMs))
|
|
|
|
|
{
|
|
|
|
|
staleListenRecovered = true;
|
2026-05-17 08:08:11 -05:00
|
|
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "glsm_stale_listen_recovered",
|
|
|
|
|
new Dictionary<string, object?>
|
|
|
|
|
{
|
|
|
|
|
["staleAgeMs"] = staleListenAgeMs,
|
|
|
|
|
["transID"] = session.TurnState.TransId,
|
|
|
|
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
|
|
|
|
}, cancellationToken);
|
2026-05-07 06:24:30 -05:00
|
|
|
}
|
|
|
|
|
|
2026-04-26 20:57:08 -05:00
|
|
|
WebSocketTurnFinalizationService.ObserveIncomingMessage(session, envelope.Text);
|
2026-04-11 21:50:26 -05:00
|
|
|
|
2026-04-26 20:57:08 -05:00
|
|
|
switch (parsedType)
|
2026-04-11 21:50:26 -05:00
|
|
|
{
|
2026-04-26 20:57:08 -05:00
|
|
|
case "CONTEXT":
|
2026-04-12 09:00:17 -05:00
|
|
|
{
|
2026-04-26 20:57:08 -05:00
|
|
|
var replies = await turnFinalizationService.HandleContextAsync(session, envelope, cancellationToken);
|
2026-05-17 08:08:11 -05:00
|
|
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "context_received",
|
|
|
|
|
new Dictionary<string, object?>
|
|
|
|
|
{
|
|
|
|
|
["transID"] = session.TurnState.TransId,
|
|
|
|
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
|
|
|
|
}, cancellationToken);
|
2026-04-26 20:57:08 -05:00
|
|
|
return replies;
|
|
|
|
|
}
|
|
|
|
|
case "LISTEN":
|
2026-04-19 07:07:54 -05:00
|
|
|
{
|
2026-04-29 09:00:04 -05:00
|
|
|
var replies = containsInlineTurnPayload
|
2026-04-26 20:57:08 -05:00
|
|
|
? await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken)
|
|
|
|
|
: WebSocketTurnFinalizationService.HandleListenSetup(session, envelope);
|
2026-05-17 08:08:11 -05:00
|
|
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed",
|
|
|
|
|
new Dictionary<string, object?>
|
|
|
|
|
{
|
|
|
|
|
["messageType"] = parsedType,
|
|
|
|
|
["replyCount"] = replies.Count,
|
|
|
|
|
["transcript"] = session.LastTranscript,
|
|
|
|
|
["intent"] = session.LastIntent,
|
|
|
|
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session),
|
|
|
|
|
["staleListenRecovered"] = staleListenRecovered,
|
|
|
|
|
["staleListenAgeMs"] = staleListenAgeMs
|
|
|
|
|
}, cancellationToken);
|
2026-04-26 20:57:08 -05:00
|
|
|
return replies;
|
|
|
|
|
}
|
2026-05-09 23:46:00 -05:00
|
|
|
case "CLIENT_NLU" or "CLIENT_ASR" or "TRIGGER":
|
2026-04-12 09:00:17 -05:00
|
|
|
{
|
2026-05-17 08:08:11 -05:00
|
|
|
var replies =
|
|
|
|
|
await turnFinalizationService.HandleTurnAsync(session, envelope, parsedType, cancellationToken);
|
|
|
|
|
await telemetrySink.RecordTurnEventAsync(envelope, session, "turn_processed",
|
|
|
|
|
new Dictionary<string, object?>
|
|
|
|
|
{
|
|
|
|
|
["messageType"] = parsedType,
|
|
|
|
|
["replyCount"] = replies.Count,
|
|
|
|
|
["transcript"] = session.LastTranscript,
|
|
|
|
|
["intent"] = session.LastIntent,
|
|
|
|
|
["glsmPhase"] = WebSocketTurnFinalizationService.ResolveGlsmPhase(session)
|
|
|
|
|
}, cancellationToken);
|
2026-04-26 20:57:08 -05:00
|
|
|
return replies;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
return [];
|
2026-04-11 07:12:57 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string ReadMessageType(string? text)
|
|
|
|
|
{
|
2026-05-17 08:08:11 -05:00
|
|
|
if (string.IsNullOrWhiteSpace(text)) return "UNKNOWN";
|
2026-04-11 07:12:57 -05:00
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
using var document = JsonDocument.Parse(text);
|
|
|
|
|
if (document.RootElement.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String)
|
|
|
|
|
return type.GetString() ?? "UNKNOWN";
|
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
|
|
|
|
return "TEXT";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return "UNKNOWN";
|
|
|
|
|
}
|
2026-04-19 07:07:54 -05:00
|
|
|
|
|
|
|
|
private static bool ContainsInlineTurnPayload(string? text)
|
|
|
|
|
{
|
2026-05-17 08:08:11 -05:00
|
|
|
if (string.IsNullOrWhiteSpace(text)) return false;
|
2026-04-19 07:07:54 -05:00
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
using var document = JsonDocument.Parse(text);
|
2026-05-17 08:08:11 -05:00
|
|
|
if (!document.RootElement.TryGetProperty("data", out var data) ||
|
|
|
|
|
data.ValueKind != JsonValueKind.Object) return false;
|
2026-04-19 07:07:54 -05:00
|
|
|
|
|
|
|
|
if (data.TryGetProperty("text", out var transcript) &&
|
|
|
|
|
transcript.ValueKind == JsonValueKind.String &&
|
|
|
|
|
!string.IsNullOrWhiteSpace(transcript.GetString()))
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
return data.TryGetProperty("asr", out var asr) &&
|
|
|
|
|
asr.ValueKind == JsonValueKind.Object &&
|
|
|
|
|
asr.TryGetProperty("text", out var asrText) &&
|
|
|
|
|
asrText.ValueKind == JsonValueKind.String &&
|
|
|
|
|
!string.IsNullOrWhiteSpace(asrText.GetString());
|
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-05-06 20:10:31 -05:00
|
|
|
|
|
|
|
|
private static (string TransId, IReadOnlyList<string> Rules) ResolveLateListenNoInputPayload(
|
|
|
|
|
CloudSession session,
|
|
|
|
|
string? text)
|
|
|
|
|
{
|
|
|
|
|
var transId = session.TurnState.TransId ?? session.LastTransId ?? string.Empty;
|
|
|
|
|
var rules = session.TurnState.ListenRules;
|
|
|
|
|
|
2026-05-17 08:08:11 -05:00
|
|
|
if (string.IsNullOrWhiteSpace(text)) return (transId, rules);
|
2026-05-06 20:10:31 -05:00
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
using var document = JsonDocument.Parse(text);
|
|
|
|
|
var root = document.RootElement;
|
|
|
|
|
|
|
|
|
|
if (root.TryGetProperty("transID", out var transIdValue) &&
|
|
|
|
|
transIdValue.ValueKind == JsonValueKind.String &&
|
|
|
|
|
!string.IsNullOrWhiteSpace(transIdValue.GetString()))
|
|
|
|
|
transId = transIdValue.GetString()!;
|
|
|
|
|
|
|
|
|
|
if (root.TryGetProperty("data", out var data) &&
|
|
|
|
|
data.ValueKind == JsonValueKind.Object &&
|
|
|
|
|
data.TryGetProperty("rules", out var ruleValues) &&
|
|
|
|
|
ruleValues.ValueKind == JsonValueKind.Array)
|
|
|
|
|
{
|
|
|
|
|
var parsedRules = ruleValues.EnumerateArray()
|
|
|
|
|
.Where(static item => item.ValueKind == JsonValueKind.String)
|
|
|
|
|
.Select(static item => item.GetString() ?? string.Empty)
|
|
|
|
|
.Where(static rule => !string.IsNullOrWhiteSpace(rule))
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
2026-05-17 08:08:11 -05:00
|
|
|
if (parsedRules.Length > 0) rules = parsedRules;
|
2026-05-06 20:10:31 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
|
|
|
|
// Best effort parsing for late-listen cleanup.
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return (transId, rules);
|
|
|
|
|
}
|
2026-05-17 08:08:11 -05:00
|
|
|
}
|