Mirrors/JiboExperiments
1
1
Fork 0
mirror of https://kevinblog.sytes.net/Code/Jibo-Revival-Group/JiboExperiments.git synced 2026-06-18 20:06:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

more jibo fixes for word of the day and hey jibo

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-18 21:05:23 -05:00
parent 54d0a10175
commit 5dbe16a0e1
16 changed files with 4156 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/DemoConversationBroker.cs
View File

@@ -7,6 +7,7 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
public async Task<ResponsePlan> HandleTurnAsync(TurnContext turn, CancellationToken cancellationToken = default)
{
var decision = await interactionService.BuildDecisionAsync(turn, cancellationToken);
var keepMicOpen = ShouldKeepMicOpen(decision.IntentName);
var plan = new ResponsePlan
{
@@ -24,20 +25,16 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
Sequence = 0,
Text = decision.ReplyText,
Voice = "griffin"
},
new ListenAction
{
Sequence = 1,
Timeout = TimeSpan.FromSeconds(12),
Mode = "follow-up"
}
},
FollowUp = new FollowUpPolicy
{
KeepMicOpen = true,
Timeout = TimeSpan.FromSeconds(12),
ExpectedTopic = "conversation"
},
FollowUp = keepMicOpen
? new FollowUpPolicy
{
KeepMicOpen = true,
Timeout = TimeSpan.FromSeconds(12),
ExpectedTopic = "conversation"
}
: FollowUpPolicy.None,
ProtocolMetadata = new Dictionary<string, object?>
{
["host"] = turn.HostName,
@@ -46,6 +43,16 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
}
};
if (keepMicOpen)
{
plan.Actions.Add(new ListenAction
{
Sequence = 1,
Timeout = TimeSpan.FromSeconds(12),
Mode = "follow-up"
});
}
if (!string.IsNullOrWhiteSpace(decision.SkillName))
{
plan.Actions.Add(new InvokeNativeSkillAction
@@ -58,4 +65,14 @@ public sealed class DemoConversationBroker(JiboInteractionService interactionSer
return plan;
}
private static bool ShouldKeepMicOpen(string? intentName)
{
return intentName switch
{
"word_of_the_day" => false,
"word_of_the_day_guess" => false,
_ => true
};
}
}

3
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -227,8 +227,7 @@ public sealed class JiboInteractionService(
return new JiboInteractionDecision(
"word_of_the_day",
"Starting word of the day.",
"@be/word-of-the-day",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
SkillPayload: new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["destination"] = "word-of-the-day"
});

2
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ProtocolToTurnContextMapper.cs
View File

@@ -26,6 +26,8 @@ public sealed class ProtocolToTurnContextMapper
attributes["context"] = turnState.ContextPayload;
}
attributes["listenHotphrase"] = turnState.ListenHotphrase;
if (turnState.ListenRules.Count > 0)
{
attributes["listenRules"] = turnState.ListenRules;

100
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -235,7 +235,7 @@ public sealed class WebSocketTurnFinalizationService(
if (root.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Object)
{
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
if (data.TryGetProperty("rules", out var rules) && rules.ValueKind == JsonValueKind.Array)
{
turnState.ListenRules = rules.EnumerateArray()
.Select(item => item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString())
@@ -256,6 +256,12 @@ public sealed class WebSocketTurnFinalizationService(
.ToArray();
}
if (data.TryGetProperty("hotphrase", out var hotphrase) &&
(hotphrase.ValueKind == JsonValueKind.True || hotphrase.ValueKind == JsonValueKind.False))
{
turnState.ListenHotphrase = hotphrase.GetBoolean();
}
if (data.TryGetProperty("intent", out var intent) && intent.ValueKind == JsonValueKind.String)
{
session.LastIntent = intent.GetString();
@@ -303,6 +309,7 @@ public sealed class WebSocketTurnFinalizationService(
turnState.AwaitingTurnCompletion = false;
turnState.SawListen = false;
turnState.SawContext = false;
turnState.ListenHotphrase = false;
turnState.ListenRules = [];
turnState.ListenAsrHints = [];
}
@@ -350,6 +357,18 @@ public sealed class WebSocketTurnFinalizationService(
}
var turnState = session.TurnState;
if (ShouldIgnoreCompletedWordOfDayTurn(finalizedTurn))
{
turnState.AwaitingTurnCompletion = false;
ResetBufferedAudio(session);
return [];
}
if (ShouldTreatEmptyHotphraseTurnAsGreeting(finalizedTurn))
{
finalizedTurn = WithSyntheticTranscript(finalizedTurn, "hello");
}
if (ShouldIgnoreLateEmptyTurn(finalizedTurn, session, messageType))
{
turnState.AwaitingTurnCompletion = false;
@@ -413,7 +432,9 @@ public sealed class WebSocketTurnFinalizationService(
: null;
turnState.AwaitingTurnCompletion = false;
var emitSkillActions = messageType != "CLIENT_NLU";
var emitSkillActions = messageType != "CLIENT_NLU" &&
!string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var replies = ResponsePlanToSocketMessagesMapper.Map(plan, finalizedTurn, session, emitSkillActions).Select(map => new WebSocketReply
{
Text = map.Text,
@@ -640,4 +661,79 @@ public sealed class WebSocketTurnFinalizationService(
string.Equals(turnTransId, session.LastTransId, StringComparison.Ordinal) &&
!string.IsNullOrWhiteSpace(session.LastIntent);
}
private static bool ShouldIgnoreCompletedWordOfDayTurn(TurnContext turn)
{
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript))
{
return false;
}
return ReadRules(turn, "listenRules")
.Any(static rule => string.Equals(rule, "word-of-the-day/right_word", StringComparison.OrdinalIgnoreCase));
}
private static bool ShouldTreatEmptyHotphraseTurnAsGreeting(TurnContext turn)
{
if (!string.IsNullOrWhiteSpace(turn.NormalizedTranscript) || !string.IsNullOrWhiteSpace(turn.RawTranscript))
{
return false;
}
if (!ReadBoolAttribute(turn, "listenHotphrase"))
{
return false;
}
return ReadRules(turn, "listenRules")
.Any(static rule => string.Equals(rule, "launch", StringComparison.OrdinalIgnoreCase));
}
private static TurnContext WithSyntheticTranscript(TurnContext turn, string transcript)
{
var attributes = new Dictionary<string, object?>(turn.Attributes, StringComparer.OrdinalIgnoreCase)
{
["syntheticTranscript"] = true
};
return new TurnContext
{
TurnId = turn.TurnId,
SessionId = turn.SessionId,
TimestampUtc = turn.TimestampUtc,
InputMode = turn.InputMode,
SourceKind = turn.SourceKind,
WakePhrase = turn.WakePhrase,
RawTranscript = transcript,
NormalizedTranscript = transcript,
DeviceId = turn.DeviceId,
HostName = turn.HostName,
RequestId = turn.RequestId,
ProtocolService = turn.ProtocolService,
ProtocolOperation = turn.ProtocolOperation,
FirmwareVersion = turn.FirmwareVersion,
ApplicationVersion = turn.ApplicationVersion,
Locale = turn.Locale,
TimeZone = turn.TimeZone,
IsFollowUpEligible = turn.IsFollowUpEligible,
Attributes = attributes
};
}
private static bool ReadBoolAttribute(TurnContext turn, string key)
{
if (!turn.Attributes.TryGetValue(key, out var value) || value is null)
{
return false;
}
return value switch
{
bool boolValue => boolValue,
JsonElement { ValueKind: JsonValueKind.True } => true,
JsonElement { ValueKind: JsonValueKind.False } => false,
_ when bool.TryParse(value.ToString(), out var parsed) => parsed,
_ => false
};
}
}

1
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Domain/Models/WebSocketTurnState.cs
View File

@@ -4,6 +4,7 @@ public sealed class WebSocketTurnState
{
public string? TransId { get; set; }
public string? ContextPayload { get; set; }
public bool ListenHotphrase { get; set; }
public string? AudioTranscriptHint { get; set; }
public string? LastSttError { get; set; }
public DateTimeOffset? LastSttErrorUtc { get; set; }

1
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs
View File

@@ -144,6 +144,7 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("word_of_the_day", decision.IntentName);
Assert.Equal("Starting word of the day.", decision.ReplyText);
Assert.Equal("word-of-the-day", decision.SkillPayload!["destination"]);
Assert.Null(decision.SkillName);
}
[Fact]

70
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -426,7 +426,7 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-spoken-guess","data":{"text":"pastoral"}}"""
});
Assert.Equal(3, replies.Count);
Assert.Equal(2, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
@@ -458,7 +458,7 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-line-guess","data":{"text":"Two."}}"""
});
Assert.Equal(3, replies.Count);
Assert.Equal(2, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("guess", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
@@ -486,11 +486,16 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-launch","data":{"text":"Play word of the day."}}"""
});
Assert.Equal(3, replies.Count);
Assert.Equal(2, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("loadMenu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("Play word of the day.", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("word-of-the-day", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("destination").GetString());
Assert.Equal("main-menu/execute_fun_stuff", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString());
var session = _store.FindSessionByToken("hub-wod-launch-token");
Assert.NotNull(session);
Assert.False(session.FollowUpOpen);
}
[Fact]
@@ -528,6 +533,30 @@ public sealed class JiboWebSocketServiceTests
Assert.Empty(lateReplies);
}
[Fact]
public async Task EmptyClientAsr_AfterWordOfDayRightWordListen_IsIgnored()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-wod-right-word-token",
Text = """{"type":"LISTEN","transID":"trans-wod-right-word","data":{"rules":["word-of-the-day/right_word","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"]}}"""
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-wod-right-word-token",
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-right-word","data":{}}"""
});
Assert.Empty(replies);
}
[Fact]
public async Task BlankAudioHotphraseTurn_IsIgnored()
{
@@ -543,6 +572,41 @@ public sealed class JiboWebSocketServiceTests
Assert.Empty(replies);
}
[Fact]
public async Task EmptyHotphraseTurn_BecomesGreetingAndKeepsFollowUpOpen()
{
await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-empty-hotphrase-token",
Text = """{"type":"LISTEN","transID":"trans-empty-hotphrase","data":{"hotphrase":true,"rules":["launch","globals/global_commands_launch"]}}"""
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-empty-hotphrase-token",
Text = """{"type":"CLIENT_ASR","transID":"trans-empty-hotphrase","data":{}}"""
});
Assert.Equal(3, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("hello", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
var session = _store.FindSessionByToken("hub-empty-hotphrase-token");
Assert.NotNull(session);
Assert.True(session.FollowUpOpen);
}
[Fact]
public async Task BufferedAudio_WithSyntheticTranscriptHint_FinalizesThroughSttSeam()
{