Mirrors/JiboExperiments
1
1
Fork 0
mirror of https://kevinblog.sytes.net/Code/Jibo-Revival-Group/JiboExperiments.git synced 2026-06-19 02:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

more fixes for wod

Browse Source
This commit is contained in:
Jacob Dubin
2026-04-19 07:34:54 -05:00
parent 35ab65f764
commit 17583e3cdc
16 changed files with 5948 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
OpenJibo/docs/development-plan.md
View File

@@ -107,6 +107,7 @@ Evidence from the smaller `2026-04-18/19` hotphrase and word-of-the-day verifica
- the `jibo test 3` bundle confirms Nimbus rejects `REDIRECT` in that cloud-skill slot, so the better next experiment is to hint the on-robot target skill directly on the synthetic `LISTEN` result and skip Nimbus `SKILL_ACTION` entirely for word-of-the-day launch
- the same bundle also shows `word-of-the-day/right_word` cleanup turns need a short ignore window for trailing audio or the robot can stay stuck in a blue-ring listening state
- the `jibo test 4` bundle exposed a broader websocket issue: inbound robot `LISTEN` setup packets were still being routed through turn finalization instead of just priming pending state, which can corrupt menu and word-of-the-day flows by treating setup turns like resolved intents
- the `jibo test 5` bundle suggests the remaining WOD launch and post-win cleanup bugs share the same root cause: we were leaving the robot-side `cloudSkillResponse` promise unresolved on `word_of_the_day`, `word_of_the_day_guess`, and `word-of-the-day/right_word`, so the latest .NET pass now emits a completion-only silent `SKILL_ACTION` for those paths instead of stopping at `LISTEN` + `EOS` or going fully silent
- the local buffered-audio seam is still producing repeated `whisper.cpp returned no transcript` and `ffmpeg ... Codec not found` failures, so lightweight waveform or energy screening is worth considering once the core launch flow is stable
Near-term interaction work should now prioritize:

9
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.cs
View File

@@ -231,7 +231,8 @@ public sealed class JiboInteractionService(
SkillPayload: new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["destination"] = "word-of-the-day",
["skillId"] = "@be/word-of-the-day"
["skillId"] = "@be/word-of-the-day",
["cloudResponseMode"] = "completion_only"
});
}
@@ -249,10 +250,12 @@ public sealed class JiboInteractionService(
return new JiboInteractionDecision(
"word_of_the_day_guess",
reply,
null,
"@be/word-of-the-day",
new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["guess"] = guess
["guess"] = guess,
["skillId"] = "@be/word-of-the-day",
["cloudResponseMode"] = "completion_only"
});
}

59
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/ResponsePlanToSocketMessagesMapper.cs
View File

@@ -175,6 +175,14 @@ public sealed class ResponsePlanToSocketMessagesMapper
];
}
public static IReadOnlyList<SocketReplyPlan> MapCompletionOnly(string transId, string skillId, int delayMs = 0)
{
return
[
new SocketReplyPlan(JsonSerializer.Serialize(BuildCompletionOnlySkillPayload(transId, skillId)), delayMs)
];
}
private static IReadOnlyList<string> ReadRules(TurnContext turn, string? messageType)
{
var attributeName = string.Equals(messageType, "CLIENT_NLU", StringComparison.OrdinalIgnoreCase)
@@ -340,6 +348,13 @@ public sealed class ResponsePlanToSocketMessagesMapper
private static object BuildSkillPayload(ResponsePlan plan, TurnContext turn, string transId, SpeakAction speak, InvokeNativeSkillAction? skill)
{
var skillPayload = skill?.Payload;
if (string.Equals(ReadPayloadString(skillPayload, "cloudResponseMode"), "completion_only", StringComparison.OrdinalIgnoreCase))
{
return BuildCompletionOnlySkillPayload(
transId,
ReadPayloadString(skillPayload, "skillId") ?? skill?.SkillName ?? "chitchat-skill");
}
var isJoke = string.Equals(plan.IntentName, "joke", StringComparison.OrdinalIgnoreCase) ||
string.Equals(skill?.SkillName, "@be/joke", StringComparison.OrdinalIgnoreCase);
var isDance = string.Equals(plan.IntentName, "dance", StringComparison.OrdinalIgnoreCase);
@@ -439,6 +454,50 @@ public sealed class ResponsePlanToSocketMessagesMapper
};
}
private static object BuildCompletionOnlySkillPayload(string transId, string skillId)
{
return new
{
type = "SKILL_ACTION",
ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
msgID = CreateHubMessageId(),
transID = transId,
data = new
{
skill = new
{
id = skillId
},
action = new
{
config = new
{
jcp = new
{
type = "SLIM",
config = new
{
play = new
{
esml = "<speak><break time='1ms'/></speak>",
meta = new
{
prompt_id = "RUNTIME_PROMPT",
prompt_sub_category = "AN",
mim_id = "runtime-silent-complete",
mim_type = "announcement"
}
}
}
}
}
},
analytics = new Dictionary<string, object?>(),
final = true
}
};
}
private static string EscapeXml(string value)
{
return value

16
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/WebSocketTurnFinalizationService.cs
View File

@@ -140,7 +140,15 @@ public sealed class WebSocketTurnFinalizationService(
session.TurnState.IgnoreAdditionalAudioUntilUtc = DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
session.FollowUpExpiresUtc = null;
ResetBufferedAudio(session);
return [];
return ResponsePlanToSocketMessagesMapper.MapCompletionOnly(
session.TurnState.TransId ?? session.LastTransId ?? string.Empty,
"@be/word-of-the-day")
.Select(map => new WebSocketReply
{
Text = map.Text,
DelayMs = map.DelayMs
})
.ToArray();
}
session.TurnState.AwaitingTurnCompletion = true;
@@ -513,9 +521,9 @@ public sealed class WebSocketTurnFinalizationService(
? null
: DateTimeOffset.UtcNow.Add(WebSocketTurnState.DefaultLateAudioIgnoreWindow);
var emitSkillActions = messageType != "CLIENT_NLU" &&
!string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var emitSkillActions = messageType != "CLIENT_NLU" ||
string.Equals(plan.IntentName, "word_of_the_day", StringComparison.OrdinalIgnoreCase) ||
string.Equals(plan.IntentName, "word_of_the_day_guess", StringComparison.OrdinalIgnoreCase);
var replies = ResponsePlanToSocketMessagesMapper.Map(plan, finalizedTurn, session, emitSkillActions).Select(map => new WebSocketReply
{
Text = map.Text,

4
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs
View File

@@ -146,6 +146,7 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("@be/word-of-the-day", decision.SkillName);
Assert.Equal("word-of-the-day", decision.SkillPayload!["destination"]);
Assert.Equal("@be/word-of-the-day", decision.SkillPayload["skillId"]);
Assert.Equal("completion_only", decision.SkillPayload["cloudResponseMode"]);
}
[Fact]
@@ -166,7 +167,10 @@ public sealed class JiboInteractionServiceTests
Assert.Equal("word_of_the_day_guess", decision.IntentName);
Assert.Equal("I heard pastoral.", decision.ReplyText);
Assert.Equal("@be/word-of-the-day", decision.SkillName);
Assert.Equal("pastoral", decision.SkillPayload!["guess"]);
Assert.Equal("@be/word-of-the-day", decision.SkillPayload["skillId"]);
Assert.Equal("completion_only", decision.SkillPayload["cloudResponseMode"]);
}
private static JiboInteractionService CreateService()

31
OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboWebSocketServiceTests.cs
View File

@@ -394,9 +394,10 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_NLU","transID":"trans-wod-guess","data":{"entities":{"guess":"pastoral"},"intent":"guess","rules":["word-of-the-day/puzzle"]}}"""
});
Assert.Equal(2, replies.Count);
Assert.Equal(3, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
@@ -426,9 +427,10 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-spoken-guess","data":{"text":"pastoral"}}"""
});
Assert.Equal(2, replies.Count);
Assert.Equal(3, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
@@ -458,11 +460,12 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-line-guess","data":{"text":"Two."}}"""
});
Assert.Equal(2, replies.Count);
Assert.Equal(3, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
Assert.Equal("guess", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("pastoral", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("entities").GetProperty("guess").GetString());
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
}
[Fact]
@@ -486,7 +489,7 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_ASR","transID":"trans-wod-launch","data":{"text":"Play word of the day."}}"""
});
Assert.Equal(2, replies.Count);
Assert.Equal(3, replies.Count);
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("loadMenu", listenPayload.RootElement.GetProperty("data").GetProperty("nlu").GetProperty("intent").GetString());
Assert.Equal("Play word of the day.", listenPayload.RootElement.GetProperty("data").GetProperty("asr").GetProperty("text").GetString());
@@ -494,6 +497,7 @@ public sealed class JiboWebSocketServiceTests
Assert.Equal("main-menu/execute_fun_stuff", listenPayload.RootElement.GetProperty("data").GetProperty("match").GetProperty("rule").GetString());
Assert.Equal("@be/word-of-the-day", listenPayload.RootElement.GetProperty("skillID").GetString());
Assert.True(listenPayload.RootElement.GetProperty("onRobot").GetBoolean());
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
var session = _store.FindSessionByToken("hub-wod-launch-token");
Assert.NotNull(session);
@@ -549,9 +553,10 @@ public sealed class JiboWebSocketServiceTests
Binary = new byte[3000]
});
Assert.Equal(2, replies.Count);
Assert.Equal(3, replies.Count);
Assert.Equal("LISTEN", ReadReplyType(replies[0]));
Assert.Equal("EOS", ReadReplyType(replies[1]));
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[2]));
using var listenPayload = JsonDocument.Parse(replies[0].Text!);
Assert.Equal("@be/word-of-the-day", listenPayload.RootElement.GetProperty("skillID").GetString());
@@ -591,7 +596,7 @@ public sealed class JiboWebSocketServiceTests
Text = """{"type":"CLIENT_NLU","transID":"trans-wod-late-empty","data":{"entities":{"guess":"pastoral"},"intent":"guess","rules":["word-of-the-day/puzzle"]}}"""
});
Assert.Equal(2, winReplies.Count);
Assert.Equal(3, winReplies.Count);
var lateReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
@@ -663,6 +668,18 @@ public sealed class JiboWebSocketServiceTests
});
var replies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
Kind = "neo-hub-listen",
Token = "hub-wod-right-word-audio-token",
Text = """{"type":"LISTEN","transID":"trans-wod-right-word-audio","data":{"rules":["word-of-the-day/right_word","globals/gui_nav","globals/mim_repeat","globals/global_commands_launch"]}}"""
});
Assert.Single(replies);
Assert.Equal("SKILL_ACTION", ReadReplyType(replies[0]));
var binaryReplies = await _service.HandleMessageAsync(new WebSocketMessageEnvelope
{
HostName = "neo-hub.jibo.com",
Path = "/listen",
@@ -671,7 +688,7 @@ public sealed class JiboWebSocketServiceTests
Binary = new byte[4096]
});
Assert.Empty(replies);
Assert.Empty(binaryReplies);
var session = _store.FindSessionByToken("hub-wod-right-word-audio-token");
Assert.NotNull(session);