From c36a01b1421b10d973ca7bc87075dd9c48e7d67a Mon Sep 17 00:00:00 2001 From: Jacob Dubin Date: Sat, 23 May 2026 20:49:09 -0500 Subject: [PATCH] Add support voice routes and short-answer STT handling --- OpenJibo/docs/feature-backlog.md | 1 + OpenJibo/docs/release-1.0.19-plan.md | 1 + .../IJiboExperienceContentRepository.cs | 4 + ...JiboInteractionService.DecisionDispatch.cs | 19 +++++ .../JiboInteractionService.IntentRouting.cs | 38 +++++++++ ...InteractionService.PersonalityDecisions.cs | 27 +++++++ ...LocalWhisperCppBufferedAudioSttStrategy.cs | 56 +++++++++++-- ...InMemoryJiboExperienceContentRepository.cs | 17 ++++ .../Content/LegacyMimCatalogImporter.cs | 38 ++++++++- .../Content/LegacyMimCatalogImporterTests.cs | 23 ++++++ .../WebSockets/JiboInteractionServiceTests.cs | 23 ++++++ ...WhisperCppBufferedAudioSttStrategyTests.cs | 78 ++++++++++++++++++- 12 files changed, 318 insertions(+), 7 deletions(-) diff --git a/OpenJibo/docs/feature-backlog.md b/OpenJibo/docs/feature-backlog.md index fa52715..ef3f13f 100644 --- a/OpenJibo/docs/feature-backlog.md +++ b/OpenJibo/docs/feature-backlog.md @@ -1005,3 +1005,4 @@ For `1.0.20` and beyond: 6. Advanced Jibo features such as pizza delivery, Uber/Lyft integration, calendar management, smart home control (Home Assistant), etc. can be added after the conversion process is smooth and stable, with a focus on features that leverage the new cloud capabilities and content personalization enabled by Open Jibo 7. LLM integration for more natural dialog, question answering, and content generation can be explored as a longer-term goal after the core platform is stable and has a growing user base to provide feedback and use cases for LLM-powered features 8. Tiered Jibo brain/orchestration plan from README.md can be implemented in parallel with the above, starting with the simplest cloud features and gradually adding more complex capabilities as the platform matures and user feedback is collected, always preserving his unique charm and original features. +9. Accessibility-first voice parity for menu actions, starting with backup / restore / update and extending to other critical app flows so menu functionality remains available through voice in a later release diff --git a/OpenJibo/docs/release-1.0.19-plan.md b/OpenJibo/docs/release-1.0.19-plan.md index df24496..9c479e6 100644 --- a/OpenJibo/docs/release-1.0.19-plan.md +++ b/OpenJibo/docs/release-1.0.19-plan.md @@ -382,6 +382,7 @@ First completed slice in this personal-report parity track: 9. Provider-backed news expansion and deeper weather parity 10. Capture indexing and retention boundary for group testing, including a lightweight manifest beside raw capture files 11. Binary-safe media storage seam with file and Azure Blob adapters, ready for original/thumbnails follow-up +12. Accessibility voice parity planning for menu-equivalent flows, starting with backup / restore / update voice coverage and broader critical-path accessibility in a later release For slice 1, use the new import ladder above to keep the work grounded in what OpenJibo can already render today versus what needs new scaffolding. For slices 2-5, use Pegasus phrase lists, MIM IDs, and behavior patterns as the source anchor before broadening into OpenJibo-native improvements. diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs index 7230c36..30f8185 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs @@ -40,6 +40,10 @@ public sealed class JiboExperienceCatalog public IReadOnlyList PersonalReportKickOffReplies { get; init; } = []; public IReadOnlyList PersonalReportOutroReplies { get; init; } = []; public IReadOnlyList ReportSkillTemplates { get; init; } = []; + public IReadOnlyList BackupHowReplies { get; init; } = []; + public IReadOnlyList RestoreHowReplies { get; init; } = []; + public IReadOnlyList UpdateNextReplies { get; init; } = []; + public IReadOnlyList UpdateLastReplies { get; init; } = []; public IReadOnlyList WeatherIntroReplies { get; init; } = []; public IReadOnlyList WeatherTomorrowIntroReplies { get; init; } = []; public IReadOnlyList WeatherTodayHighLowReplies { get; init; } = []; diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.DecisionDispatch.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.DecisionDispatch.cs index 713bb4f..75bfe5a 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.DecisionDispatch.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.DecisionDispatch.cs @@ -119,6 +119,25 @@ public sealed partial class JiboInteractionService "day" => BuildClockLaunchDecision("day", "clock", "askForDay", "Showing the day."), "current_location" => BuildCurrentLocationDecision(turn), "cloud_version" => BuildCloudVersionDecision(), + "backup_help" => BuildScriptedSupportDecision( + catalog.BackupHowReplies, + "backup_help", + "cloud backup", + "back up", + "restore"), + "restore_backup" => BuildScriptedSupportDecision( + catalog.RestoreHowReplies, + "restore_backup", + "restore you from a backup", + "restore from a backup"), + "update_next" => BuildScriptedSupportDecision( + catalog.UpdateNextReplies, + "update_next", + "next update"), + "update_last" => BuildScriptedSupportDecision( + catalog.UpdateLastReplies, + "update_last", + "last update"), "radio" => BuildRadioLaunchDecision(), "radio_genre" => BuildRadioGenreLaunchDecision(lowered), "stop" => BuildStopDecision(), diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs index 7476bf7..bd4cba3 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs @@ -1028,6 +1028,44 @@ public sealed partial class JiboInteractionService if (MatchesAny(loweredTranscript, "commute", "traffic", "drive to work", "how long to work")) return "commute"; + if (MatchesAny( + loweredTranscript, + "can i backup my jibo", + "can i back up my jibo", + "how can i backup my jibo", + "how can i back up my jibo", + "how do i backup my jibo", + "how do i back up my jibo", + "can you be backed up", + "how can i store you in the cloud", + "how can i store you online", + "how do i store you in the cloud", + "how do i store you online")) + return "backup_help"; + + if (MatchesAny( + loweredTranscript, + "can i restore you from a backup", + "how can i restore you from a backup", + "how do i restore you from a backup", + "restore you from a backup", + "restore from a backup")) + return "restore_backup"; + + if (MatchesAny( + loweredTranscript, + "when is your next update", + "when is my next update", + "when's your next update", + "when s your next update", + "when was your last update", + "when was my last update", + "when's your last update", + "when s your last update")) + return loweredTranscript.Contains("last update", StringComparison.OrdinalIgnoreCase) + ? "update_last" + : "update_next"; + if (MatchesAny(loweredTranscript, "news", "headlines", "news update", "tell me the news")) return "news"; if (IsWelcomeBackGreeting(loweredTranscript) || diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs index d85a5b9..7de1686 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs @@ -714,6 +714,21 @@ public sealed partial class JiboInteractionService preferredSnippets); } + private JiboInteractionDecision BuildScriptedSupportDecision( + IReadOnlyList replies, + string intentName, + params string[] preferredSnippets) + { + var selected = SelectLegacyReply(replies, preferredSnippets); + if (string.IsNullOrWhiteSpace(selected)) + selected = GetSupportFallbackReply(intentName); + + return new JiboInteractionDecision( + intentName, + selected, + ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates()); + } + private JiboInteractionDecision BuildScriptedHolidayGreetingDecision( JiboExperienceCatalog catalog, string intentName, @@ -758,6 +773,18 @@ public sealed partial class JiboInteractionService return ScriptedResponseDecisionBuilder.SelectLegacyReply(replies, randomizer, preferredSnippets); } + private static string GetSupportFallbackReply(string intentName) + { + return intentName switch + { + "backup_help" => "That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com.", + "restore_backup" => "That sounds a little too complicated for me, I think your best bet is to get some guidance from Jibo Customer Care. Check the Help section of the Jibo App, or go to the website, support dot jibo dot com.", + "update_next" => "That's a good question. I think they've been coming every few weeks.", + "update_last" => "Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates.", + _ => string.Empty + }; + } + private string RenderHolidayTemplate(string template, TurnContext turn, GreetingPresenceProfile presence) { var ownerName = ResolvePreferredGreetingName(turn, presence); diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs index 641e2e9..ec8330f 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs @@ -8,6 +8,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( IExternalProcessRunner processRunner) : ISttStrategy { private const int MinimumBufferedAudioBytes = 64; + private const int ShortAnswerBufferedAudioBytes = 16; public string Name => "local-whispercpp-buffered-audio"; @@ -18,7 +19,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( IsConfiguredPathAvailable(options.WhisperCliPath, true) && IsConfiguredPathAvailable(options.WhisperModelPath, true) && ReadBufferedAudioFrames(turn).Any(ContainsOpusIdentificationHeader) && - !IsBelowNoiseFloor(ReadBufferedAudioBytes(turn)); + !IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn)); } public async Task TranscribeAsync(TurnContext turn, CancellationToken cancellationToken = default) @@ -31,7 +32,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( throw new InvalidOperationException( "Local whisper.cpp STT requires buffered Ogg/Opus audio with an Opus identification header."); - if (IsBelowNoiseFloor(ReadBufferedAudioBytes(turn))) + if (IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn))) throw new InvalidOperationException( "Local whisper.cpp STT rejected buffered audio as too short or noisy for transcription."); @@ -119,9 +120,54 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( : 0; } - private static bool IsBelowNoiseFloor(int bufferedAudioBytes) + private static bool IsBelowNoiseFloor(TurnContext turn, int bufferedAudioBytes) { - return bufferedAudioBytes > 0 && bufferedAudioBytes < MinimumBufferedAudioBytes; + if (bufferedAudioBytes <= 0) return false; + + var minimumBufferedAudioBytes = IsShortAnswerTurn(turn) + ? ShortAnswerBufferedAudioBytes + : MinimumBufferedAudioBytes; + + return bufferedAudioBytes < minimumBufferedAudioBytes; + } + + private static bool IsShortAnswerTurn(TurnContext turn) + { + var rules = ReadRules(turn, "listenRules") + .Concat(ReadRules(turn, "clientRules")) + .Concat(ReadRules(turn, "listenAsrHints")); + + return rules.Any(IsShortAnswerRule); + } + + private static bool IsShortAnswerRule(string rule) + { + return string.Equals(rule, "$YESNO", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "clock/alarm_timer_change", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "clock/alarm_timer_none_set", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "create/is_it_a_keeper", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "settings/download_now_later", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "shared/yes_no", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "surprises-date/offer_date_fact", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "surprises-ota/want_to_download_now", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "word-of-the-day/surprise", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "word-of-the-day/right_word", StringComparison.OrdinalIgnoreCase) || + string.Equals(rule, "word-of-the-day/puzzle", StringComparison.OrdinalIgnoreCase); + } + + private static IEnumerable ReadRules(TurnContext turn, string key) + { + if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return []; + + return value switch + { + IReadOnlyList typed => typed, + IEnumerable enumerable => enumerable, + JsonElement { ValueKind: JsonValueKind.Array } jsonElement => jsonElement.EnumerateArray() + .Where(static item => item.ValueKind == JsonValueKind.String) + .Select(static item => item.GetString() ?? string.Empty), + _ => [] + }; } private static bool ContainsOpusIdentificationHeader(byte[] frame) @@ -167,4 +213,4 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy( return !checkFileExists || File.Exists(path); } -} \ No newline at end of file +} diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs index 45dd475..1af5400 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs @@ -195,6 +195,23 @@ public sealed class InMemoryJiboExperienceContentRepository : IJiboExperienceCon [ "The report-skill templates are loaded and waiting to be rendered." ], + BackupHowReplies = + [ + "That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com." + ], + RestoreHowReplies = + [ + "That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com." + ], + UpdateNextReplies = + [ + "That's a good question. I think they've been coming every few weeks.", + "I never know exactly when my next update is coming, but they do seem to come pretty regularly." + ], + UpdateLastReplies = + [ + "Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates." + ], WeatherIntroReplies = [ "For your weather.", diff --git a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs index 8d38fcb..34d6354 100644 --- a/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs +++ b/OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs @@ -241,6 +241,18 @@ public static class LegacyMimCatalogImporter string.Equals(fileName, "WetNowDryLater", StringComparison.OrdinalIgnoreCase)) return LegacyMimBucket.ReportSkillTemplate; + if (fileName.StartsWith("SUP_GEN_HowBackUpData", StringComparison.OrdinalIgnoreCase)) + return LegacyMimBucket.BackupHow; + + if (fileName.StartsWith("SUP_GEN_HowRestoreBackup", StringComparison.OrdinalIgnoreCase)) + return LegacyMimBucket.RestoreHow; + + if (fileName.StartsWith("SUP_UPDATE_WhenIsNextUpdate", StringComparison.OrdinalIgnoreCase)) + return LegacyMimBucket.UpdateNext; + + if (fileName.StartsWith("SUP_UPDATE_WhenWasLastUpdate", StringComparison.OrdinalIgnoreCase)) + return LegacyMimBucket.UpdateLast; + if (fileName.StartsWith("PersonalReportKickOff", StringComparison.OrdinalIgnoreCase)) return LegacyMimBucket.PersonalReportKickOff; @@ -515,6 +527,10 @@ public static class LegacyMimCatalogImporter HolidayGift, HolidayTracker, BirthdayCelebration, + BackupHow, + RestoreHow, + UpdateNext, + UpdateLast, Jokes, RobotFacts, HumanFacts, @@ -583,6 +599,7 @@ public static class LegacyMimCatalogImporter private readonly List _commuteTransportHurryReplies = []; private readonly List _commuteTransportLateReplies = []; private readonly List _commuteTransportNormalReplies = []; + private readonly List _backupHowReplies = []; private readonly List _emotionReplies = []; private readonly List _fallbacks = []; private readonly List _favoriteAnimalReplies = []; @@ -603,6 +620,9 @@ public static class LegacyMimCatalogImporter private readonly List _newsCategoryIntroReplies = []; private readonly List _newsIntroReplies = []; private readonly List _newsOutroReplies = []; + private readonly List _restoreHowReplies = []; + private readonly List _updateLastReplies = []; + private readonly List _updateNextReplies = []; private readonly List _personalities = []; private readonly List _personalReportKickOffReplies = []; private readonly List _personalReportOutroReplies = []; @@ -681,6 +701,18 @@ public static class LegacyMimCatalogImporter case LegacyMimBucket.BirthdayCelebration: AddDistinct(_birthdayCelebrationReplies, text); return; + case LegacyMimBucket.BackupHow: + AddDistinct(_backupHowReplies, text); + return; + case LegacyMimBucket.RestoreHow: + AddDistinct(_restoreHowReplies, text); + return; + case LegacyMimBucket.UpdateNext: + AddDistinct(_updateNextReplies, text); + return; + case LegacyMimBucket.UpdateLast: + AddDistinct(_updateLastReplies, text); + return; case LegacyMimBucket.Personality: if (_personalities.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase))) return; @@ -835,6 +867,7 @@ public static class LegacyMimCatalogImporter HolidayGiftReplies = [.. _holidayGiftReplies], HolidayTrackerReplies = [.. _holidayTrackerReplies], BirthdayCelebrationReplies = [.. _birthdayCelebrationReplies], + BackupHowReplies = [.. _backupHowReplies], HowAreYouReplies = [.. _howAreYous], EmotionReplies = [.. _emotionReplies], PersonalityReplies = [.. _personalities], @@ -869,7 +902,10 @@ public static class LegacyMimCatalogImporter CommuteServiceDownReplies = [.. _commuteServiceDownReplies], NewsIntroReplies = [.. _newsIntroReplies], NewsCategoryIntroReplies = [.. _newsCategoryIntroReplies], - NewsOutroReplies = [.. _newsOutroReplies] + NewsOutroReplies = [.. _newsOutroReplies], + RestoreHowReplies = [.. _restoreHowReplies], + UpdateNextReplies = [.. _updateNextReplies], + UpdateLastReplies = [.. _updateLastReplies] }; } diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs index 5687b73..6128f93 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/Content/LegacyMimCatalogImporterTests.cs @@ -190,6 +190,29 @@ public sealed class LegacyMimCatalogImporterTests reply.Contains("north Pole", StringComparison.OrdinalIgnoreCase)); } + [Fact] + public void ImportCatalog_ImportsBuildBSupportResponsesIntoDedicatedBuckets() + { + var rootDirectory = Path.Combine( + AppContext.BaseDirectory, + "Content", + "LegacyMims", + "BuildB"); + + var catalog = LegacyMimCatalogImporter.ImportCatalog(rootDirectory); + + Assert.Contains(catalog.BackupHowReplies, reply => + reply.Contains("Help section of the Jibo App", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.RestoreHowReplies, reply => + reply.Contains("Jibo Customer Care", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.UpdateNextReplies, reply => + reply.Contains("coming every few weeks", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.UpdateNextReplies, reply => + reply.Contains("pretty regularly", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.UpdateLastReplies, reply => + reply.Contains("release notes page", StringComparison.OrdinalIgnoreCase)); + } + [Fact] public void ImportCatalog_ImportsBuildBFriendshipResponsesIntoFriendBuckets() { diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs index ab61387..421c881 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/JiboInteractionServiceTests.cs @@ -712,6 +712,29 @@ public sealed class JiboInteractionServiceTests Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); } + [Theory] + [InlineData("can i backup my jibo", "backup_help", "Help section of the Jibo App")] + [InlineData("how can i restore you from a backup", "restore_backup", "Jibo Customer Care")] + [InlineData("when is your next update", "update_next", "coming every few weeks")] + [InlineData("when was your last update", "update_last", "release notes page")] + public async Task BuildDecisionAsync_SupportHelpQuestions_UseImportedReplies( + string transcript, + string expectedIntent, + string expectedReplySnippet) + { + var service = CreateService(); + + var decision = await service.BuildDecisionAsync(new TurnContext + { + RawTranscript = transcript, + NormalizedTranscript = transcript + }); + + Assert.Equal(expectedIntent, decision.IntentName); + Assert.Contains(expectedReplySnippet, decision.ReplyText, StringComparison.OrdinalIgnoreCase); + Assert.Equal("ScriptedResponse", decision.ContextUpdates![ChitchatRouteKey]); + } + [Theory] [InlineData("what do you want to talk about", "robot_want_to_talk_about", "surprise me")] [InlineData("what would you like to talk about", "robot_want_to_talk_about", "surprise me")] diff --git a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs index d4f88af..243edeb 100644 --- a/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs +++ b/OpenJibo/tests/Jibo.Cloud.Tests/WebSockets/LocalWhisperCppBufferedAudioSttStrategyTests.cs @@ -102,6 +102,34 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests Assert.False(strategy.CanHandle(turn)); } + [Theory] + [InlineData("shared/yes_no")] + [InlineData("word-of-the-day/surprise")] + public void CanHandle_ReturnsTrue_WhenShortAnswerTurnsStayUnderTheStandardNoiseFloor(string listenRule) + { + var strategy = new LocalWhisperCppBufferedAudioSttStrategy( + new BufferedAudioSttOptions + { + EnableLocalWhisperCpp = true, + FfmpegPath = "ffmpeg", + WhisperCliPath = "whisper-cli", + WhisperModelPath = "model.bin" + }, + new FakeExternalProcessRunner()); + + var turn = new TurnContext + { + Attributes = new Dictionary + { + ["bufferedAudioBytes"] = 47, + ["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() }, + ["listenRules"] = new[] { listenRule } + } + }; + + Assert.True(strategy.CanHandle(turn)); + } + [Fact] public async Task TranscribeAsync_UsesFfmpegAndWhisperCpp_WhenConfigured() { @@ -148,6 +176,54 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests } } + [Theory] + [InlineData("shared/yes_no")] + [InlineData("word-of-the-day/surprise")] + public async Task TranscribeAsync_HandlesShortAnswerTurnsWithoutHittingTheStandardNoiseFloor(string listenRule) + { + var tempDirectory = Path.Combine(Path.GetTempPath(), $"openjibo-stt-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDirectory); + + try + { + var runner = new FakeExternalProcessRunner("[00:00:00.000 --> 00:00:01.000] yes."); + var strategy = new LocalWhisperCppBufferedAudioSttStrategy( + new BufferedAudioSttOptions + { + EnableLocalWhisperCpp = true, + FfmpegPath = "ffmpeg", + WhisperCliPath = "whisper-cli", + WhisperModelPath = "model.bin", + TempDirectory = tempDirectory + }, + runner); + + var turn = new TurnContext + { + TurnId = listenRule == "shared/yes_no" + ? "turn-short-yes-no" + : "turn-short-word-of-the-day", + Locale = "en-US", + Attributes = new Dictionary + { + ["bufferedAudioBytes"] = 47, + ["bufferedAudioFrames"] = new[] { BuildMinimalOggPage() }, + ["listenRules"] = new[] { listenRule } + } + }; + + var result = await strategy.TranscribeAsync(turn); + + Assert.Equal("yes", result.Text); + Assert.Equal("local-whispercpp-buffered-audio", result.Provider); + Assert.Equal(2, runner.Calls.Count); + } + finally + { + if (Directory.Exists(tempDirectory)) Directory.Delete(tempDirectory, true); + } + } + [Fact] public async Task TranscribeAsync_NormalizesLoosePunctuationFromWhisperOutput() { @@ -275,4 +351,4 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategyTests return Task.FromResult(new ExternalProcessResult(0, string.Empty, string.Empty)); } } -} \ No newline at end of file +}