Mirrors/JiboExperiments
1
1
Fork 0
mirror of https://kevinblog.sytes.net/Code/Jibo-Revival-Group/JiboExperiments.git synced 2026-06-17 20:06:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support voice routes and short-answer STT handling

Browse Source
This commit is contained in:
Jacob Dubin
2026-05-23 20:49:09 -05:00
parent b017fd9f60
commit c36a01b142
12 changed files with 318 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Abstractions/IJiboExperienceContentRepository.cs
View File

@@ -40,6 +40,10 @@ public sealed class JiboExperienceCatalog
public IReadOnlyList<string> PersonalReportKickOffReplies { get; init; } = [];
public IReadOnlyList<string> PersonalReportOutroReplies { get; init; } = [];
public IReadOnlyList<string> ReportSkillTemplates { get; init; } = [];
public IReadOnlyList<string> BackupHowReplies { get; init; } = [];
public IReadOnlyList<string> RestoreHowReplies { get; init; } = [];
public IReadOnlyList<string> UpdateNextReplies { get; init; } = [];
public IReadOnlyList<string> UpdateLastReplies { get; init; } = [];
public IReadOnlyList<string> WeatherIntroReplies { get; init; } = [];
public IReadOnlyList<string> WeatherTomorrowIntroReplies { get; init; } = [];
public IReadOnlyList<string> WeatherTodayHighLowReplies { get; init; } = [];

19
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.DecisionDispatch.cs
View File

@@ -119,6 +119,25 @@ public sealed partial class JiboInteractionService
"day" => BuildClockLaunchDecision("day", "clock", "askForDay", "Showing the day."),
"current_location" => BuildCurrentLocationDecision(turn),
"cloud_version" => BuildCloudVersionDecision(),
"backup_help" => BuildScriptedSupportDecision(
catalog.BackupHowReplies,
"backup_help",
"cloud backup",
"back up",
"restore"),
"restore_backup" => BuildScriptedSupportDecision(
catalog.RestoreHowReplies,
"restore_backup",
"restore you from a backup",
"restore from a backup"),
"update_next" => BuildScriptedSupportDecision(
catalog.UpdateNextReplies,
"update_next",
"next update"),
"update_last" => BuildScriptedSupportDecision(
catalog.UpdateLastReplies,
"update_last",
"last update"),
"radio" => BuildRadioLaunchDecision(),
"radio_genre" => BuildRadioGenreLaunchDecision(lowered),
"stop" => BuildStopDecision(),

38
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.IntentRouting.cs
View File

@@ -1028,6 +1028,44 @@ public sealed partial class JiboInteractionService
if (MatchesAny(loweredTranscript, "commute", "traffic", "drive to work", "how long to work")) return "commute";
if (MatchesAny(
loweredTranscript,
"can i backup my jibo",
"can i back up my jibo",
"how can i backup my jibo",
"how can i back up my jibo",
"how do i backup my jibo",
"how do i back up my jibo",
"can you be backed up",
"how can i store you in the cloud",
"how can i store you online",
"how do i store you in the cloud",
"how do i store you online"))
return "backup_help";
if (MatchesAny(
loweredTranscript,
"can i restore you from a backup",
"how can i restore you from a backup",
"how do i restore you from a backup",
"restore you from a backup",
"restore from a backup"))
return "restore_backup";
if (MatchesAny(
loweredTranscript,
"when is your next update",
"when is my next update",
"when's your next update",
"when s your next update",
"when was your last update",
"when was my last update",
"when's your last update",
"when s your last update"))
return loweredTranscript.Contains("last update", StringComparison.OrdinalIgnoreCase)
? "update_last"
: "update_next";
if (MatchesAny(loweredTranscript, "news", "headlines", "news update", "tell me the news")) return "news";
if (IsWelcomeBackGreeting(loweredTranscript) ||

27
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Application/Services/JiboInteractionService.PersonalityDecisions.cs
View File

@@ -714,6 +714,21 @@ public sealed partial class JiboInteractionService
preferredSnippets);
}
private JiboInteractionDecision BuildScriptedSupportDecision(
IReadOnlyList<string> replies,
string intentName,
params string[] preferredSnippets)
{
var selected = SelectLegacyReply(replies, preferredSnippets);
if (string.IsNullOrWhiteSpace(selected))
selected = GetSupportFallbackReply(intentName);
return new JiboInteractionDecision(
intentName,
selected,
ContextUpdates: ScriptedResponseDecisionBuilder.BuildScriptedResponseContextUpdates());
}
private JiboInteractionDecision BuildScriptedHolidayGreetingDecision(
JiboExperienceCatalog catalog,
string intentName,
@@ -758,6 +773,18 @@ public sealed partial class JiboInteractionService
return ScriptedResponseDecisionBuilder.SelectLegacyReply(replies, randomizer, preferredSnippets);
}
private static string GetSupportFallbackReply(string intentName)
{
return intentName switch
{
"backup_help" => "That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com.",
"restore_backup" => "That sounds a little too complicated for me, I think your best bet is to get some guidance from Jibo Customer Care. Check the Help section of the Jibo App, or go to the website, support dot jibo dot com.",
"update_next" => "That's a good question. I think they've been coming every few weeks.",
"update_last" => "Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates.",
_ => string.Empty
};
}
private string RenderHolidayTemplate(string template, TurnContext turn, GreetingPresenceProfile presence)
{
var ownerName = ResolvePreferredGreetingName(turn, presence);

56
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Audio/LocalWhisperCppBufferedAudioSttStrategy.cs
View File

@@ -8,6 +8,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
IExternalProcessRunner processRunner) : ISttStrategy
{
private const int MinimumBufferedAudioBytes = 64;
private const int ShortAnswerBufferedAudioBytes = 16;
public string Name => "local-whispercpp-buffered-audio";
@@ -18,7 +19,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
IsConfiguredPathAvailable(options.WhisperCliPath, true) &&
IsConfiguredPathAvailable(options.WhisperModelPath, true) &&
ReadBufferedAudioFrames(turn).Any(ContainsOpusIdentificationHeader) &&
!IsBelowNoiseFloor(ReadBufferedAudioBytes(turn));
!IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn));
}
public async Task<SttResult> TranscribeAsync(TurnContext turn, CancellationToken cancellationToken = default)
@@ -31,7 +32,7 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
throw new InvalidOperationException(
"Local whisper.cpp STT requires buffered Ogg/Opus audio with an Opus identification header.");
if (IsBelowNoiseFloor(ReadBufferedAudioBytes(turn)))
if (IsBelowNoiseFloor(turn, ReadBufferedAudioBytes(turn)))
throw new InvalidOperationException(
"Local whisper.cpp STT rejected buffered audio as too short or noisy for transcription.");
@@ -119,9 +120,54 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
: 0;
}
private static bool IsBelowNoiseFloor(int bufferedAudioBytes)
private static bool IsBelowNoiseFloor(TurnContext turn, int bufferedAudioBytes)
{
return bufferedAudioBytes > 0 && bufferedAudioBytes < MinimumBufferedAudioBytes;
if (bufferedAudioBytes <= 0) return false;
var minimumBufferedAudioBytes = IsShortAnswerTurn(turn)
? ShortAnswerBufferedAudioBytes
: MinimumBufferedAudioBytes;
return bufferedAudioBytes < minimumBufferedAudioBytes;
}
private static bool IsShortAnswerTurn(TurnContext turn)
{
var rules = ReadRules(turn, "listenRules")
.Concat(ReadRules(turn, "clientRules"))
.Concat(ReadRules(turn, "listenAsrHints"));
return rules.Any(IsShortAnswerRule);
}
private static bool IsShortAnswerRule(string rule)
{
return string.Equals(rule, "$YESNO", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "clock/alarm_timer_change", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "clock/alarm_timer_none_set", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "create/is_it_a_keeper", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "settings/download_now_later", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "shared/yes_no", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "surprises-date/offer_date_fact", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "surprises-ota/want_to_download_now", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/surprise", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/right_word", StringComparison.OrdinalIgnoreCase) ||
string.Equals(rule, "word-of-the-day/puzzle", StringComparison.OrdinalIgnoreCase);
}
private static IEnumerable<string> ReadRules(TurnContext turn, string key)
{
if (!turn.Attributes.TryGetValue(key, out var value) || value is null) return [];
return value switch
{
IReadOnlyList<string> typed => typed,
IEnumerable<string> enumerable => enumerable,
JsonElement { ValueKind: JsonValueKind.Array } jsonElement => jsonElement.EnumerateArray()
.Where(static item => item.ValueKind == JsonValueKind.String)
.Select(static item => item.GetString() ?? string.Empty),
_ => []
};
}
private static bool ContainsOpusIdentificationHeader(byte[] frame)
@@ -167,4 +213,4 @@ public sealed class LocalWhisperCppBufferedAudioSttStrategy(
return !checkFileExists || File.Exists(path);
}
}
}

17
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/InMemoryJiboExperienceContentRepository.cs
View File

@@ -195,6 +195,23 @@ public sealed class InMemoryJiboExperienceContentRepository : IJiboExperienceCon
[
"The report-skill templates are loaded and waiting to be rendered."
],
BackupHowReplies =
[
"That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com."
],
RestoreHowReplies =
[
"That sounds a little bit out of my area of expertise. You can get info on that in the Help section of the Jibo App. Or try the website, support dot jibo dot com."
],
UpdateNextReplies =
[
"That's a good question. I think they've been coming every few weeks.",
"I never know exactly when my next update is coming, but they do seem to come pretty regularly."
],
UpdateLastReplies =
[
"Good question. The release notes page on the website support dot jibo dot com, will tell you the dates of all my past software updates."
],
WeatherIntroReplies =
[
"For your weather.",

38
OpenJibo/src/Jibo.Cloud/dotnet/src/Jibo.Cloud.Infrastructure/Content/LegacyMimCatalogImporter.cs
View File

@@ -241,6 +241,18 @@ public static class LegacyMimCatalogImporter
string.Equals(fileName, "WetNowDryLater", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.ReportSkillTemplate;
if (fileName.StartsWith("SUP_GEN_HowBackUpData", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.BackupHow;
if (fileName.StartsWith("SUP_GEN_HowRestoreBackup", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.RestoreHow;
if (fileName.StartsWith("SUP_UPDATE_WhenIsNextUpdate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.UpdateNext;
if (fileName.StartsWith("SUP_UPDATE_WhenWasLastUpdate", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.UpdateLast;
if (fileName.StartsWith("PersonalReportKickOff", StringComparison.OrdinalIgnoreCase))
return LegacyMimBucket.PersonalReportKickOff;
@@ -515,6 +527,10 @@ public static class LegacyMimCatalogImporter
HolidayGift,
HolidayTracker,
BirthdayCelebration,
BackupHow,
RestoreHow,
UpdateNext,
UpdateLast,
Jokes,
RobotFacts,
HumanFacts,
@@ -583,6 +599,7 @@ public static class LegacyMimCatalogImporter
private readonly List<string> _commuteTransportHurryReplies = [];
private readonly List<string> _commuteTransportLateReplies = [];
private readonly List<string> _commuteTransportNormalReplies = [];
private readonly List<string> _backupHowReplies = [];
private readonly List<JiboConditionedReply> _emotionReplies = [];
private readonly List<string> _fallbacks = [];
private readonly List<string> _favoriteAnimalReplies = [];
@@ -603,6 +620,9 @@ public static class LegacyMimCatalogImporter
private readonly List<string> _newsCategoryIntroReplies = [];
private readonly List<string> _newsIntroReplies = [];
private readonly List<string> _newsOutroReplies = [];
private readonly List<string> _restoreHowReplies = [];
private readonly List<string> _updateLastReplies = [];
private readonly List<string> _updateNextReplies = [];
private readonly List<string> _personalities = [];
private readonly List<string> _personalReportKickOffReplies = [];
private readonly List<string> _personalReportOutroReplies = [];
@@ -681,6 +701,18 @@ public static class LegacyMimCatalogImporter
case LegacyMimBucket.BirthdayCelebration:
AddDistinct(_birthdayCelebrationReplies, text);
return;
case LegacyMimBucket.BackupHow:
AddDistinct(_backupHowReplies, text);
return;
case LegacyMimBucket.RestoreHow:
AddDistinct(_restoreHowReplies, text);
return;
case LegacyMimBucket.UpdateNext:
AddDistinct(_updateNextReplies, text);
return;
case LegacyMimBucket.UpdateLast:
AddDistinct(_updateLastReplies, text);
return;
case LegacyMimBucket.Personality:
if (_personalities.Any(value => string.Equals(value, text, StringComparison.OrdinalIgnoreCase)))
return;
@@ -835,6 +867,7 @@ public static class LegacyMimCatalogImporter
HolidayGiftReplies = [.. _holidayGiftReplies],
HolidayTrackerReplies = [.. _holidayTrackerReplies],
BirthdayCelebrationReplies = [.. _birthdayCelebrationReplies],
BackupHowReplies = [.. _backupHowReplies],
HowAreYouReplies = [.. _howAreYous],
EmotionReplies = [.. _emotionReplies],
PersonalityReplies = [.. _personalities],
@@ -869,7 +902,10 @@ public static class LegacyMimCatalogImporter
CommuteServiceDownReplies = [.. _commuteServiceDownReplies],
NewsIntroReplies = [.. _newsIntroReplies],
NewsCategoryIntroReplies = [.. _newsCategoryIntroReplies],
NewsOutroReplies = [.. _newsOutroReplies]
NewsOutroReplies = [.. _newsOutroReplies],
RestoreHowReplies = [.. _restoreHowReplies],
UpdateNextReplies = [.. _updateNextReplies],
UpdateLastReplies = [.. _updateLastReplies]
};
}