planting runtime seeds

OpenJibo/src/Jibo.Runtime.Abstractions/AnimateAction.cs

@@ -0,0 +1,8 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class AnimateAction : PlanAction
+{
+    public override PlanActionType Type => PlanActionType.Animate;
+    public string AnimationId { get; init; } = string.Empty;
+    public IDictionary<string, object?> Parameters { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/BrainDecision.cs

@@ -0,0 +1,14 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class BrainDecision
+{
+    public BrainRoute Route { get; init; }
+    public string IntentName { get; init; } = string.Empty;
+    public float Confidence { get; init; }
+
+    public string? CapabilityName { get; init; }
+    public string? NativeSkillName { get; init; }
+
+    public IDictionary<string, object?> Slots { get; init; } = new Dictionary<string, object?>();
+    public IDictionary<string, object?> ContextUpdates { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/BrainRoute.cs

@@ -0,0 +1,10 @@
+namespace Jibo.Runtime.Abstractions;
+
+public enum BrainRoute
+{
+    Rules = 0,
+    NativeSkill = 1,
+    LocalAi = 2,
+    CloudAi = 3,
+    Hybrid = 4
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ConversationSession.cs

@@ -0,0 +1,15 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class ConversationSession
+{
+    public string SessionId { get; init; } = Guid.NewGuid().ToString("N");
+    public DateTimeOffset StartedUtc { get; init; } = DateTimeOffset.UtcNow;
+    public DateTimeOffset LastActivityUtc { get; set; } = DateTimeOffset.UtcNow;
+
+    public string? ActiveTopic { get; set; }
+    public string? LastIntent { get; set; }
+    public IDictionary<string, object?> Slots { get; } = new Dictionary<string, object?>();
+
+    public DateTimeOffset? FollowUpExpiresUtc { get; set; }
+    public bool FollowUpOpen => FollowUpExpiresUtc.HasValue && FollowUpExpiresUtc > DateTimeOffset.UtcNow;
+}

OpenJibo/src/Jibo.Runtime.Abstractions/FollowUpPolicy.cs

@@ -0,0 +1,11 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class FollowUpPolicy
+{
+    public static FollowUpPolicy None => new() { KeepMicOpen = false, Timeout = TimeSpan.Zero };
+
+    public bool KeepMicOpen { get; init; }
+    public TimeSpan Timeout { get; init; }
+    public string? ExpectedTopic { get; init; }
+    public IDictionary<string, object?> Hints { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IBrainStrategy.cs

@@ -0,0 +1,11 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IBrainStrategy
+{
+    string Name { get; }
+    bool CanHandle(TurnContext turn, ConversationSession session);
+    Task<BrainDecision> DecideAsync(
+        TurnContext turn,
+        ConversationSession session,
+        CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IBrainStrategySelector.cs

@@ -0,0 +1,9 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IBrainStrategySelector
+{
+    Task<IBrainStrategy> SelectAsync(
+        TurnContext turn,
+        ConversationSession session,
+        CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ICapability.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface ICapability
+{
+    string Name { get; }
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ICapabilityRegistry.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface ICapabilityRegistry
+{
+    TCapability? Get<TCapability>(string name) where TCapability : class, ICapability;
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IConversationBroker.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IConversationBroker
+{
+    Task<ResponsePlan> HandleTurnAsync(TurnContext turn, CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IResponsePlanner.cs

@@ -0,0 +1,10 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IResponsePlanner
+{
+    Task<ResponsePlan> BuildPlanAsync(
+        TurnContext turn,
+        ConversationSession session,
+        BrainDecision decision,
+        CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IRobotAdapter.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IRobotAdapter
+{
+    Task PublishPlanAsync(ResponsePlan plan, CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IRobotEventMapper.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IRobotEventMapper
+{
+    Task<TurnContext> MapToTurnContextAsync(RobotEvent robotEvent, CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IRobotEventSource.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IRobotEventSource
+{
+    IAsyncEnumerable<RobotEvent> ReadEventsAsync(CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ISttStrategy.cs

@@ -0,0 +1,8 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface ISttStrategy
+{
+    string Name { get; }
+    bool CanHandle(TurnContext turn);
+    Task<SttResult> TranscribeAsync(TurnContext turn, CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ISttStrategySelector.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface ISttStrategySelector
+{
+    Task<ISttStrategy> SelectAsync(TurnContext turn, CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/IWeatherCapability.cs

@@ -0,0 +1,6 @@
+namespace Jibo.Runtime.Abstractions;
+
+public interface IWeatherCapability : ICapability
+{
+    Task<WeatherResult> GetWeatherAsync(WeatherRequest request, CancellationToken cancellationToken = default);
+}

OpenJibo/src/Jibo.Runtime.Abstractions/InvokeNativeSkillAction.cs

@@ -0,0 +1,8 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class InvokeNativeSkillAction : PlanAction
+{
+    public override PlanActionType Type => PlanActionType.InvokeNativeSkill;
+    public string SkillName { get; init; } = string.Empty;
+    public IDictionary<string, object?> Payload { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/Jibo.Runtime.Abstractions.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

OpenJibo/src/Jibo.Runtime.Abstractions/ListenAction.cs

@@ -0,0 +1,8 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class ListenAction : PlanAction
+{
+    public override PlanActionType Type => PlanActionType.Listen;
+    public TimeSpan Timeout { get; init; }
+    public string? Mode { get; init; } // follow-up, open-mic, command
+}

OpenJibo/src/Jibo.Runtime.Abstractions/PlanAction.cs

@@ -0,0 +1,7 @@
+namespace Jibo.Runtime.Abstractions;
+
+public abstract class PlanAction
+{
+    public abstract PlanActionType Type { get; }
+    public int Sequence { get; init; }
+}

OpenJibo/src/Jibo.Runtime.Abstractions/PlanActionType.cs

@@ -0,0 +1,12 @@
+namespace Jibo.Runtime.Abstractions;
+
+public enum PlanActionType
+{
+    Speak = 0,
+    Listen = 1,
+    ShowVisual = 2,
+    Animate = 3,
+    InvokeNativeSkill = 4,
+    SetContext = 5,
+    EmitEvent = 6
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ResponsePlan.cs

@@ -0,0 +1,18 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class ResponsePlan
+{
+    public string PlanId { get; init; } = Guid.NewGuid().ToString("N");
+    public string SessionId { get; init; } = string.Empty;
+    public ResponseStatus Status { get; init; } = ResponseStatus.Succeeded;
+
+    public string? IntentName { get; init; }
+    public string? Topic { get; init; }
+    public IList<PlanAction> Actions { get; init; } = new List<PlanAction>();
+
+    public FollowUpPolicy FollowUp { get; init; } = FollowUpPolicy.None;
+    public IDictionary<string, object?> ContextUpdates { get; init; } = new Dictionary<string, object?>();
+
+    public string? DebugRoute { get; init; }
+    public IDictionary<string, object?> Diagnostics { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ResponseStatus.cs

@@ -0,0 +1,9 @@
+namespace Jibo.Runtime.Abstractions;
+
+public enum ResponseStatus
+{
+    Succeeded = 0,
+    Failed = 1,
+    Escalated = 2,
+    NoMatch = 3
+}

OpenJibo/src/Jibo.Runtime.Abstractions/RobotEvent.cs

@@ -0,0 +1,14 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class RobotEvent
+{
+    public string EventId { get; init; } = Guid.NewGuid().ToString("N");
+    public DateTimeOffset TimestampUtc { get; init; } = DateTimeOffset.UtcNow;
+
+    public string EventType { get; init; } = string.Empty;
+    public string? SessionId { get; init; }
+    public string? Transcript { get; init; }
+    public string? WakePhrase { get; init; }
+
+    public IDictionary<string, object?> Payload { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/SetContextAction.cs

@@ -0,0 +1,7 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class SetContextAction : PlanAction
+{
+    public override PlanActionType Type => PlanActionType.SetContext;
+    public IDictionary<string, object?> Values { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/ShowVisualAction.cs

@@ -0,0 +1,8 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class ShowVisualAction : PlanAction
+{
+    public override PlanActionType Type => PlanActionType.ShowVisual;
+    public string VisualId { get; init; } = string.Empty;
+    public IDictionary<string, object?> Parameters { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/SpeakAction.cs

@@ -0,0 +1,10 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class SpeakAction : PlanAction
+{
+    public override PlanActionType Type => PlanActionType.Speak;
+    public string Text { get; init; } = string.Empty;
+    public string? Voice { get; init; }
+    public string? Style { get; init; }
+    public bool CanBeInterrupted { get; init; } = true;
+}

OpenJibo/src/Jibo.Runtime.Abstractions/SttResult.cs

@@ -0,0 +1,10 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class SttResult
+{
+    public string Text { get; init; } = string.Empty;
+    public string? Provider { get; init; }
+    public float? Confidence { get; init; }
+    public string? Locale { get; init; }
+    public IDictionary<string, object?> Metadata { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/TurnContext.cs

@@ -0,0 +1,21 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class TurnContext
+{
+    public string TurnId { get; init; } = Guid.NewGuid().ToString("N");
+    public string SessionId { get; init; } = string.Empty;
+    public DateTimeOffset TimestampUtc { get; init; } = DateTimeOffset.UtcNow;
+
+    public TurnInputMode InputMode { get; init; }
+    public TurnSourceKind SourceKind { get; init; }
+
+    public string? WakePhrase { get; init; }
+    public string? RawTranscript { get; init; }
+    public string? NormalizedTranscript { get; init; }
+
+    public string? Locale { get; init; } = "en-US";
+    public string? TimeZone { get; init; }
+
+    public bool IsFollowUpEligible { get; init; }
+    public IDictionary<string, object?> Attributes { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Jibo.Runtime.Abstractions/TurnInputMode.cs

@@ -0,0 +1,9 @@
+namespace Jibo.Runtime.Abstractions;
+
+public enum TurnInputMode
+{
+    WakeWord = 0,
+    FollowUp = 1,
+    DirectText = 2,
+    System = 3
+}

OpenJibo/src/Jibo.Runtime.Abstractions/TurnSourceKind.cs

@@ -0,0 +1,9 @@
+namespace Jibo.Runtime.Abstractions;
+
+public enum TurnSourceKind
+{
+    NativeJibo = 0,
+    Simulator = 1,
+    TestHarness = 2,
+    Api = 3
+}

OpenJibo/src/Jibo.Runtime.Abstractions/WeatherRequest.cs

@@ -0,0 +1,9 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class WeatherRequest
+{
+    public string? LocationName { get; init; }
+    public string? TimeZone { get; init; }
+    public bool IncludeHourly { get; init; }
+    public bool IncludeDaily { get; init; }
+}

OpenJibo/src/Jibo.Runtime.Abstractions/WeatherResult.cs

@@ -0,0 +1,12 @@
+namespace Jibo.Runtime.Abstractions;
+
+public sealed class WeatherResult
+{
+    public string Summary { get; init; } = string.Empty;
+    public int? CurrentTemperatureF { get; init; }
+    public int? HighTodayF { get; init; }
+    public int? LowTonightF { get; init; }
+    public string? Conditions { get; init; }
+    public string? LocationLabel { get; init; }
+    public IDictionary<string, object?> Raw { get; init; } = new Dictionary<string, object?>();
+}

OpenJibo/src/Playground/Playground.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

OpenJibo/src/Playground/Program.cs

@@ -0,0 +1,247 @@
+using System.Net.Http.Json;
+using System.Net.WebSockets;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+Console.Write("Enter Jibo IP: ");
+var jiboIp = (Console.ReadLine() ?? "").Trim();
+
+if (string.IsNullOrWhiteSpace(jiboIp))
+{
+    Console.WriteLine("No IP entered.");
+    return;
+}
+
+var baseHttp = $"http://{jiboIp}:8088";
+var ttsHttp = $"http://{jiboIp}:8089";
+var wsUri = new Uri($"ws://{jiboIp}:8088/simple_port");
+
+using var http = new HttpClient();
+using var cts = new CancellationTokenSource();
+
+Console.WriteLine($"Connecting to Jibo at {jiboIp}...");
+Console.WriteLine("Press Ctrl+C to quit.");
+
+Console.CancelKeyPress += (_, e) =>
+{
+    e.Cancel = true;
+    cts.Cancel();
+};
+
+while (!cts.IsCancellationRequested)
+{
+    var taskId = $"DEBUG:demo-{Guid.NewGuid():N}";
+    var requestId = $"stt_start_{Guid.NewGuid():N}";
+
+    try
+    {
+        using var ws = new ClientWebSocket();
+        await ws.ConnectAsync(wsUri, cts.Token);
+        Console.WriteLine("WebSocket connected.");
+
+        var utteranceTcs = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        var wsReaderTask = Task.Run(async () =>
+        {
+            var buffer = new byte[8192];
+
+            while (ws.State == WebSocketState.Open && !cts.Token.IsCancellationRequested)
+            {
+                WebSocketReceiveResult result;
+                using var ms = new MemoryStream();
+
+                do
+                {
+                    result = await ws.ReceiveAsync(new ArraySegment<byte>(buffer), cts.Token);
+
+                    if (result.MessageType == WebSocketMessageType.Close)
+                    {
+                        Console.WriteLine("WebSocket closed by server.");
+                        return;
+                    }
+
+                    ms.Write(buffer, 0, result.Count);
+                }
+                while (!result.EndOfMessage);
+
+                var json = Encoding.UTF8.GetString(ms.ToArray());
+
+                AsrEvent? evt = null;
+                try
+                {
+                    evt = JsonSerializer.Deserialize<AsrEvent>(json);
+                }
+                catch
+                {
+                    Console.WriteLine($"Non-JSON WS message: {json}");
+                    continue;
+                }
+
+                if (evt == null)
+                    continue;
+
+                if (evt.TaskId != taskId)
+                    continue;
+
+                Console.WriteLine($"[{evt.EventType}] {json}");
+
+                if (evt.EventType == "speech_to_text_final")
+                {
+                    var best = PickBestUtterance(evt.Utterances);
+                    if (!string.IsNullOrWhiteSpace(best))
+                    {
+                        utteranceTcs.TrySetResult(best);
+                        return;
+                    }
+                }
+            }
+        }, cts.Token);
+
+        var startPayload = new
+        {
+            command = "start",
+            task_id = taskId,
+            audio_source_id = "alsa1",
+            hotphrase = "none",
+            speech_to_text = true,
+            request_id = requestId
+        };
+
+        var startResp = await http.PostAsJsonAsync($"{baseHttp}/asr_simple_interface", startPayload, cts.Token);
+        var startBody = await startResp.Content.ReadAsStringAsync(cts.Token);
+
+        Console.WriteLine($"ASR start: {(int)startResp.StatusCode} {startResp.ReasonPhrase}");
+        Console.WriteLine(startBody);
+
+        if (!startResp.IsSuccessStatusCode)
+            continue;
+
+        Console.WriteLine("Speak now...");
+
+        var completed = await Task.WhenAny(utteranceTcs.Task, Task.Delay(TimeSpan.FromSeconds(15), cts.Token));
+
+        if (completed != utteranceTcs.Task)
+        {
+            Console.WriteLine("Timed out waiting for speech_to_text_final.");
+        }
+        else
+        {
+            var heard = utteranceTcs.Task.Result;
+            Console.WriteLine($"Heard: {heard}");
+
+            var reply = BuildReply(heard);
+            Console.WriteLine($"Reply: {reply}");
+
+            var ttsPayload = new
+            {
+                prompt = reply,
+                locale = "en-us",
+                voice = "griffin",
+                mode = "text",
+                outputMode = "stream"
+            };
+
+            var ttsResp = await http.PostAsJsonAsync($"{ttsHttp}/tts_speak", ttsPayload, cts.Token);
+            var ttsBody = await ttsResp.Content.ReadAsStringAsync(cts.Token);
+
+            Console.WriteLine($"TTS: {(int)ttsResp.StatusCode} {ttsResp.ReasonPhrase}");
+            if (!string.IsNullOrWhiteSpace(ttsBody))
+                Console.WriteLine(ttsBody);
+        }
+
+        var stopPayload = new
+        {
+            command = "stop",
+            task_id = taskId,
+            request_id = $"stt_stop_{Guid.NewGuid():N}"
+        };
+
+        var stopResp = await http.PostAsJsonAsync($"{baseHttp}/asr_simple_interface", stopPayload, cts.Token);
+        _ = await stopResp.Content.ReadAsStringAsync(cts.Token);
+
+        Console.WriteLine("STT task stopped.");
+        Console.WriteLine();
+        Console.WriteLine("Press Enter to run another round, or Ctrl+C to quit.");
+        Console.ReadLine();
+    }
+    catch (OperationCanceledException)
+    {
+        break;
+    }
+    catch (Exception ex)
+    {
+        Console.WriteLine($"Error: {ex.Message}");
+        Console.WriteLine("Retrying in 2 seconds...");
+        await Task.Delay(2000, cts.Token);
+    }
+}
+
+static string PickBestUtterance(List<AsrUtterance>? utterances)
+{
+    if (utterances == null || utterances.Count == 0)
+        return "";
+
+    var cleaned = utterances
+        .Select(u => NormalizeUtterance(u.Utterance))
+        .Where(s => !string.IsNullOrWhiteSpace(s))
+        .Distinct(StringComparer.OrdinalIgnoreCase)
+        .OrderBy(s => s.Length)
+        .ToList();
+
+    return cleaned.FirstOrDefault() ?? "";
+}
+
+static string NormalizeUtterance(string? text)
+{
+    if (string.IsNullOrWhiteSpace(text))
+        return "";
+
+    var s = text.Trim();
+
+    // Very light cleanup for occasional weird leading duplication like "wWhat"
+    if (s.Length >= 2 && char.ToLowerInvariant(s[0]) == char.ToLowerInvariant(s[1]))
+        s = s.Substring(1);
+
+    return s;
+}
+
+static string BuildReply(string heard)
+{
+    var text = heard.Trim().ToLowerInvariant();
+
+    if (text.Contains("time"))
+        return $"It is {DateTime.Now:hh:mm tt}.";
+
+    if (text.Contains("hello") || text.Contains("hi"))
+        return "Hello! I heard you loud and clear.";
+
+    if (text.Contains("your name"))
+        return "I am Jibo, running with a local demo bridge.";
+
+    return $"You said: {heard}";
+}
+
+public sealed class AsrEvent
+{
+    [JsonPropertyName("event_type")]
+    public string? EventType { get; set; }
+
+    [JsonPropertyName("task_id")]
+    public string? TaskId { get; set; }
+
+    [JsonPropertyName("request_id")]
+    public string? RequestId { get; set; }
+
+    [JsonPropertyName("utterances")]
+    public List<AsrUtterance>? Utterances { get; set; }
+}
+
+public sealed class AsrUtterance
+{
+    [JsonPropertyName("utterance")]
+    public string? Utterance { get; set; }
+
+    [JsonPropertyName("score")]
+    public double Score { get; set; }
+}