Overlay rewrite (#148)

* Switching out overlay app, add attempt to download overlay in github action
* Adjust github action,
* Add advanced settings option
* Save capture bins separately
* Add multiple eye calibration modes
* Added check for model file existing
* Add trainer fetch to action
* Copy calibration folders to output
* Integrate trainer into calibration routine, cleanup
* Add Linux espflash binary, removed trainer files that have been moved, remove unused video bin that is included in the new overlay
* Add more platform conditions
* Move baseline checkpoints to correct directories
* Put output bins in the model 
* Move bin files into their own directory
* Fix issue where blink data was malformed
* Set up ARM builds

---------

Co-authored-by: RedInJector <[email protected]>
Co-authored-by: dfgHiatus <[email protected]>
Co-authored-by: Summer <[email protected]>
This commit is contained in:
authored and GitHub committed 2025-11-14 15:17:06 -05:00
1 parent 456ed12815
commit c848ff5c99
69 files changed
+1491 -2126

No files matched your search

+45 -10
View File
@@ -24,12 +24,17 @@ on:
required: false
jobs:
build-linux-x64:
build-linux:
defaults:
run:
shell: bash
strategy:
matrix:
include:
- { result: Baballonia.x64.AppImage, runs-on: ubuntu-22.04, publish: publish_linux.py, artifact-path: src/Baballonia.Desktop/installers/linux-x64, upload-path: src/Baballonia.Desktop/installers/linux-x64/Baballonia.x64.AppImage }
- { result: Baballonia.arm64.AppImage, runs-on: ubuntu-22.04-arm, publish: publish_linux_arm64.py, artifact-path: src/Baballonia.Desktop/installers/linux-arm64, upload-path: src/Baballonia.Desktop/installers/linux-arm64/Baballonia.arm64.AppImage }
runs-on: ubuntu-22.04
runs-on: ${{ matrix.runs-on }}
steps:
- name: Checkout latest
@@ -37,6 +42,9 @@ jobs:
with:
submodules: 'recursive'
- name: Download dependencies
run: ./download_dependencies.sh
- name: Install dependencies
run: |
dotnet tool install -g vpk
@@ -44,13 +52,18 @@ jobs:
- name: Run publish script
run: |
cd src/Baballonia.Desktop/Publish
python publish_linux.py
python ${{ matrix.publish }}
- name: Rename artifact
run: |
cd ${{ matrix.artifact-path }}
mv Baballonia.AppImage ${{ matrix.result }}
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: Baballonia.AppImage
path: src/Baballonia.Desktop/installers/linux-x64/Baballonia.AppImage
name: ${{ matrix.result }}
path: ${{ matrix.upload-path }}
build-windows-x64:
defaults:
@@ -66,10 +79,30 @@ jobs:
submodules: 'recursive'
lfs: true
- name: Fetch internal zip
run: |
cd src/Baballonia.Desktop
powershell ./fetch_internal.ps1
- name: Download dependencies
shell: pwsh
run: ./download_dependencies.ps1
#- name: Download Calibration Overlay artifacts
# env:
# GH_TOKEN: ${{ github.token }}
# run: |
# RUN_ID=$(gh run list -R Project-Babble/BabbleCalibration --workflow=export.yml --status=success --limit=1 --json databaseId | jq -r '.[0].databaseId')
# gh run download -R Project-Babble/BabbleCalibration $RUN_ID --name Windows --dir src/Baballonia.Desktop/Calibration/Windows/Overlay
# ls -R src/Baballonia.Desktop/Calibration/Windows/Overlay
#- name: Download Calibration Trainer artifacts
# env:
# GH_TOKEN: ${{ github.token }}
# run: |
# RUN_ID=$(gh run list -R Project-Babble/BabbleTrainer --workflow=build.yml --status=success --limit=1 --json databaseId | jq -r '.[0].databaseId')
# gh run download -R Project-Babble/BabbleTrainer $RUN_ID --name Windows --dir src/Baballonia.Desktop/Calibration/Windows/Trainer
# ls -R src/Baballonia.Desktop/Calibration/Windows/Trainer
#- name: Fetch internal zip
# run: |
# cd src/Baballonia.Desktop
# powershell ./fetch_internal.ps1
- name: Build project
run: |
@@ -88,7 +121,7 @@ jobs:
path: "src/Baballonia.Desktop/Baballonia Setup.exe"
publish-release:
#needs: [build-linux-x64, build-windows-x64]
#needs: [build-linux, build-windows-x64]
#for now, releases only include windows
needs: build-windows-x64
if: ${{ github.event.inputs.publish-release == 'true' }}
@@ -97,6 +130,8 @@ jobs:
shell: bash
runs-on: ubuntu-22.04
permissions:
contents: write
steps:
- name: Validate version number exists
+10
View File
@@ -404,5 +404,15 @@ FodyWeavers.xsd
# macOS files
.DS_Store
# this chonky bastard
src/Baballonia.Desktop/_internal.zip
/src/Baballonia.Desktop/python
src/Baballonia.Desktop/Calibration/Windows/Overlay/*
src/Baballonia.Desktop/Calibration/Linux/Overlay/*
src/Baballonia/Firmware/Windows/espflash.exe
src/Baballonia/Firmware/Linux/espflash
src/Baballonia.Desktop/Calibration/Windows/Trainer/BabbleTrainer.exe
src/Baballonia.Desktop/Calibration/Linux/Trainer/BabbleTrainer
-1
View File
@@ -131,7 +131,6 @@ Global
{7DBED48D-9A54-4D44-BE5B-0538B0B31A99} = {82C6A0B4-5014-4BE4-9F52-7DC1989134E3}
{2E6C572A-2499-45DC-8A65-7EBA9706A25A} = {82C6A0B4-5014-4BE4-9F52-7DC1989134E3}
{BA0A656D-FF1D-4E15-B43C-9BE5AE692256} = {82C6A0B4-5014-4BE4-9F52-7DC1989134E3}
{42922296-1CEC-4136-9D6B-1396CF84E95A} = {82C6A0B4-5014-4BE4-9F52-7DC1989134E3}
{D8C25797-49BC-4024-A812-F44AB0B77BD1} = {82C6A0B4-5014-4BE4-9F52-7DC1989134E3}
{575BB586-5394-4F94-A178-DF4062EC44A1} = {82C6A0B4-5014-4BE4-9F52-7DC1989134E3}
{4B08BDE8-099A-405F-AB71-094E8FC24AB1} = {82C6A0B4-5014-4BE4-9F52-7DC1989134E3}
+11
View File
@@ -0,0 +1,11 @@
Invoke-WebRequest -Uri "https://github.com/Project-Babble/BabbleTrainer/releases/latest/download/BabbleTrainer-x64.exe" -OutFile "src/Baballonia.Desktop/Calibration/Windows/Trainer/BabbleTrainer.exe"
Invoke-WebRequest -Uri "https://github.com/Project-Babble/BabbleCalibration/releases/latest/download/Windows.zip" -OutFile "src/Baballonia.Desktop/Calibration/Windows/Overlay/Windows.zip"
Invoke-WebRequest -Uri "https://github.com/esp-rs/espflash/releases/latest/download/espflash-x86_64-pc-windows-msvc.zip" -OutFile "src/Baballonia/Firmware/Windows/espflash.zip"
Expand-Archive -Path "src/Baballonia.Desktop/Calibration/Windows/Overlay/Windows.zip" -DestinationPath "src/Baballonia.Desktop/Calibration/Windows/Overlay"
Remove-Item "src/Baballonia.Desktop/Calibration/Windows/Overlay/Windows.zip"
Expand-Archive -Path "src/Baballonia/Firmware/Windows/espflash.zip" -DestinationPath "src/Baballonia/Firmware/Windows"
Remove-Item "src/Baballonia/Firmware/Windows/espflash.zip"
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
calibrationDownloadPath="https://github.com/Project-Babble/BabbleCalibration/releases/latest/download/Linux.zip"
trainerDownloadPath="https://github.com/Project-Babble/BabbleTrainer/releases/latest/download/BabbleTrainer-x64"
espflashDownloadPath="https://github.com/esp-rs/espflash/releases/latest/download/espflash-x86_64-unknown-linux-gnu.zip"
isArm=$(uname -m)
echo $isArm
if [[ $isArm == *"arm"* || $isArm == "aarch64" ]]; then
calibrationDownloadPath="https://github.com/Project-Babble/BabbleCalibration/releases/latest/download/Linux-ARM.zip"
trainerDownloadPath="https://github.com/Project-Babble/BabbleTrainer/releases/latest/download/BabbleTrainer-arm64"
espflashDownloadPath="https://github.com/esp-rs/espflash/releases/latest/download/espflash-aarch64-unknown-linux-gnu.zip"
fi
curl -Z -L $trainerDownloadPath -L $calibrationDownloadPath -L $espflashDownloadPath -o "src/Baballonia.Desktop/Calibration/Linux/Trainer/BabbleTrainer" -o "src/Baballonia.Desktop/Calibration/Linux/Overlay/Linux.zip" -o "src/Baballonia/Firmware/Linux/espflash.zip"
unzip "src/Baballonia.Desktop/Calibration/Linux/Overlay/Linux.zip" -d "src/Baballonia.Desktop/Calibration/Linux/Overlay"
rm "src/Baballonia.Desktop/Calibration/Linux/Overlay/Linux.zip"
unzip "src/Baballonia/Firmware/Linux/espflash.zip" -d "src/Baballonia/Firmware/Linux"
rm "src/Baballonia/Firmware/Linux/espflash.zip"
@@ -89,6 +89,8 @@ public static class CaptureBin
/// </summary>
public static void WriteAll(string path, IEnumerable<Frame> frames)
{
// this throws if for some reason there is no parent dir for "path"
Directory.CreateDirectory(Directory.GetParent(path)!.ToString());
using var fs = File.Open(path, FileMode.Create, FileAccess.Write, FileShare.None);
Write(fs, frames);
}
+26 -1
View File
@@ -6,9 +6,34 @@ namespace Baballonia.CaptureBin.IO;
/// </summary>
public static class CaptureFlags
{
public const uint FLAG_ROUTINE_BIT1 = 1U << 0;
public const uint FLAG_ROUTINE_2 = 1U << 1;
public const uint FLAG_ROUTINE_3 = 1U << 2;
public const uint FLAG_ROUTINE_4 = 1U << 3;
public const uint FLAG_ROUTINE_5 = 1U << 4;
public const uint FLAG_ROUTINE_6 = 1U << 5;
public const uint FLAG_ROUTINE_7 = 1U << 6;
public const uint FLAG_ROUTINE_8 = 1U << 7;
public const uint FLAG_ROUTINE_9 = 1U << 8;
public const uint FLAG_ROUTINE_10 = 1U << 9;
public const uint FLAG_ROUTINE_11 = 1U << 10;
public const uint FLAG_ROUTINE_12 = 1U << 11;
public const uint FLAG_ROUTINE_13 = 1U << 12;
public const uint FLAG_ROUTINE_14 = 1U << 13;
public const uint FLAG_ROUTINE_15 = 1U << 14;
public const uint FLAG_ROUTINE_16 = 1U << 15;
public const uint FLAG_ROUTINE_17 = 1U << 16;
public const uint FLAG_ROUTINE_18 = 1U << 17;
public const uint FLAG_ROUTINE_19 = 1U << 18;
public const uint FLAG_ROUTINE_20 = 1U << 19;
public const uint FLAG_VERSION_BIT1 = 1U << 20;
public const uint FLAG_VERSION_BIT2 = 1U << 21;
public const uint FLAG_VERSION_BIT3 = 1U << 22;
public const uint FLAG_VERSION_BIT4 = 1U << 23;
public const uint FLAG_CONVERGENCE = 1u << 24;
public const uint FLAG_IN_MOVEMENT = 1u << 25;
public const uint FLAG_RESTING = 1u << 26;
public const uint FLAG_RESTING = 1u << 26; // Unused right now, was used to denote tutorial sections
public const uint FLAG_DILATION_BLACK = 1u << 27; // Black screen for full dilation
public const uint FLAG_DILATION_WHITE = 1u << 28; // White screen for full constriction
public const uint FLAG_DILATION_GRADIENT = 1u << 29; // Gradient fade from white to black
@@ -41,11 +41,13 @@
<PackageReference Include="SkiaSharp.NativeAssets.macOS" Version="3.119.0" Condition="$([MSBuild]::IsOSPlatform('OSX'))" />
<!-- Linux-specific -->
<PackageReference Include="Project-Babble.OpenCvSharp4.mini.runtime.ubuntu.22.04-x64" Version="4.11.0.1" Condition="$([MSBuild]::IsOSPlatform('Linux'))" />
<PackageReference Include="Project-Babble.OpenCvSharp4.mini.runtime.ubuntu.22.04-x64" Version="4.11.0.1" Condition="$([MSBuild]::IsOSPlatform('Linux')) And !$(RuntimeIdentifier.Contains('arm'))" />
<PackageReference Include="Project-Babble.OpenCvSharp4.mini.runtime.ubuntu.22.04-arm64" Version="4.11.0.1" Condition="$([MSBuild]::IsOSPlatform('Linux')) And $(RuntimeIdentifier.Contains('arm'))" />
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.22.0" Condition="$([MSBuild]::IsOSPlatform('Linux'))" />
<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="3.119.0" Condition="$([MSBuild]::IsOSPlatform('Linux'))"/>
<PackageReference Include="SkiaSharp.NativeAssets.Linux" Version="3.119.0" Condition="$([MSBuild]::IsOSPlatform('Linux'))" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Baballonia.CaptureBin.IO\Baballonia.CaptureBin.IO.csproj" />
<ProjectReference Include="..\Baballonia.OpenCVCapture\Baballonia.OpenCVCapture.csproj"/>
<ProjectReference Include="..\Baballonia.SDK\Baballonia.SDK.csproj" />
<ProjectReference Include="..\Baballonia.SerialCameraCapture\Baballonia.SerialCameraCapture.csproj"/>
@@ -94,38 +96,15 @@
<None Update="baseline_R.pth">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Linux\gaze_overlay">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<None Update="Calibration\Linux\**" Condition="$([MSBuild]::IsOSPlatform('Linux'))">
<Link>%(RecursiveDir)\%(Filename)%(Extension)</Link>
<TargetPath>%(RecursiveDir)\%(Filename)%(Extension)</TargetPath>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Linux\trainer">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\app.rc">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\app.res">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\calibration_runner.exe">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\capture_417272897.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\gaze_overlay.exe">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\onnxruntime.dll">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\onnxruntime_providers_shared.dll">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\openvr_api.dll">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Calibration\Windows\turbojpeg.dll">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<None Update="Calibration\Windows\**" Condition="$([MSBuild]::IsOSPlatform('Windows'))">
<Link>%(RecursiveDir)\%(Filename)%(Extension)</Link>
<TargetPath>%(RecursiveDir)\%(Filename)%(Extension)</TargetPath>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="onnx_artifacts\training\checkpoint\paramfrozen_tensors.pbseq">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -148,19 +127,20 @@
<None Update="onnx_artifacts\temporal_eye_tracking_model.onnx">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="trainermin.exe">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="video.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="main.py">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<!-- SkiaSharp is currently broken on ARM for some reason -->
<Target Name="PatchSkiaSharp" Condition="$([MSBuild]::IsOSPlatform('Linux')) And $(RuntimeIdentifier.Contains('arm')) And Exists('$(OutputPath)runtimes\linux-arm64\native\libSkiaSharp.so')" AfterTargets="AfterBuild;AfterPublish">
<Exec Command="patchelf --add-needed libuuid.so.1 '$(OutputPath)runtimes\linux-arm64\native\libSkiaSharp.so'"/>
<Exec Command="patchelf --add-needed libfreetype.so.6 '$(OutputPath)runtimes\linux-arm64\native\libSkiaSharp.so'"/>
</Target>
<Target Name="PatchSkiaSharp" Condition="$([MSBuild]::IsOSPlatform('Linux')) And $(RuntimeIdentifier.Contains('arm')) And Exists('$(OutputPath)libSkiaSharp.so')" AfterTargets="AfterBuild;AfterPublish">
<Exec Command="patchelf --add-needed libuuid.so.1 '$(OutputPath)libSkiaSharp.so'"/>
<Exec Command="patchelf --add-needed libfreetype.so.6 '$(OutputPath)libSkiaSharp.so'"/>
</Target>
<Target Name="CopyModulesToFolder" AfterTargets="AfterBuild">
<Target Name="CopyModulesToFolder" AfterTargets="AfterBuild;AfterPublish">
<ItemGroup>
<ModuleDlls Include="$(OutputPath)Baballonia.SerialCameraCapture.dll" />
<ModuleDlls Include="$(OutputPath)Baballonia.OpenCVCapture.dll" />
@@ -171,34 +151,8 @@
<ModuleDlls Condition="Exists('$(OutputPath)Baballonia.IPCameraCapture.pdb')" Include="$(OutputPath)Baballonia.IPCameraCapture.pdb" />
<ModuleDlls Condition="Exists('$(OutputPath)Baballonia.VFTCapture.pdb')" Include="$(OutputPath)Baballonia.VFTCapture.pdb" />
</ItemGroup>
<MakeDir Directories="$(OutputPath)Modules" />
<Copy SourceFiles="@(ModuleDlls)" DestinationFolder="$(OutputPath)Modules\" />
<!-- Remove the DLLs from the root output directory -->
<Delete Files="@(ModuleDlls)" />
<Move SourceFiles="@(ModuleDlls)" DestinationFolder="$(OutputPath)Modules\" />
</Target>
<Target Name="ExtractZipIfNeeded" AfterTargets="Build" Condition="$([MSBuild]::IsOSPlatform('Windows'))">
<PropertyGroup>
<ZipFilePath>$(ProjectDir)_internal.zip</ZipFilePath>
<ExtractToPath>$(OutputPath)_internal</ExtractToPath>
</PropertyGroup>
<Message Text="Extracting $(ZipFilePath) to $(ExtractToPath)"
Condition="Exists('$(ZipFilePath)') AND !Exists('$(ExtractToPath)')"
Importance="high" />
<Unzip SourceFiles="$(ZipFilePath)"
DestinationFolder="$(ExtractToPath)"
Condition="Exists('$(ZipFilePath)') AND !Exists('$(ExtractToPath)')"
OverwriteReadOnlyFiles="true" />
<Message Text="Extraction folder already exists: $(ExtractToPath)"
Condition="Exists('$(ExtractToPath)')"
Importance="normal" />
<Warning Text="Zip file not found: $(ZipFilePath)"
Condition="!Exists('$(ZipFilePath)')" />
</Target>
</Project>
@@ -1,42 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Net.Http;
using Baballonia.Contracts;
using Baballonia.ViewModels.SplitViewPane;
using CommunityToolkit.Mvvm.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace Baballonia.Desktop.Calibration.Aero;
public partial class AeroOverlayTrainerCombo : IVROverlay, IVRCalibrator
{
private ILogger Logger;
private readonly Dictionary<string, Process> _activeProcesses = new();
private readonly HttpClient _httpClient = new();
private readonly string _baseUrl = "http://localhost:23951";
private static void DeleteCaptureFiles(string directoryPath)
{
// Validate directory exists
if (!Directory.Exists(directoryPath))
return;
// Get all files matching the capture pattern
string[] filesToDelete = Directory.GetFiles(directoryPath, "capture.bin");
// Delete each file
foreach (string file in filesToDelete)
{
File.Delete(file);
}
}
public void Dispose()
{
StopAllProcesses();
_httpClient?.Dispose();
}
}
@@ -1,233 +0,0 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Baballonia.Desktop.Calibration.Aero.Overlay;
using Baballonia.Models;
using Baballonia.ViewModels.SplitViewPane;
using CommunityToolkit.Mvvm.DependencyInjection;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace Baballonia.Desktop.Calibration.Aero;
public partial class AeroOverlayTrainerCombo
{
private static string Overlay { get; } = null!;
private static string OverlayPath { get; } = null!;
static AeroOverlayTrainerCombo()
{
if (OperatingSystem.IsWindows())
{
OverlayPath = Path.Combine(AppContext.BaseDirectory, "Calibration", "Windows");
Overlay = Path.Combine(OverlayPath, "gaze_overlay.exe");
}
else if (OperatingSystem.IsLinux())
{
OverlayPath = Path.Combine(AppContext.BaseDirectory, "Calibration", "Linux");
Overlay = Path.Combine(OverlayPath, "gaze_overlay");
}
}
private async Task<(bool success, string message)> StartProcess(string program,
string[]? arguments = null,
bool waitForExit = false)
{
// Make sure the overlay program exists
if (!File.Exists(program))
{
Logger.LogError(Assets.Resources.Aero_Overlay_NotFound);
return (false, Assets.Resources.Aero_Overlay_NotFound);
}
string processName = Path.GetFileNameWithoutExtension(program);
// Make sure program isn't already running
var hitList = Process.GetProcesses().Where(p => p.ProcessName == processName).ToArray();
if (hitList.Length > 0)
{
Logger.LogError(Assets.Resources.Aero_Overlay_AlreadyRunning);
foreach (var p in hitList)
{
p.Kill(true);
}
// return (false, Assets.Resources.Aero_Overlay_AlreadyRunning);
}
// Check if SteamVR is running. The overlay needs it to be running prior!
// TODO Add Monado here as well
if (!Process.GetProcesses().Any(p => p.ProcessName.ToLower().Contains("vrserver")))
{
Logger.LogError(Assets.Resources.Aero_SteamVR_NotRunning);
return (false, Assets.Resources.Aero_SteamVR_NotRunning);
}
var workingDir = AppContext.BaseDirectory;
/*if (OperatingSystem.IsWindows())
{
if (Directory.Exists("win-x64"))
workingDir = Path.Combine(AppContext.BaseDirectory, "win-x64");
}*/
var startInfo = new ProcessStartInfo
{
FileName = program,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
WorkingDirectory = workingDir,
};
if (arguments != null)
startInfo.Arguments = string.Join(" ", arguments);
var process = new Process
{
StartInfo = startInfo,
EnableRaisingEvents = true
};
process.Exited += (_, _) =>
{
_activeProcesses.Remove(processName);
};
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
_activeProcesses[processName] = process;
if (waitForExit)
{
await process.WaitForExitAsync();
}
return (true, string.Empty);
}
private bool StopProcess(string program)
{
string processName = Path.GetFileNameWithoutExtension(program);
if (_activeProcesses.TryGetValue(processName, out var process))
{
try
{
if (!process.HasExited)
{
process.Kill();
process.WaitForExit(3000); // Wait up to 3 seconds for graceful exit
}
process.Dispose();
_activeProcesses.Remove(processName);
return true;
}
catch (Exception)
{
return false;
}
}
// Check if it's running but not tracked by us
var runningProcess = Process.GetProcesses().FirstOrDefault(p => p.ProcessName == processName);
if (runningProcess != null)
{
try
{
runningProcess.Kill();
runningProcess.WaitForExit(3000);
runningProcess.Dispose();
return true;
}
catch (Exception)
{
return false;
}
}
return true; // Not running is considered success for stopping
}
private void StopAllProcesses()
{
foreach (var process in _activeProcesses.Values.ToList())
{
try
{
if (!process.HasExited)
{
process.Kill();
process.WaitForExit(3000);
}
process.Dispose();
}
catch (Exception)
{
// ignore
}
}
_activeProcesses.Clear();
}
private async Task<VrCalibrationStatus> GetStatusAsync()
{
try
{
var response = await _httpClient.GetStringAsync($"{_baseUrl}/status");
return JsonConvert.DeserializeObject<VrCalibrationStatus>(response)!;
}
catch (Exception e)
{
Logger.LogError(e.Message);
throw;
}
}
public async Task<bool> StartPreviewAsync(VrCalibration calibration)
{
try
{
var modelPath = Uri.EscapeDataString(calibration.ModelSavePath + VrCalibration.ModelName);
var url = $"{_baseUrl}/start_preview?model_path={Uri.EscapeDataString(modelPath)}";
var response = await _httpClient.GetStringAsync(url);
var result = JsonConvert.DeserializeObject<ApiResponse>(response);
return result!.Result == "ok";
}
catch (HttpRequestException e)
{
Logger.LogError(e.Message);
return false;
}
catch (Exception e)
{
Logger.LogError(e.Message);
throw;
}
}
public async Task<bool> StopPreviewAsync()
{
try
{
var response = await _httpClient.GetStringAsync($"{_baseUrl}/stop_preview");
var result = JsonConvert.DeserializeObject<ApiResponse>(response);
return result!.Result == "ok";
}
catch (HttpRequestException e)
{
Logger.LogError(e.Message);
return false;
}
catch (Exception e)
{
Logger.LogError(e.Message);
throw;
}
}
}
@@ -1,168 +0,0 @@
using System;
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
using Baballonia.Desktop.Calibration.Aero.Overlay;
using Baballonia.Models;
using Baballonia.Services;
using Baballonia.Services.events;
using Baballonia.ViewModels.SplitViewPane;
using CommunityToolkit.Mvvm.DependencyInjection;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using OpenCvSharp;
namespace Baballonia.Desktop.Calibration.Aero;
public partial class AeroOverlayTrainerCombo
{
private readonly MjpegStreamingService _leftStreamService = new();
private readonly MjpegStreamingService _rightStreamService = new();
private IEyePipelineEventBus _eyePipelineEventBus;
public async Task<(bool, string)> EyeTrackingCalibrationRequested(string calibrationRoutine)
{
// Need to pull here, the service provider isn't present until this method is called
Logger ??= Ioc.Default.GetService<ILogger<HomePageViewModel>>()!;
_eyePipelineEventBus = Ioc.Default.GetService<IEyePipelineEventBus>()!;
_eyePipelineEventBus.Subscribe<EyePipelineEvents.NewTransformedFrameEvent>(HandleEyeImageEvent);
const int leftPort = 23952;
const int rightPort = 23953;
var modelPath = Directory.GetCurrentDirectory();
var calibration = new VrCalibration
{
ModelSavePath = modelPath,
CalibrationInstructions = calibrationRoutine,
FOV = 1f,
LeftEyeMjpegSource = $"http://localhost:{leftPort}/mjpeg",
RightEyeMjpegSource = $"http://localhost:{rightPort}/mjpeg",
};
// Now for the IPC. Spool up our MJPEG streams
_leftStreamService.StartStreaming(leftPort);
_rightStreamService.StartStreaming(rightPort);
// Tell the calibrator/overlay start...
var status = await StartOverlay();
var success = status.success;
if (!success)
{
return await StopStreamingAndReturn(status.message);
}
// Then connect the camera streams...
success = await StartCamerasAsync(calibration);
if (!success)
{
return await StopStreamingAndReturn(Assets.Resources.Aero_CameraStream_Failed);
}
// If we have a good start on the overlay/streams, then start calibration
success = await StartCalibrationAsync(calibration);
if (!success)
{
return await StopStreamingAndReturn(Assets.Resources.Aero_Calibration_Failed);
}
// Wait for the process to exit
var loop = true;
while (loop)
{
var res = await GetStatusAsync();
if (res.IsTrained)
{
loop = false;
}
await Task.Delay(1000);
}
// Stop the MJPEG streams, we don't need them anymore
success = StopOverlay();
var outputMessage = success ?
Assets.Resources.Aero_Calibration_Success :
Assets.Resources.Aero_Overlay_CleanupFailed;
// Stop streaming, cleanup. No need to report an error state
await StopStreamingAndReturn(string.Empty);
// Cleanup any leftover capture.bin files
DeleteCaptureFiles(modelPath);
return await Task.FromResult((success, outputMessage));
}
private async Task<(bool, string)> StopStreamingAndReturn(string message)
{
_eyePipelineEventBus.Unsubscribe<EyePipelineEvents.NewTransformedFrameEvent>(HandleEyeImageEvent);
_leftStreamService.StopStreaming();
_rightStreamService.StopStreaming();
return await Task.FromResult((false, message));
}
private async Task<(bool success, string message)> StartOverlay(string[]? arguments = null, bool waitForExit = false)
{
return await StartProcess(Overlay, arguments, waitForExit);
}
private bool StopOverlay()
{
return StopProcess(Overlay);
}
private async Task<bool> StartCamerasAsync(VrCalibration calibration)
{
try
{
var response = await _httpClient.GetStringAsync(new Uri(
$"{_baseUrl}/start_cameras?left={calibration.LeftEyeMjpegSource}&right={calibration.RightEyeMjpegSource}"));
var result = JsonConvert.DeserializeObject<ApiResponse>(response);
return result!.Result == "ok";
}
catch (HttpRequestException e)
{
Logger.LogError(e.Message);
return false;
}
catch (Exception e)
{
Logger.LogError(e.Message);
throw;
}
}
private async Task<bool> StartCalibrationAsync(VrCalibration calibration)
{
try
{
var url = $"{_baseUrl}/start_calibration?onnx_filename={VrCalibration.ModelName}&routine_id={calibration.CalibrationInstructions}";
var response = await _httpClient.GetStringAsync(url);
var result = JsonConvert.DeserializeObject<ApiResponse>(response);
return result!.Result == "ok";
}
catch (HttpRequestException e)
{
Logger.LogError(e.Message);
return false;
}
catch (Exception e)
{
Logger.LogError(e.Message);
throw;
}
}
private void HandleEyeImageEvent(EyePipelineEvents.NewTransformedFrameEvent e)
{
var image = e.image;
int channels = image.Channels();
if (channels != 2)
return;
var images = image.Split();
_leftStreamService.UpdateMjpegFrame(images[0]);
_rightStreamService.UpdateMjpegFrame(images[1]);
}
}
@@ -0,0 +1,12 @@
using System;
using Microsoft.Extensions.Logging;
namespace Baballonia.Desktop.Calibration.Aero;
public class OverlayLogger(ILogger logger) : OverlaySDK.ILogger
{
public void Debug(string message) => logger.LogDebug(message);
public void Info(string message) => logger.LogInformation(message);
public void Warn(string message) => logger.LogWarning(message);
public void Error(string message, Exception? ex = null) => logger.LogError(ex, message);
}
@@ -1,12 +0,0 @@
using Newtonsoft.Json;
namespace Baballonia.Desktop.Calibration.Aero.Overlay;
public class ApiResponse
{
[JsonProperty("result")]
public required string Result { get; set; }
[JsonProperty("message")]
public required string Message { get; set; }
}
@@ -1,10 +0,0 @@
using System;
namespace Baballonia.Desktop.Calibration.Aero.Overlay;
public class ProcessOutputEventArgs(string output, bool isError) : EventArgs
{
public string Output { get; } = output;
public bool IsError { get; } = isError;
public DateTime Timestamp { get; } = DateTime.Now;
}
@@ -1,30 +0,0 @@
using Newtonsoft.Json;
namespace Baballonia.Desktop.Calibration.Aero.Overlay;
public class VrCalibrationStatus
{
[JsonProperty("running")]
public required string Running { get; set; }
[JsonProperty("recording")]
public required string Recording { get; set; }
[JsonProperty("calibrationComplete")]
public required string CalibrationComplete { get; set; }
[JsonProperty("isTrained")]
public required string Trained { get; set; }
[JsonProperty("currentIndex")]
public int CurrentIndex { get; set; }
[JsonProperty("maxIndex")]
public int MaxIndex { get; set; }
public bool IsRunning => Running == "1";
public bool IsRecording => Recording == "1";
public bool IsTrained => Trained == "1";
public bool IsCalibrationComplete => CalibrationComplete == "1";
public double Progress => MaxIndex > 0 ? (double)CurrentIndex / MaxIndex : 0;
}
@@ -0,0 +1,166 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Threading;
using Baballonia.CaptureBin.IO;
using OpenCvSharp;
using OverlaySDK.Packets;
namespace Baballonia.Desktop.Calibration;
public class PositionalBinCollector
{
private readonly object _lock = new();
private readonly List<Frame> _frames = new();
private HmdPositionalDataPacket? _latestPosData;
private uint _headerFlags;
public PositionalBinCollector(uint headerFlags)
{
_headerFlags = headerFlags;
}
public void UpdatePositionalData(HmdPositionalDataPacket posData)
{
Interlocked.Exchange(ref _latestPosData, posData);
}
CaptureFrameHeader GenerateHeader(HmdPositionalDataPacket positionalData)
{
var time = (ulong)DateTimeOffset.Now.ToUnixTimeMilliseconds();
return new CaptureFrameHeader
{
RoutineState = _headerFlags,
LeftEyePitch = positionalData.LeftEyePitch,
LeftEyeYaw = positionalData.LeftEyeYaw,
RightEyePitch = positionalData.RightEyePitch,
RightEyeYaw = positionalData.RightEyeYaw,
RoutinePitch = positionalData.RoutinePitch,
RoutineYaw = positionalData.RoutineYaw,
RoutineDistance = positionalData.RoutineDistance,
RoutineConvergence = positionalData.RoutineConvergence,
FovAdjustDistance = positionalData.FovAdjustDistance,
Timestamp = time,
TimestampLeft = time,
TimestampRight = time,
};
}
public Frame? AddFrame(Mat left, Mat right)
{
var posData = Interlocked.CompareExchange(ref _latestPosData, null, null);
if (posData == null)
return null;
const int jpegQuality = 50;
Cv2.ImEncode(".jpg", left, out var leftBuf, [(int)ImwriteFlags.JpegQuality, jpegQuality]);
Cv2.ImEncode(".jpg", right, out var rightBuf, [(int)ImwriteFlags.JpegQuality, jpegQuality]);
var header = GenerateHeader(posData);
var frame = new Frame
{
Header = header with
{
JpegDataLeftLength = (uint)leftBuf.Length,
JpegDataRightLength = (uint)rightBuf.Length
},
LeftJpeg = leftBuf,
RightJpeg = rightBuf
};
lock (_lock)
{
_frames.Add(frame);
}
return frame;
}
public void WriteBin(string path)
{
List<Frame> copy;
lock (_lock)
{
copy = new List<Frame>(_frames);
_frames.Clear();
}
CaptureBin.IO.CaptureBin.WriteAll(Path.Combine(Utils.ModelDataDirectory, path), copy);
}
}
public class BinCollector
{
private readonly object _lock = new();
private readonly List<Frame> _frames = new();
private uint _headerFlags;
// empty data, because bin
HmdPositionalDataPacket positionalData = new HmdPositionalDataPacket();
public BinCollector(uint headerFlags)
{
_headerFlags = headerFlags;
}
private CaptureFrameHeader GenerateHeader()
{
var time = (ulong)DateTimeOffset.Now.ToUnixTimeMilliseconds();
return new CaptureFrameHeader
{
RoutineState = _headerFlags,
LeftEyePitch = positionalData.LeftEyePitch,
LeftEyeYaw = positionalData.LeftEyeYaw,
RightEyePitch = positionalData.RightEyePitch,
RightEyeYaw = positionalData.RightEyeYaw,
RoutinePitch = positionalData.RoutinePitch,
RoutineYaw = positionalData.RoutineYaw,
RoutineDistance = positionalData.RoutineDistance,
RoutineConvergence = positionalData.RoutineConvergence,
FovAdjustDistance = positionalData.FovAdjustDistance,
Timestamp = time,
TimestampLeft = time,
TimestampRight = time,
};
}
public Frame AddFrame(Mat left, Mat right)
{
const int jpegQuality = 50;
Cv2.ImEncode(".jpg", left, out var leftBuf, [(int)ImwriteFlags.JpegQuality, jpegQuality]);
Cv2.ImEncode(".jpg", right, out var rightBuf, [(int)ImwriteFlags.JpegQuality, jpegQuality]);
var header = GenerateHeader();
var frame = new Frame
{
Header = header with
{
JpegDataLeftLength = (uint)leftBuf.Length,
JpegDataRightLength = (uint)rightBuf.Length
},
LeftJpeg = leftBuf,
RightJpeg = rightBuf
};
lock (_lock)
{
_frames.Add(frame);
}
return frame;
}
public void WriteBin(string path)
{
List<Frame> copy;
lock (_lock)
{
copy = new List<Frame>(_frames);
_frames.Clear();
}
CaptureBin.IO.CaptureBin.WriteAll(Path.Combine(Utils.ModelDataDirectory, path), copy);
}
}
@@ -0,0 +1,486 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Baballonia.CaptureBin.IO;
using Baballonia.Contracts;
using Baballonia.Desktop.Trainer;
using Baballonia.Services;
using Baballonia.Services.events;
using Google.Protobuf.WellKnownTypes;
using OpenCvSharp;
using OverlaySDK;
using OverlaySDK.Packets;
namespace Baballonia.Desktop.Calibration;
public interface ICalibrationStep
{
string Name { get; }
Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct);
}
public class BaseTutorialStep : PacketHandlerAdapter, ICalibrationStep
{
public string Name { get; }
public TimeSpan timeToRun { get; }
protected TaskCompletionSource _token = new();
public BaseTutorialStep(string name) : this(name, TimeSpan.FromSeconds(30))
{
}
public BaseTutorialStep(string name, TimeSpan time)
{
Name = name;
timeToRun = time;
}
public virtual async Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
{
dispatcher.RegisterHandler(this);
dispatcher.Dispatch(new RunVariableLenghtRoutinePacket(Name, timeToRun));
await WaitForRoutineFinishAsync(ct);
dispatcher.UnRegisterHandler(this);
}
protected async Task WaitForRoutineFinishAsync(CancellationToken ct)
{
await _token.Task.WaitAsync(ct);
}
public override void OnRoutineFinishedPacket(RoutineFinishedPacket packet)
{
_token.SetResult();
}
}
public abstract class PositionalAwareCaptureStep : PacketHandlerAdapter, ICalibrationStep
{
public string Name { get; }
public uint Flags { get; }
protected PositionalBinCollector _positionalBinCollector;
protected TaskCompletionSource _token = new();
protected bool _shouldCollect = false;
protected TimeSpan _timeToTun;
public PositionalAwareCaptureStep(string name, uint flags, TimeSpan time)
{
Name = name;
Flags = flags;
_positionalBinCollector = new PositionalBinCollector(flags);
_timeToTun = time;
}
public abstract Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct);
public override void OnHmdPositionalData(HmdPositionalDataPacket positionalData)
{
if (!_shouldCollect)
return;
_positionalBinCollector.UpdatePositionalData(positionalData);
}
public virtual void OnNewEyeFrame(EyePipelineEvents.NewTransformedFrameEvent frame)
{
if (!_shouldCollect)
return;
var images = frame.image.Split();
_positionalBinCollector.AddFrame(images[0], images[1]);
}
protected void StartCollecting()
{
_shouldCollect = true;
}
protected void StopCollecting()
{
_shouldCollect = false;
}
protected async Task WaitForRoutineFinishAsync(CancellationToken ct)
{
await _token.Task.WaitAsync(ct);
}
public override void OnRoutineFinishedPacket(RoutineFinishedPacket packet)
{
_token.SetResult();
}
public void Dispose()
{
_token.SetCanceled();
}
}
public abstract class BaseCaptureStep : PacketHandlerAdapter, ICalibrationStep
{
public string Name { get; }
public uint Flags { get; }
protected BinCollector _binCollector;
protected TaskCompletionSource _token = new();
protected bool _shouldCollect = false;
protected TimeSpan _timeToTun;
public BaseCaptureStep(string name, uint flags, TimeSpan time)
{
Name = name;
Flags = flags;
_binCollector = new BinCollector(flags);
_timeToTun = time;
}
public abstract Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct);
public virtual void OnNewEyeFrame(EyePipelineEvents.NewTransformedFrameEvent frame)
{
if (!_shouldCollect)
return;
var images = frame.image.Split();
AddFrame(images);
}
public virtual Frame AddFrame(Mat[] images)
{
return _binCollector.AddFrame(images[0], images[1]);
}
protected void StartCollecting()
{
_shouldCollect = true;
}
protected void StopCollecting()
{
_shouldCollect = false;
}
protected async Task WaitForRoutineFinishAsync(CancellationToken ct)
{
await _token.Task.WaitAsync(ct);
}
public override void OnRoutineFinishedPacket(RoutineFinishedPacket packet)
{
_token.SetResult();
}
public void Dispose()
{
_token.SetCanceled();
}
}
public class GazeCaptureStep : BasePositionalAwareEyeCaptureStep
{
private Stopwatch _posDataTimer = new();
private readonly TimeSpan _posDataTimeout = TimeSpan.FromSeconds(0.2);
public GazeCaptureStep(IEyePipelineEventBus bus) : this(bus, TimeSpan.FromSeconds(120))
{
}
public GazeCaptureStep(IEyePipelineEventBus bus, TimeSpan time) : base(bus, "gaze",
CaptureFlags.FLAG_GOOD_DATA |
CaptureFlags.FLAG_IN_MOVEMENT |
CaptureFlags.FLAG_VERSION_BIT1 |
CaptureFlags.FLAG_ROUTINE_BIT1, time)
{
}
public override void OnHmdPositionalData(HmdPositionalDataPacket positionalData)
{
if (!_shouldCollect)
return;
_positionalBinCollector.UpdatePositionalData(positionalData);
_posDataTimer.Restart();
}
public override void OnNewEyeFrame(EyePipelineEvents.NewTransformedFrameEvent frame)
{
if (!_shouldCollect)
return;
if (_posDataTimer.Elapsed <= _posDataTimeout)
{
var images = frame.image.Split();
var f = _positionalBinCollector.AddFrame(images[0], images[1]);
if (f is not null)
{
f.Header = f.Header with
{
RoutineLeftLid = 1,
RoutineRightLid = 1,
};
}
}
}
}
public class BasePositionalAwareEyeCaptureStep(
IEyePipelineEventBus eyePipelineEvent,
string name,
uint flags,
TimeSpan time)
: PositionalAwareCaptureStep(name, flags, time)
{
public override async Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
{
dispatcher.RegisterHandler(this);
eyePipelineEvent.Subscribe<EyePipelineEvents.NewTransformedFrameEvent>(OnNewEyeFrame);
dispatcher.Dispatch(new RunVariableLenghtRoutinePacket(Name, _timeToTun));
StartCollecting();
await WaitForRoutineFinishAsync(ct);
eyePipelineEvent.Unsubscribe<EyePipelineEvents.NewTransformedFrameEvent>(OnNewEyeFrame);
dispatcher.UnRegisterHandler(this);
if (ct.IsCancellationRequested)
return;
_positionalBinCollector.WriteBin(Name + ".bin");
}
}
public class BaseEyeCaptureStep(
IEyePipelineEventBus eyePipelineEvent,
string name,
uint flags,
TimeSpan time,
float lid = 0,
float browRaise = 0,
float browAngry = 0,
float widen = 0,
float squint = 0,
float dilate = 0)
: BaseCaptureStep(name, flags, time)
{
public override async Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
{
dispatcher.RegisterHandler(this);
eyePipelineEvent.Subscribe<EyePipelineEvents.NewTransformedFrameEvent>(OnNewEyeFrame);
dispatcher.Dispatch(new RunVariableLenghtRoutinePacket(Name, _timeToTun));
StartCollecting();
await WaitForRoutineFinishAsync(ct);
eyePipelineEvent.Unsubscribe<EyePipelineEvents.NewTransformedFrameEvent>(OnNewEyeFrame);
dispatcher.UnRegisterHandler(this);
if (ct.IsCancellationRequested)
return;
_binCollector.WriteBin(Name + ".bin");
}
public override Frame AddFrame(Mat[] images)
{
var frame = base.AddFrame(images);
frame.Header = frame.Header with
{
RoutineLeftLid = lid,
RoutineRightLid = lid,
RoutineBrowRaise = browRaise,
RoutineBrowAngry = browAngry,
RoutineWiden = widen,
RoutineSquint = squint,
RoutineDilate = dilate,
};
return frame;
}
}
public class CommandDispatchStep : ICalibrationStep
{
public string Name { get; }
public CommandDispatchStep(string name)
{
Name = name;
}
public Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
{
dispatcher.Dispatch(new RunFixedLenghtRoutinePacket(Name));
return Task.CompletedTask;
}
}
public class TrainerCalibrationStep : ICalibrationStep
{
public string Name { get; }
private readonly ITrainerService _trainer;
public TrainerCalibrationStep(ITrainerService overlayTrainer)
{
_trainer = overlayTrainer;
Name = "trainer";
}
public async Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
{
dispatcher.Dispatch(new RunVariableLenghtRoutinePacket(Name, TimeSpan.FromSeconds(120)));
var onProgresHandler = (TrainerProgressReportPacket packet) => { dispatcher.Dispatch(packet); };
_trainer.OnProgress += onProgresHandler;
_trainer.RunTraining(Path.Combine(Utils.ModelDataDirectory, "user_cal.bin"),
Path.Combine(Utils.ModelDataDirectory, "tuned_temporal_eye_tracking_latest.onnx"));
await _trainer.WaitAsync();
_trainer.OnProgress -= onProgresHandler;
}
}
public class EyeCaptureStepFactory
{
private readonly IEyePipelineEventBus _eyePipelineEvent;
public EyeCaptureStepFactory(IEyePipelineEventBus eyePipelineEvent)
{
_eyePipelineEvent = eyePipelineEvent;
}
public BaseEyeCaptureStep Create(string name, uint flags, TimeSpan time,
float lid = 0,
float browRaise = 0,
float browAngry = 0,
float widen = 0,
float squint = 0,
float dilate = 0) =>
new(_eyePipelineEvent, name, flags, time, lid, browRaise, browAngry, widen, squint, dilate);
}
public class MergeBinsStep : ICalibrationStep
{
public string Name { get; } = "bin_merger";
private string[] _binNames;
public MergeBinsStep(params string[] binNames)
{
_binNames = binNames;
}
public Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
{
MergeBins("user_cal.bin", _binNames);
return Task.CompletedTask;
}
private static void MergeBins(string result, params string[] inputs)
{
var resultPath = Path.Combine(Utils.ModelDataDirectory, result);
var inputPaths = inputs.Select(i => Path.Combine(Utils.ModelDataDirectory, i)).ToArray();
CaptureBin.IO.CaptureBin.Concatenate(resultPath, inputPaths);
}
}
public class EyeCalibration
{
private readonly EyeCaptureStepFactory _eyeCaptureStepFactory;
private readonly ITrainerService _trainer;
private readonly IEyePipelineEventBus _eyePipelineEventBus;
public EyeCalibration(EyeCaptureStepFactory eyeCaptureStepFactory, ITrainerService trainer,
IEyePipelineEventBus eyePipelineEventBus)
{
_eyeCaptureStepFactory = eyeCaptureStepFactory;
_trainer = trainer;
_eyePipelineEventBus = eyePipelineEventBus;
}
public IEnumerable<ICalibrationStep> BasicAllCalibration()
{
List<ICalibrationStep> steps =
[
new BaseTutorialStep("gazetutorial"),
new GazeCaptureStep(_eyePipelineEventBus),
new BaseTutorialStep("blinktutorial", TimeSpan.FromSeconds(10)),
_eyeCaptureStepFactory.Create("blink",
CaptureFlags.FLAG_GOOD_DATA |
CaptureFlags.FLAG_IN_MOVEMENT |
CaptureFlags.FLAG_VERSION_BIT1 |
CaptureFlags.FLAG_ROUTINE_BIT1,
TimeSpan.FromSeconds(20), lid: 0
),
// steps.Add(new BaseTutorialStep("dilationtutorial"));
// steps.Add(_eyeCaptureStepFactory.Create("dilation",
// CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_DILATION_BLACK));
new BaseTutorialStep("widentutorial", TimeSpan.FromSeconds(10)),
_eyeCaptureStepFactory.Create("widen",
CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20), widen: 1, lid: 1),
new BaseTutorialStep("squinttutorial", TimeSpan.FromSeconds(10)),
_eyeCaptureStepFactory.Create("squint",
CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20), squint: 1, lid: 1),
new BaseTutorialStep("browtutorial", TimeSpan.FromSeconds(10)),
_eyeCaptureStepFactory.Create("brow",
CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20), browAngry: 1, lid: 1),
// steps.Add(new BaseTutorialStep("covergencetutorial"));
// steps.Add(_eyeCaptureStepFactory.Create("covergence",
// CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_WHATEVER_NOT_IMPLEMENTED));
new MergeBinsStep("gaze.bin", "blink.bin", "widen.bin", "squint.bin", "brow.bin"),
new TrainerCalibrationStep(_trainer),
new CommandDispatchStep("close")
];
// steps.Add(new BaseTutorialStep("dilationtutorial"));
// steps.Add(_eyeCaptureStepFactory.Create("dilation",
// CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_DILATION_BLACK));
// steps.Add(new BaseTutorialStep("covergencetutorial"));
// steps.Add(_eyeCaptureStepFactory.Create("covergence",
// CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_WHATEVER_NOT_IMPLEMENTED));
return steps;
}
public IEnumerable<ICalibrationStep> BasicAllCalibrationQuick()
{
List<ICalibrationStep> steps =
[
new BaseTutorialStep("gazetutorialshort", TimeSpan.FromSeconds(5)),
new GazeCaptureStep(_eyePipelineEventBus, TimeSpan.FromSeconds(10)),
new BaseTutorialStep("blinktutorial", TimeSpan.FromSeconds(4)),
_eyeCaptureStepFactory.Create("blink",
CaptureFlags.FLAG_GOOD_DATA |
CaptureFlags.FLAG_IN_MOVEMENT |
CaptureFlags.FLAG_VERSION_BIT1 |
CaptureFlags.FLAG_ROUTINE_BIT1,
TimeSpan.FromSeconds(20)
),
new BaseTutorialStep("widentutorial", TimeSpan.FromSeconds(4)),
_eyeCaptureStepFactory.Create("widen",
CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20)),
new BaseTutorialStep("squinttutorial", TimeSpan.FromSeconds(4)),
_eyeCaptureStepFactory.Create("squint",
CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20)),
new BaseTutorialStep("browtutorial", TimeSpan.FromSeconds(4)),
_eyeCaptureStepFactory.Create("brow",
CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20)),
new MergeBinsStep("gaze.bin", "blink.bin", "widen.bin", "squint.bin", "brow.bin"),
new TrainerCalibrationStep(_trainer),
new CommandDispatchStep("close")
];
return steps;
}
}
@@ -0,0 +1,11 @@
using System;
using System.Threading.Tasks;
namespace Baballonia.Desktop.Calibration;
public interface IOverlayProgram : IDisposable
{
bool CanStart();
void Start();
Task WaitForExitAsync();
}
Whitespace-only changes.
Whitespace-only changes.
Binary file not shown.
@@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Baballonia.CaptureBin.IO;
using Baballonia.Contracts;
using Baballonia.Desktop.Calibration.Aero;
using Baballonia.Desktop.Trainer;
using Baballonia.Helpers;
using Baballonia.Services;
using Baballonia.Services.events;
using Microsoft.Extensions.Logging;
using OpenCvSharp;
using OverlaySDK;
using OverlaySDK.Packets;
namespace Baballonia.Desktop.Calibration;
public class OverlayTrainerService : IVROverlay, IDisposable
{
private ILocalSettingsService _localSettingsService;
private ILogger<OverlayTrainerService> _logger;
private IOverlayProgram _program;
private EyePipelineManager _eyePipelineManager;
private readonly EyeCalibration _eyeCalibration;
private readonly CancellationTokenSource _tokenSource = new();
public OverlayTrainerService(ILogger<OverlayTrainerService> logger, IOverlayProgram overlayProgram,
ILocalSettingsService localSettingsService, EyePipelineManager eyePipelineManager,
EyeCalibration eyeCalibration)
{
_logger = logger;
_program = overlayProgram;
_localSettingsService = localSettingsService;
_eyePipelineManager = eyePipelineManager;
_eyeCalibration = eyeCalibration;
}
public void Dispose()
{
_program.Dispose();
}
public async Task<(bool success, string status)> EyeTrackingCalibrationRequested(
CalibrationRoutine.Routines routine)
{
if (!_program.CanStart())
{
return (false, "Cannot start Overlay");
}
_program.Start();
var logger = new OverlayLogger(_logger);
var sfactory = new SocketFactory();
var sock = sfactory.CreateServer("127.0.0.1", 2425);
logger.Info("Accepted connection");
var tcp = new EventDrivenTcpClient(sock);
var client = new EventDrivenJsonClient(tcp);
var messageDispatcher = new OverlayMessageDispatcher(logger, client);
if (!Directory.Exists(Utils.ModelDataDirectory)) Directory.CreateDirectory(Utils.ModelDataDirectory);
if (!Directory.Exists(Utils.ModelsDirectory)) Directory.CreateDirectory(Utils.ModelsDirectory);
var steps = routine switch
{
CalibrationRoutine.Routines.BasicCalibration => _eyeCalibration.BasicAllCalibration(),
CalibrationRoutine.Routines.BasicCalibrationNoTutorial => _eyeCalibration.BasicAllCalibrationQuick(),
_ => _eyeCalibration.BasicAllCalibration()
};
foreach (var calibrationStep in steps)
{
await calibrationStep.ExecuteAsync(messageDispatcher, _tokenSource.Token);
}
var srcPath = Path.Combine(Utils.ModelDataDirectory, "tuned_temporal_eye_tracking_latest.onnx");
var destPath = Path.Combine(Utils.ModelsDirectory,
$"tuned_temporal_eye_tracking_{DateTime.Now:yyyyMMdd_HHmmss}.onnx");
File.Move(srcPath, destPath);
_localSettingsService.SaveSetting("EyeHome_EyeModel", destPath);
await _eyePipelineManager.LoadInferenceAsync();
await _program.WaitForExitAsync();
return (true, string.Empty);
}
}
@@ -0,0 +1,111 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Baballonia.Assets;
using Microsoft.Extensions.Logging;
namespace Baballonia.Desktop.Calibration.Aero;
public class OverlayProgram : IOverlayProgram, IDisposable
{
private ILogger<OverlayProgram> _logger;
private string? _executablePath;
private Process? _process;
public OverlayProgram(ILogger<OverlayProgram> logger)
{
var isWindows = OperatingSystem.IsWindows();
var isArm = RuntimeInformation.OSArchitecture is Architecture.Arm or Architecture.Arm64 or Architecture.Armv6;
var architectureIdentifier = isArm ? "arm64" : "x86_64";
var OverlayPath = Path.Combine(AppContext.BaseDirectory, "Calibration", isWindows ? "Windows" : "Linux",
"Overlay");
var Overlay = Path.Combine(OverlayPath,
isWindows ? $"BabbleCalibration.{architectureIdentifier}.exe" : $"BabbleCalibration.{architectureIdentifier}");
_executablePath = Overlay;
_logger = logger;
}
public bool CanStart()
{
if (!File.Exists(_executablePath))
{
_logger.LogError("Trainer program not found: {} not exists", _executablePath);
return false;
}
return true;
}
public void Start()
{
if (_executablePath == null)
return;
_process?.Kill();
var hitList = Process.GetProcesses()
.Where(p => p.ProcessName == Path.GetFileNameWithoutExtension(_executablePath)).ToArray();
if (hitList.Length > 0)
{
foreach (var p in hitList) p.Kill(true);
}
var processList = Process.GetProcesses();
var steamvr = processList.Any(p => p.ProcessName.ToLower().Contains("vrserver"));
var monado = processList.Any(p => p.ProcessName.ToLower().Contains("monado"));
var isWindows = OperatingSystem.IsWindows();
var launchArgs = "";
if (!steamvr && !monado)
{
launchArgs = "--use-debug";
}
else
{
if (isWindows)
{
if (steamvr)
launchArgs = "--use-openvr";
else if (monado) launchArgs = "--xr-mode on"; //uhhhhh?????
}
else
{
launchArgs = "--xr-mode on";
}
}
var startInfo = new ProcessStartInfo
{
FileName = _executablePath,
WindowStyle = ProcessWindowStyle.Hidden,
CreateNoWindow = true,
Arguments = launchArgs
};
_process = new Process
{
StartInfo = startInfo,
EnableRaisingEvents = true
};
_process.Start();
}
public Task WaitForExitAsync()
{
if (_process == null)
return Task.CompletedTask;
return _process.WaitForExitAsync();
}
public void Dispose()
{
_process?.Kill();
_process = null;
}
}
@@ -0,0 +1,153 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Baballonia.Contracts;
using Microsoft.Extensions.Logging;
using OverlaySDK.Packets;
namespace Baballonia.Desktop.Trainer;
public partial class TrainerService : ITrainerService
{
private readonly object _lock = new();
private readonly ILogger<TrainerService> _logger;
private Process? trainerProcess;
public event Action<TrainerProgressReportPacket>? OnProgress;
public TrainerService(ILogger<TrainerService> logger)
{
_logger = logger;
}
[GeneratedRegex(@"Batch\s+(\d+)/(\d+),\s+Loss:\s+([0-9.]+)")]
private static partial Regex ParseBatchRegex();
TrainerProgressReportPacket? ParseBatch(string line)
{
var match = ParseBatchRegex().Match(line);
if (!match.Success)
return null;
_logger.LogDebug(line);
var currentBatch = int.Parse(match.Groups[1].Value);
var totalBatches = int.Parse(match.Groups[2].Value);
var loss = double.Parse(match.Groups[3].Value);
return new TrainerProgressReportPacket("Batch", currentBatch, totalBatches, loss);
}
[GeneratedRegex(@"Epoch\s+(\d+)/(\d+)\s+completed\s+in\s+([\d.]+)s\.\s+Average\s+loss:\s+([\d.eE+-]+)")]
private static partial Regex ParseEpochRegex();
TrainerProgressReportPacket? ParseEpoch(string line)
{
var match = ParseEpochRegex().Match(line);
if (!match.Success)
return null;
_logger.LogDebug(line);
var currentEpoch = int.Parse(match.Groups[1].Value);
var totalEpochs = int.Parse(match.Groups[2].Value);
var loss = double.Parse(match.Groups[3].Value);
return new TrainerProgressReportPacket("Epoch", currentEpoch, totalEpochs, loss);
}
[GeneratedRegex(@"\s*Training\s+completed\s+successfully!\s*")]
private static partial Regex ParseTrainingCompleteRegex();
bool ParseTrainingComplete(string line)
{
var match = ParseTrainingCompleteRegex().Match(line);
_logger.LogDebug(line);
return match.Success;
}
void NewLineEventHandler(object sender, DataReceivedEventArgs dataReceivedEventArgs)
{
_logger.LogDebug(dataReceivedEventArgs.Data);
if (dataReceivedEventArgs.Data == null)
return;
var progress = ParseBatch(dataReceivedEventArgs.Data);
if (progress == null)
progress = ParseEpoch(dataReceivedEventArgs.Data);
if (progress != null)
{
OnProgress?.Invoke(progress);
}
var isCompleted = ParseTrainingComplete(dataReceivedEventArgs.Data);
if (isCompleted)
return;
}
public void RunTraining(string usercalbinPath, string outputfilePath)
{
if (!File.Exists(usercalbinPath))
throw new FileNotFoundException(usercalbinPath + " not found");
var isWindows = OperatingSystem.IsWindows();
var basePath = Path.Combine(AppContext.BaseDirectory, "Calibration", isWindows ? "Windows" : "Linux", "Trainer");
var trainerPath = Path.Combine(basePath, isWindows ? $"BabbleTrainer.exe" : $"BabbleTrainer");
if (!File.Exists(trainerPath))
throw new FileNotFoundException(trainerPath + " not found");
lock (_lock)
{
if (trainerProcess != null && trainerProcess.HasExited)
trainerProcess = null;
if (trainerProcess != null)
throw new Exception("Training process already running");
var launchArgs = $"{usercalbinPath} {outputfilePath}";
var startInfo = new ProcessStartInfo
{
FileName = trainerPath,
Arguments = launchArgs,
UseShellExecute = false,
RedirectStandardOutput = true,
WorkingDirectory = basePath,
};
trainerProcess = new Process
{
StartInfo = startInfo,
EnableRaisingEvents = true,
};
trainerProcess.OutputDataReceived += NewLineEventHandler;
trainerProcess.Exited += (sender, args) => { Interlocked.Exchange(ref trainerProcess, null); };
trainerProcess.Start();
trainerProcess.BeginOutputReadLine();
}
}
public Task WaitAsync()
{
Process? process;
lock (_lock)
{
process = trainerProcess;
}
return process != null
? process.WaitForExitAsync()
: Task.CompletedTask;
}
public void Dispose()
{
lock (_lock)
{
trainerProcess?.Kill();
}
}
}
Whitespace-only changes.
Whitespace-only changes.
@@ -1 +0,0 @@
1 ICON "app.ico"
Binary file not shown.
Whitespace-only changes.
+12 -2
View File
@@ -4,6 +4,8 @@ using Baballonia.Desktop.Captures;
using System;
using System.Threading;
using Baballonia.Contracts;
using Baballonia.Desktop.Calibration;
using Baballonia.Desktop.Trainer;
using Microsoft.Extensions.DependencyInjection;
using Velopack;
@@ -34,12 +36,20 @@ sealed class Program
VelopackApp.Build().Run();
App.RegisterPlatformServices<
AeroOverlayTrainerCombo,
App.RegisterRequiredPlatformServices<
OverlayTrainerService,
DesktopDeviceEnumerator,
DesktopConnector
>();
App.RegisterPlatformSpecifficServices(collection =>
{
collection.AddSingleton<IOverlayProgram, OverlayProgram>();
collection.AddSingleton<ITrainerService, TrainerService>();
collection.AddSingleton<EyeCaptureStepFactory>();
collection.AddSingleton<EyeCalibration>();
});
try
{
var builder = BuildAvaloniaApp();
@@ -33,7 +33,8 @@ def make_installer(output_dir, installer_dir, version):
"--packVersion", version,
"--packDir", output_dir,
"--mainExe", "Baballonia.Desktop",
"--outputDir", installer_dir
"--outputDir", installer_dir,
"--icon", "../Assets/IconOpaque_32x32.png"
]
print(f"Running: {' '.join(cmd)}")
subprocess.run(cmd, check=True)
Binary file not shown.
File diff suppressed because it is too large. Load diff
-47
View File
@@ -1,47 +0,0 @@
# -*- mode: python ; coding: utf-8 -*-
#from PyInstaller.utils.hooks import collect_dynamic_libs
#binaries = collect_dynamic_libs('torch_directml')
a = Analysis(
['trainermin.py'],
pathex=[],
#binaries=binaries,
datas=[],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=0,
)
pyz = PYZ(a.pure)
exe = EXE(
pyz,
a.scripts,
[],
exclude_binaries=False,
name='trainermin',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
console=True,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
onefile=True,
)
coll = COLLECT(
exe,
a.binaries,
a.datas,
strip=False,
upx=True,
upx_exclude=[],
name='trainermin',
onefile=True,
)
Binary file not shown.
@@ -0,0 +1,80 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Baballonia.Contracts;
using Baballonia.Desktop.Calibration;
using Baballonia.Desktop.Calibration.Aero;
using Baballonia.Desktop.Trainer;
using Baballonia.Helpers;
using Baballonia.Services;
using Baballonia.Services.events;
using JetBrains.Annotations;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using OpenCvSharp;
using OverlaySDK.Packets;
namespace Baballonia.Tests.Calibration;
[TestClass]
[TestSubject(typeof(OverlayTrainerService))]
public class OverlayTrainerServiceTest
{
[TestMethod]
public async Task Test()
{
// ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().AddDebug());
//
// EyePipelineEventBus eyePipelineEventBus = new EyePipelineEventBus();
// CancellationTokenSource tokenSource = new CancellationTokenSource();
// Task.Run(() =>
// {
// while (!tokenSource.Token.IsCancellationRequested)
// {
// Mat frame1 = new Mat(256, 256, MatType.CV_8UC1);
// Mat frame2 = new Mat(256, 256, MatType.CV_8UC1);
// Mat combiner = new Mat();
//
// Cv2.Merge([frame1, frame2], combiner);
//
// eyePipelineEventBus.Publish(new EyePipelineEvents.NewTransformedFrameEvent(combiner));
// Thread.Sleep(22);
// }
// });
//
// var mockTrainer = new Mock<ITrainerService>();
// var progressEvent = new Action<TrainerProgressReportPacket>((p) => { });
// mockTrainer.SetupAdd(m => m.OnProgress += It.IsAny<Action<TrainerProgressReportPacket>>())
// .Callback<Action<TrainerProgressReportPacket>>(h => progressEvent += h);
// mockTrainer.SetupRemove(m => m.OnProgress -= It.IsAny<Action<TrainerProgressReportPacket>>())
// .Callback<Action<TrainerProgressReportPacket>>(h => progressEvent -= h);
//
// mockTrainer.Setup(m => m.RunTraining(It.IsAny<string>(), It.IsAny<string>()))
// .Callback(() =>
// {
// progressEvent?.Invoke(new TrainerProgressReportPacket("Batch", 1, 100, 0.123));
// Thread.Sleep(2000);
// progressEvent?.Invoke(new TrainerProgressReportPacket("Epoch", 1, 10, 0.045));
// Thread.Sleep(2000);
// progressEvent?.Invoke(new TrainerProgressReportPacket("Epoch", 10, 10, 0.005));
// });
//
// mockTrainer.Setup(m => m.WaitAsync())
// .Returns(Task.CompletedTask);
//
// var mockOverlayProgram = new Mock<IOverlayProgram>();
// mockOverlayProgram.Setup(program => program.CanStart()).Returns(true);
// mockOverlayProgram.Setup(program => program.WaitForExitAsync()).Returns(Task.CompletedTask);
//
// OverlayTrainerService overlayTrainerService =
// new OverlayTrainerService(loggerFactory.CreateLogger<OverlayTrainerService>(), eyePipelineEventBus, mockTrainer.Object, mockOverlayProgram.Object);
//
// await overlayTrainerService.EyeTrackingCalibrationRequested(CalibrationRoutine.Map["BasicCalibration"]);
//
// tokenSource.Cancel();
//
//
// mockTrainer.Verify(service => service.RunTraining(It.IsAny<string>(), It.IsAny<string>()), Times.Once);
}
}
@@ -0,0 +1,25 @@
using System;
using System.Threading.Tasks;
using Baballonia.Desktop.Trainer;
using JetBrains.Annotations;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace Baballonia.Tests.Trainer;
[TestClass]
[TestSubject(typeof(TrainerService))]
public class TrainerServiceTest
{
[TestMethod]
public async Task Test()
{
var factory = LoggerFactory.Create(builder => builder.AddConsole().AddDebug());
var log = factory.CreateLogger<TrainerService>();
TrainerService service = new TrainerService(log);
service.RunTraining("test.bin", "model.onnx");
await service.WaitAsync();
}
}
@@ -20,7 +20,7 @@ public class VFTCaptureFactory : ICaptureFactory
public bool CanConnect(string address)
{
var lowered = address.ToLower();
return lowered.StartsWith("/dev/video");
return lowered.StartsWith("/dev/video") && OperatingSystem.IsLinux();
}
public string GetProviderName()
+10 -2
View File
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
@@ -33,8 +34,9 @@ public class App : Application
private IHost? _host;
private bool IsTeardDown = false;
private static Action<IServiceCollection> ConfigurePlatformServices { get; set; }
private static Action<IServiceCollection>? _platformSpecifficServices;
public static void RegisterPlatformServices<TOverlay, TDeviceEnumerator, TPlatformConnector>()
public static void RegisterRequiredPlatformServices<TOverlay, TDeviceEnumerator, TPlatformConnector>()
where TOverlay : class, IVROverlay
where TDeviceEnumerator : class, IDeviceEnumerator
where TPlatformConnector : class, IPlatformConnector
@@ -47,6 +49,11 @@ public class App : Application
};
}
public static void RegisterPlatformSpecifficServices(Action<IServiceCollection> configure)
{
_platformSpecifficServices = configure;
}
public override void Initialize()
{
@@ -149,6 +156,7 @@ public class App : Application
}
ConfigurePlatformServices.Invoke(services);
_platformSpecifficServices?.Invoke(services);
services.AddHostedService(provider => provider.GetService<OscRecvService>()!);
services.AddHostedService(provider => provider.GetService<ParameterSenderService>()!);
@@ -220,7 +228,7 @@ public class App : Application
desktop.MainWindow.Loaded += (_, _) => { desktop.MainWindow.ShowOnboardingIfNeeded(); };
desktop.Exit += (s, e) =>
{
OnShutdown(s,e);
OnShutdown(s, e);
_host.Dispose();
};
desktop.ShutdownRequested += OnShutdown;
+34 -28
View File
@@ -1,17 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
@@ -26,36 +26,36 @@
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
@@ -744,4 +744,10 @@
<data name="Settings_SteamVR_Auto_Start_Description" xml:space="preserve">
<value>Should the app launch with SteamVR? (Change requires a SteamVR restart.)</value>
</data>
</root>
<data name="Settings_Advanced_Header" xml:space="preserve">
<value>Enable advanced settings</value>
</data>
<data name="Settings_Advanced_Description" xml:space="preserve">
<value>Advanced Settings</value>
</data>
</root>
+11 -3
View File
@@ -13,8 +13,15 @@
<ItemGroup>
<AvaloniaResource Include="Assets\**" />
<None Update="Firmware\Windows\espflash.exe">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<None Update="Firmware\Windows\espflash.exe" Condition="$([MSBuild]::IsOSPlatform('Windows'))">
<Link>%(RecursiveDir)\%(Filename)%(Extension)</Link>
<TargetPath>%(RecursiveDir)\%(Filename)%(Extension)</TargetPath>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Firmware\Linux\espflash" Condition="$([MSBuild]::IsOSPlatform('Linux'))">
<Link>%(RecursiveDir)\%(Filename)%(Extension)</Link>
<TargetPath>%(RecursiveDir)\%(Filename)%(Extension)</TargetPath>
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="font.ttf">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -52,6 +59,7 @@
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="OpenCvSharp4" Version="4.11.0.20250507" />
<PackageReference Include="OscCore" Version="1.0.5" />
<PackageReference Include="Project-Babble.Baballonia.OverlaySDK" Version="1.0.5-alpha" />
<PackageReference Include="SkiaSharp" Version="3.119.0" />
<PackageReference Include="SoundFlow" Version="1.2.1" />
<PackageReference Include="System.IO.Ports" Version="9.0.8" />
@@ -104,7 +112,7 @@
<EmbeddedResource Include="faceModel.onnx">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="SteamVR\manifest.vrmanifest" Link ="manifest.vrmanifest">
<EmbeddedResource Include="SteamVR\manifest.vrmanifest" Link="manifest.vrmanifest">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="SteamVR\openvr_api.dll" Link="openvr_api.dll">
@@ -0,0 +1,14 @@
using System;
using System.Diagnostics;
using System.Threading.Tasks;
using OverlaySDK.Packets;
namespace Baballonia.Contracts;
public interface ITrainerService : IDisposable
{
public event Action<TrainerProgressReportPacket>? OnProgress;
public void RunTraining(string usercalbinPath, string outputfilePath);
public Task WaitAsync();
}
+2 -1
View File
@@ -1,9 +1,10 @@
using System;
using System.Threading.Tasks;
using Baballonia.Helpers;
namespace Baballonia.Contracts;
public interface IVROverlay : IDisposable
{
public Task<(bool success, string status)> EyeTrackingCalibrationRequested(string calibrationRoutine);
public Task<(bool success, string status)> EyeTrackingCalibrationRequested(CalibrationRoutine.Routines calibrationRoutine);
}
View File
Whitespace-only changes.
Binary file not shown.
Whitespace-only changes.
+13 -1
View File
@@ -1,10 +1,21 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using Tmds.DBus.Protocol;
namespace Baballonia.Helpers;
public static class CalibrationRoutine
{
public enum Routines
{
BasicCalibration,
BasicCalibrationNoTutorial,
GazeOnly,
BlinkOnly,
}
public static readonly Dictionary<string, Routines> Map = Enum.GetValues<Routines>().ToDictionary(i => i.ToString(), i => i);
/*
public static readonly Dictionary<string, string> Map = new()
{
{ "BasicCalibration", "0" },
@@ -31,4 +42,5 @@ public static class CalibrationRoutine
{ "DepthSimulation", "21" },
{ "QuickCalibrationNoTutorial", "22" },
};
*/
}
+22 -12
View File
@@ -32,34 +32,44 @@ public class EyePipelineEventBus : GenericEventBus, IEyePipelineEventBus
public class GenericEventBus : IEventBus
{
private readonly Dictionary<Type, List<Delegate>> _subscribers = new();
private object _lock = new();
public void Subscribe<T>(Action<T> callback)
{
if (!_subscribers.TryGetValue(typeof(T), out var list))
lock (_lock)
{
list = new List<Delegate>();
_subscribers[typeof(T)] = list;
}
if (!_subscribers.TryGetValue(typeof(T), out var list))
{
list = new List<Delegate>();
_subscribers[typeof(T)] = list;
}
list.Add(callback);
list.Add(callback);
}
}
public void Unsubscribe<T>(Action<T> callback)
{
if (_subscribers.TryGetValue(typeof(T), out var list))
lock (_lock)
{
list.Remove(callback);
if (list.Count == 0)
_subscribers.Remove(typeof(T));
if (_subscribers.TryGetValue(typeof(T), out var list))
{
list.Remove(callback);
if (list.Count == 0)
_subscribers.Remove(typeof(T));
}
}
}
public void Publish<T>(T data)
{
if (_subscribers.TryGetValue(typeof(T), out var list))
lock (_lock)
{
foreach (var callback in list.Cast<Action<T>>())
callback(data);
if (_subscribers.TryGetValue(typeof(T), out var list))
{
foreach (var callback in list.Cast<Action<T>>())
callback(data);
}
}
}
}
+18 -2
View File
@@ -141,9 +141,17 @@ public class EyePipelineManager
public async Task<bool> StartLeftVideoSource(string cameraAddress, string preferredBackend)
{
if (string.IsNullOrEmpty(cameraAddress))
return false;
if (_pipeline.VideoSource == null)
{
var cam = await _singleCameraSourceFactory.CreateStart(cameraAddress, preferredBackend);
SingleCameraSource cam;
if (string.IsNullOrEmpty(preferredBackend))
cam = await _singleCameraSourceFactory.CreateStart(cameraAddress);
else
cam = await _singleCameraSourceFactory.CreateStart(cameraAddress, preferredBackend);
if (cam == null)
return false;
@@ -202,9 +210,17 @@ public class EyePipelineManager
public async Task<bool> StartRightVideoSource(string cameraAddress, string preferredBackend)
{
if (string.IsNullOrEmpty(cameraAddress))
return false;
if (_pipeline.VideoSource == null)
{
var cam = await _singleCameraSourceFactory.CreateStart(cameraAddress, preferredBackend);
SingleCameraSource cam;
if (string.IsNullOrEmpty(preferredBackend))
cam = await _singleCameraSourceFactory.CreateStart(cameraAddress);
else
cam = await _singleCameraSourceFactory.CreateStart(cameraAddress, preferredBackend);
if (cam == null)
return false;
@@ -4,6 +4,7 @@ using System.Threading.Tasks;
using Baballonia.Contracts;
using Baballonia.Services.Inference.Filters;
using Baballonia.Services.Inference.Models;
using Baballonia.Services.Inference.VideoSources;
using Microsoft.Extensions.Logging;
namespace Baballonia.Services.Inference;
@@ -98,13 +99,21 @@ public class FacePipelineManager
public async Task<bool> StartVideoSource(string cameraAddress, string preferredBackend)
{
if (string.IsNullOrEmpty(cameraAddress))
return false;
if (_pipeline.VideoSource != null)
{
_pipeline.VideoSource.Dispose();
_pipeline.VideoSource = null;
}
var cam = await _singleCameraSourceFactory.CreateStart(cameraAddress, preferredBackend);
SingleCameraSource cam;
if (string.IsNullOrEmpty(preferredBackend))
cam = await _singleCameraSourceFactory.CreateStart(cameraAddress);
else
cam = await _singleCameraSourceFactory.CreateStart(cameraAddress, preferredBackend);
if (cam == null)
return false;
+4
View File
@@ -62,6 +62,10 @@ public static class Utils
? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "ProjectBabble", "Models")
: AppContext.BaseDirectory;
public static readonly string ModelDataDirectory = IsSupportedDesktopOS
? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "ProjectBabble", "ModelData")
: AppContext.BaseDirectory;
public static readonly string VrcftLibsDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"VRCFaceTracking",
"CustomLibs");
@@ -65,6 +65,10 @@ public partial class AppSettingsViewModel : ViewModelBase
[property: SavedSetting("AppSettings_LogLevel", "Debug")]
private string _logLevel;
[ObservableProperty]
[property: SavedSetting("AppSettings_AdvancedOptions", false)]
private bool _advancedOptions;
[ObservableProperty]
[property: SavedSetting("AppSettings_StabilizeEyes", true)]
private bool _stabilizeEyes;
@@ -112,7 +116,7 @@ public partial class AppSettingsViewModel : ViewModelBase
SettingsService.Save(this);
_facePipelineManager.LoadFilter();
_eyePipelineManager.LoadFilter();
if (p.PropertyName == nameof(StabilizeEyes))
{
_eyePipelineManager.LoadEyeStabilization();
@@ -97,7 +97,9 @@ public partial class FirmwareViewModel : ViewModelBase, IDisposable
public FirmwareViewModel()
{
AvailableFirmwareTypes.Clear();
var binaries = Directory.GetFiles(_bundledFirmwarePath, "*.bin");
var binaries = Directory.
GetFiles(_bundledFirmwarePath, "*.bin").
OrderByDescending(x => x.Length);
foreach (var bin in binaries)
{
AvailableFirmwareTypes.Add(Path.GetFileName(bin));
@@ -108,13 +108,24 @@ public partial class HomePageViewModel : ViewModelBase, IDisposable
partial void OnDisplayAddressChanged(string value)
{
// "it looks like shit" be damned, i've seen *way* too many instances of people needing to force
// a capture backend and this stupid visual preference garbage is getting in the way of it.
// we're going to get more support tickets from people who can't find this when they're on linux or
// using vive face trackers.
// if you think it looks bad, suck it, make it look better yourself, it has to be here,
// the NEED take precedent over visual preferences.
//var advancedEnabled = _localSettingsService.ReadSetting<bool>("AppSettings_AdvancedOptions");
var advancedEnabled = true;
var availableCaptureFactories = _platformConnector.GetCaptureFactories()
.Where(factory => factory.CanConnect(value)).ToArray();
var shouldShow = availableCaptureFactories.Length >= 2;
var shouldShow = advancedEnabled && availableCaptureFactories.Length >= 2;
CaptureMethodVisible = shouldShow;
CaptureMethods.Clear();
// IF YOU'RE TRYING TO ENABLE A SPECIFIC BACKEND FOR A CAPTURE, DO NOT TOUCH THIS!
// THIS IS NOT WHERE YOU FORCE A BACKEND
// OVERRIDE THE CANCAPTURE() METHOD INSIDE THE CAPTURE'S CAPTURE FACTORY
if (shouldShow)
{
CaptureMethods.Add(Assets.Resources.Home_Backend_Default);
@@ -127,7 +138,6 @@ public partial class HomePageViewModel : ViewModelBase, IDisposable
{
SelectedCaptureMethod = "";
}
}
partial void OnSelectedCaptureMethodChanged(string value)
@@ -395,7 +405,7 @@ public partial class HomePageViewModel : ViewModelBase, IDisposable
private readonly ILogger<HomePageViewModel> _logger;
private readonly IPlatformConnector _platformConnector;
public string RequestedVRCalibration = CalibrationRoutine.Map["QuickCalibration"];
public CalibrationRoutine.Routines RequestedVRCalibration = CalibrationRoutine.Map["BasicCalibration"];
public HomePageViewModel(FacePipelineManager facePipelineManager,
@@ -653,29 +663,30 @@ public partial class HomePageViewModel : ViewModelBase, IDisposable
[RelayCommand]
private async Task RequestVRCalibration()
{
var res = await _vrOverlay.EyeTrackingCalibrationRequested(RequestedVRCalibration);
if (res.success)
{
if (!Directory.Exists(Utils.ModelsDirectory))
var res = await Task.Run(async () =>
{
Directory.CreateDirectory(Utils.ModelsDirectory);
try
{
return await _vrOverlay.EyeTrackingCalibrationRequested(RequestedVRCalibration);
}
catch (Exception ex)
{
return (false, ex.Message);
}
}
var destPath = Path.Combine(Utils.ModelsDirectory,
$"tuned_temporal_eye_tracking_{DateTime.Now.ToString("yyyyMMdd_HHmmss")}.onnx");
File.Move("tuned_temporal_eye_tracking.onnx", destPath);
_localSettings.SaveSetting("EyeHome_EyeModel", destPath);
await _eyePipelineManager.LoadInferenceAsync();
);
if (res.Item1)
{
SelectedCalibrationTextBlock.Foreground = new SolidColorBrush(Colors.Green);
}
else
{
SelectedCalibrationTextBlock.Foreground = new SolidColorBrush(Colors.Red);
_logger.LogError(res.status);
_logger.LogError(res.Item2);
}
var previousText = SelectedCalibrationTextBlock.Text;
SelectedCalibrationTextBlock.Text = res.status;
SelectedCalibrationTextBlock.Text = res.Item2;
await Task.Delay(5000);
SelectedCalibrationTextBlock.Text = previousText;
SelectedCalibrationTextBlock.Foreground = new SolidColorBrush(GetBaseHighColor());
@@ -369,6 +369,12 @@
IconPath="PowerRegular"
SettingToggled="{Binding SteamvrAutoStart, Mode=TwoWay}" />
<controls:SettingsToggle
Title="Advanced settings"
Description="Enable advanced settings"
IconPath="SettingsRegular"
SettingToggled="{Binding AdvancedOptions, Mode=TwoWay}" />
<Border
Padding="15,5,15,5"
Background="{DynamicResource SystemChromeMediumLowColor}"
+15 -3
View File
@@ -56,17 +56,29 @@
</StackPanel>
</SplitButton.Content>
<SplitButton.Flyout>
<MenuFlyout> <!-- Names come from CalibrationRoutine.cs -->
<MenuItem Click="OnCalibrationMenuItemClick" x:Name="QuickCalibration" Header="{x:Static assets:Resources.Home_Eye_Calibration}">
<MenuItem Click="OnCalibrationMenuItemClick" x:Name="BasicCalibration" Header="{x:Static assets:Resources.Home_Eye_Calibration}">
<MenuItem.Icon>
<PathIcon Data="{StaticResource HeadsetVrRegular}" />
</MenuItem.Icon>
</MenuItem>
<!--<MenuItem Click="OnCalibrationMenuItemClick" x:Name="QuickCalibrationNoTutorial" Header="Eye Calibration (No Tutorial)">
<MenuItem Click="OnCalibrationMenuItemClick" x:Name="BasicCalibrationNoTutorial" Header="Eye Calibration (No Tutorial)">
<MenuItem.Icon>
<PathIcon Data="{StaticResource HeadsetVrRegular}" />
</MenuItem.Icon>
</MenuItem>-->
</MenuItem>
<MenuItem Click="OnCalibrationMenuItemClick" x:Name="GazeOnly" Header="Gaze Calibration">
<MenuItem.Icon>
<PathIcon Data="{StaticResource HeadsetVrRegular}" />
</MenuItem.Icon>
</MenuItem>
<MenuItem Click="OnCalibrationMenuItemClick" x:Name="BlinkOnly" Header="Blink Calibration">
<MenuItem.Icon>
<PathIcon Data="{StaticResource HeadsetVrRegular}" />
</MenuItem.Icon>
</MenuItem>
</MenuFlyout>
</SplitButton.Flyout>
</SplitButton>