livekit · ladvoc · May 29, 2025 · May 24, 2025 · May 24, 2025 · May 24, 2025
diff --git a/Runtime/Scripts/AudioFrame.cs b/Runtime/Scripts/AudioFrame.cs
@@ -2,7 +2,6 @@
 using LiveKit.Proto;
 using LiveKit.Internal;
 using Unity.Collections;
-using Unity.Collections.LowLevel.Unsafe;
 
 namespace LiveKit
 {
@@ -39,16 +38,6 @@ internal AudioFrame(OwnedAudioFrameBuffer info)
             _dataPtr = (IntPtr)_info.DataPtr;
         }
 
-        internal AudioFrame(uint sampleRate, uint numChannels, uint samplesPerChannel) {
-            _sampleRate = sampleRate;
-            _numChannels = numChannels;
-            _samplesPerChannel = samplesPerChannel;
-            unsafe
-            {
-                _allocatedData = new NativeArray<byte>(Length, Allocator.Persistent);
-                _dataPtr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(_allocatedData);
-            }
-        }
         ~AudioFrame()
         {
             Dispose(false);

diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs
@@ -46,6 +46,7 @@ internal sealed class FfiClient : IFFIClient
         // participant events are not allowed in the fii protocol public event ParticipantEventReceivedDelegate ParticipantEventReceived;
         public event VideoStreamEventReceivedDelegate? VideoStreamEventReceived;
         public event AudioStreamEventReceivedDelegate? AudioStreamEventReceived;
+        public event CaptureAudioFrameReceivedDelegate? CaptureAudioFrameReceived;
 
         public event PerformRpcReceivedDelegate? PerformRpcReceived;
 
@@ -287,6 +288,7 @@ static unsafe void FFICallback(UIntPtr data, UIntPtr size)
                         Instance.AudioStreamEventReceived?.Invoke(r.AudioStreamEvent!);
                         break;
                     case FfiEvent.MessageOneofCase.CaptureAudioFrame:
+                         Instance.CaptureAudioFrameReceived?.Invoke(r.CaptureAudioFrame!);
                         break;
                     case FfiEvent.MessageOneofCase.PerformRpc:
                         Instance.PerformRpcReceived?.Invoke(r.PerformRpc!);

diff --git a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs
@@ -48,6 +48,8 @@ namespace LiveKit.Internal
 
     internal delegate void SendTextReceivedDelegate(StreamSendTextCallback e);
 
+    internal delegate void CaptureAudioFrameReceivedDelegate(CaptureAudioFrameCallback e);
+
     // Events
     internal delegate void RoomEventReceivedDelegate(RoomEvent e);
 

diff --git a/Runtime/Scripts/MicrophoneSource.cs b/Runtime/Scripts/MicrophoneSource.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections;
 using UnityEngine;
 
 namespace LiveKit
@@ -19,12 +20,6 @@ sealed public class MicrophoneSource : RtcAudioSource
         private bool _disposed = false;
         private bool _started = false;
 
-        /// <summary>
-        /// True indicates the capture has started but is temporarily suspended
-        /// due to the application entering the background.
-        /// </summary>
-        private bool _suspended = false;
-
         /// <summary>
         /// Creates a new microphone source for the given device.
         /// </summary>
@@ -36,7 +31,6 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(2, Rt
         {
             _deviceName = deviceName;
             _sourceObject = sourceObject;
-            MonoBehaviourContext.OnApplicationPauseEvent += OnApplicationPause;
         }
 
         /// <summary>
@@ -54,10 +48,19 @@ public override void Start()
             base.Start();
             if (_started) return;
 
+
             if (!Application.HasUserAuthorization(mode: UserAuthorization.Microphone))
                 throw new InvalidOperationException("Microphone access not authorized");
 
-            var clip = Microphone.Start(
+            MonoBehaviourContext.OnApplicationPauseEvent += OnApplicationPause;
+            MonoBehaviourContext.RunCoroutine(StartMicrophone());
+
+            _started = true;
+        }
+
+        private IEnumerator StartMicrophone()
+        {
+             var clip = Microphone.Start(
                 _deviceName,
                 loop: true,
                 lengthSec: 1,
@@ -76,9 +79,8 @@ public override void Start()
             probe.AudioRead += OnAudioRead;
 
             var waitUntilReady = new WaitUntil(() => Microphone.GetPosition(_deviceName) > 0);
-            MonoBehaviourContext.RunCoroutine(waitUntilReady, () => source?.Play());
-
-            _started = true;
+            yield return waitUntilReady;
+            source.Play();
         }
 
         /// <summary>
@@ -87,8 +89,13 @@ public override void Start()
         public override void Stop()
         {
             base.Stop();
-            if (!_started) return;
+            MonoBehaviourContext.RunCoroutine(StopMicrophone());
+            MonoBehaviourContext.OnApplicationPauseEvent -= OnApplicationPause;
+            _started = false;
+        }
 
+        private IEnumerator StopMicrophone()
+        {
             if (Microphone.IsRecording(_deviceName))
                 Microphone.End(_deviceName);
 
@@ -98,8 +105,7 @@ public override void Stop()
 
             var source = _sourceObject.GetComponent<AudioSource>();
             UnityEngine.Object.Destroy(source);
-
-            _started = false;
+            yield return null;
         }
 
         private void OnAudioRead(float[] data, int channels, int sampleRate)
@@ -109,20 +115,14 @@ private void OnAudioRead(float[] data, int channels, int sampleRate)
 
         private void OnApplicationPause(bool pause)
         {
-            // When the application is paused (i.e. enters the background), place
-            // the microphone capture in a suspended state. This prevents stale audio
-            // samples from being captured and sent to the server when the application
-            // is resumed.
-            if (_suspended && !pause)
-            {
-                Start();
-                _suspended = false;
-            }
-            else if (!_suspended && pause)
-            {
-                Stop();
-                _suspended = true;
-            }
+            if (!pause && _started)
+                MonoBehaviourContext.RunCoroutine(RestartMicrophone());
+        }
+
+        private IEnumerator RestartMicrophone()
+        {
+            yield return StopMicrophone();
+            yield return StartMicrophone();
         }
 
         protected override void Dispose(bool disposing)

diff --git a/Runtime/Scripts/RtcAudioSource.cs b/Runtime/Scripts/RtcAudioSource.cs
@@ -2,9 +2,10 @@
 using System.Collections;
 using LiveKit.Proto;
 using LiveKit.Internal;
-using System.Threading;
 using LiveKit.Internal.FFIClients.Requests;
-using UnityEngine;
+using Unity.Collections;
+using Unity.Collections.LowLevel.Unsafe;
+using System.Diagnostics;
 
 namespace LiveKit
 {
@@ -31,7 +32,7 @@ public abstract class RtcAudioSource : IRtcSource, IDisposable
         /// </remarks>
         public abstract event Action<float[], int, int> AudioRead;
 
-#if UNITY_IOS
+#if UNITY_IOS && !UNITY_EDITOR
         // iOS microphone sample rate is 24k
         public static uint DefaultMicrophoneSampleRate = 24000;
 
@@ -48,10 +49,10 @@ public abstract class RtcAudioSource : IRtcSource, IDisposable
         internal readonly FfiHandle Handle;
         protected AudioSourceInfo _info;
 
-        // Possibly used on the AudioThread
-        private CancellationTokenSource _cts;
-        private Thread _readAudioThread;
-        private ThreadSafeQueue<AudioFrame> _frameQueue = new ThreadSafeQueue<AudioFrame>();
+        /// <summary>
+        /// Temporary frame buffer for invoking the FFI capture method.
+        /// </summary>
+        private NativeArray<short> _frameData;
 
         private bool _muted = false;
         public override bool Muted => _muted;
@@ -70,6 +71,8 @@ protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType =
             newAudioSource.SampleRate = _sourceType == RtcAudioSourceType.AudioSourceMicrophone ?
                 DefaultMicrophoneSampleRate : DefaultSampleRate;
 
+            UnityEngine.Debug.Log($"NewAudioSource: {newAudioSource.NumChannels} {newAudioSource.SampleRate}");
+
             newAudioSource.Options = request.TempResource<AudioSourceOptions>();
             newAudioSource.Options.EchoCancellation = true;
             newAudioSource.Options.AutoGainControl = true;
@@ -86,11 +89,6 @@ protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType =
         public virtual void Start()
         {
             if (_started) return;
-            _frameQueue.Clear();
-            _cts = new CancellationTokenSource();
-            var cancellationToken = _cts.Token;
-            _readAudioThread = new Thread(() => Update(cancellationToken));
-            _readAudioThread.Start();
             AudioRead += OnAudioRead;
             _started = true;
         }
@@ -101,78 +99,62 @@ public virtual void Start()
         public virtual void Stop()
         {
             if (!_started) return;
-            _cts.Cancel();
-            _readAudioThread.Join();
             AudioRead -= OnAudioRead;
             _started = false;
         }
 
-        private void Update(CancellationToken token)
+        private void OnAudioRead(float[] data, int channels, int sampleRate)
         {
-            while (!token.IsCancellationRequested)
+            if (_muted) return;
+
+            // The length of the data buffer corresponds to the DSP buffer size.
+            if (_frameData.Length != data.Length)
             {
-                ReadAudio();
-                Thread.Sleep(Constants.TASK_DELAY);
+                if (_frameData.IsCreated) _frameData.Dispose();
+                 _frameData = new NativeArray<short>(data.Length, Allocator.Persistent);
             }
-        }
-
-        private void OnAudioRead(float[] data, int channels, int sampleRate)
-        {
-            var samplesPerChannel = data.Length / channels;
-            var frame = new AudioFrame((uint)sampleRate, (uint)channels, (uint)samplesPerChannel);
 
+            // Copy from the audio read buffer into the frame buffer, converting
+            // each sample to a 16-bit signed integer.
             static short FloatToS16(float v)
             {
                 v *= 32768f;
                 v = Math.Min(v, 32767f);
                 v = Math.Max(v, -32768f);
                 return (short)(v + Math.Sign(v) * 0.5f);
             }
+            for (int i = 0; i < data.Length; i++)
+                _frameData[i] = FloatToS16(data[i]);
+
+            // Capture the frame.
+            using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
+            using var audioFrameBufferInfo = request.TempResource<AudioFrameBufferInfo>();
+
+            var pushFrame = request.request;
+            pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
+            pushFrame.Buffer = audioFrameBufferInfo;
             unsafe
             {
-                var frameData = new Span<short>(frame.Data.ToPointer(), frame.Length / sizeof(short));
-                for (int i = 0; i < data.Length; i++)
-                {
-                    frameData[i] = FloatToS16(data[i]);
-                }
+                 pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility
+                    .GetUnsafePtr(_frameData);
             }
-            _frameQueue.Enqueue(frame);
-        }
+            pushFrame.Buffer.NumChannels = (uint)channels;
+            pushFrame.Buffer.SampleRate = (uint)sampleRate;
+            pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels;
 
-        private void ReadAudio()
-        {
-            while (_frameQueue.Count > 0)
+            using var response = request.Send();
+            FfiResponse res = response;
+
+            // Wait for async callback, log an error if the capture fails.
+            var asyncId = res.CaptureAudioFrame.AsyncId;
+            void Callback(CaptureAudioFrameCallback callback)
             {
-                try
-                {
-                    AudioFrame frame = _frameQueue.Dequeue();
-
-                    if(_muted)
-                    {
-                        continue;
-                    }
-                    unsafe
-                    {
-                        using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
-                        using var audioFrameBufferInfo = request.TempResource<AudioFrameBufferInfo>();
-
-                        var pushFrame = request.request;
-                        pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
-
-                        pushFrame.Buffer = audioFrameBufferInfo;
-                        pushFrame.Buffer.DataPtr = (ulong)frame.Data;
-                        pushFrame.Buffer.NumChannels = frame.NumChannels;
-                        pushFrame.Buffer.SampleRate = frame.SampleRate;
-                        pushFrame.Buffer.SamplesPerChannel = frame.SamplesPerChannel;
-
-                        using var response = request.Send();
-                    }
-                }
-                catch (Exception e)
-                {
-                    Utils.Error("Audio Framedata error: " + e.Message);
-                }
+                if (callback.AsyncId != asyncId) return;
+                if (callback.HasError)
+                    Utils.Error($"Audio capture failed: {callback.Error}");
+                FfiClient.Instance.CaptureAudioFrameReceived -= Callback;
             }
+            FfiClient.Instance.CaptureAudioFrameReceived += Callback;
         }
 
         /// <summary>
@@ -195,6 +177,7 @@ public void Dispose()
         protected virtual void Dispose(bool disposing)
         {
             if (!_disposed && disposing) Stop();
+            if (_frameData.IsCreated) _frameData.Dispose();
             _disposed = true;
         }
 

diff --git a/Runtime/Scripts/WebCameraSource.cs b/Runtime/Scripts/WebCameraSource.cs
@@ -62,7 +62,6 @@ protected override bool ReadBuffer()
                 _previewTexture.width != width ||
                 _previewTexture.height != height)
             {
-                Debug.Log("Creating new texture");
                 // Required when using Allocator.Persistent
                 if (_captureBuffer.IsCreated)
                     _captureBuffer.Dispose();