diff --git a/Runtime/Scripts/AudioFrame.cs b/Runtime/Scripts/AudioFrame.cs index 5b3441a5..9be0a3c9 100644 --- a/Runtime/Scripts/AudioFrame.cs +++ b/Runtime/Scripts/AudioFrame.cs @@ -2,7 +2,6 @@ using LiveKit.Proto; using LiveKit.Internal; using Unity.Collections; -using Unity.Collections.LowLevel.Unsafe; namespace LiveKit { @@ -39,16 +38,6 @@ internal AudioFrame(OwnedAudioFrameBuffer info) _dataPtr = (IntPtr)_info.DataPtr; } - internal AudioFrame(uint sampleRate, uint numChannels, uint samplesPerChannel) { - _sampleRate = sampleRate; - _numChannels = numChannels; - _samplesPerChannel = samplesPerChannel; - unsafe - { - _allocatedData = new NativeArray(Length, Allocator.Persistent); - _dataPtr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(_allocatedData); - } - } ~AudioFrame() { Dispose(false); diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs index 9e8a1065..d037a3bb 100644 --- a/Runtime/Scripts/Internal/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClient.cs @@ -46,6 +46,7 @@ internal sealed class FfiClient : IFFIClient // participant events are not allowed in the fii protocol public event ParticipantEventReceivedDelegate ParticipantEventReceived; public event VideoStreamEventReceivedDelegate? VideoStreamEventReceived; public event AudioStreamEventReceivedDelegate? AudioStreamEventReceived; + public event CaptureAudioFrameReceivedDelegate? CaptureAudioFrameReceived; public event PerformRpcReceivedDelegate? PerformRpcReceived; @@ -287,6 +288,7 @@ static unsafe void FFICallback(UIntPtr data, UIntPtr size) Instance.AudioStreamEventReceived?.Invoke(r.AudioStreamEvent!); break; case FfiEvent.MessageOneofCase.CaptureAudioFrame: + Instance.CaptureAudioFrameReceived?.Invoke(r.CaptureAudioFrame!); break; case FfiEvent.MessageOneofCase.PerformRpc: Instance.PerformRpcReceived?.Invoke(r.PerformRpc!); diff --git a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs index eb9005aa..94f0b1da 100644 --- a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs +++ b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs @@ -48,6 +48,8 @@ namespace LiveKit.Internal internal delegate void SendTextReceivedDelegate(StreamSendTextCallback e); + internal delegate void CaptureAudioFrameReceivedDelegate(CaptureAudioFrameCallback e); + // Events internal delegate void RoomEventReceivedDelegate(RoomEvent e); diff --git a/Runtime/Scripts/MicrophoneSource.cs b/Runtime/Scripts/MicrophoneSource.cs index f49da5b7..923df981 100644 --- a/Runtime/Scripts/MicrophoneSource.cs +++ b/Runtime/Scripts/MicrophoneSource.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using UnityEngine; namespace LiveKit @@ -19,12 +20,6 @@ sealed public class MicrophoneSource : RtcAudioSource private bool _disposed = false; private bool _started = false; - /// - /// True indicates the capture has started but is temporarily suspended - /// due to the application entering the background. - /// - private bool _suspended = false; - /// /// Creates a new microphone source for the given device. /// @@ -36,7 +31,6 @@ public MicrophoneSource(string deviceName, GameObject sourceObject) : base(2, Rt { _deviceName = deviceName; _sourceObject = sourceObject; - MonoBehaviourContext.OnApplicationPauseEvent += OnApplicationPause; } /// @@ -54,10 +48,19 @@ public override void Start() base.Start(); if (_started) return; + if (!Application.HasUserAuthorization(mode: UserAuthorization.Microphone)) throw new InvalidOperationException("Microphone access not authorized"); - var clip = Microphone.Start( + MonoBehaviourContext.OnApplicationPauseEvent += OnApplicationPause; + MonoBehaviourContext.RunCoroutine(StartMicrophone()); + + _started = true; + } + + private IEnumerator StartMicrophone() + { + var clip = Microphone.Start( _deviceName, loop: true, lengthSec: 1, @@ -76,9 +79,8 @@ public override void Start() probe.AudioRead += OnAudioRead; var waitUntilReady = new WaitUntil(() => Microphone.GetPosition(_deviceName) > 0); - MonoBehaviourContext.RunCoroutine(waitUntilReady, () => source?.Play()); - - _started = true; + yield return waitUntilReady; + source.Play(); } /// @@ -87,8 +89,13 @@ public override void Start() public override void Stop() { base.Stop(); - if (!_started) return; + MonoBehaviourContext.RunCoroutine(StopMicrophone()); + MonoBehaviourContext.OnApplicationPauseEvent -= OnApplicationPause; + _started = false; + } + private IEnumerator StopMicrophone() + { if (Microphone.IsRecording(_deviceName)) Microphone.End(_deviceName); @@ -98,8 +105,7 @@ public override void Stop() var source = _sourceObject.GetComponent(); UnityEngine.Object.Destroy(source); - - _started = false; + yield return null; } private void OnAudioRead(float[] data, int channels, int sampleRate) @@ -109,20 +115,14 @@ private void OnAudioRead(float[] data, int channels, int sampleRate) private void OnApplicationPause(bool pause) { - // When the application is paused (i.e. enters the background), place - // the microphone capture in a suspended state. This prevents stale audio - // samples from being captured and sent to the server when the application - // is resumed. - if (_suspended && !pause) - { - Start(); - _suspended = false; - } - else if (!_suspended && pause) - { - Stop(); - _suspended = true; - } + if (!pause && _started) + MonoBehaviourContext.RunCoroutine(RestartMicrophone()); + } + + private IEnumerator RestartMicrophone() + { + yield return StopMicrophone(); + yield return StartMicrophone(); } protected override void Dispose(bool disposing) diff --git a/Runtime/Scripts/RtcAudioSource.cs b/Runtime/Scripts/RtcAudioSource.cs index 4e2d25f1..ef512733 100644 --- a/Runtime/Scripts/RtcAudioSource.cs +++ b/Runtime/Scripts/RtcAudioSource.cs @@ -2,9 +2,10 @@ using System.Collections; using LiveKit.Proto; using LiveKit.Internal; -using System.Threading; using LiveKit.Internal.FFIClients.Requests; -using UnityEngine; +using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; +using System.Diagnostics; namespace LiveKit { @@ -31,7 +32,7 @@ public abstract class RtcAudioSource : IRtcSource, IDisposable /// public abstract event Action AudioRead; -#if UNITY_IOS +#if UNITY_IOS && !UNITY_EDITOR // iOS microphone sample rate is 24k public static uint DefaultMicrophoneSampleRate = 24000; @@ -48,10 +49,10 @@ public abstract class RtcAudioSource : IRtcSource, IDisposable internal readonly FfiHandle Handle; protected AudioSourceInfo _info; - // Possibly used on the AudioThread - private CancellationTokenSource _cts; - private Thread _readAudioThread; - private ThreadSafeQueue _frameQueue = new ThreadSafeQueue(); + /// + /// Temporary frame buffer for invoking the FFI capture method. + /// + private NativeArray _frameData; private bool _muted = false; public override bool Muted => _muted; @@ -70,6 +71,8 @@ protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType = newAudioSource.SampleRate = _sourceType == RtcAudioSourceType.AudioSourceMicrophone ? DefaultMicrophoneSampleRate : DefaultSampleRate; + UnityEngine.Debug.Log($"NewAudioSource: {newAudioSource.NumChannels} {newAudioSource.SampleRate}"); + newAudioSource.Options = request.TempResource(); newAudioSource.Options.EchoCancellation = true; newAudioSource.Options.AutoGainControl = true; @@ -86,11 +89,6 @@ protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType = public virtual void Start() { if (_started) return; - _frameQueue.Clear(); - _cts = new CancellationTokenSource(); - var cancellationToken = _cts.Token; - _readAudioThread = new Thread(() => Update(cancellationToken)); - _readAudioThread.Start(); AudioRead += OnAudioRead; _started = true; } @@ -101,26 +99,23 @@ public virtual void Start() public virtual void Stop() { if (!_started) return; - _cts.Cancel(); - _readAudioThread.Join(); AudioRead -= OnAudioRead; _started = false; } - private void Update(CancellationToken token) + private void OnAudioRead(float[] data, int channels, int sampleRate) { - while (!token.IsCancellationRequested) + if (_muted) return; + + // The length of the data buffer corresponds to the DSP buffer size. + if (_frameData.Length != data.Length) { - ReadAudio(); - Thread.Sleep(Constants.TASK_DELAY); + if (_frameData.IsCreated) _frameData.Dispose(); + _frameData = new NativeArray(data.Length, Allocator.Persistent); } - } - - private void OnAudioRead(float[] data, int channels, int sampleRate) - { - var samplesPerChannel = data.Length / channels; - var frame = new AudioFrame((uint)sampleRate, (uint)channels, (uint)samplesPerChannel); + // Copy from the audio read buffer into the frame buffer, converting + // each sample to a 16-bit signed integer. static short FloatToS16(float v) { v *= 32768f; @@ -128,51 +123,38 @@ static short FloatToS16(float v) v = Math.Max(v, -32768f); return (short)(v + Math.Sign(v) * 0.5f); } + for (int i = 0; i < data.Length; i++) + _frameData[i] = FloatToS16(data[i]); + + // Capture the frame. + using var request = FFIBridge.Instance.NewRequest(); + using var audioFrameBufferInfo = request.TempResource(); + + var pushFrame = request.request; + pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); + pushFrame.Buffer = audioFrameBufferInfo; unsafe { - var frameData = new Span(frame.Data.ToPointer(), frame.Length / sizeof(short)); - for (int i = 0; i < data.Length; i++) - { - frameData[i] = FloatToS16(data[i]); - } + pushFrame.Buffer.DataPtr = (ulong)NativeArrayUnsafeUtility + .GetUnsafePtr(_frameData); } - _frameQueue.Enqueue(frame); - } + pushFrame.Buffer.NumChannels = (uint)channels; + pushFrame.Buffer.SampleRate = (uint)sampleRate; + pushFrame.Buffer.SamplesPerChannel = (uint)data.Length / (uint)channels; - private void ReadAudio() - { - while (_frameQueue.Count > 0) + using var response = request.Send(); + FfiResponse res = response; + + // Wait for async callback, log an error if the capture fails. + var asyncId = res.CaptureAudioFrame.AsyncId; + void Callback(CaptureAudioFrameCallback callback) { - try - { - AudioFrame frame = _frameQueue.Dequeue(); - - if(_muted) - { - continue; - } - unsafe - { - using var request = FFIBridge.Instance.NewRequest(); - using var audioFrameBufferInfo = request.TempResource(); - - var pushFrame = request.request; - pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle(); - - pushFrame.Buffer = audioFrameBufferInfo; - pushFrame.Buffer.DataPtr = (ulong)frame.Data; - pushFrame.Buffer.NumChannels = frame.NumChannels; - pushFrame.Buffer.SampleRate = frame.SampleRate; - pushFrame.Buffer.SamplesPerChannel = frame.SamplesPerChannel; - - using var response = request.Send(); - } - } - catch (Exception e) - { - Utils.Error("Audio Framedata error: " + e.Message); - } + if (callback.AsyncId != asyncId) return; + if (callback.HasError) + Utils.Error($"Audio capture failed: {callback.Error}"); + FfiClient.Instance.CaptureAudioFrameReceived -= Callback; } + FfiClient.Instance.CaptureAudioFrameReceived += Callback; } /// @@ -195,6 +177,7 @@ public void Dispose() protected virtual void Dispose(bool disposing) { if (!_disposed && disposing) Stop(); + if (_frameData.IsCreated) _frameData.Dispose(); _disposed = true; } diff --git a/Runtime/Scripts/WebCameraSource.cs b/Runtime/Scripts/WebCameraSource.cs index 639f87a0..efe2098a 100644 --- a/Runtime/Scripts/WebCameraSource.cs +++ b/Runtime/Scripts/WebCameraSource.cs @@ -62,7 +62,6 @@ protected override bool ReadBuffer() _previewTexture.width != width || _previewTexture.height != height) { - Debug.Log("Creating new texture"); // Required when using Allocator.Persistent if (_captureBuffer.IsCreated) _captureBuffer.Dispose();