livekit · cloudwebrtc · Apr 8, 2024 · Dec 16, 2023 · Dec 16, 2023 · Mar 29, 2024
diff --git a/README.md.meta b/README.md.meta
diff --git a/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta b/Runtime/Plugins/ffi-android-arm64/liblivekit_ffi.so.meta
diff --git a/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta b/Runtime/Plugins/ffi-linux-arm64/liblivekit_ffi.so.meta
diff --git a/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta b/Runtime/Plugins/ffi-windows-x86_64/livekit_ffi.dll.meta
diff --git a/Runtime/Plugins/libwebrtc.jar.meta b/Runtime/Plugins/libwebrtc.jar.meta
diff --git a/Runtime/Scripts/AudioFrame.cs b/Runtime/Scripts/AudioFrame.cs
@@ -1,5 +1,6 @@
 using System;
 using LiveKit.Proto;
+using LiveKit.Internal;
 using Unity.Collections;
 using Unity.Collections.LowLevel.Unsafe;
 
@@ -9,7 +10,7 @@ public class AudioFrame : IDisposable
     {
         private AudioFrameBufferInfo _info;
 
-        private FfiOwnedHandle _handle;
+        private FfiHandle _handle;
 
         private bool _disposed = false;
 
@@ -27,14 +28,14 @@ public class AudioFrame : IDisposable
 
         public int Length => (int) (SamplesPerChannel * NumChannels * sizeof(short));
 
-        internal AudioFrame(FfiOwnedHandle handle, AudioFrameBufferInfo info)
+        internal AudioFrame(OwnedAudioFrameBuffer info)
         {
-            _handle = handle;
-            _info = info;
+            _handle = FfiHandle.FromOwnedHandle(info.Handle);
+            _info = info.Info;
             _sampleRate = _info.SampleRate;
             _numChannels = _info.NumChannels;
             _samplesPerChannel = _info.SamplesPerChannel;
-            _dataPtr = (IntPtr)info.DataPtr;
+            _dataPtr = (IntPtr)_info.DataPtr;
         }
 
         internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel) {
@@ -66,4 +67,4 @@ protected virtual void Dispose(bool disposing)
             }
         }
     }
-}
+}
diff --git a/Runtime/Scripts/AudioSource.cs b/Runtime/Scripts/AudioSource.cs
@@ -2,6 +2,8 @@
 using UnityEngine;
 using LiveKit.Proto;
 using LiveKit.Internal;
+using System.Threading;
+using LiveKit.Internal.FFIClients.Requests;
 
 namespace LiveKit
 {
@@ -10,86 +12,143 @@ public class RtcAudioSource
         private AudioSource _audioSource;
         private AudioFilter _audioFilter;
 
-        internal readonly FfiOwnedHandle Handle;
+        internal readonly FfiHandle Handle;
         protected AudioSourceInfo _info;
 
         // Used on the AudioThread
         private AudioFrame _frame;
+        private Thread _readAudioThread;
+        private object _lock = new object();
+        private float[] _data;
+        private int _channels;
+        private int _sampleRate;
+        private volatile bool _pending = false;
 
         public RtcAudioSource(AudioSource source)
         {
-            var newAudioSource = new NewAudioSourceRequest();
+            using var request = FFIBridge.Instance.NewRequest<NewAudioSourceRequest>();
+            var newAudioSource = request.request;
             newAudioSource.Type = AudioSourceType.AudioSourceNative;
             newAudioSource.NumChannels = 2;
             newAudioSource.SampleRate = 48000;
-
-            var request = new FfiRequest();
-            request.NewAudioSource = newAudioSource;
-
-            var resp = FfiClient.SendRequest(request);
-            var respSource = resp.NewAudioSource.Source;
-            _info = respSource.Info;
-
-            Handle = respSource.Handle;
+            using var response = request.Send();
+            FfiResponse res = response;
+            _info = res.NewAudioSource.Source.Info;
+            //TODO pooling handles
+            Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle);
             UpdateSource(source);
         }
 
-        private void UpdateSource(AudioSource source)
+        public void Start()
         {
-            _audioSource = source;
-            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
-            //_audioFilter.hideFlags = HideFlags.HideInInspector;
+            Stop();
+            _readAudioThread = new Thread(Update);
+            _readAudioThread.Start();
+
             _audioFilter.AudioRead += OnAudioRead;
-            source.Play();
+            _audioSource.Play();
         }
 
-
-        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        public void Stop()
         {
-            var samplesPerChannel = data.Length / channels;
-            if (_frame == null || _frame.NumChannels != channels 
-                || _frame.SampleRate != sampleRate 
-                || _frame.SamplesPerChannel != samplesPerChannel)
+            if (_readAudioThread != null)
             {
-                _frame = new AudioFrame(sampleRate, channels, samplesPerChannel);
-            }
-
-            static short FloatToS16(float v) {
-                v *= 32768f;
-                v = Math.Min(v, 32767f);
-                v = Math.Max(v, -32768f);
-                return (short)(v + Math.Sign(v) * 0.5f);
+                _readAudioThread.Abort();
+                _readAudioThread = null;
             }
+            if(_audioFilter) _audioFilter.AudioRead -= OnAudioRead;
+            if(_audioSource) _audioSource.Stop();
+        }
 
-            unsafe {
-                var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
-                for (int i = 0; i < data.Length; i++)
+        private void Update()
+        {
+            while (true)
+            {
+                Thread.Sleep(Constants.TASK_DELAY);
+                if (_pending)
                 {
-                    frameData[i] = FloatToS16(data[i]);
+                    ReadAudio();
                 }
             }
+        }
 
-            // Don't play the audio locally
-            Array.Clear(data, 0, data.Length);
-
-            var audioFrameBufferInfo = new AudioFrameBufferInfo();
-
-            audioFrameBufferInfo.DataPtr = (ulong)_frame.Data;
-            audioFrameBufferInfo.NumChannels = _frame.NumChannels;
-            audioFrameBufferInfo.SampleRate = _frame.SampleRate;
-            audioFrameBufferInfo.SamplesPerChannel = _frame.SamplesPerChannel;
-
-            var pushFrame = new CaptureAudioFrameRequest();
-            pushFrame.SourceHandle = Handle.Id;
-            pushFrame.Buffer = audioFrameBufferInfo;
-
-            var request = new FfiRequest();
-            request.CaptureAudioFrame = pushFrame;
+        private void ReadAudio()
+        {
+            _pending = false;
+            lock (_lock)
+            {
+                var samplesPerChannel = _data.Length / _channels;
+                if (_frame == null
+                    || _frame.NumChannels != _channels
+                    || _frame.SampleRate != _sampleRate
+                    || _frame.SamplesPerChannel != samplesPerChannel)
+                {
+                    _frame = new AudioFrame(_sampleRate, _channels, samplesPerChannel);
+                }
+                try
+                {
 
-            FfiClient.SendRequest(request);
+                    static short FloatToS16(float v)
+                    {
+                        v *= 32768f;
+                        v = Math.Min(v, 32767f);
+                        v = Math.Max(v, -32768f);
+                        return (short)(v + Math.Sign(v) * 0.5f);
+                    }
+
+                    unsafe
+                    {
+                        var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
+                        for (int i = 0; i < _data.Length; i++)
+                        {
+                            frameData[i] = FloatToS16(_data[i]); 
+                        }
+
+                        // Don't play the audio locally
+                        Array.Clear(_data, 0, _data.Length);
+
+                        using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
+                        using var audioFrameBufferInfo = request.TempResource<AudioFrameBufferInfo>();
+
+                        var pushFrame = request.request;
+                        pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
+
+                        pushFrame.Buffer = audioFrameBufferInfo;
+                        pushFrame.Buffer.DataPtr = (ulong)_frame.Data;
+                        pushFrame.Buffer.NumChannels = _frame.NumChannels;
+                        pushFrame.Buffer.SampleRate = _frame.SampleRate;
+                        pushFrame.Buffer.SamplesPerChannel = _frame.SamplesPerChannel;
+
+                        using var response = request.Send();
+
+                        pushFrame.Buffer.DataPtr = 0;
+                        pushFrame.Buffer.NumChannels = 0;
+                        pushFrame.Buffer.SampleRate = 0;
+                        pushFrame.Buffer.SamplesPerChannel = 0;
+                    }
+                }
+                catch (Exception e)
+                {
+                    Utils.Error("Audio Framedata error: " + e.Message);
+                }
+            }
+        }
 
+        private void UpdateSource(AudioSource source)
+        {
+            _audioSource = source;
+            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
+        }
 
-            //Debug.Log($"Pushed audio frame with {data.Length} samples");
+        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        {
+            lock (_lock)
+            {
+                _data = data;
+                _channels = channels;
+                _sampleRate = sampleRate;
+                _pending = true;
+            }
         }
     }
 }