livekit · markgrossnickle · Dec 15, 2023 · Dec 16, 2023 · Dec 16, 2023 · Dec 16, 2023
diff --git a/.vs/VSWorkspaceState.json b/.vs/VSWorkspaceState.json
@@ -0,0 +1,7 @@
+{
+  "ExpandedNodes": [
+    ""
+  ],
+  "SelectedNode": "\\C:\\Users\\ashka\\Source\\Repos\\client-sdk-unity",
+  "PreviewInSolutionExplorer": false
+}
diff --git a/.vs/client-sdk-unity/v17/.wsuo b/.vs/client-sdk-unity/v17/.wsuo
diff --git a/README.md.meta b/README.md.meta
diff --git a/Runtime/Plugins/x86_64/livekit_ffi.dll b/Runtime/Plugins/x86_64/livekit_ffi.dll
diff --git a/Runtime/Scripts/AudioFrame.cs b/Runtime/Scripts/AudioFrame.cs
@@ -8,7 +8,12 @@ public class AudioFrame : IDisposable
     {
         private AudioFrameBufferInfo _info;
 
-        internal readonly FfiHandle Handle;
+        //internal readonly FfiHandle Handle;
+        private FfiHandle _handle;
+        internal FfiHandle Handle
+        {
+            get { return _handle; }
+        }
         private bool _disposed = false;
 
         public uint NumChannels => _info.NumChannels;
@@ -20,7 +25,7 @@ public class AudioFrame : IDisposable
 
         internal AudioFrame(FfiHandle handle, AudioFrameBufferInfo info)
         {
-            Handle = handle;
+            _handle = handle;
             _info = info;
         }
 
@@ -30,14 +35,20 @@ internal AudioFrame(int sampleRate, int numChannels, int samplesPerChannel) {
             alloc.NumChannels = (uint) numChannels;
             alloc.SamplesPerChannel = (uint) samplesPerChannel;
 
-            var request = new FFIRequest();
+            var request = new FfiRequest();
             request.AllocAudioBuffer = alloc;
 
+            Init(request);
+        }
+
+        void Init(FfiRequest request)
+        {
             var res = FfiClient.SendRequest(request);
-            var bufferInfo = res.AllocAudioBuffer.Buffer;
+            var bufferInfo = res.AllocAudioBuffer.Buffer.Info;
 
-            Handle = new FfiHandle((IntPtr)bufferInfo.Handle.Id);
+            _handle = new FfiHandle((IntPtr)res.AllocAudioBuffer.Buffer.Handle.Id);
             _info = bufferInfo;
+
         }
 
         ~AudioFrame()

diff --git a/Runtime/Scripts/AudioSource.cs b/Runtime/Scripts/AudioSource.cs
@@ -6,6 +6,8 @@
 using UnityEngine.Rendering;
 using Unity.Collections;
 using Unity.Collections.LowLevel.Unsafe;
+using System.Threading.Tasks;
+using System.Threading;
 
 namespace LiveKit
 {
@@ -14,74 +16,121 @@ public class RtcAudioSource
         private AudioSource _audioSource;
         private AudioFilter _audioFilter;
 
-        internal readonly FfiHandle Handle;
+        //internal readonly FfiHandle Handle;
+        private FfiHandle _handle;
+        internal FfiHandle Handle
+        {
+            get { return _handle; }
+        }
         protected AudioSourceInfo _info;
 
         // Used on the AudioThread
         private AudioFrame _frame;
+        private Thread _readAudioThread;
+        private bool _pending = false;
+        private object _lock = new object();
 
         public RtcAudioSource(AudioSource source)
         {
             var newAudioSource = new NewAudioSourceRequest();
             newAudioSource.Type = AudioSourceType.AudioSourceNative;
 
-            var request = new FFIRequest();
+            var request = new FfiRequest();
             request.NewAudioSource = newAudioSource;
 
             var resp = FfiClient.SendRequest(request);
-            _info = resp.NewAudioSource.Source;
-            Handle = new FfiHandle((IntPtr)_info.Handle.Id);
+
+            _info = resp.NewAudioSource.Source.Info;
+            _handle = new FfiHandle((IntPtr)resp.NewAudioSource.Source.Handle.Id);
             UpdateSource(source);
+
         }
 
-        private void UpdateSource(AudioSource source)
+        public void Start()
         {
-            _audioSource = source;
-            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
-            //_audioFilter.hideFlags = HideFlags.HideInInspector;
-            _audioFilter.AudioRead += OnAudioRead;
-            source.Play();
+            Stop();
+            _readAudioThread = new Thread(async () => await Update());
+            _readAudioThread.Start();
         }
 
+        public void Stop()
+        {
+            if (_readAudioThread != null) _readAudioThread.Abort();
+        }
 
-        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        async private Task Update()
         {
-            var samplesPerChannel = data.Length / channels;
-            if (_frame == null || _frame.NumChannels != channels 
-                || _frame.SampleRate != sampleRate 
-                || _frame.SamplesPerChannel != samplesPerChannel)
+            while(true)
             {
-                _frame = new AudioFrame(sampleRate, channels, samplesPerChannel);
-            }
-
-            static short FloatToS16(float v) {
-                v *= 32768f;
-                v = Math.Min(v, 32767f);
-                v = Math.Max(v, -32768f);
-                return (short)(v + Math.Sign(v) * 0.5f);
+                await Task.Delay(Constants.TASK_DELAY);
+                if(_pending)
+                {
+                    ReadAudio();
+                }
             }
+        }
 
-            unsafe {
-                var frameData = new Span<short>(_frame.Data.ToPointer(), _frame.Length / sizeof(short));
-                for (int i = 0; i < data.Length; i++)
+        private void ReadAudio()
+        {
+            _pending = false;
+            lock (_lock)
+            {
+                var samplesPerChannel = _data.Length / _channels;
+                if (_frame == null || _frame.NumChannels != _channels
+                    || _frame.SampleRate != _sampleRate
+                    || _frame.SamplesPerChannel != samplesPerChannel)
                 {
-                    frameData[i] = FloatToS16(data[i]);
+                    _frame = new AudioFrame(_sampleRate, _channels, samplesPerChannel);
                 }
-            }
 
-            // Don't play the audio locally
-            Array.Clear(data, 0, data.Length);
+                try
+                {
+                    // Don't play the audio locally
+                    Array.Clear(_data, 0, _data.Length);
+
+                    var pushFrame = new CaptureAudioFrameRequest();
+                    pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
+                    pushFrame.Buffer = new AudioFrameBufferInfo() { DataPtr = (ulong)_frame.Handle.DangerousGetHandle() };
 
-            var pushFrame = new CaptureAudioFrameRequest();
-            pushFrame.SourceHandle = new FFIHandleId { Id = (ulong)Handle.DangerousGetHandle() };
-            pushFrame.BufferHandle = new FFIHandleId { Id = (ulong)_frame.Handle.DangerousGetHandle() };
+                    var request = new FfiRequest();
+                    request.CaptureAudioFrame = pushFrame;
 
-            var request = new FFIRequest();
-            request.CaptureAudioFrame = pushFrame;
+                    FfiClient.SendRequest(request);
 
-            FfiClient.SendRequest(request);
+                    Utils.Debug($"Pushed audio frame with {_data.Length} samples");
 
-            Debug.Log($"Pushed audio frame with {data.Length} samples");
+                }
+                catch (Exception e)
+                {
+
+                    Utils.Error("Audio Framedata error: " + e.Message);
+                }
+            }
+        }
+
+        private void UpdateSource(AudioSource source)
+        {
+            _audioSource = source;
+            _audioFilter = source.gameObject.AddComponent<AudioFilter>();
+            //_audioFilter.hideFlags = HideFlags.HideInInspector;
+            _audioFilter.AudioRead += OnAudioRead;
+            source.Play();
         }
+
+        private float[] _data;
+        private int _channels;
+        private int _sampleRate;
+
+        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        {
+            lock (_lock)
+            {
+                _data = data;
+                _channels = channels;
+                _sampleRate = sampleRate;
+                _pending = true;
+            }
+        }
+
     }
 }