Forum Discussion
YanaArtis
4 years agoProtege
Recording sound from mic during Oculus LipSync.
I use Oculus LipSync in my application for Quest and Rift. Articulation works fine. I can record facial blendshapes with my code and then playback the recorded articulations via C# code later. Now I am trying to record the speech while Oculus asset performs lipsync. But I cannot get the sound from microphone because OVRLipSyncMicInput.cs starts capturing sound from microphone into temporary audioclip with 1-second duration, therefore my recording from microphone conflicts with Oculus code.
How can I record the sound during Oculus lipsync to play it later simultaneously with recorded lipsync?
I use Unity 2019.4.25f1, Oculus Integration 28.0.
I have found the solution working on Rift and Quest:
// I have created my own class OculusLipSyncMicInput, copied into it source code from // Oculus' OVRLipSyncMicInput.cs, and made several modifications: public class OculusLipSyncMicInput : MonoBehaviour { ... private int _micNdx = 0; private AudioClip _recordedClip; private AudioClip _croppedClip; private bool _isRecordingSpeech = false; ... void Start() { audioSource.loop = true; audioSource.mute = false; bool micSelected = false; _micNdx = 0; if (Microphone.devices.Length > 1) { for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++) { if (Microphone.devices[i].ToString().Contains("Rift")) { _micNdx = i; micSelected = true; } } for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++) { if (Microphone.devices[i].ToString().Contains("Oculus")) { _micNdx = i; micSelected = true; } } } InitializeMicrophone(); } ... private void InitializeMicrophone() { if (initialized) { return; } if (Microphone.devices.Length == 0) { return; } // selectedDevice = Microphone.devices[0].ToString(); // Oculus' code selectedDevice = Microphone.devices[_micNdx].ToString(); // My code micSelected = true; GetMicCaps(); initialized = true; } ... // public void StartMicrophone() // Oculus' code public void StartMicrophone(int soundLenSec = 1) // My code { if (micSelected == false) return; //Starts recording // audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code Stopwatch timer = Stopwatch.StartNew(); // Wait until the recording has started while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000) { Thread.Sleep(50); } if (Microphone.GetPosition(selectedDevice) <= 0) { throw new Exception("Timeout initializing microphone " + selectedDevice); } // Play the audio source audioSource.Play(); } ... // And I added several methods at the end public void StartMicrophoneRecord (int soundLenSec) { StopMicrophone(); StartMicrophone(soundLenSec); _isRecordingSpeech = true; } public void EndMicrophoneRecord () { int recordedSamples = Microphone.GetPosition(null); _recordedClip = (audioSource == null) ? null : audioSource.clip; StopMicrophone(); if (_isRecordingSpeech) { if ((audioSource == null) || (audioSource.clip == null)) { _recordedClip = null; } else { float[] croppedData = new float[recordedSamples * _recordedClip.channels]; _recordedClip.GetData(croppedData, 0); if (_croppedClip != null) { _croppedClip.UnloadAudioData(); Destroy(_croppedClip); } _croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false); _croppedClip.SetData(croppedData, 0); } _isRecordingSpeech = false; } StartMicrophone(1); } public AudioClip GetMicrophoneRecord () { return _croppedClip; } public void Mute () { micInputVolume = 0; } public void Unmute () { micInputVolume = 100; } }
9 Replies
Replies have been turned off for this discussion
- YanaArtisProtege
I have found the solution. I created OculusLipSyncMicInput.cs - my own version of OVRLipSyncMicInput.cs that permits me to record the sound.
- jynxiwinExplorer
Hi, I am interested in what you ended up doing. Can you share?
- YanaArtisProtege
// I have created my own class OculusLipSyncMicInput, copied into it source code from // Oculus' OVRLipSyncMicInput.cs, and made several modifications: public class OculusLipSyncMicInput : MonoBehaviour { ... private int _micNdx = 0; private AudioClip _recordedClip; private AudioClip _croppedClip; private bool _isRecordingSpeech = false; ... void Start() { audioSource.loop = true; // Set the AudioClip to loop audioSource.mute = false; // Automatically choose suitable microphone. Required for Oculus Rift. _micNdx = 0; if (Microphone.devices.Length > 1) { for (int i = 0; i < Microphone.devices.Length; i++) { if (Microphone.devices[i].ToString().Contains("Oculus")) { _micNdx = i; break; } } } InitializeMicrophone(); } ... private void InitializeMicrophone() { if (initialized) { return; } if (Microphone.devices.Length == 0) { return; } // selectedDevice = Microphone.devices[0].ToString(); // Oculus' code selectedDevice = Microphone.devices[_micNdx].ToString(); // My code micSelected = true; GetMicCaps(); initialized = true; } ... // public void StartMicrophone() // Oculus' code public void StartMicrophone(int soundLenSec = 1) // My code { if (micSelected == false) return; //Starts recording // audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code Stopwatch timer = Stopwatch.StartNew(); // Wait until the recording has started while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000) { Thread.Sleep(50); } if (Microphone.GetPosition(selectedDevice) <= 0) { throw new Exception("Timeout initializing microphone " + selectedDevice); } // Play the audio source audioSource.Play(); } ... // And I added several methods at the end public void StartMicrophoneRecord (int soundLenSec) { StopMicrophone(); StartMicrophone(soundLenSec); _isRecordingSpeech = true; } public void EndMicrophoneRecord () { int recordedSamples = Microphone.GetPosition(null); _recordedClip = (audioSource == null) ? null : audioSource.clip; StopMicrophone(); if (_isRecordingSpeech) { if ((audioSource == null) || (audioSource.clip == null)) { _recordedClip = null; } else { float[] croppedData = new float[recordedSamples * _recordedClip.channels]; _recordedClip.GetData(croppedData, 0); if (_croppedClip != null) { _croppedClip.UnloadAudioData(); Destroy(_croppedClip); } _croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false); _croppedClip.SetData(croppedData, 0); } _isRecordingSpeech = false; } StartMicrophone(1); } public AudioClip GetMicrophoneRecord () { return _croppedClip; } public void Mute () { micInputVolume = 0; } public void Unmute () { micInputVolume = 100; } }
- jynxiwinExplorer
Thank you. I am doing something similar and getting 10 seconds of silence in a .wav I'm saving. Are you creating a file with this voice data?
- YanaArtisProtege
Yes, and I saved real voice. I even process it raising the pitch 🙂 There is the result of my voice capturing/processing:
- YanaArtisProtege
I have found the solution working on Rift and Quest:
// I have created my own class OculusLipSyncMicInput, copied into it source code from // Oculus' OVRLipSyncMicInput.cs, and made several modifications: public class OculusLipSyncMicInput : MonoBehaviour { ... private int _micNdx = 0; private AudioClip _recordedClip; private AudioClip _croppedClip; private bool _isRecordingSpeech = false; ... void Start() { audioSource.loop = true; audioSource.mute = false; bool micSelected = false; _micNdx = 0; if (Microphone.devices.Length > 1) { for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++) { if (Microphone.devices[i].ToString().Contains("Rift")) { _micNdx = i; micSelected = true; } } for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++) { if (Microphone.devices[i].ToString().Contains("Oculus")) { _micNdx = i; micSelected = true; } } } InitializeMicrophone(); } ... private void InitializeMicrophone() { if (initialized) { return; } if (Microphone.devices.Length == 0) { return; } // selectedDevice = Microphone.devices[0].ToString(); // Oculus' code selectedDevice = Microphone.devices[_micNdx].ToString(); // My code micSelected = true; GetMicCaps(); initialized = true; } ... // public void StartMicrophone() // Oculus' code public void StartMicrophone(int soundLenSec = 1) // My code { if (micSelected == false) return; //Starts recording // audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code Stopwatch timer = Stopwatch.StartNew(); // Wait until the recording has started while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000) { Thread.Sleep(50); } if (Microphone.GetPosition(selectedDevice) <= 0) { throw new Exception("Timeout initializing microphone " + selectedDevice); } // Play the audio source audioSource.Play(); } ... // And I added several methods at the end public void StartMicrophoneRecord (int soundLenSec) { StopMicrophone(); StartMicrophone(soundLenSec); _isRecordingSpeech = true; } public void EndMicrophoneRecord () { int recordedSamples = Microphone.GetPosition(null); _recordedClip = (audioSource == null) ? null : audioSource.clip; StopMicrophone(); if (_isRecordingSpeech) { if ((audioSource == null) || (audioSource.clip == null)) { _recordedClip = null; } else { float[] croppedData = new float[recordedSamples * _recordedClip.channels]; _recordedClip.GetData(croppedData, 0); if (_croppedClip != null) { _croppedClip.UnloadAudioData(); Destroy(_croppedClip); } _croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false); _croppedClip.SetData(croppedData, 0); } _isRecordingSpeech = false; } StartMicrophone(1); } public AudioClip GetMicrophoneRecord () { return _croppedClip; } public void Mute () { micInputVolume = 0; } public void Unmute () { micInputVolume = 100; } }
- jynxiwinExplorer
Is this on Quest2?
- YanaArtisProtege
Yes. On Quest 1 and Quest 2 it works fine both as standalone Quest app, and as PC VR app. I just received message from my Rift tester. There are errors on the end of recording when running with Rift:
C:\buildslave\unity\build\Modules/Audio/Public/sound/SoundManager.cpp(729) : Error executing instance->m_Sound->unlock(ptr1, ptr2, len1, len2) (An invalid parameter was passed to this function. )
net.kiborgov.vrss.OculusLipSyncMicInput:EndMicrophoneRecord()ArgumentException: Length of created clip must be larger than 0
at UnityEngine.AudioClip.Create (System.String name, System.Int32 lengthSamples, System.Int32 channels, System.Int32 frequency, System.Boolean stream, UnityEngine.AudioClip+PCMReaderCallback pcmreadercallback, UnityEngine.AudioClip+PCMSetPositionCallback pcmsetpositioncallback) [0x00000] in <00000000000000000000000000000000>:0
at UnityEngine.AudioClip.Create (System.String name, System.Int32 lengthSamples, System.Int32 channels, System.Int32 frequency, System.Boolean stream) [0x00000] in <00000000000000000000000000000000>:0
at net.kiborgov.vrss.OculusLipSyncMicInput.EndMicrophoneRecord () [0x00000] in <00000000000000000000000000000000>:0
- jynxiwinExplorer
Yea, I worked out a similar solution. Thanks for the help....
Now to get TTS wotking - got RTVoice
Quick Links
- Horizon Developer Support
- Quest User Forums
- Troubleshooting Forum for problems with a game or app
- Quest Support for problems with your device