Forum Discussion
vivekrajagopal
4 years agoMember
Meta Avatars 2 Lipsync - PUN 2 & Photon Voice issue!
Dear Devs, I'm struggling with problem since a week. I use PUN 2 and Photon Voice to bring Meta Avatars 2 in a multiplayer environment. Below are my issues, 1. When there is no Photon Voice se...
- 3 years ago
Dear Developers,
I've found a proper fix for this issue. I've explained it in my blog that might help you.
Please check this blog: https://medium.com/@vivekrajagopal_84414/meta-avatars-2-and-photon-unity-voice-issue-lipsync-pun-voice-resolved-6a7c5d229b1c
- 3 years ago
Dear Developers,
Hi, I've found a proper fix for this issue. I've explained it in my blog that might help you.
Please check this blog: https://medium.com/@vivekrajagopal_84414/meta-avatars-2-and-photon-unity-voice-issue-lipsync-pun-voice-resolved-6a7c5d229b1c
olaysia
4 years agoExpert Protege
Sure, which part would you like me to share? The Central Microphone, the custom photon recorder input or the lips sync clip updater?
ErickMarinC
4 years agoExplorer
It would be amazing to have:
- Custom Photon Recorder Input
- Central Microphone
Thank you so much!
- olaysia4 years agoExpert Protege
This is the script for my custom Photon Recorder Input. I basically just copied Photon's MicrophoneWrapper except, as you can see, instead of calling Microphone.Start() I instead fetch the audio clip from my central microphone with GGMicrophone.GetMicrophoneAudioClip(). 'GG' is just a marker for classes that my company writes.
using UnityEngine; using Photon.Voice; public class GGMicrophoneInputForPhoton : IAudioReader<float> { public GGMicrophoneInputForPhoton() { GGMicrophone.Instance.StartMicrophone(); } public int SamplingRate { get { AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip(); return clip.frequency; } } public int Channels { get { AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip(); return clip.channels; } } public string Error { get; private set; } public void Dispose() { GGMicrophone.Instance.StopMicrophone(); } private int micPrevPos; private int micLoopCnt; private int readAbsPos; public bool Read(float[] buffer) { if (Error != null) { return false; } int micPos = GGMicrophone.Instance.GetPosition(); // loop detection if (micPos < micPrevPos) { micLoopCnt++; } micPrevPos = micPos; AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip(); var micAbsPos = micLoopCnt * clip.samples + micPos; if (clip.channels == 0) { Error = "Number of channels is 0 in Read()"; //logger.LogError("[PV] MicWrapper: " + Error); return false; } var bufferSamplesCount = buffer.Length / clip.channels; var nextReadPos = this.readAbsPos + bufferSamplesCount; if (nextReadPos < micAbsPos) { clip.GetData(buffer, this.readAbsPos % clip.samples); this.readAbsPos = nextReadPos; return true; } else { return false; } } }To initialise the custom input I use this very simple script:
using GGChassis.GGAvatars; using Oculus.Platform; using Photon.Pun; using Photon.Voice.PUN; using Photon.Voice.Unity; using System.Collections; using System.Collections.Generic; using UnityEngine; public class GGMicrophoneInputForPhotonInitializer : MonoBehaviour { private void Start() { Recorder recorder = PhotonVoiceNetwork.Instance.PrimaryRecorder; if (recorder != null) { recorder.SourceType = Recorder.InputSourceType.Factory; recorder.InputFactory = () => new GGMicrophoneInputForPhoton(); } else { Debug.LogError("Could not set recorder's input source type because no recorder was found."); } } }And here is my central microphone script:
public class GGMicrophone : MonoBehaviour { [SerializeField] AudioClip m_audioClip; [Tooltip("Enable to see the Mic Input Volume")][SerializeField] bool m_debugMicrophone = true; [Range(0f, 1f)] [SerializeField] float m_micInputVolume = 0f; [Range(0, 1000000)] [SerializeField] int m_microphonePosition; [SerializeField] int m_audioClipId; [SerializeField] float m_microphoneSensitivity = 50f; [SerializeField] float m_threshold = 0.1f; int m_samplingRate = 48000; int m_sampleWindow = 64; int m_audioClipLength = 1; static private GGMicrophone m_instance; static public GGMicrophone Instance { get { return m_instance; } } private void Awake() { if (m_instance == null) { m_instance = this; } else { Destroy(this.gameObject); } } [ContextMenu("Start Microphone")] public void StartMicrophone() { string microphoneName = GetMicrophoneDeviceName(); m_audioClip = Microphone.Start(microphoneName, true, m_audioClipLength, m_samplingRate); string clipId = m_audioClip.GetInstanceID().ToString(); m_audioClip.name = "GGMicAudioClip_" + clipId; } [ContextMenu("Stop Microphone")] public void StopMicrophone() { string microphoneName = GetMicrophoneDeviceName(); Microphone.End(microphoneName); } public bool IsRecording() { string microphoneName = GetMicrophoneDeviceName(); return Microphone.IsRecording(microphoneName); } public int GetPosition() { string microphoneName = GetMicrophoneDeviceName(); return Microphone.GetPosition(microphoneName); } public string GetMicrophoneDeviceName() { return Microphone.devices[0]; } public AudioClip GetMicrophoneAudioClip() { return m_audioClip; } private void Start() { StartMicrophone(); SceneManager.sceneLoaded += OnSceneLoaded; } void OnSceneLoaded(Scene scene, LoadSceneMode mode) { StartMicrophone(); } private void Update() { if (m_debugMicrophone == true) { if ((m_audioClip != null)) { float micInputVolume = GetVolumeFromMicrophone() * m_microphoneSensitivity; if (micInputVolume < m_threshold) { micInputVolume = 0; } m_micInputVolume = micInputVolume; m_audioClipId = m_audioClip.GetInstanceID(); } m_microphonePosition = GetPosition(); } } public float GetVolumeFromMicrophone() { return GetVolumeFromAudioClip(Microphone.GetPosition(Microphone.devices[0]), m_audioClip); } float GetVolumeFromAudioClip(int clipPosition, AudioClip audioClip) { if (clipPosition - m_sampleWindow < 0) { return 0; } float[] data = new float[m_sampleWindow]; audioClip.GetData(data, clipPosition - m_sampleWindow); float totalVolume = 0; for (int i = 0; i < data.Length; i++) { totalVolume += Mathf.Abs(data[i]); } float averageVolume = totalVolume / m_sampleWindow; return averageVolume; } public void EnableMicrophoneDebugger(bool enabled) { m_debugMicrophone = enabled; } public void UpdateAudioClipLength(float clipLength) { m_audioClipLength = (int)clipLength; } public void SetSamplingRate(int samplingRate) { m_samplingRate = samplingRate; } } - olaysia4 years agoExpert Protege
Here's my custom photon recorder input. I basically copied it off Photon's MicWrapper script except instead of calling Microphone.Start(), which steals the mic input from the lip sync, I simply get the audio clip from my central microphone with the call GGMicrophone.Instance.GetMicrophoneAudioClip()
using UnityEngine; using Photon.Voice; public class GGMicrophoneInputForPhoton : IAudioReader<float> { public GGMicrophoneInputForPhoton() { GGMicrophone.Instance.StartMicrophone(); } public int SamplingRate { get { AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip(); return clip.frequency; } } public int Channels { get { AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip(); return clip.channels; } } public string Error { get; private set; } public void Dispose() { GGMicrophone.Instance.StopMicrophone(); } private int micPrevPos; private int micLoopCnt; private int readAbsPos; public bool Read(float[] buffer) { if (Error != null) { return false; } int micPos = GGMicrophone.Instance.GetPosition(); // loop detection if (micPos < micPrevPos) { micLoopCnt++; } micPrevPos = micPos; AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip(); var micAbsPos = micLoopCnt * clip.samples + micPos; if (clip.channels == 0) { Error = "Number of channels is 0 in Read()"; //logger.LogError("[PV] MicWrapper: " + Error); return false; } var bufferSamplesCount = buffer.Length / clip.channels; var nextReadPos = this.readAbsPos + bufferSamplesCount; if (nextReadPos < micAbsPos) { clip.GetData(buffer, this.readAbsPos % clip.samples); this.readAbsPos = nextReadPos; return true; } else { return false; } } }To initialise this custom input I simply pop this script on a GameObject in my scene:
public class GGMicrophoneInputForPhotonInitializer : MonoBehaviour { private void Start() { Recorder recorder = PhotonVoiceNetwork.Instance.PrimaryRecorder; if (recorder != null) { recorder.SourceType = Recorder.InputSourceType.Factory; recorder.InputFactory = () => new GGMicrophoneInputForPhoton(); } else { Debug.LogError("Could not set recorder's input source type because no recorder was found."); } } }And here is my central Microphone:
public class GGMicrophone : MonoBehaviour { [SerializeField] AudioClip m_audioClip; [Tooltip("Enable to see the Mic Input Volume")][SerializeField] bool m_debugMicrophone = true; [Range(0f, 1f)] [SerializeField] float m_micInputVolume = 0f; [Range(0, 1000000)] [SerializeField] int m_microphonePosition; [SerializeField] int m_audioClipId; [SerializeField] float m_microphoneSensitivity = 50f; [SerializeField] float m_threshold = 0.1f; int m_samplingRate = 16000; int m_sampleWindow = 64; int m_audioClipLength = 1; static private GGMicrophone m_instance; static public GGMicrophone Instance { get { return m_instance; } } private void Awake() { if (m_instance == null) { m_instance = this; } else { Destroy(this.gameObject); } } [ContextMenu("Start Microphone")] public void StartMicrophone() { string microphoneName = GetMicrophoneDeviceName(); m_audioClip = Microphone.Start(microphoneName, true, m_audioClipLength, m_samplingRate); string clipId = m_audioClip.GetInstanceID().ToString(); m_audioClip.name = "GGMicAudioClip_" + clipId; } [ContextMenu("Stop Microphone")] public void StopMicrophone() { string microphoneName = GetMicrophoneDeviceName(); Microphone.End(microphoneName); } public bool IsRecording() { string microphoneName = GetMicrophoneDeviceName(); return Microphone.IsRecording(microphoneName); } public int GetPosition() { string microphoneName = GetMicrophoneDeviceName(); return Microphone.GetPosition(microphoneName); } public string GetMicrophoneDeviceName() { return Microphone.devices[0]; } public AudioClip GetMicrophoneAudioClip() { return m_audioClip; } private void Start() { StartMicrophone(); SceneManager.sceneLoaded += OnSceneLoaded; } void OnSceneLoaded(Scene scene, LoadSceneMode mode) { StartMicrophone(); } private void Update() { if (m_debugMicrophone == true) { if ((m_audioClip != null)) { float micInputVolume = GetVolumeFromMicrophone() * m_microphoneSensitivity; if (micInputVolume < m_threshold) { micInputVolume = 0; } m_micInputVolume = micInputVolume; m_audioClipId = m_audioClip.GetInstanceID(); } m_microphonePosition = GetPosition(); } } public float GetVolumeFromMicrophone() { return GetVolumeFromAudioClip(Microphone.GetPosition(Microphone.devices[0]), m_audioClip); } float GetVolumeFromAudioClip(int clipPosition, AudioClip audioClip) { if (clipPosition - m_sampleWindow < 0) { return 0; } float[] data = new float[m_sampleWindow]; audioClip.GetData(data, clipPosition - m_sampleWindow); float totalVolume = 0; for (int i = 0; i < data.Length; i++) { totalVolume += Mathf.Abs(data[i]); } float averageVolume = totalVolume / m_sampleWindow; return averageVolume; } public void EnableMicrophoneDebugger(bool enabled) { m_debugMicrophone = enabled; } public void UpdateAudioClipLength(float clipLength) { m_audioClipLength = (int)clipLength; } public void SetSamplingRate(int samplingRate) { m_samplingRate = samplingRate; } }- ErickMarinC4 years agoExplorer
Thank you so much!
Quick Links
- Horizon Developer Support
- Quest User Forums
- Troubleshooting Forum for problems with a game or app
- Quest Support for problems with your device
Other Meta Support
Related Content
- 6 months ago
- 8 days ago
- 7 months ago