Meta Avatars 2 Lipsync - PUN 2 & Photon Voice issue!

Expert Protege

4 years ago

This is the script for my custom Photon Recorder Input. I basically just copied Photon's MicrophoneWrapper except, as you can see, instead of calling Microphone.Start() I instead fetch the audio clip from my central microphone with GGMicrophone.GetMicrophoneAudioClip(). 'GG' is just a marker for classes that my company writes.

using UnityEngine;
using Photon.Voice;

public class GGMicrophoneInputForPhoton : IAudioReader<float>
{
    public GGMicrophoneInputForPhoton()
    {
        GGMicrophone.Instance.StartMicrophone();
    }

    public int SamplingRate
    {
        get
        {
            AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip();
            return clip.frequency;
        }
    }

    public int Channels
    {
        get
        {
            AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip();
            return clip.channels;
        }
    }

    public string Error { get; private set; }

    public void Dispose()
    {
        GGMicrophone.Instance.StopMicrophone();
    }

    private int micPrevPos;
    private int micLoopCnt;
    private int readAbsPos;

    public bool Read(float[] buffer)
    {
        if (Error != null)
        {
            return false;
        }
        int micPos = GGMicrophone.Instance.GetPosition();
        // loop detection
        if (micPos < micPrevPos)
        {
            micLoopCnt++;
        }
        micPrevPos = micPos;

        AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip();

        var micAbsPos = micLoopCnt * clip.samples + micPos;

        if (clip.channels == 0)
        {
            Error = "Number of channels is 0 in Read()";
            //logger.LogError("[PV] MicWrapper: " + Error);
            return false;
        }
        var bufferSamplesCount = buffer.Length / clip.channels;

        var nextReadPos = this.readAbsPos + bufferSamplesCount;
        if (nextReadPos < micAbsPos)
        {
            clip.GetData(buffer, this.readAbsPos % clip.samples);
            this.readAbsPos = nextReadPos;
            return true;
        }
        else
        {
            return false;
        }
    }
}

To initialise the custom input I use this very simple script:

using GGChassis.GGAvatars;
using Oculus.Platform;
using Photon.Pun;
using Photon.Voice.PUN;
using Photon.Voice.Unity;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class GGMicrophoneInputForPhotonInitializer : MonoBehaviour
{
    private void Start()
    {
        Recorder recorder = PhotonVoiceNetwork.Instance.PrimaryRecorder;
        if (recorder != null)
        {
            recorder.SourceType = Recorder.InputSourceType.Factory;
            recorder.InputFactory = () => new GGMicrophoneInputForPhoton();
        }
        else
        {
            Debug.LogError("Could not set recorder's input source type because no recorder was found.");
        }
    }
}

And here is my central microphone script:

public class GGMicrophone : MonoBehaviour
{
    [SerializeField] AudioClip m_audioClip;
    [Tooltip("Enable to see the Mic Input Volume")][SerializeField] bool m_debugMicrophone = true;
    [Range(0f, 1f)] [SerializeField] float m_micInputVolume = 0f;
    [Range(0, 1000000)] [SerializeField] int m_microphonePosition;
    [SerializeField] int m_audioClipId;
    [SerializeField] float m_microphoneSensitivity = 50f;
    [SerializeField] float m_threshold = 0.1f;
    int m_samplingRate = 48000;
    int m_sampleWindow = 64;
    int m_audioClipLength = 1;

    static private GGMicrophone m_instance;
    static public GGMicrophone Instance
    {
        get 
        { 
            return m_instance; 
        }
    }

    private void Awake()
    {
        if (m_instance == null)
        {
            m_instance = this;
        }
        else
        {
            Destroy(this.gameObject);
        }
    }

    [ContextMenu("Start Microphone")]
    public void StartMicrophone()
    {
        string microphoneName = GetMicrophoneDeviceName();
        m_audioClip = Microphone.Start(microphoneName, true, m_audioClipLength, m_samplingRate);
        string clipId = m_audioClip.GetInstanceID().ToString();
        m_audioClip.name = "GGMicAudioClip_" + clipId;
    }

    [ContextMenu("Stop Microphone")]
    public void StopMicrophone()
    {
        string microphoneName = GetMicrophoneDeviceName();
        Microphone.End(microphoneName);
    }

    public bool IsRecording()
    {
        string microphoneName = GetMicrophoneDeviceName();
        return Microphone.IsRecording(microphoneName);
    }

    public int GetPosition()
    {
        string microphoneName = GetMicrophoneDeviceName();
        return Microphone.GetPosition(microphoneName);
    }

    public string GetMicrophoneDeviceName()
    {
        return Microphone.devices[0];
    }

    public AudioClip GetMicrophoneAudioClip()
    {
        return m_audioClip;
    }

    private void Start()
    {
        StartMicrophone();
        SceneManager.sceneLoaded += OnSceneLoaded;
    }

    void OnSceneLoaded(Scene scene, LoadSceneMode mode)
    {
        StartMicrophone();
    }

    private void Update()
    {
        if (m_debugMicrophone == true)
        {
            if ((m_audioClip != null))
            {
                float micInputVolume = GetVolumeFromMicrophone() * m_microphoneSensitivity;
                if (micInputVolume < m_threshold)
                {
                    micInputVolume = 0;
                }
                m_micInputVolume = micInputVolume;
                m_audioClipId = m_audioClip.GetInstanceID();
            }
            m_microphonePosition = GetPosition();
        }
    }

    public float GetVolumeFromMicrophone()
    {
        return GetVolumeFromAudioClip(Microphone.GetPosition(Microphone.devices[0]), m_audioClip);
    }

    float GetVolumeFromAudioClip(int clipPosition, AudioClip audioClip)
    {
        if (clipPosition - m_sampleWindow < 0)
        {
            return 0;
        }

        float[] data = new float[m_sampleWindow];
        audioClip.GetData(data, clipPosition - m_sampleWindow);

        float totalVolume = 0;
        for (int i = 0; i < data.Length; i++)
        {
            totalVolume += Mathf.Abs(data[i]);
        }

        float averageVolume = totalVolume / m_sampleWindow;
        return averageVolume;
    }

    public void EnableMicrophoneDebugger(bool enabled)
    {
        m_debugMicrophone = enabled;
    }

    public void UpdateAudioClipLength(float clipLength)
    {
        m_audioClipLength = (int)clipLength;
    }

    public void SetSamplingRate(int samplingRate)
    {
        m_samplingRate = samplingRate;
    }
}

olaysia

Expert Protege

4 years ago

Here's my custom photon recorder input. I basically copied it off Photon's MicWrapper script except instead of calling Microphone.Start(), which steals the mic input from the lip sync, I simply get the audio clip from my central microphone with the call GGMicrophone.Instance.GetMicrophoneAudioClip()

using UnityEngine;
using Photon.Voice;

public class GGMicrophoneInputForPhoton : IAudioReader<float>
{
    public GGMicrophoneInputForPhoton()
    {
        GGMicrophone.Instance.StartMicrophone();
    }

    public int SamplingRate
    {
        get
        {
            AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip();
            return clip.frequency;
        }
    }

    public int Channels
    {
        get
        {
            AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip();
            return clip.channels;
        }
    }

    public string Error { get; private set; }

    public void Dispose()
    {
        GGMicrophone.Instance.StopMicrophone();
    }

    private int micPrevPos;
    private int micLoopCnt;
    private int readAbsPos;

    public bool Read(float[] buffer)
    {
        if (Error != null)
        {
            return false;
        }
        int micPos = GGMicrophone.Instance.GetPosition();
        // loop detection
        if (micPos < micPrevPos)
        {
            micLoopCnt++;
        }
        micPrevPos = micPos;

        AudioClip clip = GGMicrophone.Instance.GetMicrophoneAudioClip();

        var micAbsPos = micLoopCnt * clip.samples + micPos;

        if (clip.channels == 0)
        {
            Error = "Number of channels is 0 in Read()";
            //logger.LogError("[PV] MicWrapper: " + Error);
            return false;
        }
        var bufferSamplesCount = buffer.Length / clip.channels;

        var nextReadPos = this.readAbsPos + bufferSamplesCount;
        if (nextReadPos < micAbsPos)
        {
            clip.GetData(buffer, this.readAbsPos % clip.samples);
            this.readAbsPos = nextReadPos;
            return true;
        }
        else
        {
            return false;
        }
    }
}

To initialise this custom input I simply pop this script on a GameObject in my scene:

public class GGMicrophoneInputForPhotonInitializer : MonoBehaviour
{
    private void Start()
    {
        Recorder recorder = PhotonVoiceNetwork.Instance.PrimaryRecorder;
        if (recorder != null)
        {
            recorder.SourceType = Recorder.InputSourceType.Factory;
            recorder.InputFactory = () => new GGMicrophoneInputForPhoton();
        }
        else
        {
            Debug.LogError("Could not set recorder's input source type because no recorder was found.");
        }
    }
}

And here is my central Microphone:

public class GGMicrophone : MonoBehaviour
{
    [SerializeField] AudioClip m_audioClip;
    [Tooltip("Enable to see the Mic Input Volume")][SerializeField] bool m_debugMicrophone = true;
    [Range(0f, 1f)] [SerializeField] float m_micInputVolume = 0f;
    [Range(0, 1000000)] [SerializeField] int m_microphonePosition;
    [SerializeField] int m_audioClipId;
    [SerializeField] float m_microphoneSensitivity = 50f;
    [SerializeField] float m_threshold = 0.1f;
    int m_samplingRate = 16000;
    int m_sampleWindow = 64;
    int m_audioClipLength = 1;

    static private GGMicrophone m_instance;
    static public GGMicrophone Instance
    {
        get 
        { 
            return m_instance; 
        }
    }

    private void Awake()
    {
        if (m_instance == null)
        {
            m_instance = this;
        }
        else
        {
            Destroy(this.gameObject);
        }
    }

    [ContextMenu("Start Microphone")]
    public void StartMicrophone()
    {
        string microphoneName = GetMicrophoneDeviceName();
        m_audioClip = Microphone.Start(microphoneName, true, m_audioClipLength, m_samplingRate);
        string clipId = m_audioClip.GetInstanceID().ToString();
        m_audioClip.name = "GGMicAudioClip_" + clipId;
    }

    [ContextMenu("Stop Microphone")]
    public void StopMicrophone()
    {
        string microphoneName = GetMicrophoneDeviceName();
        Microphone.End(microphoneName);
    }

    public bool IsRecording()
    {
        string microphoneName = GetMicrophoneDeviceName();
        return Microphone.IsRecording(microphoneName);
    }

    public int GetPosition()
    {
        string microphoneName = GetMicrophoneDeviceName();
        return Microphone.GetPosition(microphoneName);
    }

    public string GetMicrophoneDeviceName()
    {
        return Microphone.devices[0];
    }

    public AudioClip GetMicrophoneAudioClip()
    {
        return m_audioClip;
    }

    private void Start()
    {
        StartMicrophone();
        SceneManager.sceneLoaded += OnSceneLoaded;
    }

    void OnSceneLoaded(Scene scene, LoadSceneMode mode)
    {
        StartMicrophone();
    }

    private void Update()
    {
        if (m_debugMicrophone == true)
        {
            if ((m_audioClip != null))
            {
                float micInputVolume = GetVolumeFromMicrophone() * m_microphoneSensitivity;
                if (micInputVolume < m_threshold)
                {
                    micInputVolume = 0;
                }
                m_micInputVolume = micInputVolume;
                m_audioClipId = m_audioClip.GetInstanceID();
            }
            m_microphonePosition = GetPosition();
        }
    }

    public float GetVolumeFromMicrophone()
    {
        return GetVolumeFromAudioClip(Microphone.GetPosition(Microphone.devices[0]), m_audioClip);
    }

    float GetVolumeFromAudioClip(int clipPosition, AudioClip audioClip)
    {
        if (clipPosition - m_sampleWindow < 0)
        {
            return 0;
        }

        float[] data = new float[m_sampleWindow];
        audioClip.GetData(data, clipPosition - m_sampleWindow);

        float totalVolume = 0;
        for (int i = 0; i < data.Length; i++)
        {
            totalVolume += Mathf.Abs(data[i]);
        }

        float averageVolume = totalVolume / m_sampleWindow;
        return averageVolume;
    }

    public void EnableMicrophoneDebugger(bool enabled)
    {
        m_debugMicrophone = enabled;
    }

    public void UpdateAudioClipLength(float clipLength)
    {
        m_audioClipLength = (int)clipLength;
    }

    public void SetSamplingRate(int samplingRate)
    {
        m_samplingRate = samplingRate;
    }
}

ErickMarinC
Explorer
4 years ago
Thank you so much!

Forum Discussion

🚨 This forum is archived and read-only. To submit a forum post, please visit our new Developer Forum. 🚨

Meta Avatars 2 Lipsync - PUN 2 & Photon Voice issue!

Quick Links

Other Meta Support

Related Content

Avatar clothing creation

avatar clothing

Problems with Meta Avatars SDK (v31.0.0)

Selling avatar clothing

Custom Avatar Items not loading. Generic Purple avatar.