// -----------------------------------------------------------------------
//
// Photon Voice API Framework for Photon - Copyright (C) 2017 Exit Games GmbH
//
//
// Photon data streaming support.
//
// developer@photonengine.com
// ----------------------------------------------------------------------------
using System;
using System.Linq;
using System.Collections.Generic;
namespace Photon.Voice
{
public interface ILogger
{
void LogError(string fmt, params object[] args);
void LogWarning(string fmt, params object[] args);
void LogInfo(string fmt, params object[] args);
void LogDebug(string fmt, params object[] args);
}
public interface IVoiceTransport
{
bool IsChannelJoined(int channelId);
// targetPlayerId: to all if 0, to myself if -1
void SendVoicesInfo(IEnumerable voices, int channelId, int targetPlayerId);
// targetPlayerId: to all if 0, to myself if -1
void SendVoiceRemove(LocalVoice voice, int channelId, int targetPlayerId);
// targetPlayerId: to all if 0, to myself if -1
void SendFrame(ArraySegment data, FrameFlags flags, byte evNumber, byte voiceId, int channelId, int targetPlayerId, bool reliable, LocalVoice localVoice);
string ChannelIdStr(int channelId);
string PlayerIdStr(int playerId);
}
///
/// Voice client interact with other clients on network via IVoiceTransport.
///
public class VoiceClient : IDisposable
{
internal IVoiceTransport transport;
internal ILogger logger;
/// Lost frames counter.
public int FramesLost { get; internal set; }
/// Received frames counter.
public int FramesReceived { get; private set; }
/// Sent frames counter.
public int FramesSent { get { int x = 0; foreach (var v in this.localVoices) { x += v.Value.FramesSent; } return x; } }
/// Sent frames bytes counter.
public int FramesSentBytes { get { int x = 0; foreach (var v in this.localVoices) { x += v.Value.FramesSentBytes; } return x; } }
/// Average time required voice packet to return to sender.
public int RoundTripTime { get; private set; }
/// Average round trip time variation.
public int RoundTripTimeVariance { get; private set; }
/// Do not log warning when duplicate info received.
public bool SuppressInfoDuplicateWarning { get; set; }
/// Remote voice info event delegate.
public delegate void RemoteVoiceInfoDelegate(int channelId, int playerId, byte voiceId, VoiceInfo voiceInfo, ref RemoteVoiceOptions options);
///
/// Register a method to be called when remote voice info arrived (after join or new new remote voice creation).
/// Metod parameters: (int channelId, int playerId, byte voiceId, VoiceInfo voiceInfo, ref RemoteVoiceOptions options);
///
public RemoteVoiceInfoDelegate OnRemoteVoiceInfoAction { get; set; }
/// Lost frames simulation ratio.
public int DebugLostPercent { get; set; }
private int prevRtt = 0;
/// Iterates through copy of all local voices list.
public IEnumerable LocalVoices
{
get
{
var res = new LocalVoice[this.localVoices.Count];
this.localVoices.Values.CopyTo(res, 0);
return res;
}
}
/// Iterates through copy of all local voices list of given channel.
public IEnumerable LocalVoicesInChannel(int channelId)
{
List channelVoices;
if (this.localVoicesPerChannel.TryGetValue(channelId, out channelVoices))
{
var res = new LocalVoice[channelVoices.Count];
channelVoices.CopyTo(res, 0);
return res;
}
else
{
return new LocalVoice[0];
}
}
/// Iterates through all remote voices infos.
public IEnumerable RemoteVoiceInfos
{
get
{
foreach (var playerVoices in this.remoteVoices)
{
foreach (var voice in playerVoices.Value)
{
yield return new RemoteVoiceInfo(voice.Value.channelId, playerVoices.Key, voice.Key, voice.Value.Info);
}
}
}
}
public void LogSpacingProfiles()
{
foreach (var voice in this.localVoices)
{
voice.Value.SendSpacingProfileStart(); // in case it's not started yet
this.logger.LogInfo(voice.Value.LogPrefix + " ev. prof.: " + voice.Value.SendSpacingProfileDump);
}
foreach (var playerVoices in this.remoteVoices)
{
foreach (var voice in playerVoices.Value)
{
voice.Value.ReceiveSpacingProfileStart(); // in case it's not started yet
this.logger.LogInfo(voice.Value.LogPrefix + " ev. prof.: " + voice.Value.ReceiveSpacingProfileDump);
}
}
}
public void LogStats()
{
int dc = FrameBuffer.statDisposerCreated;
int dd = FrameBuffer.statDisposerDisposed;
int pp = FrameBuffer.statPinned;
int pu = FrameBuffer.statUnpinned;
this.logger.LogInfo("[PV] FrameBuffer stats Disposer: " + dc + " - " + dd + " = " + (dc - dd));
this.logger.LogInfo("[PV] FrameBuffer stats Pinned: " + pp + " - " + pu + " = " + (pp - pu));
}
public void SetRemoteVoiceDelayFrames(Codec codec, int delayFrames)
{
remoteVoiceDelayFrames[codec] = delayFrames;
foreach (var playerVoices in this.remoteVoices)
{
foreach (var voice in playerVoices.Value)
{
if (codec == voice.Value.Info.Codec)
{
voice.Value.DelayFrames = delayFrames;
}
}
}
}
// store delay to apply on new remote voices
private Dictionary remoteVoiceDelayFrames = new Dictionary();
public struct CreateOptions
{
public byte VoiceIDMin;
public byte VoiceIDMax;
static public CreateOptions Default = new CreateOptions()
{
VoiceIDMin = 1, // 0 means invalid id
VoiceIDMax = 15 // preserve ids for other clients creating voices for the same player (server plugin)
};
}
/// Creates VoiceClient instance
public VoiceClient(IVoiceTransport transport, ILogger logger, CreateOptions opt = default(CreateOptions))
{
this.transport = transport;
this.logger = logger;
if (opt.Equals(default(CreateOptions)))
{
opt = CreateOptions.Default;
}
this.voiceIDMin = opt.VoiceIDMin;
this.voiceIDMax = opt.VoiceIDMax;
this.voiceIdLast = this.voiceIDMax;
}
///
/// This method dispatches all available incoming commands and then sends this client's outgoing commands.
/// Call this method regularly (2..20 times a second).
///
public void Service()
{
foreach (var v in localVoices)
{
v.Value.service();
}
}
private LocalVoice createLocalVoice(int channelId, Func voiceFactory)
{
var newId = getNewVoiceId();
if (newId != 0)
{
LocalVoice v = voiceFactory(newId, channelId);
if (v != null)
{
addVoice(newId, channelId, v);
this.logger.LogInfo(v.LogPrefix + " added enc: " + v.Info.ToString());
return v;
}
}
return null;
}
///
/// Creates basic outgoing stream w/o data processing support. Provided encoder should generate output data stream.
///
/// Outgoing stream parameters.
/// Transport channel specific to transport.
/// Encoder producing the stream.
/// Outgoing stream handler.
public LocalVoice CreateLocalVoice(VoiceInfo voiceInfo, int channelId = 0, IEncoder encoder = null)
{
return (LocalVoice)createLocalVoice(channelId, (vId, chId) => new LocalVoice(this, encoder, vId, voiceInfo, chId));
}
///
/// Creates outgoing stream consuming sequence of values passed in array buffers of arbitrary length which repacked in frames of constant length for further processing and encoding.
///
/// Type of data consumed by outgoing stream (element type of array buffers).
/// Outgoing stream parameters.
/// Size of buffer LocalVoiceFramed repacks input data stream to.
/// Transport channel specific to transport.
/// Encoder compressing data stream in pipeline.
/// Outgoing stream handler.
public LocalVoiceFramed CreateLocalVoiceFramed(VoiceInfo voiceInfo, int frameSize, int channelId = 0, IEncoder encoder = null)
{
return (LocalVoiceFramed)createLocalVoice(channelId, (vId, chId) => new LocalVoiceFramed(this, encoder, vId, voiceInfo, chId, frameSize));
}
public LocalVoiceAudio CreateLocalVoiceAudio(VoiceInfo voiceInfo, IAudioDesc audioSourceDesc, IEncoder encoder, int channelId)
{
return (LocalVoiceAudio)createLocalVoice(channelId, (vId, chId) => LocalVoiceAudio.Create(this, vId, encoder, voiceInfo, audioSourceDesc, chId));
}
///
/// Creates outgoing audio stream of type automatically assigned and adds procedures (callback or serviceable) for consuming given audio source data.
/// Adds audio specific features (e.g. resampling, level meter) to processing pipeline and to returning stream handler.
///
/// Outgoing stream parameters.
/// Streaming audio source.
/// Voice's audio sample type. If does not match source audio sample type, conversion will occur.
/// Transport channel specific to transport.
/// Audio encoder. Set to null to use default Opus encoder.
/// Outgoing stream handler.
///
/// audioSourceDesc.SamplingRate and voiceInfo.SamplingRate may do not match. Automatic resampling will occur in this case.
///
public LocalVoice CreateLocalVoiceAudioFromSource(VoiceInfo voiceInfo, IAudioDesc source, AudioSampleType sampleType, IEncoder encoder = null, int channelId = 0)
{
// resolve AudioSampleType.Source to concrete type for encoder creation
if (sampleType == AudioSampleType.Source)
{
if (source is IAudioPusher || source is IAudioReader)
{
sampleType = AudioSampleType.Float;
}
else if (source is IAudioPusher || source is IAudioReader)
{
sampleType = AudioSampleType.Short;
}
}
if (encoder == null)
{
switch (sampleType)
{
case AudioSampleType.Float:
encoder = Platform.CreateDefaultAudioEncoder(logger, voiceInfo);
break;
case AudioSampleType.Short:
encoder = Platform.CreateDefaultAudioEncoder(logger, voiceInfo);
break;
}
}
if (source is IAudioPusher)
{
if (sampleType == AudioSampleType.Short)
{
logger.LogInfo("[PV] Creating local voice with source samples type conversion from IAudioPusher float to short.");
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
// we can safely reuse the same buffer in callbacks from native code
//
var bufferFactory = new FactoryReusableArray(0);
((IAudioPusher)source).SetCallback(buf => {
var shortBuf = localVoice.BufferFactory.New(buf.Length);
AudioUtil.Convert(buf, shortBuf, buf.Length);
localVoice.PushDataAsync(shortBuf);
}, bufferFactory);
return localVoice;
}
else
{
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
((IAudioPusher)source).SetCallback(buf => localVoice.PushDataAsync(buf), localVoice.BufferFactory);
return localVoice;
}
}
else if (source is IAudioPusher)
{
if (sampleType == AudioSampleType.Float)
{
logger.LogInfo("[PV] Creating local voice with source samples type conversion from IAudioPusher short to float.");
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
// we can safely reuse the same buffer in callbacks from native code
//
var bufferFactory = new FactoryReusableArray(0);
((IAudioPusher)source).SetCallback(buf =>
{
var floatBuf = localVoice.BufferFactory.New(buf.Length);
AudioUtil.Convert(buf, floatBuf, buf.Length);
localVoice.PushDataAsync(floatBuf);
}, bufferFactory);
return localVoice;
}
else
{
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
((IAudioPusher)source).SetCallback(buf => localVoice.PushDataAsync(buf), localVoice.BufferFactory);
return localVoice;
}
}
else if (source is IAudioReader)
{
if (sampleType == AudioSampleType.Short)
{
logger.LogInfo("[PV] Creating local voice with source samples type conversion from IAudioReader float to short.");
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
localVoice.LocalUserServiceable = new BufferReaderPushAdapterAsyncPoolFloatToShort(localVoice, source as IAudioReader);
return localVoice;
}
else
{
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
localVoice.LocalUserServiceable = new BufferReaderPushAdapterAsyncPool(localVoice, source as IAudioReader);
return localVoice;
}
}
else if (source is IAudioReader)
{
if (sampleType == AudioSampleType.Float)
{
logger.LogInfo("[PV] Creating local voice with source samples type conversion from IAudioReader short to float.");
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
localVoice.LocalUserServiceable = new BufferReaderPushAdapterAsyncPoolShortToFloat(localVoice, source as IAudioReader);
return localVoice;
}
else
{
var localVoice = CreateLocalVoiceAudio(voiceInfo, source, encoder, channelId);
localVoice.LocalUserServiceable = new BufferReaderPushAdapterAsyncPool(localVoice, source as IAudioReader);
return localVoice;
}
}
else
{
logger.LogError("[PV] CreateLocalVoiceAudioFromSource does not support Voice.IAudioDesc of type {0}", source.GetType());
return LocalVoiceAudioDummy.Dummy;
}
}
#if PHOTON_VOICE_VIDEO_ENABLE
///
/// Creates outgoing video stream consuming sequence of image buffers.
///
/// Outgoing stream parameters.
/// Video recorder.
/// Transport channel specific to transport.
/// Outgoing stream handler.
public LocalVoiceVideo CreateLocalVoiceVideo(VoiceInfo voiceInfo, IVideoRecorder recorder, int channelId = 0)
{
var lv = (LocalVoiceVideo)createLocalVoice(channelId, (vId, chId) => new LocalVoiceVideo(this, recorder.Encoder, vId, voiceInfo, chId));
if (recorder is IVideoRecorderPusher)
{
(recorder as IVideoRecorderPusher).VideoSink = lv;
}
return lv;
}
#endif
private byte voiceIDMin;
private byte voiceIDMax;
private byte voiceIdLast; // inited with voiceIDMax: the first id will be voiceIDMin
private byte idInc(byte id)
{
return id == voiceIDMax ? voiceIDMin : (byte)(id + 1);
}
private byte getNewVoiceId()
{
var used = new bool[256];
foreach (var v in localVoices)
{
used[v.Value.id] = true;
}
for (byte id = idInc(voiceIdLast); id != voiceIdLast; id = idInc(id))
{
if (!used[id])
{
voiceIdLast = id;
return id;
}
}
return 0;
}
void addVoice(byte newId, int channelId, LocalVoice v)
{
localVoices[newId] = v;
List voiceList;
if (!localVoicesPerChannel.TryGetValue(channelId, out voiceList))
{
voiceList = new List();
localVoicesPerChannel[channelId] = voiceList;
}
voiceList.Add(v);
if (this.transport.IsChannelJoined(channelId))
{
sendVoicesInfoAndConfigFrame(new List() { v }, channelId, 0); // broadcast if joined
}
v.InterestGroup = this.GlobalInterestGroup;
}
///
/// Removes local voice (outgoing data stream).
/// Handler of outgoing stream to be removed.
///
public void RemoveLocalVoice(LocalVoice voice)
{
this.localVoices.Remove(voice.id);
this.localVoicesPerChannel[voice.channelId].Remove(voice);
if (this.transport.IsChannelJoined(voice.channelId))
{
this.transport.SendVoiceRemove(voice, voice.channelId, 0);
}
voice.Dispose();
this.logger.LogInfo(voice.LogPrefix + " removed");
}
private void sendChannelVoicesInfo(int channelId, int targetPlayerId)
{
if (this.transport.IsChannelJoined(channelId))
{
List voiceList;
if (this.localVoicesPerChannel.TryGetValue(channelId, out voiceList))
{
sendVoicesInfoAndConfigFrame(voiceList, channelId, targetPlayerId);
}
}
}
internal void sendVoicesInfoAndConfigFrame(IEnumerable voiceList, int channelId, int targetPlayerId)
{
this.transport.SendVoicesInfo(voiceList, channelId, targetPlayerId);
foreach (var v in voiceList)
{
v.sendConfigFrame(targetPlayerId);
}
// send debug echo infos to myself if broadcast requested
if (targetPlayerId == 0)
{
var debugEchoVoices = localVoices.Values.Where(x => x.DebugEchoMode);
if (debugEchoVoices.Count() > 0)
{
this.transport.SendVoicesInfo(debugEchoVoices, channelId, -1);
}
}
}
internal byte GlobalInterestGroup
{
get { return this.globalInterestGroup; }
set
{
this.globalInterestGroup = value;
foreach (var v in this.localVoices)
{
v.Value.InterestGroup = this.globalInterestGroup;
}
}
}
#region nonpublic
private byte globalInterestGroup;
private Dictionary localVoices = new Dictionary();
private Dictionary> localVoicesPerChannel = new Dictionary>();
// player id -> voice id -> voice
private Dictionary> remoteVoices = new Dictionary>();
private void clearRemoteVoices()
{
foreach (var playerVoices in remoteVoices)
{
foreach (var voice in playerVoices.Value)
{
voice.Value.removeAndDispose();
}
}
remoteVoices.Clear();
this.logger.LogInfo("[PV] Remote voices cleared");
}
private void clearRemoteVoicesInChannel(int channelId)
{
foreach (var playerVoices in remoteVoices)
{
List toRemove = new List();
foreach (var voice in playerVoices.Value)
{
if (voice.Value.channelId == channelId)
{
voice.Value.removeAndDispose();
toRemove.Add(voice.Key);
}
}
foreach (var id in toRemove)
{
playerVoices.Value.Remove(id);
}
}
this.logger.LogInfo("[PV] Remote voices for channel " + this.channelStr(channelId) + " cleared");
}
private void clearRemoteVoicesInChannelForPlayer(int channelId, int playerId)
{
Dictionary playerVoices = null;
if (remoteVoices.TryGetValue(playerId, out playerVoices))
{
List toRemove = new List();
foreach (var v in playerVoices)
{
if (v.Value.channelId == channelId)
{
v.Value.removeAndDispose();
toRemove.Add(v.Key);
}
}
foreach (var id in toRemove)
{
playerVoices.Remove(id);
}
}
}
public void onJoinChannel(int channel)
{
sendChannelVoicesInfo(channel, 0);// my join, broadcast
}
public void onLeaveChannel(int channel)
{
clearRemoteVoicesInChannel(channel);
}
public void onLeaveAllChannels()
{
clearRemoteVoices();
}
public void onPlayerJoin(int channelId, int playerId)
{
sendChannelVoicesInfo(channelId, playerId);// send to new joined only
}
public void onPlayerLeave(int channelId, int playerId)
{
clearRemoteVoicesInChannelForPlayer(channelId, playerId);
}
public void onVoiceInfo(int channelId, int playerId, byte voiceId, byte eventNumber, VoiceInfo info)
{
Dictionary playerVoices = null;
if (!remoteVoices.TryGetValue(playerId, out playerVoices))
{
playerVoices = new Dictionary();
remoteVoices[playerId] = playerVoices;
}
if (!playerVoices.ContainsKey(voiceId))
{
var voiceStr = " p#" + this.playerStr(playerId) + " v#" + voiceId + " ch#" + channelStr(channelId);
this.logger.LogInfo("[PV] " + voiceStr + " Info received: " + info.ToString() + " ev=" + eventNumber);
var logPrefix = "[PV] Remote " + info.Codec + voiceStr;
RemoteVoiceOptions options = new RemoteVoiceOptions(logger, logPrefix, info);
if (this.OnRemoteVoiceInfoAction != null)
{
this.OnRemoteVoiceInfoAction(channelId, playerId, voiceId, info, ref options);
}
var rv = new RemoteVoice(this, options, channelId, playerId, voiceId, info, eventNumber);
playerVoices[voiceId] = rv;
int delayFrames;
if (remoteVoiceDelayFrames.TryGetValue(info.Codec, out delayFrames))
{
rv.DelayFrames = delayFrames;
}
}
else
{
if (!this.SuppressInfoDuplicateWarning)
{
this.logger.LogWarning("[PV] Info duplicate for voice #" + voiceId + " of player " + this.playerStr(playerId) + " at channel " + this.channelStr(channelId));
}
}
}
public void onVoiceRemove(int channelId, int playerId, byte[] voiceIds)
{
Dictionary playerVoices = null;
if (remoteVoices.TryGetValue(playerId, out playerVoices))
{
foreach (var voiceId in voiceIds)
{
RemoteVoice voice;
if (playerVoices.TryGetValue(voiceId, out voice))
{
playerVoices.Remove(voiceId);
this.logger.LogInfo("[PV] Remote voice #" + voiceId + " of player " + this.playerStr(playerId) + " at channel " + this.channelStr(channelId) + " removed");
voice.removeAndDispose();
}
else
{
this.logger.LogWarning("[PV] Remote voice #" + voiceId + " of player " + this.playerStr(playerId) + " at channel " + this.channelStr(channelId) + " not found when trying to remove");
}
}
}
else
{
this.logger.LogWarning("[PV] Remote voice list of player " + this.playerStr(playerId) + " at channel " + this.channelStr(channelId) + " not found when trying to remove voice(s)");
}
}
Random rnd = new Random();
public void onFrame(int channelId, int playerId, byte voiceId, byte evNumber, ref FrameBuffer receivedBytes, bool isLocalPlayer)
{
if (isLocalPlayer)
{
// rtt measurement in debug echo mode
LocalVoice voice;
if (this.localVoices.TryGetValue(voiceId, out voice))
{
int sendTime;
if (voice.eventTimestamps.TryGetValue(evNumber, out sendTime))
{
int rtt = Environment.TickCount - sendTime;
int rttvar = rtt - prevRtt;
prevRtt = rtt;
if (rttvar < 0) rttvar = -rttvar;
this.RoundTripTimeVariance = (rttvar + RoundTripTimeVariance * 19) / 20;
this.RoundTripTime = (rtt + RoundTripTime * 19) / 20;
}
}
//internal Dictionary localEventTimestamps = new Dictionary();
}
if (this.DebugLostPercent > 0 && rnd.Next(100) < this.DebugLostPercent)
{
this.logger.LogWarning("[PV] Debug Lost Sim: 1 packet dropped");
return;
}
FramesReceived++;
Dictionary playerVoices = null;
if (remoteVoices.TryGetValue(playerId, out playerVoices))
{
RemoteVoice voice = null;
if (playerVoices.TryGetValue(voiceId, out voice))
{
voice.receiveBytes(ref receivedBytes, evNumber);
}
else
{
this.logger.LogWarning("[PV] Frame event for not inited voice #" + voiceId + " of player " + this.playerStr(playerId) + " at channel " + this.channelStr(channelId));
}
}
else
{
this.logger.LogWarning("[PV] Frame event for voice #" + voiceId + " of not inited player " + this.playerStr(playerId) + " at channel " + this.channelStr(channelId));
}
}
internal string channelStr(int channelId)
{
var str = this.transport.ChannelIdStr(channelId);
if (str != null)
{
return channelId + "(" + str + ")";
}
else
{
return channelId.ToString();
}
}
internal string playerStr(int playerId)
{
var str = this.transport.PlayerIdStr(playerId);
if (str != null)
{
return playerId + "(" + str + ")";
}
else
{
return playerId.ToString();
}
}
//public string ToStringFull()
//{
// return string.Format("Photon.Voice.Client, local: {0}, remote: {1}", localVoices.Count, remoteVoices.Count);
//}
#endregion
public void Dispose()
{
foreach (var v in this.localVoices)
{
v.Value.Dispose();
}
foreach (var playerVoices in remoteVoices)
{
foreach (var voice in playerVoices.Value)
{
voice.Value.Dispose();
}
}
}
}
}