From 21f2b1224a677a510cb0ffc53fc5ffa4c2be8750 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Wed, 1 May 2019 21:27:41 -0700 Subject: [PATCH] Decode directly into the audio renderer's buffer to avoid a copy --- app/streaming/audio/audio.cpp | 19 ++++---- app/streaming/audio/renderers/renderer.h | 4 +- app/streaming/audio/renderers/sdl.h | 5 ++- app/streaming/audio/renderers/sdlaud.cpp | 24 +++++++++-- app/streaming/audio/renderers/slaud.cpp | 43 +++++-------------- app/streaming/audio/renderers/slaud.h | 6 +-- .../audio/renderers/soundioaudiorenderer.cpp | 27 +++++++----- .../audio/renderers/soundioaudiorenderer.h | 4 +- app/streaming/session.h | 1 - 9 files changed, 70 insertions(+), 63 deletions(-) diff --git a/app/streaming/audio/audio.cpp b/app/streaming/audio/audio.cpp index a304efee..8af95673 100644 --- a/app/streaming/audio/audio.cpp +++ b/app/streaming/audio/audio.cpp @@ -143,19 +143,22 @@ void Session::arDecodeAndPlaySample(char* sampleData, int sampleLength) s_ActiveSession->m_AudioSampleCount++; if (s_ActiveSession->m_AudioRenderer != nullptr) { + int desiredSize = sizeof(short) * SAMPLES_PER_FRAME * s_ActiveSession->m_AudioConfig.channelCount; + void* buffer = s_ActiveSession->m_AudioRenderer->getAudioBuffer(&desiredSize); + if (buffer == nullptr) { + return; + } + samplesDecoded = opus_multistream_decode(s_ActiveSession->m_OpusDecoder, (unsigned char*)sampleData, sampleLength, - s_ActiveSession->m_OpusDecodeBuffer, - SAMPLES_PER_FRAME, + (short*)buffer, + desiredSize / sizeof(short) / s_ActiveSession->m_AudioConfig.channelCount, 0); if (samplesDecoded > 0) { - if (!s_ActiveSession->m_AudioRenderer->submitAudio(s_ActiveSession->m_OpusDecodeBuffer, - static_cast( - sizeof(short) * - samplesDecoded * - s_ActiveSession->m_AudioConfig.channelCount))) { - + SDL_assert(desiredSize >= sizeof(short) * samplesDecoded * s_ActiveSession->m_AudioConfig.channelCount); + desiredSize = sizeof(short) * samplesDecoded * s_ActiveSession->m_AudioConfig.channelCount; + if (!s_ActiveSession->m_AudioRenderer->submitAudio(desiredSize)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Reinitializing audio renderer after failure"); diff --git a/app/streaming/audio/renderers/renderer.h b/app/streaming/audio/renderers/renderer.h index 0fbdad4f..25059cca 100644 --- a/app/streaming/audio/renderers/renderer.h +++ b/app/streaming/audio/renderers/renderer.h @@ -12,6 +12,8 @@ public: virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig) = 0; + virtual void* getAudioBuffer(int* size) = 0; + // Return false if an unrecoverable error has occurred and the renderer must be reinitialized - virtual bool submitAudio(short* audioBuffer, int audioSize) = 0; + virtual bool submitAudio(int bytesWritten) = 0; }; diff --git a/app/streaming/audio/renderers/sdl.h b/app/streaming/audio/renderers/sdl.h index 552a6e4c..0a186096 100644 --- a/app/streaming/audio/renderers/sdl.h +++ b/app/streaming/audio/renderers/sdl.h @@ -12,8 +12,11 @@ public: virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig); - virtual bool submitAudio(short* audioBuffer, int audioSize); + virtual void* getAudioBuffer(int* size); + + virtual bool submitAudio(int bytesWritten); private: SDL_AudioDeviceID m_AudioDevice; + void* m_AudioBuffer; }; diff --git a/app/streaming/audio/renderers/sdlaud.cpp b/app/streaming/audio/renderers/sdlaud.cpp index ea572f3c..955bca08 100644 --- a/app/streaming/audio/renderers/sdlaud.cpp +++ b/app/streaming/audio/renderers/sdlaud.cpp @@ -6,7 +6,8 @@ #include SdlAudioRenderer::SdlAudioRenderer() - : m_AudioDevice(0) + : m_AudioDevice(0), + m_AudioBuffer(nullptr) { SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO)); if (SDL_InitSubSystem(SDL_INIT_AUDIO) != 0) { @@ -37,7 +38,13 @@ bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to open audio device: %s", SDL_GetError()); - SDL_QuitSubSystem(SDL_INIT_AUDIO); + return false; + } + + m_AudioBuffer = malloc(SAMPLES_PER_FRAME * sizeof(short) * opusConfig->channelCount); + if (m_AudioBuffer == nullptr) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to allocate audio buffer"); return false; } @@ -65,13 +72,22 @@ SdlAudioRenderer::~SdlAudioRenderer() SDL_CloseAudioDevice(m_AudioDevice); } + if (m_AudioBuffer != nullptr) { + free(m_AudioBuffer); + } + SDL_QuitSubSystem(SDL_INIT_AUDIO); SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO)); } -bool SdlAudioRenderer::submitAudio(short* audioBuffer, int audioSize) +void* SdlAudioRenderer::getAudioBuffer(int*) { - if (SDL_QueueAudio(m_AudioDevice, audioBuffer, audioSize) < 0) { + return m_AudioBuffer; +} + +bool SdlAudioRenderer::submitAudio(int bytesWritten) +{ + if (SDL_QueueAudio(m_AudioDevice, m_AudioBuffer, bytesWritten) < 0) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to queue audio sample: %s", SDL_GetError()); diff --git a/app/streaming/audio/renderers/slaud.cpp b/app/streaming/audio/renderers/slaud.cpp index e478101c..b5e89de4 100644 --- a/app/streaming/audio/renderers/slaud.cpp +++ b/app/streaming/audio/renderers/slaud.cpp @@ -9,9 +9,7 @@ SLAudioRenderer::SLAudioRenderer() : m_AudioContext(nullptr), - m_AudioStream(nullptr), - m_AudioBuffer(nullptr), - m_AudioBufferBytesFilled(0) + m_AudioStream(nullptr) { SLAudio_SetLogFunction(SLAudioRenderer::slLogCallback, nullptr); } @@ -25,7 +23,7 @@ bool SLAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* o return false; } - m_AudioBufferSize = SAMPLES_PER_FRAME * sizeof(short) * opusConfig->channelCount * FRAMES_PER_SUBMISSION; + m_AudioBufferSize = SAMPLES_PER_FRAME * sizeof(short) * opusConfig->channelCount; m_AudioStream = SLAudio_CreateStream(m_AudioContext, opusConfig->sampleRate, opusConfig->channelCount, @@ -43,16 +41,14 @@ bool SLAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* o return true; } +void* SLAudioRenderer::getAudioBuffer(int* size) +{ + SDL_assert(*size == m_AudioBufferSize); + return SLAudio_BeginFrame(m_AudioStream); +} + SLAudioRenderer::~SLAudioRenderer() { - if (m_AudioBufferBytesFilled != 0) { - // We had a buffer in flight when we quit. Just in case - // SLAudio doesn't handle this properly, we'll zero and submit - // it just to be safe. - memset(m_AudioBuffer, 0, m_AudioBufferSize); - SLAudio_SubmitFrame(m_AudioStream); - } - if (m_AudioStream != nullptr) { SLAudio_FreeStream(m_AudioStream); } @@ -62,28 +58,9 @@ SLAudioRenderer::~SLAudioRenderer() } } -bool SLAudioRenderer::submitAudio(short* audioBuffer, int audioSize) +bool SLAudioRenderer::submitAudio(int) { - if (m_AudioBufferBytesFilled == 0) { - // Get a new audio buffer from SLAudio - m_AudioBuffer = (char*)SLAudio_BeginFrame(m_AudioStream); - if (m_AudioBuffer == nullptr) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "SLAudio_BeginFrame() failed"); - return true; - } - } - - // Accumulate several frames of audio before submitting to reduce CPU load - SDL_assert(audioSize <= m_AudioBufferSize - m_AudioBufferBytesFilled); - memcpy(&m_AudioBuffer[m_AudioBufferBytesFilled], audioBuffer, audioSize); - m_AudioBufferBytesFilled += audioSize; - - // Submit the buffer when it's full - if (m_AudioBufferBytesFilled == m_AudioBufferSize) { - SLAudio_SubmitFrame(m_AudioStream); - m_AudioBufferBytesFilled = 0; - } - + SLAudio_SubmitFrame(m_AudioStream); return true; } diff --git a/app/streaming/audio/renderers/slaud.h b/app/streaming/audio/renderers/slaud.h index c6ac3709..869a9103 100644 --- a/app/streaming/audio/renderers/slaud.h +++ b/app/streaming/audio/renderers/slaud.h @@ -12,14 +12,14 @@ public: virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig); - virtual bool submitAudio(short* audioBuffer, int audioSize); + virtual void* getAudioBuffer(int* size); + + virtual bool submitAudio(int bytesWritten); private: static void slLogCallback(void* context, ESLAudioLog logLevel, const char* message); CSLAudioContext* m_AudioContext; CSLAudioStream* m_AudioStream; - char* m_AudioBuffer; int m_AudioBufferSize; - int m_AudioBufferBytesFilled; }; diff --git a/app/streaming/audio/renderers/soundioaudiorenderer.cpp b/app/streaming/audio/renderers/soundioaudiorenderer.cpp index 5ed7d9e4..e96b2b7b 100644 --- a/app/streaming/audio/renderers/soundioaudiorenderer.cpp +++ b/app/streaming/audio/renderers/soundioaudiorenderer.cpp @@ -284,7 +284,20 @@ bool SoundIoAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATI return true; } -bool SoundIoAudioRenderer::submitAudio(short* audioBuffer, int audioSize) +void* SoundIoAudioRenderer::getAudioBuffer(int* size) +{ + // We must always write a full frame of audio. If we don't, + // the reader will get out of sync with the writer and our + // channels will get all mixed up. To ensure this is always + // the case, round our bytes free down to the next multiple + // of our frame size. + int bytesFree = soundio_ring_buffer_free_count(m_RingBuffer); + int bytesPerFrame = m_OpusChannelCount * m_OutputStream->bytes_per_sample; + *size = qMin(*size, (bytesFree / bytesPerFrame) * bytesPerFrame); + return soundio_ring_buffer_write_ptr(m_RingBuffer); +} + +bool SoundIoAudioRenderer::submitAudio(int bytesWritten) { if (m_Errored) { return false; @@ -293,16 +306,8 @@ bool SoundIoAudioRenderer::submitAudio(short* audioBuffer, int audioSize) // Flush events to update with new device arrivals soundio_flush_events(m_SoundIo); - // We must always write a full frame of audio. If we don't, - // the reader will get out of sync with the writer and our - // channels will get all mixed up. To ensure this is always - // the case, round our bytes free down to the next multiple - // of our frame size. - int bytesFree = soundio_ring_buffer_free_count(m_RingBuffer); - int bytesPerFrame = m_OpusChannelCount * m_OutputStream->bytes_per_sample; - int bytesToWrite = qMin(audioSize, (bytesFree / bytesPerFrame) * bytesPerFrame); - memcpy(soundio_ring_buffer_write_ptr(m_RingBuffer), audioBuffer, bytesToWrite); - soundio_ring_buffer_advance_write_ptr(m_RingBuffer, bytesToWrite); + // Advance the write pointer + soundio_ring_buffer_advance_write_ptr(m_RingBuffer, bytesWritten); return true; } diff --git a/app/streaming/audio/renderers/soundioaudiorenderer.h b/app/streaming/audio/renderers/soundioaudiorenderer.h index 24160ee9..ad29bb04 100644 --- a/app/streaming/audio/renderers/soundioaudiorenderer.h +++ b/app/streaming/audio/renderers/soundioaudiorenderer.h @@ -13,7 +13,9 @@ public: virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig); - virtual bool submitAudio(short* audioBuffer, int audioSize); + virtual void* getAudioBuffer(int* size); + + virtual bool submitAudio(int bytesWritten); private: int scoreChannelLayout(const struct SoundIoChannelLayout* layout, const OPUS_MULTISTREAM_CONFIGURATION* opusConfig); diff --git a/app/streaming/session.h b/app/streaming/session.h index af2490e4..15d116c9 100644 --- a/app/streaming/session.h +++ b/app/streaming/session.h @@ -148,7 +148,6 @@ private: int m_ActiveVideoFrameRate; OpusMSDecoder* m_OpusDecoder; - short m_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME]; IAudioRenderer* m_AudioRenderer; OPUS_MULTISTREAM_CONFIGURATION m_AudioConfig; int m_AudioSampleCount;