From 339f0075c8a6d3431ad08be9e0f8c85fa48556b1 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Mon, 19 Mar 2018 16:14:09 +0100 Subject: [PATCH 01/16] Use game category for audio output to reduce audio latency --- src/autoapp/Projection/AudioOutput.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/autoapp/Projection/AudioOutput.cpp b/src/autoapp/Projection/AudioOutput.cpp index 7b309d5..f211987 100644 --- a/src/autoapp/Projection/AudioOutput.cpp +++ b/src/autoapp/Projection/AudioOutput.cpp @@ -52,8 +52,9 @@ void AudioOutput::createAudioOutput() OPENAUTO_LOG(debug) << "[AudioOutput] create."; audioOutput_ = std::make_unique(QAudioDeviceInfo::defaultOutputDevice(), audioFormat_); - // Default volume level (max) produces crackles - audioOutput_->setVolume(static_cast(0.90)); + // Setting this category switching to the low latency mode + // See: http://code.qt.io/cgit/qt/qtmultimedia.git/tree/src/plugins/pulseaudio/qaudiooutput_pulse.cpp?h=5.7#n58 + audioOutput_->setCategory("game"); } bool AudioOutput::open() @@ -103,15 +104,12 @@ void AudioOutput::onStartPlayback() audioOutput_->start(&audioBuffer_); playbackStarted_ = true; } - else - { - audioOutput_->resume(); - } } void AudioOutput::onSuspendPlayback() { - audioOutput_->suspend(); + // QAudioOutput is in pull mode so suspending/resuming are not needed. + // Keep this interface for any further purposes } void AudioOutput::onStopPlayback() From e4771d0d1cf139b7bbe1ea0b6f27f596875dbb67 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 01:04:26 +0100 Subject: [PATCH 02/16] Add Qt prefixes to audio class implementations --- .../{AudioInput.hpp => QtAudioInput.hpp} | 4 +- .../{AudioOutput.hpp => QtAudioOutput.hpp} | 4 +- .../{AudioInput.cpp => QtAudioInput.cpp} | 34 +++++++------- .../{AudioOutput.cpp => QtAudioOutput.cpp} | 47 +++++++++---------- src/autoapp/Projection/ServiceFactory.cpp | 12 ++--- 5 files changed, 50 insertions(+), 51 deletions(-) rename include/f1x/openauto/autoapp/Projection/{AudioInput.hpp => QtAudioInput.hpp} (92%) rename include/f1x/openauto/autoapp/Projection/{AudioOutput.hpp => QtAudioOutput.hpp} (92%) rename src/autoapp/Projection/{AudioInput.cpp => QtAudioInput.cpp} (77%) rename src/autoapp/Projection/{AudioOutput.cpp => QtAudioOutput.cpp} (62%) diff --git a/include/f1x/openauto/autoapp/Projection/AudioInput.hpp b/include/f1x/openauto/autoapp/Projection/QtAudioInput.hpp similarity index 92% rename from include/f1x/openauto/autoapp/Projection/AudioInput.hpp rename to include/f1x/openauto/autoapp/Projection/QtAudioInput.hpp index 1dd7db0..97816ea 100644 --- a/include/f1x/openauto/autoapp/Projection/AudioInput.hpp +++ b/include/f1x/openauto/autoapp/Projection/QtAudioInput.hpp @@ -32,11 +32,11 @@ namespace autoapp namespace projection { -class AudioInput: public QObject, public IAudioInput +class QtAudioInput: public QObject, public IAudioInput { Q_OBJECT public: - AudioInput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate); + QtAudioInput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate); bool open() override; bool isActive() const override; diff --git a/include/f1x/openauto/autoapp/Projection/AudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/QtAudioOutput.hpp similarity index 92% rename from include/f1x/openauto/autoapp/Projection/AudioOutput.hpp rename to include/f1x/openauto/autoapp/Projection/QtAudioOutput.hpp index 9aa4d41..e026a91 100644 --- a/include/f1x/openauto/autoapp/Projection/AudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/QtAudioOutput.hpp @@ -32,12 +32,12 @@ namespace autoapp namespace projection { -class AudioOutput: public QObject, public IAudioOutput +class QtAudioOutput: public QObject, public IAudioOutput { Q_OBJECT public: - AudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate); + QtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate); bool open() override; void write(const aasdk::common::DataConstBuffer& buffer) override; void start() override; diff --git a/src/autoapp/Projection/AudioInput.cpp b/src/autoapp/Projection/QtAudioInput.cpp similarity index 77% rename from src/autoapp/Projection/AudioInput.cpp rename to src/autoapp/Projection/QtAudioInput.cpp index 2fe3fbf..1558430 100644 --- a/src/autoapp/Projection/AudioInput.cpp +++ b/src/autoapp/Projection/QtAudioInput.cpp @@ -17,7 +17,7 @@ */ #include -#include +#include #include namespace f1x @@ -29,7 +29,7 @@ namespace autoapp namespace projection { -AudioInput::AudioInput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate) +QtAudioInput::QtAudioInput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate) : ioDevice_(nullptr) { qRegisterMetaType("StartPromise::Pointer"); @@ -42,32 +42,32 @@ AudioInput::AudioInput(uint32_t channelCount, uint32_t sampleSize, uint32_t samp audioFormat_.setSampleType(QAudioFormat::SignedInt); this->moveToThread(QApplication::instance()->thread()); - connect(this, &AudioInput::startRecording, this, &AudioInput::onStartRecording, Qt::QueuedConnection); - connect(this, &AudioInput::stopRecording, this, &AudioInput::onStopRecording, Qt::QueuedConnection); + connect(this, &QtAudioInput::startRecording, this, &QtAudioInput::onStartRecording, Qt::QueuedConnection); + connect(this, &QtAudioInput::stopRecording, this, &QtAudioInput::onStopRecording, Qt::QueuedConnection); QMetaObject::invokeMethod(this, "createAudioInput", Qt::BlockingQueuedConnection); } -void AudioInput::createAudioInput() +void QtAudioInput::createAudioInput() { OPENAUTO_LOG(debug) << "[AudioInput] create."; audioInput_ = (std::make_unique(QAudioDeviceInfo::defaultInputDevice(), audioFormat_)); } -bool AudioInput::open() +bool QtAudioInput::open() { std::lock_guard lock(mutex_); return ioDevice_ == nullptr; } -bool AudioInput::isActive() const +bool QtAudioInput::isActive() const { std::lock_guard lock(mutex_); return ioDevice_ != nullptr; } -void AudioInput::read(ReadPromise::Pointer promise) +void QtAudioInput::read(ReadPromise::Pointer promise) { std::lock_guard lock(mutex_); @@ -85,32 +85,32 @@ void AudioInput::read(ReadPromise::Pointer promise) } } -void AudioInput::start(StartPromise::Pointer promise) +void QtAudioInput::start(StartPromise::Pointer promise) { emit startRecording(std::move(promise)); } -void AudioInput::stop() +void QtAudioInput::stop() { emit stopRecording(); } -uint32_t AudioInput::getSampleSize() const +uint32_t QtAudioInput::getSampleSize() const { return audioFormat_.sampleSize(); } -uint32_t AudioInput::getChannelCount() const +uint32_t QtAudioInput::getChannelCount() const { return audioFormat_.channelCount(); } -uint32_t AudioInput::getSampleRate() const +uint32_t QtAudioInput::getSampleRate() const { return audioFormat_.sampleRate(); } -void AudioInput::onStartRecording(StartPromise::Pointer promise) +void QtAudioInput::onStartRecording(StartPromise::Pointer promise) { std::lock_guard lock(mutex_); @@ -118,7 +118,7 @@ void AudioInput::onStartRecording(StartPromise::Pointer promise) if(ioDevice_ != nullptr) { - connect(ioDevice_, &QIODevice::readyRead, this, &AudioInput::onReadyRead, Qt::QueuedConnection); + connect(ioDevice_, &QIODevice::readyRead, this, &QtAudioInput::onReadyRead, Qt::QueuedConnection); promise->resolve(); } else @@ -127,7 +127,7 @@ void AudioInput::onStartRecording(StartPromise::Pointer promise) } } -void AudioInput::onStopRecording() +void QtAudioInput::onStopRecording() { std::lock_guard lock(mutex_); @@ -147,7 +147,7 @@ void AudioInput::onStopRecording() audioInput_->stop(); } -void AudioInput::onReadyRead() +void QtAudioInput::onReadyRead() { std::lock_guard lock(mutex_); diff --git a/src/autoapp/Projection/AudioOutput.cpp b/src/autoapp/Projection/QtAudioOutput.cpp similarity index 62% rename from src/autoapp/Projection/AudioOutput.cpp rename to src/autoapp/Projection/QtAudioOutput.cpp index f211987..959ca1f 100644 --- a/src/autoapp/Projection/AudioOutput.cpp +++ b/src/autoapp/Projection/QtAudioOutput.cpp @@ -17,7 +17,7 @@ */ #include -#include +#include #include namespace f1x @@ -29,7 +29,7 @@ namespace autoapp namespace projection { -AudioOutput::AudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate) +QtAudioOutput::QtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate) : playbackStarted_(false) { audioFormat_.setChannelCount(channelCount); @@ -40,79 +40,78 @@ AudioOutput::AudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sa audioFormat_.setSampleType(QAudioFormat::SignedInt); this->moveToThread(QApplication::instance()->thread()); - connect(this, &AudioOutput::startPlayback, this, &AudioOutput::onStartPlayback); - connect(this, &AudioOutput::suspendPlayback, this, &AudioOutput::onSuspendPlayback); - connect(this, &AudioOutput::stopPlayback, this, &AudioOutput::onStopPlayback); + connect(this, &QtAudioOutput::startPlayback, this, &QtAudioOutput::onStartPlayback); + connect(this, &QtAudioOutput::suspendPlayback, this, &QtAudioOutput::onSuspendPlayback); + connect(this, &QtAudioOutput::stopPlayback, this, &QtAudioOutput::onStopPlayback); QMetaObject::invokeMethod(this, "createAudioOutput", Qt::BlockingQueuedConnection); } -void AudioOutput::createAudioOutput() +void QtAudioOutput::createAudioOutput() { - OPENAUTO_LOG(debug) << "[AudioOutput] create."; + OPENAUTO_LOG(debug) << "[QtAudioOutput] create."; audioOutput_ = std::make_unique(QAudioDeviceInfo::defaultOutputDevice(), audioFormat_); - - // Setting this category switching to the low latency mode - // See: http://code.qt.io/cgit/qt/qtmultimedia.git/tree/src/plugins/pulseaudio/qaudiooutput_pulse.cpp?h=5.7#n58 - audioOutput_->setCategory("game"); } -bool AudioOutput::open() +bool QtAudioOutput::open() { return audioBuffer_.open(QIODevice::ReadWrite); } -void AudioOutput::write(const aasdk::common::DataConstBuffer& buffer) +void QtAudioOutput::write(const aasdk::common::DataConstBuffer& buffer) { audioBuffer_.write(reinterpret_cast(buffer.cdata), buffer.size); } -void AudioOutput::start() +void QtAudioOutput::start() { emit startPlayback(); } -void AudioOutput::stop() +void QtAudioOutput::stop() { emit stopPlayback(); } -void AudioOutput::suspend() +void QtAudioOutput::suspend() { emit suspendPlayback(); } -uint32_t AudioOutput::getSampleSize() const +uint32_t QtAudioOutput::getSampleSize() const { return audioFormat_.sampleSize(); } -uint32_t AudioOutput::getChannelCount() const +uint32_t QtAudioOutput::getChannelCount() const { return audioFormat_.channelCount(); } -uint32_t AudioOutput::getSampleRate() const +uint32_t QtAudioOutput::getSampleRate() const { return audioFormat_.sampleRate(); } -void AudioOutput::onStartPlayback() +void QtAudioOutput::onStartPlayback() { if(!playbackStarted_) { audioOutput_->start(&audioBuffer_); playbackStarted_ = true; } + else + { + audioOutput_->resume(); + } } -void AudioOutput::onSuspendPlayback() +void QtAudioOutput::onSuspendPlayback() { - // QAudioOutput is in pull mode so suspending/resuming are not needed. - // Keep this interface for any further purposes + audioOutput_->suspend(); } -void AudioOutput::onStopPlayback() +void QtAudioOutput::onStopPlayback() { if(playbackStarted_) { diff --git a/src/autoapp/Projection/ServiceFactory.cpp b/src/autoapp/Projection/ServiceFactory.cpp index ade1e38..9205b25 100644 --- a/src/autoapp/Projection/ServiceFactory.cpp +++ b/src/autoapp/Projection/ServiceFactory.cpp @@ -32,8 +32,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -59,22 +59,22 @@ ServiceList ServiceFactory::create(aasdk::messenger::IMessenger::Pointer messeng { ServiceList serviceList; - IAudioInput::Pointer audioInput(new AudioInput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + IAudioInput::Pointer audioInput(new QtAudioInput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(audioInput))); if(configuration_->musicAudioChannelEnabled()) { - IAudioOutput::Pointer mediaAudioOutput(new AudioOutput(2, 16, 48000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + IAudioOutput::Pointer mediaAudioOutput(new QtAudioOutput(2, 16, 48000), std::bind(&QObject::deleteLater, std::placeholders::_1)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(mediaAudioOutput))); } if(configuration_->speechAudioChannelEnabled()) { - IAudioOutput::Pointer speechAudioOutput(new AudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + IAudioOutput::Pointer speechAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(speechAudioOutput))); } - IAudioOutput::Pointer systemAudioOutput(new AudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + IAudioOutput::Pointer systemAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(systemAudioOutput))); serviceList.emplace_back(std::make_shared(ioService_, messenger)); From 023c7619c7680ca3d69500e662c2fb396a3b6384 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 01:16:04 +0100 Subject: [PATCH 03/16] Add cmake to find rtaudio library --- CMakeLists.txt | 3 ++ cmake_modules/Findrtaudio.cmake | 70 +++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 cmake_modules/Findrtaudio.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a530db..fc87b2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,7 @@ find_package(libusb-1.0 REQUIRED) find_package(Qt5 COMPONENTS Multimedia MultimediaWidgets Bluetooth) find_package(Protobuf REQUIRED) find_package(OpenSSL REQUIRED) +find_package(rtaudio REQUIRED) if(WIN32) set(WINSOCK2_LIBRARIES "ws2_32") @@ -54,6 +55,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR} ${LIBUSB_1_INCLUDE_DIRS} ${PROTOBUF_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR} + ${RTAUDIO_INCLUDE_DIRS} ${AASDK_PROTO_INCLUDE_DIRS} ${AASDK_INCLUDE_DIRS} ${BCM_HOST_INCLUDE_DIRS} @@ -80,6 +82,7 @@ target_link_libraries(autoapp ${BCM_HOST_LIBRARIES} ${ILCLIENT_LIBRARIES} ${WINSOCK2_LIBRARIES} + ${RTAUDIO_LIBRARIES} ${AASDK_PROTO_LIBRARIES} ${AASDK_LIBRARIES}) diff --git a/cmake_modules/Findrtaudio.cmake b/cmake_modules/Findrtaudio.cmake new file mode 100644 index 0000000..60b9384 --- /dev/null +++ b/cmake_modules/Findrtaudio.cmake @@ -0,0 +1,70 @@ +# +# This file is part of openauto project. +# Copyright (C) 2018 f1x.studio (Michal Szwaj) +# +# openauto is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# openauto is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with openauto. If not, see . +# + +if (RTAUDIO_LIBRARIES AND RTAUDIO_INCLUDE_DIRS) + # in cache already + set(RTAUDIO_FOUND TRUE) +else (RTAUDIO_LIBRARIES AND RTAUDIO_INCLUDE_DIRS) + find_path(RTAUDIO_INCLUDE_DIR + NAMES + RtAudio.h + PATHS + /usr/include + /usr/local/include + /opt/local/include + /sw/include + PATH_SUFFIXES + rtaudio + ) + + find_library(RTAUDIO_LIBRARY + NAMES + rtaudio + PATHS + /usr/lib + /usr/local/lib + /opt/local/lib + /sw/lib + ) + + set(RTAUDIO_INCLUDE_DIRS + ${RTAUDIO_INCLUDE_DIR} + ) + set(RTAUDIO_LIBRARIES + ${RTAUDIO_LIBRARY} +) + + if (RTAUDIO_INCLUDE_DIRS AND RTAUDIO_LIBRARIES) + set(RTAUDIO_FOUND TRUE) + endif (RTAUDIO_INCLUDE_DIRS AND RTAUDIO_LIBRARIES) + + if (RTAUDIO_FOUND) + if (NOT rtaudio_FIND_QUIETLY) + message(STATUS "Found rtaudio:") + message(STATUS " - Includes: ${RTAUDIO_INCLUDE_DIRS}") + message(STATUS " - Libraries: ${RTAUDIO_LIBRARIES}") + endif (NOT rtaudio_FIND_QUIETLY) + else (RTAUDIO_FOUND) + if (rtaudio_FIND_REQUIRED) + message(FATAL_ERROR "Could not find rtaudio") + endif (rtaudio_FIND_REQUIRED) + endif (RTAUDIO_FOUND) + + mark_as_advanced(RTAUDIO_INCLUDE_DIRS RTAUDIO_LIBRARIES) + +endif (RTAUDIO_LIBRARIES AND RTAUDIO_INCLUDE_DIRS) From bd6013081ae20b752911a8c33d4e6b2185566727 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 02:52:03 +0100 Subject: [PATCH 04/16] Implement RtAudio backend --- .../autoapp/Projection/RtAudioOutput.hpp | 63 ++++++++ src/autoapp/Projection/AudioService.cpp | 6 + src/autoapp/Projection/RtAudioOutput.cpp | 148 ++++++++++++++++++ src/autoapp/Projection/ServiceFactory.cpp | 6 +- 4 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp create mode 100644 src/autoapp/Projection/RtAudioOutput.cpp diff --git a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp new file mode 100644 index 0000000..1e111b8 --- /dev/null +++ b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp @@ -0,0 +1,63 @@ +/* +* This file is part of openauto project. +* Copyright (C) 2018 f1x.studio (Michal Szwaj) +* +* openauto is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 3 of the License, or +* (at your option) any later version. + +* openauto is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with openauto. If not, see . +*/ + +#pragma once + +#include +#include +#include + +namespace f1x +{ +namespace openauto +{ +namespace autoapp +{ +namespace projection +{ + +class RtAudioOutput: public IAudioOutput +{ +public: + RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate); + bool open() override; + void write(const aasdk::common::DataConstBuffer& buffer) override; + void start() override; + void stop() override; + void suspend() override; + uint32_t getSampleSize() const override; + uint32_t getChannelCount() const override; + uint32_t getSampleRate() const override; + +private: + static int audioBufferReadHandler(void* outputBuffer, void* inputBuffer, unsigned int nBufferFrames, + double streamTime, RtAudioStreamStatus status, void* userData); + + uint32_t channelCount_; + uint32_t sampleSize_; + uint32_t sampleRate_; + SequentialBuffer audioBuffer_; + bool playbackStarted_; + RtAudio dac_; + std::mutex mutex_; +}; + +} +} +} +} diff --git a/src/autoapp/Projection/AudioService.cpp b/src/autoapp/Projection/AudioService.cpp index 3e68c11..d5d2da5 100644 --- a/src/autoapp/Projection/AudioService.cpp +++ b/src/autoapp/Projection/AudioService.cpp @@ -93,6 +93,12 @@ void AudioService::onChannelOpenRequest(const aasdk::proto::messages::ChannelOpe OPENAUTO_LOG(info) << "[AudioService] open request" << ", channel: " << aasdk::messenger::channelIdToString(channel_->getId()) << ", priority: " << request.priority(); + + OPENAUTO_LOG(debug) << "[AudioService] channel: " << aasdk::messenger::channelIdToString(channel_->getId()) + << " audio output sample rate: " << audioOutput_->getSampleRate() + << ", sample size: " << audioOutput_->getSampleSize() + << ", channel count: " << audioOutput_->getChannelCount(); + const aasdk::proto::enums::Status::Enum status = audioOutput_->open() ? aasdk::proto::enums::Status::OK : aasdk::proto::enums::Status::FAIL; OPENAUTO_LOG(info) << "[AudioService] open status: " << status << ", channel: " << aasdk::messenger::channelIdToString(channel_->getId()); diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp new file mode 100644 index 0000000..68a1756 --- /dev/null +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -0,0 +1,148 @@ +/* +* This file is part of openauto project. +* Copyright (C) 2018 f1x.studio (Michal Szwaj) +* +* openauto is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 3 of the License, or +* (at your option) any later version. + +* openauto is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with openauto. If not, see . +*/ + +#include +#include + +namespace f1x +{ +namespace openauto +{ +namespace autoapp +{ +namespace projection +{ + +RtAudioOutput::RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate) + : channelCount_(channelCount) + , sampleSize_(sampleSize) + , sampleRate_(sampleRate) + , playbackStarted_(false) +{ +} + +bool RtAudioOutput::open() +{ + std::lock_guard lock(mutex_); + + if(dac_.getDeviceCount() > 0) + { + RtAudio::StreamParameters parameters; + parameters.deviceId = dac_.getDefaultOutputDevice(); + parameters.nChannels = channelCount_; + parameters.firstChannel = 0; + + try + { + uint32_t bufferFrames = 256; + dac_.openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this)); + + return audioBuffer_.open(QIODevice::ReadWrite); + } + catch(const RtAudioError& e) + { + OPENAUTO_LOG(error) << "[RtAudioOutput] Failed to open audio output, what: " << e.what(); + } + } + else + { + OPENAUTO_LOG(error) << "[RtAudioOutput] No output devices found."; + } + + return false; +} + +void RtAudioOutput::write(const aasdk::common::DataConstBuffer& buffer) +{ + audioBuffer_.write(reinterpret_cast(buffer.cdata), buffer.size); +} + +void RtAudioOutput::start() +{ + std::lock_guard lock(mutex_); + + if(dac_.isStreamOpen() && !dac_.isStreamRunning()) + { + try + { + dac_.startStream(); + } + catch(const RtAudioError& e) + { + OPENAUTO_LOG(error) << "[RtAudioOutput] Failed to start audio output, what: " << e.what(); + } + } +} + +void RtAudioOutput::stop() +{ + std::lock_guard lock(mutex_); + + this->suspend(); + + if(dac_.isStreamOpen()) + { + dac_.closeStream(); + } +} + +void RtAudioOutput::suspend() +{ + std::lock_guard lock(mutex_); + + if(!dac_.isStreamOpen() && !dac_.isStreamRunning()) + { + try + { + dac_.stopStream(); + } + catch(const RtAudioError& e) + { + OPENAUTO_LOG(error) << "[RtAudioOutput] Failed to suspend audio output, what: " << e.what(); + } + } +} + +uint32_t RtAudioOutput::getSampleSize() const +{ + return sampleSize_; +} + +uint32_t RtAudioOutput::getChannelCount() const +{ + return channelCount_; +} + +uint32_t RtAudioOutput::getSampleRate() const +{ + return sampleRate_; +} + +int RtAudioOutput::audioBufferReadHandler(void* outputBuffer, void* inputBuffer, unsigned int nBufferFrames, + double streamTime, RtAudioStreamStatus status, void* userData) +{ + RtAudioOutput* self = static_cast(userData); + const auto bufferSize = nBufferFrames * (self->sampleSize_ / 8) * self->channelCount_; + self->audioBuffer_.read(reinterpret_cast(outputBuffer), bufferSize); + return 0; +} + +} +} +} +} diff --git a/src/autoapp/Projection/ServiceFactory.cpp b/src/autoapp/Projection/ServiceFactory.cpp index 9205b25..2e54710 100644 --- a/src/autoapp/Projection/ServiceFactory.cpp +++ b/src/autoapp/Projection/ServiceFactory.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -64,17 +65,20 @@ ServiceList ServiceFactory::create(aasdk::messenger::IMessenger::Pointer messeng if(configuration_->musicAudioChannelEnabled()) { - IAudioOutput::Pointer mediaAudioOutput(new QtAudioOutput(2, 16, 48000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + //IAudioOutput::Pointer mediaAudioOutput(new QtAudioOutput(2, 16, 48000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + auto mediaAudioOutput(std::make_shared(2, 16, 48000)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(mediaAudioOutput))); } if(configuration_->speechAudioChannelEnabled()) { IAudioOutput::Pointer speechAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + //auto speechAudioOutput(std::make_shared(1, 16, 1600)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(speechAudioOutput))); } IAudioOutput::Pointer systemAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + //auto systemAudioOutput(std::make_shared(1, 16, 1600)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(systemAudioOutput))); serviceList.emplace_back(std::make_shared(ioService_, messenger)); From 66610efd33bdadf131958039e4a97b43b1932974 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 03:28:31 +0100 Subject: [PATCH 05/16] Use RtAudio for all audio channels --- .../autoapp/Projection/RtAudioOutput.hpp | 3 +- src/autoapp/Projection/RtAudioOutput.cpp | 31 ++++++++++++------- src/autoapp/Projection/ServiceFactory.cpp | 8 ++--- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp index 1e111b8..7bfba51 100644 --- a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp @@ -52,8 +52,7 @@ private: uint32_t sampleSize_; uint32_t sampleRate_; SequentialBuffer audioBuffer_; - bool playbackStarted_; - RtAudio dac_; + std::unique_ptr dac_; std::mutex mutex_; }; diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index 68a1756..d84428b 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -32,25 +32,34 @@ RtAudioOutput::RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_ : channelCount_(channelCount) , sampleSize_(sampleSize) , sampleRate_(sampleRate) - , playbackStarted_(false) + , dac_() { + try + { + dac_ = std::make_unique(RtAudio::LINUX_PULSE); + } + catch(...) + { + // fallback + dac_ = std::make_unique(); + } } bool RtAudioOutput::open() { std::lock_guard lock(mutex_); - if(dac_.getDeviceCount() > 0) + if(dac_->getDeviceCount() > 0) { RtAudio::StreamParameters parameters; - parameters.deviceId = dac_.getDefaultOutputDevice(); + parameters.deviceId = dac_->getDefaultOutputDevice(); parameters.nChannels = channelCount_; parameters.firstChannel = 0; try { - uint32_t bufferFrames = 256; - dac_.openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this)); + uint32_t bufferFrames = 1024; + dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this)); return audioBuffer_.open(QIODevice::ReadWrite); } @@ -76,11 +85,11 @@ void RtAudioOutput::start() { std::lock_guard lock(mutex_); - if(dac_.isStreamOpen() && !dac_.isStreamRunning()) + if(dac_->isStreamOpen() && !dac_->isStreamRunning()) { try { - dac_.startStream(); + dac_->startStream(); } catch(const RtAudioError& e) { @@ -95,9 +104,9 @@ void RtAudioOutput::stop() this->suspend(); - if(dac_.isStreamOpen()) + if(dac_->isStreamOpen()) { - dac_.closeStream(); + dac_->closeStream(); } } @@ -105,11 +114,11 @@ void RtAudioOutput::suspend() { std::lock_guard lock(mutex_); - if(!dac_.isStreamOpen() && !dac_.isStreamRunning()) + if(!dac_->isStreamOpen() && !dac_->isStreamRunning()) { try { - dac_.stopStream(); + dac_->stopStream(); } catch(const RtAudioError& e) { diff --git a/src/autoapp/Projection/ServiceFactory.cpp b/src/autoapp/Projection/ServiceFactory.cpp index 2e54710..82ede87 100644 --- a/src/autoapp/Projection/ServiceFactory.cpp +++ b/src/autoapp/Projection/ServiceFactory.cpp @@ -72,13 +72,13 @@ ServiceList ServiceFactory::create(aasdk::messenger::IMessenger::Pointer messeng if(configuration_->speechAudioChannelEnabled()) { - IAudioOutput::Pointer speechAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); - //auto speechAudioOutput(std::make_shared(1, 16, 1600)); + //IAudioOutput::Pointer speechAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + auto speechAudioOutput(std::make_shared(1, 16, 16000)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(speechAudioOutput))); } - IAudioOutput::Pointer systemAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); - //auto systemAudioOutput(std::make_shared(1, 16, 1600)); + //IAudioOutput::Pointer systemAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + auto systemAudioOutput(std::make_shared(1, 16, 16000)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(systemAudioOutput))); serviceList.emplace_back(std::make_shared(ioService_, messenger)); From 205ed34468d4afb30897d16bbe21b0e72333cfb8 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 03:33:32 +0100 Subject: [PATCH 06/16] Decrease audio buffer size --- src/autoapp/Projection/RtAudioOutput.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index d84428b..ca94bfc 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -58,7 +58,7 @@ bool RtAudioOutput::open() try { - uint32_t bufferFrames = 1024; + uint32_t bufferFrames = 128; dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this)); return audioBuffer_.open(QIODevice::ReadWrite); From 24bc8fda9d2b559b8563a61c556e84db57400ef5 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 03:55:20 +0100 Subject: [PATCH 07/16] Synchronize audio buffer callback --- src/autoapp/Projection/RtAudioOutput.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index ca94bfc..eb0d493 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -32,7 +32,6 @@ RtAudioOutput::RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_ : channelCount_(channelCount) , sampleSize_(sampleSize) , sampleRate_(sampleRate) - , dac_() { try { @@ -146,6 +145,8 @@ int RtAudioOutput::audioBufferReadHandler(void* outputBuffer, void* inputBuffer, double streamTime, RtAudioStreamStatus status, void* userData) { RtAudioOutput* self = static_cast(userData); + std::lock_guardmutex_)> lock(self->mutex_); + const auto bufferSize = nBufferFrames * (self->sampleSize_ / 8) * self->channelCount_; self->audioBuffer_.read(reinterpret_cast(outputBuffer), bufferSize); return 0; From 493b68e2a454f747e18691246a8ac5c6724c5281 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 03:56:39 +0100 Subject: [PATCH 08/16] Update readme --- Readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Readme.md b/Readme.md index aaf9217..fa43395 100644 --- a/Readme.md +++ b/Readme.md @@ -42,6 +42,7 @@ Copyrights (c) 2018 f1x.studio (Michal Szwaj) - [Boost libraries](http://www.boost.org/) - [Qt libraries](https://www.qt.io/) - [CMake](https://cmake.org/) + - [RtAudio](https://www.music.mcgill.ca/~gary/rtaudio/playback.html) - Broadcom ilclient from RaspberryPI 3 firmware - OpenMAX IL API From 8900c180e36c4a23f017ecc9bf4d9611e60b835d Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sat, 24 Mar 2018 11:40:28 +0100 Subject: [PATCH 09/16] Fix deadlock when stopping audio output --- .../autoapp/Projection/RtAudioOutput.hpp | 1 + src/autoapp/Projection/RtAudioOutput.cpp | 30 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp index 7bfba51..f3d69d5 100644 --- a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp @@ -45,6 +45,7 @@ public: uint32_t getSampleRate() const override; private: + void doSuspend(); static int audioBufferReadHandler(void* outputBuffer, void* inputBuffer, unsigned int nBufferFrames, double streamTime, RtAudioStreamStatus status, void* userData); diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index eb0d493..416b58b 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -101,7 +101,7 @@ void RtAudioOutput::stop() { std::lock_guard lock(mutex_); - this->suspend(); + this->doSuspend(); if(dac_->isStreamOpen()) { @@ -112,18 +112,7 @@ void RtAudioOutput::stop() void RtAudioOutput::suspend() { std::lock_guard lock(mutex_); - - if(!dac_->isStreamOpen() && !dac_->isStreamRunning()) - { - try - { - dac_->stopStream(); - } - catch(const RtAudioError& e) - { - OPENAUTO_LOG(error) << "[RtAudioOutput] Failed to suspend audio output, what: " << e.what(); - } - } + this->doSuspend(); } uint32_t RtAudioOutput::getSampleSize() const @@ -141,6 +130,21 @@ uint32_t RtAudioOutput::getSampleRate() const return sampleRate_; } +void RtAudioOutput::doSuspend() +{ + if(!dac_->isStreamOpen() && !dac_->isStreamRunning()) + { + try + { + dac_->stopStream(); + } + catch(const RtAudioError& e) + { + OPENAUTO_LOG(error) << "[RtAudioOutput] Failed to suspend audio output, what: " << e.what(); + } + } +} + int RtAudioOutput::audioBufferReadHandler(void* outputBuffer, void* inputBuffer, unsigned int nBufferFrames, double streamTime, RtAudioStreamStatus status, void* userData) { From 488d70c201bcd912a8c3570f2b391b5c47a974eb Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sun, 25 Mar 2018 04:42:32 +0200 Subject: [PATCH 10/16] Make audio output more realtime --- src/autoapp/Projection/RtAudioOutput.cpp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index 416b58b..e0d7037 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -33,15 +33,9 @@ RtAudioOutput::RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_ , sampleSize_(sampleSize) , sampleRate_(sampleRate) { - try - { - dac_ = std::make_unique(RtAudio::LINUX_PULSE); - } - catch(...) - { - // fallback - dac_ = std::make_unique(); - } + std::vector apis; + RtAudio::getCompiledApi(apis); + dac_ = std::find(apis.begin(), apis.end(), RtAudio::LINUX_PULSE) == apis.end() ? std::make_unique() : std::make_unique(RtAudio::LINUX_PULSE); } bool RtAudioOutput::open() @@ -57,9 +51,11 @@ bool RtAudioOutput::open() try { - uint32_t bufferFrames = 128; - dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this)); - + RtAudio::StreamOptions streamOptions; + streamOptions.numberOfBuffers = 1; + streamOptions.flags = RTAUDIO_MINIMIZE_LATENCY | RTAUDIO_SCHEDULE_REALTIME; + uint32_t bufferFrames = 64; + dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this), &streamOptions); return audioBuffer_.open(QIODevice::ReadWrite); } catch(const RtAudioError& e) From 18fa86b53290ba02706be7687d5c50f77af55fc3 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sun, 25 Mar 2018 11:45:47 +0200 Subject: [PATCH 11/16] Enable playback only when pcm data are available --- .../autoapp/Projection/RtAudioOutput.hpp | 1 + src/autoapp/Projection/RtAudioOutput.cpp | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp index f3d69d5..4aae81b 100644 --- a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp @@ -55,6 +55,7 @@ private: SequentialBuffer audioBuffer_; std::unique_ptr dac_; std::mutex mutex_; + bool playbackRequested_; }; } diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index e0d7037..0da5cfb 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -32,6 +32,7 @@ RtAudioOutput::RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_ : channelCount_(channelCount) , sampleSize_(sampleSize) , sampleRate_(sampleRate) + , playbackRequested_(false) { std::vector apis; RtAudio::getCompiledApi(apis); @@ -54,7 +55,7 @@ bool RtAudioOutput::open() RtAudio::StreamOptions streamOptions; streamOptions.numberOfBuffers = 1; streamOptions.flags = RTAUDIO_MINIMIZE_LATENCY | RTAUDIO_SCHEDULE_REALTIME; - uint32_t bufferFrames = 64; + uint32_t bufferFrames = 256; dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this), &streamOptions); return audioBuffer_.open(QIODevice::ReadWrite); } @@ -74,13 +75,10 @@ bool RtAudioOutput::open() void RtAudioOutput::write(const aasdk::common::DataConstBuffer& buffer) { audioBuffer_.write(reinterpret_cast(buffer.cdata), buffer.size); -} -void RtAudioOutput::start() -{ std::lock_guard lock(mutex_); - if(dac_->isStreamOpen() && !dac_->isStreamRunning()) + if(playbackRequested_ && dac_->isStreamOpen() && !dac_->isStreamRunning()) { try { @@ -91,6 +89,15 @@ void RtAudioOutput::start() OPENAUTO_LOG(error) << "[RtAudioOutput] Failed to start audio output, what: " << e.what(); } } + + playbackRequested_ = false; +} + +void RtAudioOutput::start() +{ + std::lock_guard lock(mutex_); + + playbackRequested_ = true; } void RtAudioOutput::stop() From f9692b76a1d23e26e9c7f5ac175f1cf2a3346c84 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sun, 25 Mar 2018 11:51:17 +0200 Subject: [PATCH 12/16] Revert last commit --- .../autoapp/Projection/RtAudioOutput.hpp | 1 - src/autoapp/Projection/RtAudioOutput.cpp | 17 +++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp index 4aae81b..f3d69d5 100644 --- a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp @@ -55,7 +55,6 @@ private: SequentialBuffer audioBuffer_; std::unique_ptr dac_; std::mutex mutex_; - bool playbackRequested_; }; } diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index 0da5cfb..e0d7037 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -32,7 +32,6 @@ RtAudioOutput::RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_ : channelCount_(channelCount) , sampleSize_(sampleSize) , sampleRate_(sampleRate) - , playbackRequested_(false) { std::vector apis; RtAudio::getCompiledApi(apis); @@ -55,7 +54,7 @@ bool RtAudioOutput::open() RtAudio::StreamOptions streamOptions; streamOptions.numberOfBuffers = 1; streamOptions.flags = RTAUDIO_MINIMIZE_LATENCY | RTAUDIO_SCHEDULE_REALTIME; - uint32_t bufferFrames = 256; + uint32_t bufferFrames = 64; dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this), &streamOptions); return audioBuffer_.open(QIODevice::ReadWrite); } @@ -75,10 +74,13 @@ bool RtAudioOutput::open() void RtAudioOutput::write(const aasdk::common::DataConstBuffer& buffer) { audioBuffer_.write(reinterpret_cast(buffer.cdata), buffer.size); +} +void RtAudioOutput::start() +{ std::lock_guard lock(mutex_); - if(playbackRequested_ && dac_->isStreamOpen() && !dac_->isStreamRunning()) + if(dac_->isStreamOpen() && !dac_->isStreamRunning()) { try { @@ -89,15 +91,6 @@ void RtAudioOutput::write(const aasdk::common::DataConstBuffer& buffer) OPENAUTO_LOG(error) << "[RtAudioOutput] Failed to start audio output, what: " << e.what(); } } - - playbackRequested_ = false; -} - -void RtAudioOutput::start() -{ - std::lock_guard lock(mutex_); - - playbackRequested_ = true; } void RtAudioOutput::stop() From b550104e181c3f820bfa128f200f6fad43e78498 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sun, 25 Mar 2018 14:57:23 +0200 Subject: [PATCH 13/16] Revert to use pulse --- src/autoapp/Projection/RtAudioOutput.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index e0d7037..a8a43ef 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -52,9 +52,8 @@ bool RtAudioOutput::open() try { RtAudio::StreamOptions streamOptions; - streamOptions.numberOfBuffers = 1; streamOptions.flags = RTAUDIO_MINIMIZE_LATENCY | RTAUDIO_SCHEDULE_REALTIME; - uint32_t bufferFrames = 64; + uint32_t bufferFrames = 256; dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this), &streamOptions); return audioBuffer_.open(QIODevice::ReadWrite); } From 00acf0ee3eebebb4f0ff8be28cd5722575445202 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sun, 25 Mar 2018 15:15:03 +0200 Subject: [PATCH 14/16] Use timestamp of audio output --- .../openauto/autoapp/Projection/IAudioOutput.hpp | 3 ++- .../openauto/autoapp/Projection/QtAudioOutput.hpp | 2 +- .../openauto/autoapp/Projection/RtAudioOutput.hpp | 2 +- src/autoapp/Projection/AudioService.cpp | 14 +++++++------- src/autoapp/Projection/QtAudioOutput.cpp | 2 +- src/autoapp/Projection/RtAudioOutput.cpp | 9 ++++++++- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/include/f1x/openauto/autoapp/Projection/IAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/IAudioOutput.hpp index 383ad28..c41de31 100644 --- a/include/f1x/openauto/autoapp/Projection/IAudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/IAudioOutput.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include namespace f1x @@ -39,7 +40,7 @@ public: virtual ~IAudioOutput() = default; virtual bool open() = 0; - virtual void write(const aasdk::common::DataConstBuffer& buffer) = 0; + virtual void write(aasdk::messenger::Timestamp::ValueType timestamp, const aasdk::common::DataConstBuffer& buffer) = 0; virtual void start() = 0; virtual void stop() = 0; virtual void suspend() = 0; diff --git a/include/f1x/openauto/autoapp/Projection/QtAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/QtAudioOutput.hpp index e026a91..742dbf8 100644 --- a/include/f1x/openauto/autoapp/Projection/QtAudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/QtAudioOutput.hpp @@ -39,7 +39,7 @@ class QtAudioOutput: public QObject, public IAudioOutput public: QtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate); bool open() override; - void write(const aasdk::common::DataConstBuffer& buffer) override; + void write(aasdk::messenger::Timestamp::ValueType, const aasdk::common::DataConstBuffer& buffer) override; void start() override; void stop() override; void suspend() override; diff --git a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp index f3d69d5..063d5b3 100644 --- a/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp +++ b/include/f1x/openauto/autoapp/Projection/RtAudioOutput.hpp @@ -36,7 +36,7 @@ class RtAudioOutput: public IAudioOutput public: RtAudioOutput(uint32_t channelCount, uint32_t sampleSize, uint32_t sampleRate); bool open() override; - void write(const aasdk::common::DataConstBuffer& buffer) override; + void write(aasdk::messenger::Timestamp::ValueType timestamp, const aasdk::common::DataConstBuffer& buffer) override; void start() override; void stop() override; void suspend() override; diff --git a/src/autoapp/Projection/AudioService.cpp b/src/autoapp/Projection/AudioService.cpp index d5d2da5..d09001f 100644 --- a/src/autoapp/Projection/AudioService.cpp +++ b/src/autoapp/Projection/AudioService.cpp @@ -152,14 +152,9 @@ void AudioService::onAVChannelStopIndication(const aasdk::proto::messages::AVCha channel_->receive(this->shared_from_this()); } -void AudioService::onAVMediaWithTimestampIndication(aasdk::messenger::Timestamp::ValueType, const aasdk::common::DataConstBuffer& buffer) +void AudioService::onAVMediaWithTimestampIndication(aasdk::messenger::Timestamp::ValueType timestamp, const aasdk::common::DataConstBuffer& buffer) { - this->onAVMediaIndication(buffer); -} - -void AudioService::onAVMediaIndication(const aasdk::common::DataConstBuffer& buffer) -{ - audioOutput_->write(buffer); + audioOutput_->write(timestamp, buffer); aasdk::proto::messages::AVMediaAckIndication indication; indication.set_session(session_); indication.set_value(1); @@ -170,6 +165,11 @@ void AudioService::onAVMediaIndication(const aasdk::common::DataConstBuffer& buf channel_->receive(this->shared_from_this()); } +void AudioService::onAVMediaIndication(const aasdk::common::DataConstBuffer& buffer) +{ + this->onAVMediaWithTimestampIndication(0, buffer); +} + void AudioService::onChannelError(const aasdk::error::Error& e) { OPENAUTO_LOG(error) << "[AudioService] channel error: " << e.what() diff --git a/src/autoapp/Projection/QtAudioOutput.cpp b/src/autoapp/Projection/QtAudioOutput.cpp index 959ca1f..dff8049 100644 --- a/src/autoapp/Projection/QtAudioOutput.cpp +++ b/src/autoapp/Projection/QtAudioOutput.cpp @@ -58,7 +58,7 @@ bool QtAudioOutput::open() return audioBuffer_.open(QIODevice::ReadWrite); } -void QtAudioOutput::write(const aasdk::common::DataConstBuffer& buffer) +void QtAudioOutput::write(aasdk::messenger::Timestamp::ValueType, const aasdk::common::DataConstBuffer& buffer) { audioBuffer_.write(reinterpret_cast(buffer.cdata), buffer.size); } diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index a8a43ef..b03ef59 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -70,9 +70,16 @@ bool RtAudioOutput::open() return false; } -void RtAudioOutput::write(const aasdk::common::DataConstBuffer& buffer) +void RtAudioOutput::write(aasdk::messenger::Timestamp::ValueType timestamp, const aasdk::common::DataConstBuffer& buffer) { audioBuffer_.write(reinterpret_cast(buffer.cdata), buffer.size); + + std::lock_guard lock(mutex_); + + if(dac_->isStreamOpen()) + { + dac_->setStreamTime(timestamp / 1000000); + } } void RtAudioOutput::start() From b0bb0c7b4fab0442251ebcd88d549114c26046f4 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sun, 25 Mar 2018 16:05:25 +0200 Subject: [PATCH 15/16] Remove stream options usage --- src/autoapp/Projection/RtAudioOutput.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/autoapp/Projection/RtAudioOutput.cpp b/src/autoapp/Projection/RtAudioOutput.cpp index b03ef59..eb05420 100644 --- a/src/autoapp/Projection/RtAudioOutput.cpp +++ b/src/autoapp/Projection/RtAudioOutput.cpp @@ -51,10 +51,8 @@ bool RtAudioOutput::open() try { - RtAudio::StreamOptions streamOptions; - streamOptions.flags = RTAUDIO_MINIMIZE_LATENCY | RTAUDIO_SCHEDULE_REALTIME; - uint32_t bufferFrames = 256; - dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this), &streamOptions); + uint32_t bufferFrames = 128; + dac_->openStream(¶meters, nullptr, RTAUDIO_SINT16, sampleRate_, &bufferFrames, &RtAudioOutput::audioBufferReadHandler, static_cast(this), nullptr); return audioBuffer_.open(QIODevice::ReadWrite); } catch(const RtAudioError& e) @@ -73,13 +71,6 @@ bool RtAudioOutput::open() void RtAudioOutput::write(aasdk::messenger::Timestamp::ValueType timestamp, const aasdk::common::DataConstBuffer& buffer) { audioBuffer_.write(reinterpret_cast(buffer.cdata), buffer.size); - - std::lock_guard lock(mutex_); - - if(dac_->isStreamOpen()) - { - dac_->setStreamTime(timestamp / 1000000); - } } void RtAudioOutput::start() From 4d5327776309694b9480ddfb87ed5186015cefe7 Mon Sep 17 00:00:00 2001 From: "michal.szwaj" Date: Sun, 25 Mar 2018 17:30:57 +0200 Subject: [PATCH 16/16] Add settings for selection of audio output backend --- .../Configuration/AudioOutputBackendType.hpp | 39 +++++++++++++++ .../autoapp/Configuration/Configuration.hpp | 4 ++ .../autoapp/Configuration/IConfiguration.hpp | 3 ++ .../autoapp/Projection/ServiceFactory.hpp | 1 + src/autoapp/Configuration/Configuration.cpp | 14 ++++++ src/autoapp/Projection/ServiceFactory.cpp | 47 +++++++++++-------- src/autoapp/UI/SettingsWindow.cpp | 5 ++ src/autoapp/UI/settingswindow.ui | 41 ++++++++++++++++ 8 files changed, 135 insertions(+), 19 deletions(-) create mode 100644 include/f1x/openauto/autoapp/Configuration/AudioOutputBackendType.hpp diff --git a/include/f1x/openauto/autoapp/Configuration/AudioOutputBackendType.hpp b/include/f1x/openauto/autoapp/Configuration/AudioOutputBackendType.hpp new file mode 100644 index 0000000..e954874 --- /dev/null +++ b/include/f1x/openauto/autoapp/Configuration/AudioOutputBackendType.hpp @@ -0,0 +1,39 @@ +/* +* This file is part of openauto project. +* Copyright (C) 2018 f1x.studio (Michal Szwaj) +* +* openauto is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 3 of the License, or +* (at your option) any later version. + +* openauto is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with openauto. If not, see . +*/ + +#pragma once + +namespace f1x +{ +namespace openauto +{ +namespace autoapp +{ +namespace configuration +{ + +enum class AudioOutputBackendType +{ + RTAUDIO, + QT +}; + +} +} +} +} diff --git a/include/f1x/openauto/autoapp/Configuration/Configuration.hpp b/include/f1x/openauto/autoapp/Configuration/Configuration.hpp index 93d44b3..f4a0649 100644 --- a/include/f1x/openauto/autoapp/Configuration/Configuration.hpp +++ b/include/f1x/openauto/autoapp/Configuration/Configuration.hpp @@ -69,6 +69,8 @@ public: void setMusicAudioChannelEnabled(bool value) override; bool speechAudioChannelEnabled() const override; void setSpeechAudioChannelEnabled(bool value) override; + AudioOutputBackendType getAudioOutputBackendType() const override; + void setAudioOutputBackendType(AudioOutputBackendType value) override; private: void readButtonCodes(boost::property_tree::ptree& iniConfig); @@ -88,6 +90,7 @@ private: std::string bluetoothRemoteAdapterAddress_; bool musicAudioChannelEnabled_; bool speechAudiochannelEnabled_; + AudioOutputBackendType audioOutputBackendType_; static const std::string cConfigFileName; @@ -103,6 +106,7 @@ private: static const std::string cAudioMusicAudioChannelEnabled; static const std::string cAudioSpeechAudioChannelEnabled; + static const std::string cAudioOutputBackendType; static const std::string cBluetoothAdapterTypeKey; static const std::string cBluetoothRemoteAdapterAddressKey; diff --git a/include/f1x/openauto/autoapp/Configuration/IConfiguration.hpp b/include/f1x/openauto/autoapp/Configuration/IConfiguration.hpp index 2f086cd..2165a68 100644 --- a/include/f1x/openauto/autoapp/Configuration/IConfiguration.hpp +++ b/include/f1x/openauto/autoapp/Configuration/IConfiguration.hpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace f1x { @@ -77,6 +78,8 @@ public: virtual void setMusicAudioChannelEnabled(bool value) = 0; virtual bool speechAudioChannelEnabled() const = 0; virtual void setSpeechAudioChannelEnabled(bool value) = 0; + virtual AudioOutputBackendType getAudioOutputBackendType() const = 0; + virtual void setAudioOutputBackendType(AudioOutputBackendType value) = 0; }; } diff --git a/include/f1x/openauto/autoapp/Projection/ServiceFactory.hpp b/include/f1x/openauto/autoapp/Projection/ServiceFactory.hpp index 0d929b6..f7fdbd2 100644 --- a/include/f1x/openauto/autoapp/Projection/ServiceFactory.hpp +++ b/include/f1x/openauto/autoapp/Projection/ServiceFactory.hpp @@ -40,6 +40,7 @@ private: IService::Pointer createVideoService(aasdk::messenger::IMessenger::Pointer messenger); IService::Pointer createBluetoothService(aasdk::messenger::IMessenger::Pointer messenger); IService::Pointer createInputService(aasdk::messenger::IMessenger::Pointer messenger); + void createAudioServices(ServiceList& serviceList, aasdk::messenger::IMessenger::Pointer messenger); boost::asio::io_service& ioService_; configuration::IConfiguration::Pointer configuration_; diff --git a/src/autoapp/Configuration/Configuration.cpp b/src/autoapp/Configuration/Configuration.cpp index 72e51e1..0e79129 100644 --- a/src/autoapp/Configuration/Configuration.cpp +++ b/src/autoapp/Configuration/Configuration.cpp @@ -42,6 +42,7 @@ const std::string Configuration::cVideoMarginHeight = "Video.MarginHeight"; const std::string Configuration::cAudioMusicAudioChannelEnabled = "Audio.MusicAudioChannelEnabled"; const std::string Configuration::cAudioSpeechAudioChannelEnabled = "Audio.SpeechAudioChannelEnabled"; +const std::string Configuration::cAudioOutputBackendType = "Audio.OutputBackendType"; const std::string Configuration::cBluetoothAdapterTypeKey = "Bluetooth.AdapterType"; const std::string Configuration::cBluetoothRemoteAdapterAddressKey = "Bluetooth.RemoteAdapterAddress"; @@ -101,6 +102,7 @@ void Configuration::load() musicAudioChannelEnabled_ = iniConfig.get(cAudioMusicAudioChannelEnabled, true); speechAudiochannelEnabled_ = iniConfig.get(cAudioSpeechAudioChannelEnabled, true); + audioOutputBackendType_ = static_cast(iniConfig.get(cAudioOutputBackendType, static_cast(AudioOutputBackendType::RTAUDIO))); } catch(const boost::property_tree::ini_parser_error& e) { @@ -126,6 +128,7 @@ void Configuration::reset() bluetoothRemoteAdapterAddress_ = ""; musicAudioChannelEnabled_ = true; speechAudiochannelEnabled_ = true; + audioOutputBackendType_ = AudioOutputBackendType::RTAUDIO; } void Configuration::save() @@ -149,6 +152,7 @@ void Configuration::save() iniConfig.put(cAudioMusicAudioChannelEnabled, musicAudioChannelEnabled_); iniConfig.put(cAudioSpeechAudioChannelEnabled, speechAudiochannelEnabled_); + iniConfig.put(cAudioOutputBackendType, static_cast(audioOutputBackendType_)); boost::property_tree::ini_parser::write_ini(cConfigFileName, iniConfig); } @@ -282,6 +286,16 @@ void Configuration::setSpeechAudioChannelEnabled(bool value) speechAudiochannelEnabled_ = value; } +AudioOutputBackendType Configuration::getAudioOutputBackendType() const +{ + return audioOutputBackendType_; +} + +void Configuration::setAudioOutputBackendType(AudioOutputBackendType value) +{ + audioOutputBackendType_ = value; +} + void Configuration::readButtonCodes(boost::property_tree::ptree& iniConfig) { this->insertButtonCode(iniConfig, cInputPlayButtonKey, aasdk::proto::enums::ButtonCode::PLAY); diff --git a/src/autoapp/Projection/ServiceFactory.cpp b/src/autoapp/Projection/ServiceFactory.cpp index 82ede87..d10ddff 100644 --- a/src/autoapp/Projection/ServiceFactory.cpp +++ b/src/autoapp/Projection/ServiceFactory.cpp @@ -62,25 +62,7 @@ ServiceList ServiceFactory::create(aasdk::messenger::IMessenger::Pointer messeng IAudioInput::Pointer audioInput(new QtAudioInput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(audioInput))); - - if(configuration_->musicAudioChannelEnabled()) - { - //IAudioOutput::Pointer mediaAudioOutput(new QtAudioOutput(2, 16, 48000), std::bind(&QObject::deleteLater, std::placeholders::_1)); - auto mediaAudioOutput(std::make_shared(2, 16, 48000)); - serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(mediaAudioOutput))); - } - - if(configuration_->speechAudioChannelEnabled()) - { - //IAudioOutput::Pointer speechAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); - auto speechAudioOutput(std::make_shared(1, 16, 16000)); - serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(speechAudioOutput))); - } - - //IAudioOutput::Pointer systemAudioOutput(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); - auto systemAudioOutput(std::make_shared(1, 16, 16000)); - serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(systemAudioOutput))); - + this->createAudioServices(serviceList, messenger); serviceList.emplace_back(std::make_shared(ioService_, messenger)); serviceList.emplace_back(this->createVideoService(messenger)); serviceList.emplace_back(this->createBluetoothService(messenger)); @@ -145,6 +127,33 @@ IService::Pointer ServiceFactory::createInputService(aasdk::messenger::IMessenge return std::make_shared(ioService_, messenger, std::move(inputDevice)); } +void ServiceFactory::createAudioServices(ServiceList& serviceList, aasdk::messenger::IMessenger::Pointer messenger) +{ + if(configuration_->musicAudioChannelEnabled()) + { + auto mediaAudioOutput = configuration_->getAudioOutputBackendType() == configuration::AudioOutputBackendType::RTAUDIO ? + std::make_shared(2, 16, 48000) : + IAudioOutput::Pointer(new QtAudioOutput(2, 16, 48000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + + serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(mediaAudioOutput))); + } + + if(configuration_->speechAudioChannelEnabled()) + { + auto speechAudioOutput = configuration_->getAudioOutputBackendType() == configuration::AudioOutputBackendType::RTAUDIO ? + std::make_shared(1, 16, 16000) : + IAudioOutput::Pointer(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + + serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(speechAudioOutput))); + } + + auto systemAudioOutput = configuration_->getAudioOutputBackendType() == configuration::AudioOutputBackendType::RTAUDIO ? + std::make_shared(1, 16, 16000) : + IAudioOutput::Pointer(new QtAudioOutput(1, 16, 16000), std::bind(&QObject::deleteLater, std::placeholders::_1)); + + serviceList.emplace_back(std::make_shared(ioService_, messenger, std::move(systemAudioOutput))); +} + } } } diff --git a/src/autoapp/UI/SettingsWindow.cpp b/src/autoapp/UI/SettingsWindow.cpp index 3117cf3..5f64b3e 100644 --- a/src/autoapp/UI/SettingsWindow.cpp +++ b/src/autoapp/UI/SettingsWindow.cpp @@ -97,6 +97,7 @@ void SettingsWindow::onSave() configuration_->setMusicAudioChannelEnabled(ui_->checkBoxMusicAudioChannel->isChecked()); configuration_->setSpeechAudioChannelEnabled(ui_->checkBoxSpeechAudioChannel->isChecked()); + configuration_->setAudioOutputBackendType(ui_->radioButtonRtAudio->isChecked() ? configuration::AudioOutputBackendType::RTAUDIO : configuration::AudioOutputBackendType::QT); configuration_->save(); this->close(); @@ -149,6 +150,10 @@ void SettingsWindow::load() ui_->checkBoxMusicAudioChannel->setChecked(configuration_->musicAudioChannelEnabled()); ui_->checkBoxSpeechAudioChannel->setChecked(configuration_->speechAudioChannelEnabled()); + + const auto& audioOutputBackendType = configuration_->getAudioOutputBackendType(); + ui_->radioButtonRtAudio->setChecked(audioOutputBackendType == configuration::AudioOutputBackendType::RTAUDIO); + ui_->radioButtonQtAudio->setChecked(audioOutputBackendType == configuration::AudioOutputBackendType::QT); } void SettingsWindow::loadButtonCheckBoxes() diff --git a/src/autoapp/UI/settingswindow.ui b/src/autoapp/UI/settingswindow.ui index b066a78..4028487 100644 --- a/src/autoapp/UI/settingswindow.ui +++ b/src/autoapp/UI/settingswindow.ui @@ -486,6 +486,45 @@ color: rgb(238, 238, 236); + + + + 0 + 130 + 621 + 61 + + + + Output backend + + + + + 10 + 30 + 112 + 23 + + + + RT audio + + + + + + 140 + 30 + 112 + 23 + + + + Qt + + + @@ -969,6 +1008,8 @@ color: rgb(238, 238, 236); horizontalSliderScreenDPI checkBoxMusicAudioChannel checkBoxSpeechAudioChannel + radioButtonRtAudio + radioButtonQtAudio checkBoxEnableTouchscreen listWidgetButtons checkBoxPlayButton