From 9887bf14ee751a2d3299bd066567e901d45fd248 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sat, 21 Dec 2019 09:30:16 -0600 Subject: [PATCH 01/45] Initial work on Opus audio plugin This commit is made up of changes to VCPKG and CMake to include the Opus libraries, as well as a skeleton project for an Opus audio plugin. --- cmake/macros/TargetOpus.cmake | 18 ++++++ cmake/ports/hifi-deps/CONTROL | 4 +- cmake/ports/opus/CONTROL | 4 ++ cmake/ports/opus/portfile.cmake | 28 +++++++++ plugins/CMakeLists.txt | 2 + plugins/opusCodec/CMakeLists.txt | 15 +++++ plugins/opusCodec/src/OpusCodecManager.cpp | 63 +++++++++++++++++++++ plugins/opusCodec/src/OpusCodecManager.h | 54 ++++++++++++++++++ plugins/opusCodec/src/OpusCodecProvider.cpp | 45 +++++++++++++++ plugins/opusCodec/src/plugin.json | 4 ++ 10 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 cmake/macros/TargetOpus.cmake create mode 100644 cmake/ports/opus/CONTROL create mode 100644 cmake/ports/opus/portfile.cmake create mode 100644 plugins/opusCodec/CMakeLists.txt create mode 100644 plugins/opusCodec/src/OpusCodecManager.cpp create mode 100644 plugins/opusCodec/src/OpusCodecManager.h create mode 100644 plugins/opusCodec/src/OpusCodecProvider.cpp create mode 100644 plugins/opusCodec/src/plugin.json diff --git a/cmake/macros/TargetOpus.cmake b/cmake/macros/TargetOpus.cmake new file mode 100644 index 0000000000..20fcadfc57 --- /dev/null +++ b/cmake/macros/TargetOpus.cmake @@ -0,0 +1,18 @@ +# +# Created by Michael Bailey on 12/20/2019 +# Copyright 2019 Michael Bailey +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_opus) + if (ANDROID) + # no idea if this is correct + target_link_libraries(${TARGET_NAME}) + else() + # using VCPKG for opus + find_package(OPUS REQUIRED) + target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${OPUS_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${OPUS_LIBRARIES}) + endif() +endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 4cf952ccf0..b1a7f96a00 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps -Version: 0.3 +Version: 0.4 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), opus, tbb (!android&!osx), zlib, webrtc (!android) diff --git a/cmake/ports/opus/CONTROL b/cmake/ports/opus/CONTROL new file mode 100644 index 0000000000..c7b8d246f5 --- /dev/null +++ b/cmake/ports/opus/CONTROL @@ -0,0 +1,4 @@ +Source: opus +Version: 1.3.1 +Homepage: https://github.com/xiph/opus +Description: Totally open, royalty-free, highly versatile audio codec diff --git a/cmake/ports/opus/portfile.cmake b/cmake/ports/opus/portfile.cmake new file mode 100644 index 0000000000..bf23718df8 --- /dev/null +++ b/cmake/ports/opus/portfile.cmake @@ -0,0 +1,28 @@ +include(vcpkg_common_functions) + +vcpkg_from_github( + OUT_SOURCE_PATH + SOURCE_PATH + REPO + xiph/opus + REF + e85ed7726db5d677c9c0677298ea0cb9c65bdd23 + SHA512 + a8c7e5bf383c06f1fdffd44d9b5f658f31eb4800cb59d12da95ddaeb5646f7a7b03025f4663362b888b1374d4cc69154f006ba07b5840ec61ddc1a1af01d6c54 + HEAD_REF + master) + +vcpkg_configure_cmake(SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA) +vcpkg_install_cmake() +vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/Opus) +vcpkg_copy_pdbs() + +file(INSTALL + ${SOURCE_PATH}/COPYING + DESTINATION + ${CURRENT_PACKAGES_DIR}/share/opus + RENAME copyright) + +file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/lib/cmake + ${CURRENT_PACKAGES_DIR}/lib/cmake + ${CURRENT_PACKAGES_DIR}/debug/include) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index c9bc3e4c33..c88bb8a00d 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -43,6 +43,8 @@ set(DIR "pcmCodec") add_subdirectory(${DIR}) set(DIR "hifiCodec") add_subdirectory(${DIR}) +set(DIR "opusCodec") +add_subdirectory(${DIR}) # example plugins set(DIR "KasenAPIExample") diff --git a/plugins/opusCodec/CMakeLists.txt b/plugins/opusCodec/CMakeLists.txt new file mode 100644 index 0000000000..761e275929 --- /dev/null +++ b/plugins/opusCodec/CMakeLists.txt @@ -0,0 +1,15 @@ +# +# Created by Michael Bailey on 12/20/2019 +# Copyright 2019 Michael Bailey +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + +set(TARGET_NAME opusCodec) +setup_hifi_client_server_plugin() +link_hifi_libraries(shared plugins) + +if (BUILD_SERVER) + install_beside_console() +endif () diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp new file mode 100644 index 0000000000..3c47ac15fb --- /dev/null +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -0,0 +1,63 @@ +// +// opusCodec.cpp +// plugins/opusCodec/src +// +// Created by Michael Bailey on 12/20/2019 +// Copyright 2019 Michael Bailey +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "OpusCodecManager.h" + +#include + +#include + +// Not sure how many of these are needed, but here they are. +#include +#include +#include +#include +#include + +const char* OpusCodec::NAME { "opus" }; + +void OpusCodec::init() { +} + +void OpusCodec::deinit() { +} + +bool OpusCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void OpusCodec::deactivate() { + CodecPlugin::deactivate(); +} + + +bool OpusCodec::isSupported() const { + return true; +} + + + +Encoder* OpusCodec::createEncoder(int sampleRate, int numChannels) { + return this; +} + +Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { + return this; +} + +void OpusCodec::releaseEncoder(Encoder* encoder) { + // do nothing +} + +void OpusCodec::releaseDecoder(Decoder* decoder) { + // do nothing +} \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusCodecManager.h b/plugins/opusCodec/src/OpusCodecManager.h new file mode 100644 index 0000000000..b02a549e8c --- /dev/null +++ b/plugins/opusCodec/src/OpusCodecManager.h @@ -0,0 +1,54 @@ +// +// OpusCodecManager.h +// plugins/opusCodec/src +// +// Created by Michael Bailey on 12/20/2019 +// Copyright 2019 Michael Bailey +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi__opusCodecManager_h +#define hifi__opusCodecManager_h + +#include + +class OpusCodec : public CodecPlugin, public Encoder, public Decoder { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + +private: + static const char* NAME; +}; + +#endif // hifi__opusCodecManager_h diff --git a/plugins/opusCodec/src/OpusCodecProvider.cpp b/plugins/opusCodec/src/OpusCodecProvider.cpp new file mode 100644 index 0000000000..ab72f25023 --- /dev/null +++ b/plugins/opusCodec/src/OpusCodecProvider.cpp @@ -0,0 +1,45 @@ +// +// Created by Michael Bailey on 12/20/2019 +// Copyright 2019 Michael Bailey +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "OpusCodecManager.h" + +class OpusCodecProvider : public QObject, public CodecProvider { + Q_OBJECT + Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json") + Q_INTERFACES(CodecProvider) + +public: + OpusCodecProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~OpusCodecProvider() {} + + virtual CodecPluginList getCodecPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + + CodecPluginPointer opusCodec(new OpusCodec()); + if (opusCodec->isSupported()) { + _codecPlugins.push_back(opusCodec); + } + }); + return _codecPlugins; + } + +private: + CodecPluginList _codecPlugins; +}; + +#include "OpusCodecProvider.moc" diff --git a/plugins/opusCodec/src/plugin.json b/plugins/opusCodec/src/plugin.json new file mode 100644 index 0000000000..bd7c707f7d --- /dev/null +++ b/plugins/opusCodec/src/plugin.json @@ -0,0 +1,4 @@ +{ + "name":"Opus Codec", + "version":1 +} From e182f98a347f9555f60eabd247f8e630afb50a24 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sat, 21 Dec 2019 10:03:38 -0600 Subject: [PATCH 02/45] Fix VCPKG Opus CONTROL failure on Linux VCPKG on Linux really, really does not like the "Homepage:" line. --- cmake/ports/opus/CONTROL | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/ports/opus/CONTROL b/cmake/ports/opus/CONTROL index c7b8d246f5..4314774ab5 100644 --- a/cmake/ports/opus/CONTROL +++ b/cmake/ports/opus/CONTROL @@ -1,4 +1,3 @@ Source: opus Version: 1.3.1 -Homepage: https://github.com/xiph/opus Description: Totally open, royalty-free, highly versatile audio codec From 9b4c419b2c3d24ea29e38285f38f4a56802753d5 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sat, 21 Dec 2019 12:05:14 -0600 Subject: [PATCH 03/45] Opus cleanup and C++ wrapper Did a tiny amount of fleshing out and some cleanup. Also added a Google implemented C++ Opus wrapper in case it will be useful. --- plugins/opusCodec/src/OpusCodecManager.cpp | 29 +++- plugins/opusCodec/src/OpusCodecManager.h | 16 +- plugins/opusCodec/src/OpusWrapper.cpp | 174 +++++++++++++++++++++ plugins/opusCodec/src/OpusWrapper.h | 131 ++++++++++++++++ 4 files changed, 333 insertions(+), 17 deletions(-) create mode 100644 plugins/opusCodec/src/OpusWrapper.cpp create mode 100644 plugins/opusCodec/src/OpusWrapper.h diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp index 3c47ac15fb..8f7d4fe6e5 100644 --- a/plugins/opusCodec/src/OpusCodecManager.cpp +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -22,6 +22,15 @@ #include #include +#define FRAME_SIZE 960 +#define SAMPLE_RATE 48000 +#define CHANNELS 2 +#define APPLICATION OPUS_APPLICATION_AUDIO +#define BITRATE 64000 + +#define MAX_FRAME_SIZE 6*FRAME_SIZE +#define MAX_PACKET_SIZE 3*1276 + const char* OpusCodec::NAME { "opus" }; void OpusCodec::init() { @@ -45,9 +54,23 @@ bool OpusCodec::isSupported() const { } +class OpusEncoder : public Encoder { +public: + OpusEncoder(int sampleRate, int numChannels) { + + } + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer.resize(_encodedSize); + } + +private: + int _encodedSize; +}; + Encoder* OpusCodec::createEncoder(int sampleRate, int numChannels) { - return this; + return new OpusEncoder(sampleRate, numChannels); } Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { @@ -55,9 +78,9 @@ Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { } void OpusCodec::releaseEncoder(Encoder* encoder) { - // do nothing + delete encoder; } void OpusCodec::releaseDecoder(Decoder* decoder) { - // do nothing + delete decoder; } \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusCodecManager.h b/plugins/opusCodec/src/OpusCodecManager.h index b02a549e8c..b75e78b23c 100644 --- a/plugins/opusCodec/src/OpusCodecManager.h +++ b/plugins/opusCodec/src/OpusCodecManager.h @@ -9,8 +9,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef hifi__opusCodecManager_h -#define hifi__opusCodecManager_h +#ifndef hifi__OpusCodecManager_h +#define hifi__OpusCodecManager_h #include @@ -35,18 +35,6 @@ public: virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = decodedBuffer; - } - - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = encodedBuffer; - } - - virtual void lostFrame(QByteArray& decodedBuffer) override { - memset(decodedBuffer.data(), 0, decodedBuffer.size()); - } - private: static const char* NAME; }; diff --git a/plugins/opusCodec/src/OpusWrapper.cpp b/plugins/opusCodec/src/OpusWrapper.cpp new file mode 100644 index 0000000000..0d8ff87965 --- /dev/null +++ b/plugins/opusCodec/src/OpusWrapper.cpp @@ -0,0 +1,174 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "OpusWrapper.h" + +std::string opus::ErrorToString(int error) { + switch (error) { + case OPUS_OK: + return "OK"; + case OPUS_BAD_ARG: + return "One or more invalid/out of range arguments."; + case OPUS_BUFFER_TOO_SMALL: + return "The mode struct passed is invalid."; + case OPUS_INTERNAL_ERROR: + return "An internal error was detected."; + case OPUS_INVALID_PACKET: + return "The compressed data passed is corrupted."; + case OPUS_UNIMPLEMENTED: + return "Invalid/unsupported request number."; + case OPUS_INVALID_STATE: + return "An encoder or decoder structure is invalid or already freed."; + default: + return "Unknown error code: " + std::to_string(error); + } +} + +void opus::internal::OpusDestroyer::operator()(OpusEncoder* encoder) const +noexcept { + opus_encoder_destroy(encoder); +} + +void opus::internal::OpusDestroyer::operator()(OpusDecoder* decoder) const +noexcept { + opus_decoder_destroy(decoder); +} + +opus::Encoder::Encoder(opus_int32 sample_rate, int num_channels, + int application, int expected_loss_percent) + : num_channels_{ num_channels } { + int error{}; + encoder_.reset( + opus_encoder_create(sample_rate, num_channels, application, &error)); + valid_ = error == OPUS_OK; + if (!valid()) { + // LOG(INFO) << "Could not construct encoder. Error: " << ErrorToString(error); + return; + } + if (expected_loss_percent > 0) { + // LOG(INFO) << "Enabling FEC in the encoder."; + Ctl(OPUS_SET_INBAND_FEC(1)); + Ctl(OPUS_SET_PACKET_LOSS_PERC(expected_loss_percent)); + } +} + +bool opus::Encoder::ResetState() { + valid_ = Ctl(OPUS_RESET_STATE) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetBitrate(int bitrate) { + valid_ = Ctl(OPUS_SET_BITRATE(bitrate)) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetVariableBitrate(int vbr) { + valid_ = Ctl(OPUS_SET_VBR(vbr)) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetComplexity(int complexity) { + valid_ = Ctl(OPUS_SET_COMPLEXITY(complexity)) == OPUS_OK; + return valid_; +} + +int opus::Encoder::GetLookahead() { + opus_int32 skip{}; + valid_ = Ctl(OPUS_GET_LOOKAHEAD(&skip)) == OPUS_OK; + return skip; +} + +std::vector> opus::Encoder::Encode( + const std::vector& pcm, int frame_size) { + constexpr auto sample_size = sizeof(pcm[0]); + const auto frame_length = frame_size * num_channels_ * sample_size; + auto data_length = pcm.size() * sample_size; + if (data_length % frame_length != 0u) { + // LOG(WARNING) << "PCM samples contain an incomplete frame. Ignoring the " + // "incomplete frame."; + data_length -= (data_length % frame_length); + } + + std::vector> encoded; + for (std::size_t i{}; i < data_length; i += frame_length) { + encoded.push_back(EncodeFrame(pcm.begin() + (i / sample_size), frame_size)); + } + return encoded; +} + +std::vector opus::Encoder::EncodeFrame( + const std::vector::const_iterator& frame_start, + int frame_size) { + const auto frame_length = (frame_size * num_channels_ * sizeof(*frame_start)); + std::vector encoded(frame_length); + auto num_bytes = opus_encode(encoder_.get(), &*frame_start, frame_size, + encoded.data(), encoded.size()); + if (num_bytes < 0) { + // LOG(ERROR) << "Encode error: " << opus::ErrorToString(num_bytes); + return {}; + } + encoded.resize(num_bytes); + return encoded; +} + +opus::Decoder::Decoder(opus_uint32 sample_rate, int num_channels) + : num_channels_(num_channels) { + int error{}; + decoder_.reset(opus_decoder_create(sample_rate, num_channels, &error)); + valid_ = error == OPUS_OK; +} + +std::vector opus::Decoder::Decode( + const std::vector>& packets, int frame_size, + bool decode_fec) { + std::vector decoded; + for (const auto& enc : packets) { + auto just_decoded = Decode(enc, frame_size, decode_fec); + decoded.insert(std::end(decoded), std::begin(just_decoded), + std::end(just_decoded)); + } + return decoded; +} + +std::vector opus::Decoder::Decode( + const std::vector& packet, int frame_size, bool decode_fec) { + const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); + std::vector decoded(frame_length); + auto num_samples = opus_decode(decoder_.get(), packet.data(), packet.size(), + decoded.data(), frame_size, decode_fec); + if (num_samples < 0) { + // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); + return {}; + } + decoded.resize(num_samples * num_channels_); + return decoded; +} + +std::vector opus::Decoder::DecodeDummy(int frame_size) { + const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); + std::vector decoded(frame_length); + auto num_samples = + opus_decode(decoder_.get(), nullptr, 0, decoded.data(), frame_size, true); + if (num_samples < 0) { + // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); + return {}; + } + decoded.resize(num_samples * num_channels_); + return decoded; +} \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusWrapper.h b/plugins/opusCodec/src/OpusWrapper.h new file mode 100644 index 0000000000..713dff87ff --- /dev/null +++ b/plugins/opusCodec/src/OpusWrapper.h @@ -0,0 +1,131 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPUSCPP_OPUS_WRAPPER_H_ +#define OPUSCPP_OPUS_WRAPPER_H_ + +#include +#include +#include + +#include "opus/opus.h" + +namespace opus { + + std::string ErrorToString(int error); + + namespace internal { + // Deleter for OpusEncoders and OpusDecoders + struct OpusDestroyer { + void operator()(OpusEncoder* encoder) const noexcept; + void operator()(OpusDecoder* decoder) const noexcept; + }; + template + using opus_uptr = std::unique_ptr; + } // namespace internal + + class Encoder { + public: + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gaa89264fd93c9da70362a0c9b96b9ca88 + // Fs corresponds to sample_rate + // + // If expected_loss_percent is positive, FEC will be enabled + Encoder(opus_int32 sample_rate, int num_channels, int application, + int expected_loss_percent = 0); + + // Resets internal state of encoder. This should be called between encoding + // different streams so that back-to-back decoding and one-at-a-time decoding + // give the same result. Returns true on success. + bool ResetState(); + + // Sets the desired bitrate. Rates from 500 to 512000 are meaningful as well + // as the special values OPUS_AUTO and OPUS_BITRATE_MAX. If this method + // is not called, the default value of OPUS_AUTO is used. + // Returns true on success. + bool SetBitrate(int bitrate); + + // Enables or disables variable bitrate in the encoder. By default, variable + // bitrate is enabled. Returns true on success. + bool SetVariableBitrate(int vbr); + + // Sets the computational complexity of the encoder, in the range of 0 to 10, + // inclusive, with 10 being the highest complexity. Returns true on success. + bool SetComplexity(int complexity); + + // Gets the total samples of delay added by the entire codec. This value + // is the minimum amount of 'preskip' that has to be specified in an + // ogg-stream that encapsulates the encoded audio. + int GetLookahead(); + + // Takes audio data and encodes it. Returns a sequence of encoded packets. + // pcm.size() must be divisible by frame_size * (number of channels); + // pcm must not contain any incomplete packets. + // see documentation for pcm and frame_size at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gad2d6bf6a9ffb6674879d7605ed073e25 + std::vector> Encode( + const std::vector& pcm, int frame_size); + + int valid() const { return valid_; } + + private: + std::vector EncodeFrame( + const std::vector::const_iterator& frame_start, + int frame_size); + + template + int Ctl(int request, Ts... args) const { + return opus_encoder_ctl(encoder_.get(), request, args...); + } + + int num_channels_{}; + bool valid_{}; + internal::opus_uptr encoder_; + }; + + class Decoder { + public: + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga753f6fe0b699c81cfd47d70c8e15a0bd + // Fs corresponds to sample_rate + Decoder(opus_uint32 sample_rate, int num_channels); + + // Takes a sequence of encoded packets and decodes them. Returns the decoded + // audio. + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 + std::vector Decode( + const std::vector>& packets, int frame_size, + bool decode_fec); + + int valid() const { return valid_; } + + // Takes an encoded packet and decodes it. Returns the decoded audio + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 + std::vector Decode(const std::vector& packet, + int frame_size, bool decode_fec); + + // Generates a dummy frame by passing nullptr to the underlying opus decode. + std::vector DecodeDummy(int frame_size); + + private: + int num_channels_{}; + bool valid_{}; + internal::opus_uptr decoder_; + }; + +} // namespace opus + +#endif \ No newline at end of file From 3d8d359c350d11cdb6022f38f955ee3bd1bdbb5d Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sat, 21 Dec 2019 09:30:16 -0600 Subject: [PATCH 04/45] Initial work on Opus audio plugin This commit is made up of changes to VCPKG and CMake to include the Opus libraries, as well as a skeleton project for an Opus audio plugin. --- cmake/macros/TargetOpus.cmake | 18 ++++++ cmake/ports/hifi-deps/CONTROL | 4 +- cmake/ports/opus/CONTROL | 4 ++ cmake/ports/opus/portfile.cmake | 28 +++++++++ plugins/CMakeLists.txt | 2 + plugins/opusCodec/CMakeLists.txt | 15 +++++ plugins/opusCodec/src/OpusCodecManager.cpp | 63 +++++++++++++++++++++ plugins/opusCodec/src/OpusCodecManager.h | 54 ++++++++++++++++++ plugins/opusCodec/src/OpusCodecProvider.cpp | 45 +++++++++++++++ plugins/opusCodec/src/plugin.json | 4 ++ 10 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 cmake/macros/TargetOpus.cmake create mode 100644 cmake/ports/opus/CONTROL create mode 100644 cmake/ports/opus/portfile.cmake create mode 100644 plugins/opusCodec/CMakeLists.txt create mode 100644 plugins/opusCodec/src/OpusCodecManager.cpp create mode 100644 plugins/opusCodec/src/OpusCodecManager.h create mode 100644 plugins/opusCodec/src/OpusCodecProvider.cpp create mode 100644 plugins/opusCodec/src/plugin.json diff --git a/cmake/macros/TargetOpus.cmake b/cmake/macros/TargetOpus.cmake new file mode 100644 index 0000000000..20fcadfc57 --- /dev/null +++ b/cmake/macros/TargetOpus.cmake @@ -0,0 +1,18 @@ +# +# Created by Michael Bailey on 12/20/2019 +# Copyright 2019 Michael Bailey +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +# +macro(TARGET_opus) + if (ANDROID) + # no idea if this is correct + target_link_libraries(${TARGET_NAME}) + else() + # using VCPKG for opus + find_package(OPUS REQUIRED) + target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${OPUS_INCLUDE_DIRS}) + target_link_libraries(${TARGET_NAME} ${OPUS_LIBRARIES}) + endif() +endmacro() diff --git a/cmake/ports/hifi-deps/CONTROL b/cmake/ports/hifi-deps/CONTROL index 4cf952ccf0..b1a7f96a00 100644 --- a/cmake/ports/hifi-deps/CONTROL +++ b/cmake/ports/hifi-deps/CONTROL @@ -1,4 +1,4 @@ Source: hifi-deps -Version: 0.3 +Version: 0.4 Description: Collected dependencies for High Fidelity applications -Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), tbb (!android&!osx), zlib, webrtc (!android) +Build-Depends: bullet3, draco, etc2comp, glm, nvtt, openexr (!android), openssl (windows), opus, tbb (!android&!osx), zlib, webrtc (!android) diff --git a/cmake/ports/opus/CONTROL b/cmake/ports/opus/CONTROL new file mode 100644 index 0000000000..c7b8d246f5 --- /dev/null +++ b/cmake/ports/opus/CONTROL @@ -0,0 +1,4 @@ +Source: opus +Version: 1.3.1 +Homepage: https://github.com/xiph/opus +Description: Totally open, royalty-free, highly versatile audio codec diff --git a/cmake/ports/opus/portfile.cmake b/cmake/ports/opus/portfile.cmake new file mode 100644 index 0000000000..bf23718df8 --- /dev/null +++ b/cmake/ports/opus/portfile.cmake @@ -0,0 +1,28 @@ +include(vcpkg_common_functions) + +vcpkg_from_github( + OUT_SOURCE_PATH + SOURCE_PATH + REPO + xiph/opus + REF + e85ed7726db5d677c9c0677298ea0cb9c65bdd23 + SHA512 + a8c7e5bf383c06f1fdffd44d9b5f658f31eb4800cb59d12da95ddaeb5646f7a7b03025f4663362b888b1374d4cc69154f006ba07b5840ec61ddc1a1af01d6c54 + HEAD_REF + master) + +vcpkg_configure_cmake(SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA) +vcpkg_install_cmake() +vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/Opus) +vcpkg_copy_pdbs() + +file(INSTALL + ${SOURCE_PATH}/COPYING + DESTINATION + ${CURRENT_PACKAGES_DIR}/share/opus + RENAME copyright) + +file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/lib/cmake + ${CURRENT_PACKAGES_DIR}/lib/cmake + ${CURRENT_PACKAGES_DIR}/debug/include) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index c9bc3e4c33..c88bb8a00d 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -43,6 +43,8 @@ set(DIR "pcmCodec") add_subdirectory(${DIR}) set(DIR "hifiCodec") add_subdirectory(${DIR}) +set(DIR "opusCodec") +add_subdirectory(${DIR}) # example plugins set(DIR "KasenAPIExample") diff --git a/plugins/opusCodec/CMakeLists.txt b/plugins/opusCodec/CMakeLists.txt new file mode 100644 index 0000000000..761e275929 --- /dev/null +++ b/plugins/opusCodec/CMakeLists.txt @@ -0,0 +1,15 @@ +# +# Created by Michael Bailey on 12/20/2019 +# Copyright 2019 Michael Bailey +# +# Distributed under the Apache License, Version 2.0. +# See the accompanying file LICENSE or http:#www.apache.org/licenses/LICENSE-2.0.html +# + +set(TARGET_NAME opusCodec) +setup_hifi_client_server_plugin() +link_hifi_libraries(shared plugins) + +if (BUILD_SERVER) + install_beside_console() +endif () diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp new file mode 100644 index 0000000000..3c47ac15fb --- /dev/null +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -0,0 +1,63 @@ +// +// opusCodec.cpp +// plugins/opusCodec/src +// +// Created by Michael Bailey on 12/20/2019 +// Copyright 2019 Michael Bailey +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "OpusCodecManager.h" + +#include + +#include + +// Not sure how many of these are needed, but here they are. +#include +#include +#include +#include +#include + +const char* OpusCodec::NAME { "opus" }; + +void OpusCodec::init() { +} + +void OpusCodec::deinit() { +} + +bool OpusCodec::activate() { + CodecPlugin::activate(); + return true; +} + +void OpusCodec::deactivate() { + CodecPlugin::deactivate(); +} + + +bool OpusCodec::isSupported() const { + return true; +} + + + +Encoder* OpusCodec::createEncoder(int sampleRate, int numChannels) { + return this; +} + +Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { + return this; +} + +void OpusCodec::releaseEncoder(Encoder* encoder) { + // do nothing +} + +void OpusCodec::releaseDecoder(Decoder* decoder) { + // do nothing +} \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusCodecManager.h b/plugins/opusCodec/src/OpusCodecManager.h new file mode 100644 index 0000000000..b02a549e8c --- /dev/null +++ b/plugins/opusCodec/src/OpusCodecManager.h @@ -0,0 +1,54 @@ +// +// OpusCodecManager.h +// plugins/opusCodec/src +// +// Created by Michael Bailey on 12/20/2019 +// Copyright 2019 Michael Bailey +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi__opusCodecManager_h +#define hifi__opusCodecManager_h + +#include + +class OpusCodec : public CodecPlugin, public Encoder, public Decoder { + Q_OBJECT + +public: + // Plugin functions + bool isSupported() const override; + const QString getName() const override { return NAME; } + + void init() override; + void deinit() override; + + /// Called when a plugin is being activated for use. May be called multiple times. + bool activate() override; + /// Called when a plugin is no longer being used. May be called multiple times. + void deactivate() override; + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void lostFrame(QByteArray& decodedBuffer) override { + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + +private: + static const char* NAME; +}; + +#endif // hifi__opusCodecManager_h diff --git a/plugins/opusCodec/src/OpusCodecProvider.cpp b/plugins/opusCodec/src/OpusCodecProvider.cpp new file mode 100644 index 0000000000..ab72f25023 --- /dev/null +++ b/plugins/opusCodec/src/OpusCodecProvider.cpp @@ -0,0 +1,45 @@ +// +// Created by Michael Bailey on 12/20/2019 +// Copyright 2019 Michael Bailey +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include + +#include +#include +#include + +#include +#include + +#include "OpusCodecManager.h" + +class OpusCodecProvider : public QObject, public CodecProvider { + Q_OBJECT + Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json") + Q_INTERFACES(CodecProvider) + +public: + OpusCodecProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~OpusCodecProvider() {} + + virtual CodecPluginList getCodecPlugins() override { + static std::once_flag once; + std::call_once(once, [&] { + + CodecPluginPointer opusCodec(new OpusCodec()); + if (opusCodec->isSupported()) { + _codecPlugins.push_back(opusCodec); + } + }); + return _codecPlugins; + } + +private: + CodecPluginList _codecPlugins; +}; + +#include "OpusCodecProvider.moc" diff --git a/plugins/opusCodec/src/plugin.json b/plugins/opusCodec/src/plugin.json new file mode 100644 index 0000000000..bd7c707f7d --- /dev/null +++ b/plugins/opusCodec/src/plugin.json @@ -0,0 +1,4 @@ +{ + "name":"Opus Codec", + "version":1 +} From 4e41c0710d8f1c2aaa0c8ce86345c2bdfdbbe208 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sat, 21 Dec 2019 10:03:38 -0600 Subject: [PATCH 05/45] Fix VCPKG Opus CONTROL failure on Linux VCPKG on Linux really, really does not like the "Homepage:" line. --- cmake/ports/opus/CONTROL | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/ports/opus/CONTROL b/cmake/ports/opus/CONTROL index c7b8d246f5..4314774ab5 100644 --- a/cmake/ports/opus/CONTROL +++ b/cmake/ports/opus/CONTROL @@ -1,4 +1,3 @@ Source: opus Version: 1.3.1 -Homepage: https://github.com/xiph/opus Description: Totally open, royalty-free, highly versatile audio codec From a39ad97a1f7aa64892a4d52b3cbe70459989fe73 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sat, 21 Dec 2019 12:05:14 -0600 Subject: [PATCH 06/45] Opus cleanup and C++ wrapper Did a tiny amount of fleshing out and some cleanup. Also added a Google implemented C++ Opus wrapper in case it will be useful. --- plugins/opusCodec/src/OpusCodecManager.cpp | 29 +++- plugins/opusCodec/src/OpusCodecManager.h | 16 +- plugins/opusCodec/src/OpusWrapper.cpp | 174 +++++++++++++++++++++ plugins/opusCodec/src/OpusWrapper.h | 131 ++++++++++++++++ 4 files changed, 333 insertions(+), 17 deletions(-) create mode 100644 plugins/opusCodec/src/OpusWrapper.cpp create mode 100644 plugins/opusCodec/src/OpusWrapper.h diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp index 3c47ac15fb..8f7d4fe6e5 100644 --- a/plugins/opusCodec/src/OpusCodecManager.cpp +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -22,6 +22,15 @@ #include #include +#define FRAME_SIZE 960 +#define SAMPLE_RATE 48000 +#define CHANNELS 2 +#define APPLICATION OPUS_APPLICATION_AUDIO +#define BITRATE 64000 + +#define MAX_FRAME_SIZE 6*FRAME_SIZE +#define MAX_PACKET_SIZE 3*1276 + const char* OpusCodec::NAME { "opus" }; void OpusCodec::init() { @@ -45,9 +54,23 @@ bool OpusCodec::isSupported() const { } +class OpusEncoder : public Encoder { +public: + OpusEncoder(int sampleRate, int numChannels) { + + } + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer.resize(_encodedSize); + } + +private: + int _encodedSize; +}; + Encoder* OpusCodec::createEncoder(int sampleRate, int numChannels) { - return this; + return new OpusEncoder(sampleRate, numChannels); } Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { @@ -55,9 +78,9 @@ Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { } void OpusCodec::releaseEncoder(Encoder* encoder) { - // do nothing + delete encoder; } void OpusCodec::releaseDecoder(Decoder* decoder) { - // do nothing + delete decoder; } \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusCodecManager.h b/plugins/opusCodec/src/OpusCodecManager.h index b02a549e8c..b75e78b23c 100644 --- a/plugins/opusCodec/src/OpusCodecManager.h +++ b/plugins/opusCodec/src/OpusCodecManager.h @@ -9,8 +9,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef hifi__opusCodecManager_h -#define hifi__opusCodecManager_h +#ifndef hifi__OpusCodecManager_h +#define hifi__OpusCodecManager_h #include @@ -35,18 +35,6 @@ public: virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = decodedBuffer; - } - - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = encodedBuffer; - } - - virtual void lostFrame(QByteArray& decodedBuffer) override { - memset(decodedBuffer.data(), 0, decodedBuffer.size()); - } - private: static const char* NAME; }; diff --git a/plugins/opusCodec/src/OpusWrapper.cpp b/plugins/opusCodec/src/OpusWrapper.cpp new file mode 100644 index 0000000000..0d8ff87965 --- /dev/null +++ b/plugins/opusCodec/src/OpusWrapper.cpp @@ -0,0 +1,174 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "OpusWrapper.h" + +std::string opus::ErrorToString(int error) { + switch (error) { + case OPUS_OK: + return "OK"; + case OPUS_BAD_ARG: + return "One or more invalid/out of range arguments."; + case OPUS_BUFFER_TOO_SMALL: + return "The mode struct passed is invalid."; + case OPUS_INTERNAL_ERROR: + return "An internal error was detected."; + case OPUS_INVALID_PACKET: + return "The compressed data passed is corrupted."; + case OPUS_UNIMPLEMENTED: + return "Invalid/unsupported request number."; + case OPUS_INVALID_STATE: + return "An encoder or decoder structure is invalid or already freed."; + default: + return "Unknown error code: " + std::to_string(error); + } +} + +void opus::internal::OpusDestroyer::operator()(OpusEncoder* encoder) const +noexcept { + opus_encoder_destroy(encoder); +} + +void opus::internal::OpusDestroyer::operator()(OpusDecoder* decoder) const +noexcept { + opus_decoder_destroy(decoder); +} + +opus::Encoder::Encoder(opus_int32 sample_rate, int num_channels, + int application, int expected_loss_percent) + : num_channels_{ num_channels } { + int error{}; + encoder_.reset( + opus_encoder_create(sample_rate, num_channels, application, &error)); + valid_ = error == OPUS_OK; + if (!valid()) { + // LOG(INFO) << "Could not construct encoder. Error: " << ErrorToString(error); + return; + } + if (expected_loss_percent > 0) { + // LOG(INFO) << "Enabling FEC in the encoder."; + Ctl(OPUS_SET_INBAND_FEC(1)); + Ctl(OPUS_SET_PACKET_LOSS_PERC(expected_loss_percent)); + } +} + +bool opus::Encoder::ResetState() { + valid_ = Ctl(OPUS_RESET_STATE) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetBitrate(int bitrate) { + valid_ = Ctl(OPUS_SET_BITRATE(bitrate)) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetVariableBitrate(int vbr) { + valid_ = Ctl(OPUS_SET_VBR(vbr)) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetComplexity(int complexity) { + valid_ = Ctl(OPUS_SET_COMPLEXITY(complexity)) == OPUS_OK; + return valid_; +} + +int opus::Encoder::GetLookahead() { + opus_int32 skip{}; + valid_ = Ctl(OPUS_GET_LOOKAHEAD(&skip)) == OPUS_OK; + return skip; +} + +std::vector> opus::Encoder::Encode( + const std::vector& pcm, int frame_size) { + constexpr auto sample_size = sizeof(pcm[0]); + const auto frame_length = frame_size * num_channels_ * sample_size; + auto data_length = pcm.size() * sample_size; + if (data_length % frame_length != 0u) { + // LOG(WARNING) << "PCM samples contain an incomplete frame. Ignoring the " + // "incomplete frame."; + data_length -= (data_length % frame_length); + } + + std::vector> encoded; + for (std::size_t i{}; i < data_length; i += frame_length) { + encoded.push_back(EncodeFrame(pcm.begin() + (i / sample_size), frame_size)); + } + return encoded; +} + +std::vector opus::Encoder::EncodeFrame( + const std::vector::const_iterator& frame_start, + int frame_size) { + const auto frame_length = (frame_size * num_channels_ * sizeof(*frame_start)); + std::vector encoded(frame_length); + auto num_bytes = opus_encode(encoder_.get(), &*frame_start, frame_size, + encoded.data(), encoded.size()); + if (num_bytes < 0) { + // LOG(ERROR) << "Encode error: " << opus::ErrorToString(num_bytes); + return {}; + } + encoded.resize(num_bytes); + return encoded; +} + +opus::Decoder::Decoder(opus_uint32 sample_rate, int num_channels) + : num_channels_(num_channels) { + int error{}; + decoder_.reset(opus_decoder_create(sample_rate, num_channels, &error)); + valid_ = error == OPUS_OK; +} + +std::vector opus::Decoder::Decode( + const std::vector>& packets, int frame_size, + bool decode_fec) { + std::vector decoded; + for (const auto& enc : packets) { + auto just_decoded = Decode(enc, frame_size, decode_fec); + decoded.insert(std::end(decoded), std::begin(just_decoded), + std::end(just_decoded)); + } + return decoded; +} + +std::vector opus::Decoder::Decode( + const std::vector& packet, int frame_size, bool decode_fec) { + const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); + std::vector decoded(frame_length); + auto num_samples = opus_decode(decoder_.get(), packet.data(), packet.size(), + decoded.data(), frame_size, decode_fec); + if (num_samples < 0) { + // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); + return {}; + } + decoded.resize(num_samples * num_channels_); + return decoded; +} + +std::vector opus::Decoder::DecodeDummy(int frame_size) { + const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); + std::vector decoded(frame_length); + auto num_samples = + opus_decode(decoder_.get(), nullptr, 0, decoded.data(), frame_size, true); + if (num_samples < 0) { + // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); + return {}; + } + decoded.resize(num_samples * num_channels_); + return decoded; +} \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusWrapper.h b/plugins/opusCodec/src/OpusWrapper.h new file mode 100644 index 0000000000..713dff87ff --- /dev/null +++ b/plugins/opusCodec/src/OpusWrapper.h @@ -0,0 +1,131 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPUSCPP_OPUS_WRAPPER_H_ +#define OPUSCPP_OPUS_WRAPPER_H_ + +#include +#include +#include + +#include "opus/opus.h" + +namespace opus { + + std::string ErrorToString(int error); + + namespace internal { + // Deleter for OpusEncoders and OpusDecoders + struct OpusDestroyer { + void operator()(OpusEncoder* encoder) const noexcept; + void operator()(OpusDecoder* decoder) const noexcept; + }; + template + using opus_uptr = std::unique_ptr; + } // namespace internal + + class Encoder { + public: + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gaa89264fd93c9da70362a0c9b96b9ca88 + // Fs corresponds to sample_rate + // + // If expected_loss_percent is positive, FEC will be enabled + Encoder(opus_int32 sample_rate, int num_channels, int application, + int expected_loss_percent = 0); + + // Resets internal state of encoder. This should be called between encoding + // different streams so that back-to-back decoding and one-at-a-time decoding + // give the same result. Returns true on success. + bool ResetState(); + + // Sets the desired bitrate. Rates from 500 to 512000 are meaningful as well + // as the special values OPUS_AUTO and OPUS_BITRATE_MAX. If this method + // is not called, the default value of OPUS_AUTO is used. + // Returns true on success. + bool SetBitrate(int bitrate); + + // Enables or disables variable bitrate in the encoder. By default, variable + // bitrate is enabled. Returns true on success. + bool SetVariableBitrate(int vbr); + + // Sets the computational complexity of the encoder, in the range of 0 to 10, + // inclusive, with 10 being the highest complexity. Returns true on success. + bool SetComplexity(int complexity); + + // Gets the total samples of delay added by the entire codec. This value + // is the minimum amount of 'preskip' that has to be specified in an + // ogg-stream that encapsulates the encoded audio. + int GetLookahead(); + + // Takes audio data and encodes it. Returns a sequence of encoded packets. + // pcm.size() must be divisible by frame_size * (number of channels); + // pcm must not contain any incomplete packets. + // see documentation for pcm and frame_size at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gad2d6bf6a9ffb6674879d7605ed073e25 + std::vector> Encode( + const std::vector& pcm, int frame_size); + + int valid() const { return valid_; } + + private: + std::vector EncodeFrame( + const std::vector::const_iterator& frame_start, + int frame_size); + + template + int Ctl(int request, Ts... args) const { + return opus_encoder_ctl(encoder_.get(), request, args...); + } + + int num_channels_{}; + bool valid_{}; + internal::opus_uptr encoder_; + }; + + class Decoder { + public: + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga753f6fe0b699c81cfd47d70c8e15a0bd + // Fs corresponds to sample_rate + Decoder(opus_uint32 sample_rate, int num_channels); + + // Takes a sequence of encoded packets and decodes them. Returns the decoded + // audio. + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 + std::vector Decode( + const std::vector>& packets, int frame_size, + bool decode_fec); + + int valid() const { return valid_; } + + // Takes an encoded packet and decodes it. Returns the decoded audio + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 + std::vector Decode(const std::vector& packet, + int frame_size, bool decode_fec); + + // Generates a dummy frame by passing nullptr to the underlying opus decode. + std::vector DecodeDummy(int frame_size); + + private: + int num_channels_{}; + bool valid_{}; + internal::opus_uptr decoder_; + }; + +} // namespace opus + +#endif \ No newline at end of file From cf56bf415f990ed6facd013ae83bd2a03c3b6b0f Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 00:18:57 +0100 Subject: [PATCH 07/45] Initial attempt at Opus compression. --- plugins/opusCodec/src/OpusCodecManager.cpp | 44 ++-- plugins/opusCodec/src/OpusCodecManager.h | 3 +- plugins/opusCodec/src/OpusCodecProvider.cpp | 8 +- plugins/opusCodec/src/OpusDecoder.cpp | 97 +++++++ plugins/opusCodec/src/OpusDecoder.h | 28 ++ plugins/opusCodec/src/OpusEncoder.cpp | 270 ++++++++++++++++++++ plugins/opusCodec/src/OpusEncoder.h | 69 +++++ 7 files changed, 486 insertions(+), 33 deletions(-) create mode 100644 plugins/opusCodec/src/OpusDecoder.cpp create mode 100644 plugins/opusCodec/src/OpusDecoder.h create mode 100644 plugins/opusCodec/src/OpusEncoder.cpp create mode 100644 plugins/opusCodec/src/OpusEncoder.h diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp index 8f7d4fe6e5..d86208c6f2 100644 --- a/plugins/opusCodec/src/OpusCodecManager.cpp +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -22,6 +22,9 @@ #include #include +#include "OpusEncoder.h" +#include "OpusDecoder.h" + #define FRAME_SIZE 960 #define SAMPLE_RATE 48000 #define CHANNELS 2 @@ -31,56 +34,41 @@ #define MAX_FRAME_SIZE 6*FRAME_SIZE #define MAX_PACKET_SIZE 3*1276 -const char* OpusCodec::NAME { "opus" }; +const char* AthenaOpusCodec::NAME { "opus" }; -void OpusCodec::init() { +void AthenaOpusCodec::init() { } -void OpusCodec::deinit() { +void AthenaOpusCodec::deinit() { } -bool OpusCodec::activate() { +bool AthenaOpusCodec::activate() { CodecPlugin::activate(); return true; } -void OpusCodec::deactivate() { +void AthenaOpusCodec::deactivate() { CodecPlugin::deactivate(); } -bool OpusCodec::isSupported() const { +bool AthenaOpusCodec::isSupported() const { return true; } -class OpusEncoder : public Encoder { -public: - OpusEncoder(int sampleRate, int numChannels) { - - } - - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer.resize(_encodedSize); - } - -private: - int _encodedSize; -}; - - -Encoder* OpusCodec::createEncoder(int sampleRate, int numChannels) { - return new OpusEncoder(sampleRate, numChannels); +Encoder* AthenaOpusCodec::createEncoder(int sampleRate, int numChannels) { + return new AthenaOpusEncoder(sampleRate, numChannels); } -Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { - return this; +Decoder* AthenaOpusCodec::createDecoder(int sampleRate, int numChannels) { + return new AthenaOpusDecoder(sampleRate, numChannels); } -void OpusCodec::releaseEncoder(Encoder* encoder) { +void AthenaOpusCodec::releaseEncoder(Encoder* encoder) { delete encoder; } -void OpusCodec::releaseDecoder(Decoder* decoder) { +void AthenaOpusCodec::releaseDecoder(Decoder* decoder) { delete decoder; -} \ No newline at end of file +} diff --git a/plugins/opusCodec/src/OpusCodecManager.h b/plugins/opusCodec/src/OpusCodecManager.h index b75e78b23c..1db1731fc0 100644 --- a/plugins/opusCodec/src/OpusCodecManager.h +++ b/plugins/opusCodec/src/OpusCodecManager.h @@ -14,7 +14,7 @@ #include -class OpusCodec : public CodecPlugin, public Encoder, public Decoder { +class AthenaOpusCodec : public CodecPlugin { Q_OBJECT public: @@ -35,6 +35,7 @@ public: virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; + private: static const char* NAME; }; diff --git a/plugins/opusCodec/src/OpusCodecProvider.cpp b/plugins/opusCodec/src/OpusCodecProvider.cpp index ab72f25023..79f01de4bd 100644 --- a/plugins/opusCodec/src/OpusCodecProvider.cpp +++ b/plugins/opusCodec/src/OpusCodecProvider.cpp @@ -17,20 +17,20 @@ #include "OpusCodecManager.h" -class OpusCodecProvider : public QObject, public CodecProvider { +class AthenaOpusCodecProvider : public QObject, public CodecProvider { Q_OBJECT Q_PLUGIN_METADATA(IID CodecProvider_iid FILE "plugin.json") Q_INTERFACES(CodecProvider) public: - OpusCodecProvider(QObject* parent = nullptr) : QObject(parent) {} - virtual ~OpusCodecProvider() {} + AthenaOpusCodecProvider(QObject* parent = nullptr) : QObject(parent) {} + virtual ~AthenaOpusCodecProvider() {} virtual CodecPluginList getCodecPlugins() override { static std::once_flag once; std::call_once(once, [&] { - CodecPluginPointer opusCodec(new OpusCodec()); + CodecPluginPointer opusCodec(new AthenaOpusCodec()); if (opusCodec->isSupported()) { _codecPlugins.push_back(opusCodec); } diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp new file mode 100644 index 0000000000..d47274e523 --- /dev/null +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -0,0 +1,97 @@ +#include +#include + + + +#include "OpusDecoder.h" + +static QLoggingCategory decoder("AthenaOpusDecoder"); + +static QString error_to_string(int error) { + switch (error) { + case OPUS_OK: + return "OK"; + case OPUS_BAD_ARG: + return "One or more invalid/out of range arguments."; + case OPUS_BUFFER_TOO_SMALL: + return "The mode struct passed is invalid."; + case OPUS_INTERNAL_ERROR: + return "An internal error was detected."; + case OPUS_INVALID_PACKET: + return "The compressed data passed is corrupted."; + case OPUS_UNIMPLEMENTED: + return "Invalid/unsupported request number."; + case OPUS_INVALID_STATE: + return "An encoder or decoder structure is invalid or already freed."; + default: + return QString("Unknown error code: %i").arg(error); + } +} + + +AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) +{ + int error; + + _opus_sample_rate = sampleRate; + _opus_num_channels = numChannels; + + _decoder = opus_decoder_create(sampleRate, numChannels, &error); + + if ( error != OPUS_OK ) { + qCCritical(decoder) << "Failed to initialize Opus encoder: " << error_to_string(error); + _decoder = nullptr; + return; + } + + + qCDebug(decoder) << "Opus decoder initialized"; +} + +AthenaOpusDecoder::~AthenaOpusDecoder() +{ + if ( _decoder ) + opus_decoder_destroy(_decoder); + +} + +void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &decodedBuffer) +{ + assert(_decoder); + + PerformanceTimer perfTimer("AthenaOpusDecoder::decode"); + + decodedBuffer.resize( 65536 ); // Brute force, yeah! + int frame_size = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); + + int frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), frame_size, 1 ); + + if ( frames >= 0 ) { + qCDebug(decoder) << "Decoded " << frames << " Opus frames"; + decodedBuffer.resize( frames * static_cast(sizeof( opus_int16 )) * _opus_num_channels ); + } else { + qCCritical(decoder) << "Failed to decode audio: " << error_to_string(frames); + } + +} + +void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) +{ + assert(_decoder); + + PerformanceTimer perfTimer("AthenaOpusDecoder::lostFrame"); + + decodedBuffer.resize( 65536 ); // Brute force, yeah! + int frame_size = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); + + int frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), frame_size, 1 ); + + if ( frames >= 0 ) { + qCDebug(decoder) << "Produced " << frames << " opus frames from a lost frame"; + decodedBuffer.resize( frames * static_cast(sizeof( opus_int16 )) * _opus_num_channels ); + } else { + qCCritical(decoder) << "Failed to decode lost frame: " << error_to_string(frames); + } +} + + diff --git a/plugins/opusCodec/src/OpusDecoder.h b/plugins/opusCodec/src/OpusDecoder.h new file mode 100644 index 0000000000..d6b869695e --- /dev/null +++ b/plugins/opusCodec/src/OpusDecoder.h @@ -0,0 +1,28 @@ +#ifndef OPUSDECODER_H +#define OPUSDECODER_H + + +#include +#include "opus/opus.h" + + +class AthenaOpusDecoder : public Decoder { +public: + AthenaOpusDecoder(int sampleRate, int numChannels); + ~AthenaOpusDecoder() override; + + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; + virtual void lostFrame( QByteArray &decodedBuffer) override; + + +private: + int _encodedSize; + + OpusDecoder *_decoder = nullptr; + int _opus_sample_rate = 0; + int _opus_num_channels = 0; +}; + + +#endif // OPUSDECODER_H diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp new file mode 100644 index 0000000000..a3a7ed62bc --- /dev/null +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -0,0 +1,270 @@ + +#include +#include + +#include "OpusEncoder.h" +#include "OpusWrapper.h" +#include "opus/opus.h" + +static QLoggingCategory encoder("AthenaOpusEncoder"); + +static QString error_to_string(int error) { + switch (error) { + case OPUS_OK: + return "OK"; + case OPUS_BAD_ARG: + return "One or more invalid/out of range arguments."; + case OPUS_BUFFER_TOO_SMALL: + return "The mode struct passed is invalid."; + case OPUS_INTERNAL_ERROR: + return "An internal error was detected."; + case OPUS_INVALID_PACKET: + return "The compressed data passed is corrupted."; + case OPUS_UNIMPLEMENTED: + return "Invalid/unsupported request number."; + case OPUS_INVALID_STATE: + return "An encoder or decoder structure is invalid or already freed."; + default: + return QString("Unknown error code: %i").arg(error); + } +} + + + +AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) +{ + _opus_sample_rate = sampleRate; + _opus_channels = numChannels; + + int error; + + _encoder = opus_encoder_create(sampleRate, numChannels, DEFAULT_APPLICATION, &error); + + if ( error != OPUS_OK ) { + qCCritical(encoder) << "Failed to initialize Opus encoder: " << error_to_string(error); + _encoder = nullptr; + return; + } + + setBitrate(DEFAULT_BITRATE); + setComplexity(DEFAULT_COMPLEXITY); + setApplication(DEFAULT_APPLICATION); + setSignal(DEFAULT_SIGNAL); + + qCDebug(encoder) << "Opus encoder initialized"; +} + +AthenaOpusEncoder::~AthenaOpusEncoder() +{ + opus_encoder_destroy(_encoder); +} + + + +void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { + + PerformanceTimer perfTimer("AthenaOpusEncoder::encode"); + assert(_encoder); + + encodedBuffer.resize( decodedBuffer.size() ); + int frame_size = decodedBuffer.length()/ _opus_channels / static_cast(sizeof(opus_int16)); + + int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.data()), frame_size, reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); + + if ( bytes >= 0 ) { + qCDebug(encoder) << "Encoded " << decodedBuffer.length() << " bytes into " << bytes << " opus bytes"; + encodedBuffer.resize(bytes); + } else { + encodedBuffer.resize(0); + + qCWarning(encoder) << "Error when encoding " << decodedBuffer.length() << " bytes of audio: " << error_to_string(bytes); + } + +} + +int AthenaOpusEncoder::getComplexity() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_COMPLEXITY(&ret)); + return ret; +} + +void AthenaOpusEncoder::setComplexity(int complexity) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_COMPLEXITY(complexity)); + + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting complexity to " << complexity << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getBitrate() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_BITRATE(&ret)); + return ret; +} + +void AthenaOpusEncoder::setBitrate(int bitrate) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_BITRATE(bitrate)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting bitrate to " << bitrate << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getVBR() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_VBR(&ret)); + return ret; +} + +void AthenaOpusEncoder::setVBR(int vbr) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_VBR(vbr)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting VBR to " << vbr << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getVBRConstraint() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_VBR_CONSTRAINT(&ret)); + return ret; +} + +void AthenaOpusEncoder::setVBRConstraint(int vbr_const) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_VBR_CONSTRAINT(vbr_const)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting VBR constraint to " << vbr_const << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getMaxBandwidth() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_MAX_BANDWIDTH(&ret)); + return ret; +} + +void AthenaOpusEncoder::setMaxBandwidth(int maxbw) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxbw)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting max bandwidth to " << maxbw << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getBandwidth() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_BANDWIDTH(&ret)); + return ret; +} + +void AthenaOpusEncoder::setBandwidth(int bw) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_BANDWIDTH(bw)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting bandwidth to " << bw << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getSignal() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_SIGNAL(&ret)); + return ret; +} + +void AthenaOpusEncoder::setSignal(int signal) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_SIGNAL(signal)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting signal to " << signal << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getApplication() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_APPLICATION(&ret)); + return ret; +} + +void AthenaOpusEncoder::setApplication(int app) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(app)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting application to " << app << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getLookahead() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_LOOKAHEAD(&ret)); + return ret; +} + +int AthenaOpusEncoder::getInbandFEC() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_INBAND_FEC(&ret)); + return ret; +} + +void AthenaOpusEncoder::setInbandFEC(int fec) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(fec)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting inband FEC to " << fec << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getExpectedPacketLossPercent() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_PACKET_LOSS_PERC(&ret)); + return ret; +} + +void AthenaOpusEncoder::setExpectedPacketLossPercent(int perc) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(perc)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting loss percent to " << perc << ": " << error_to_string(ret); +} + +int AthenaOpusEncoder::getDTX() const +{ + assert(_encoder); + int ret; + opus_encoder_ctl(_encoder, OPUS_GET_DTX(&ret)); + return ret; +} + +void AthenaOpusEncoder::setDTX(int dtx) +{ + assert(_encoder); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_DTX(dtx)); + if ( ret != OPUS_OK ) + qCWarning(encoder) << "Error when setting DTX to " << dtx << ": " << error_to_string(ret); +} + + diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h new file mode 100644 index 0000000000..859178281b --- /dev/null +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -0,0 +1,69 @@ +#ifndef OPUSENCODER_H +#define OPUSENCODER_H +#include +#include "OpusWrapper.h" +#include "opus/opus.h" + + +class AthenaOpusEncoder : public Encoder { +public: + const int DEFAULT_BITRATE = 44100; + const int DEFAULT_COMPLEXITY = 10; + const int DEFAULT_APPLICATION = OPUS_APPLICATION_VOIP; + const int DEFAULT_SIGNAL = OPUS_AUTO; + + + AthenaOpusEncoder(int sampleRate, int numChannels); + ~AthenaOpusEncoder() override; + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + + + int getComplexity() const; + void setComplexity(int complexity); + + int getBitrate() const; + void setBitrate(int bitrate); + + int getVBR() const; + void setVBR(int vbr); + + int getVBRConstraint() const; + void setVBRConstraint(int vbr_const); + + int getMaxBandwidth() const; + void setMaxBandwidth(int maxbw); + + int getBandwidth() const; + void setBandwidth(int bw); + + int getSignal() const; + void setSignal(int signal); + + int getApplication() const; + void setApplication(int app); + + int getLookahead() const; + + int getInbandFEC() const; + void setInbandFEC(int fec); + + int getExpectedPacketLossPercent() const; + void setExpectedPacketLossPercent(int perc); + + int getDTX() const; + void setDTX(int dtx); + + +private: + + int _opus_sample_rate = 0; + int _opus_channels = 0; + int _opus_expected_loss = 0; + + + OpusEncoder *_encoder = nullptr; +}; + + +#endif // OPUSENCODER_H From e39674737778eee8441ddb194c8e4e520f08e020 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 02:38:40 +0100 Subject: [PATCH 08/45] Add Opus to list of codecs --- domain-server/resources/describe-settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index b854955953..cdf92918c6 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1227,8 +1227,8 @@ "name": "codec_preference_order", "label": "Audio Codec Preference Order", "help": "List of codec names in order of preferred usage", - "placeholder": "hifiAC, zlib, pcm", - "default": "hifiAC,zlib,pcm", + "placeholder": "opus, hifiAC, zlib, pcm", + "default": "opus,hifiAC,zlib,pcm", "advanced": true } ] From 7616c9c06e18ef0faad181e61199265a9f0028bc Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 13:33:14 +0100 Subject: [PATCH 09/45] Don't attempt FEC decoding in decode(). More logging. Also use the fixed size audio buffer for decoding -- audio always works in packets of the same size, so we should always get the same amount out of decoding. --- plugins/opusCodec/CMakeLists.txt | 2 +- plugins/opusCodec/src/OpusDecoder.cpp | 14 ++++++++++---- plugins/opusCodec/src/OpusDecoder.h | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/plugins/opusCodec/CMakeLists.txt b/plugins/opusCodec/CMakeLists.txt index 761e275929..176c311fe8 100644 --- a/plugins/opusCodec/CMakeLists.txt +++ b/plugins/opusCodec/CMakeLists.txt @@ -8,7 +8,7 @@ set(TARGET_NAME opusCodec) setup_hifi_client_server_plugin() -link_hifi_libraries(shared plugins) +link_hifi_libraries(shared audio plugins) if (BUILD_SERVER) install_beside_console() diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index d47274e523..080742a127 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -1,5 +1,6 @@ #include #include +#include @@ -45,7 +46,7 @@ AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) } - qCDebug(decoder) << "Opus decoder initialized"; + qCDebug(decoder) << "Opus decoder initialized, sampleRate = " << sampleRate << "; numChannels = " << numChannels; } AthenaOpusDecoder::~AthenaOpusDecoder() @@ -61,10 +62,14 @@ void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &deco PerformanceTimer perfTimer("AthenaOpusDecoder::decode"); - decodedBuffer.resize( 65536 ); // Brute force, yeah! + + int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opus_num_channels; + + decodedBuffer.resize( buffer_size ); int frame_size = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); - int frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), frame_size, 1 ); + qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " << decodedBuffer.size() << "; frameCount = " << frame_size; + int frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), frame_size, 0 ); if ( frames >= 0 ) { qCDebug(decoder) << "Decoded " << frames << " Opus frames"; @@ -81,7 +86,8 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) PerformanceTimer perfTimer("AthenaOpusDecoder::lostFrame"); - decodedBuffer.resize( 65536 ); // Brute force, yeah! + int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opus_num_channels; + decodedBuffer.resize( buffer_size ); int frame_size = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); int frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), frame_size, 1 ); diff --git a/plugins/opusCodec/src/OpusDecoder.h b/plugins/opusCodec/src/OpusDecoder.h index d6b869695e..0aaadedc7a 100644 --- a/plugins/opusCodec/src/OpusDecoder.h +++ b/plugins/opusCodec/src/OpusDecoder.h @@ -22,6 +22,7 @@ private: OpusDecoder *_decoder = nullptr; int _opus_sample_rate = 0; int _opus_num_channels = 0; + int _decoded_size = 0; }; From fdab09c7a15091488c8a4d8a3098b502c5bfc1c7 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 16:26:59 +0100 Subject: [PATCH 10/45] use constData() for encoding, it's faster --- plugins/opusCodec/src/OpusEncoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index a3a7ed62bc..092a7a7c79 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -69,7 +69,7 @@ void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& enco encodedBuffer.resize( decodedBuffer.size() ); int frame_size = decodedBuffer.length()/ _opus_channels / static_cast(sizeof(opus_int16)); - int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.data()), frame_size, reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); + int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.constData()), frame_size, reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); if ( bytes >= 0 ) { qCDebug(encoder) << "Encoded " << decodedBuffer.length() << " bytes into " << bytes << " opus bytes"; From bf3d9cec8eda8a5f628ca6f5713df1fd7183c1f2 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 16:40:35 +0100 Subject: [PATCH 11/45] Always maintain size of audio output buffer. More debug output. --- plugins/opusCodec/src/OpusDecoder.cpp | 50 +++++++++++++++++++-------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index 080742a127..6afd50d125 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -59,23 +59,32 @@ AthenaOpusDecoder::~AthenaOpusDecoder() void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &decodedBuffer) { assert(_decoder); - PerformanceTimer perfTimer("AthenaOpusDecoder::decode"); - + // The audio system encodes and decodes always in fixed size chunks int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opus_num_channels; decodedBuffer.resize( buffer_size ); - int frame_size = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); + int buffer_frames = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); - qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " << decodedBuffer.size() << "; frameCount = " << frame_size; - int frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), frame_size, 0 ); + qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " << decodedBuffer.size() << "; frameCount = " << buffer_frames; + int decoded_frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), buffer_frames, 0 ); - if ( frames >= 0 ) { - qCDebug(decoder) << "Decoded " << frames << " Opus frames"; - decodedBuffer.resize( frames * static_cast(sizeof( opus_int16 )) * _opus_num_channels ); + if ( decoded_frames >= 0 ) { + qCDebug(decoder) << "Decoded " << decoded_frames << " Opus frames, " << buffer_frames << " expected"; + + if ( decoded_frames < buffer_frames ) { + qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames << " were expected!"; + + int start = decoded_frames * static_cast(sizeof(int16_t)) * _opus_num_channels; + memset( &decodedBuffer.data()[start], 0, static_cast(decodedBuffer.length() - start)); + } else if ( decoded_frames > buffer_frames ) { + // This should never happen + qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames << " were expected! Buffer overflow!?"; + } } else { - qCCritical(decoder) << "Failed to decode audio: " << error_to_string(frames); + qCCritical(decoder) << "Failed to decode audio: " << error_to_string(decoded_frames); + decodedBuffer.fill('\0'); } } @@ -88,15 +97,26 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opus_num_channels; decodedBuffer.resize( buffer_size ); - int frame_size = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); + int buffer_frames = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); - int frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), frame_size, 1 ); + int decoded_frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), buffer_frames, 1 ); - if ( frames >= 0 ) { - qCDebug(decoder) << "Produced " << frames << " opus frames from a lost frame"; - decodedBuffer.resize( frames * static_cast(sizeof( opus_int16 )) * _opus_num_channels ); + if ( decoded_frames >= 0 ) { + + qCDebug(decoder) << "Produced " << decoded_frames << " opus frames from a lost frame, " << buffer_frames << " expected"; + + if ( decoded_frames < buffer_frames ) { + qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames << " were expected!"; + + int start = decoded_frames * static_cast(sizeof(int16_t)) * _opus_num_channels; + memset( &decodedBuffer.data()[start], 0, static_cast(decodedBuffer.length() - start)); + } else if ( decoded_frames > buffer_frames ) { + // This should never happen + qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames << " were expected! Buffer overflow!?"; + } } else { - qCCritical(decoder) << "Failed to decode lost frame: " << error_to_string(frames); + qCCritical(decoder) << "Failed to decode lost frame: " << error_to_string(decoded_frames); + decodedBuffer.fill('\0'); } } From 91b3be036fe871d9330c70aa8f65fca5b69c7fae Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 18:50:29 +0100 Subject: [PATCH 12/45] Dump encoder arguments --- plugins/opusCodec/src/OpusEncoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 092a7a7c79..84dfcac35b 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -51,7 +51,7 @@ AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) setApplication(DEFAULT_APPLICATION); setSignal(DEFAULT_SIGNAL); - qCDebug(encoder) << "Opus encoder initialized"; + qCDebug(encoder) << "Opus encoder initialized, sampleRate = " << sampleRate << "; numChannels = " << numChannels; } AthenaOpusEncoder::~AthenaOpusEncoder() From 09aab2a2c6d14330c98a301f0802f448849cd3e7 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 23:07:40 +0100 Subject: [PATCH 13/45] Bump default quality --- plugins/opusCodec/src/OpusEncoder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index 859178281b..bae5a6402a 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -7,7 +7,7 @@ class AthenaOpusEncoder : public Encoder { public: - const int DEFAULT_BITRATE = 44100; + const int DEFAULT_BITRATE = 128000; const int DEFAULT_COMPLEXITY = 10; const int DEFAULT_APPLICATION = OPUS_APPLICATION_VOIP; const int DEFAULT_SIGNAL = OPUS_AUTO; From 18e83aa93d947446d68f97105ebb2c152baf8b98 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 9 Jan 2020 23:07:53 +0100 Subject: [PATCH 14/45] Remove encoder/decoder logging spam --- plugins/opusCodec/src/OpusDecoder.cpp | 6 +++--- plugins/opusCodec/src/OpusEncoder.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index 6afd50d125..265021c8fd 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -71,7 +71,7 @@ void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &deco int decoded_frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), buffer_frames, 0 ); if ( decoded_frames >= 0 ) { - qCDebug(decoder) << "Decoded " << decoded_frames << " Opus frames, " << buffer_frames << " expected"; + //qCDebug(decoder) << "Decoded " << decoded_frames << " Opus frames, " << buffer_frames << " expected"; if ( decoded_frames < buffer_frames ) { qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames << " were expected!"; @@ -102,8 +102,7 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) int decoded_frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), buffer_frames, 1 ); if ( decoded_frames >= 0 ) { - - qCDebug(decoder) << "Produced " << decoded_frames << " opus frames from a lost frame, " << buffer_frames << " expected"; + //qCDebug(decoder) << "Produced " << decoded_frames << " opus frames from a lost frame, " << buffer_frames << " expected"; if ( decoded_frames < buffer_frames ) { qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames << " were expected!"; @@ -118,6 +117,7 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) qCCritical(decoder) << "Failed to decode lost frame: " << error_to_string(decoded_frames); decodedBuffer.fill('\0'); } + } diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 84dfcac35b..593e74e921 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -72,7 +72,7 @@ void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& enco int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.constData()), frame_size, reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); if ( bytes >= 0 ) { - qCDebug(encoder) << "Encoded " << decodedBuffer.length() << " bytes into " << bytes << " opus bytes"; + //qCDebug(encoder) << "Encoded " << decodedBuffer.length() << " bytes into " << bytes << " opus bytes"; encodedBuffer.resize(bytes); } else { encodedBuffer.resize(0); From d030d82355a40829d36f6220f6ad2c4a47024a32 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Fri, 10 Jan 2020 12:17:22 -0600 Subject: [PATCH 15/45] Cosmetic change for TargetOpus.cmake --- cmake/macros/TargetOpus.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/macros/TargetOpus.cmake b/cmake/macros/TargetOpus.cmake index 20fcadfc57..eb72deed6a 100644 --- a/cmake/macros/TargetOpus.cmake +++ b/cmake/macros/TargetOpus.cmake @@ -5,7 +5,7 @@ # Distributed under the Apache License, Version 2.0. # See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html # -macro(TARGET_opus) +macro(TARGET_OPUS) if (ANDROID) # no idea if this is correct target_link_libraries(${TARGET_NAME}) From 32c438ffc22efabae241cb02e18817e2d38624c7 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Fri, 10 Jan 2020 12:17:53 -0600 Subject: [PATCH 16/45] Update CMakeLists.txt Fix interface linking errors on Windows. --- plugins/opusCodec/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/opusCodec/CMakeLists.txt b/plugins/opusCodec/CMakeLists.txt index 176c311fe8..06ed6eab95 100644 --- a/plugins/opusCodec/CMakeLists.txt +++ b/plugins/opusCodec/CMakeLists.txt @@ -9,6 +9,7 @@ set(TARGET_NAME opusCodec) setup_hifi_client_server_plugin() link_hifi_libraries(shared audio plugins) +target_opus() if (BUILD_SERVER) install_beside_console() From c9595016f8b5d585486b91f323137081dbc43c98 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Fri, 10 Jan 2020 12:19:19 -0600 Subject: [PATCH 17/45] DEBUG: Temp removal of hifiCodec and pcmCodec projects. This is to ensure that no other codecs are being fallen back upon when testing the Opus codec. --- plugins/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index c88bb8a00d..126f8d34ec 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -39,10 +39,10 @@ if (NOT SERVER_ONLY AND NOT ANDROID) endif() # server-side plugins -set(DIR "pcmCodec") -add_subdirectory(${DIR}) -set(DIR "hifiCodec") -add_subdirectory(${DIR}) +# set(DIR "pcmCodec") +# add_subdirectory(${DIR}) +# set(DIR "hifiCodec") +# add_subdirectory(${DIR}) set(DIR "opusCodec") add_subdirectory(${DIR}) From 00221704005d6d07e794902e1ec67bbcbf609a1e Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sun, 12 Jan 2020 18:33:46 -0600 Subject: [PATCH 18/45] Fix minor cmake oopsie. --- plugins/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index e49b847d3b..126f8d34ec 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -45,8 +45,6 @@ endif() # add_subdirectory(${DIR}) set(DIR "opusCodec") add_subdirectory(${DIR}) -set(DIR "opusCodec") -add_subdirectory(${DIR}) # example plugins set(DIR "KasenAPIExample") From 26bebba8dd14ccf4178963c3fad998038e9d8b5f Mon Sep 17 00:00:00 2001 From: Thijs Wenker Date: Mon, 13 Jan 2020 03:08:45 +0100 Subject: [PATCH 19/45] fix opus library linking in cmake Signed-off-by: Marcus Llewellyn --- cmake/macros/TargetOpus.cmake | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/cmake/macros/TargetOpus.cmake b/cmake/macros/TargetOpus.cmake index eb72deed6a..a8faf5139e 100644 --- a/cmake/macros/TargetOpus.cmake +++ b/cmake/macros/TargetOpus.cmake @@ -5,14 +5,9 @@ # Distributed under the Apache License, Version 2.0. # See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html # -macro(TARGET_OPUS) - if (ANDROID) - # no idea if this is correct - target_link_libraries(${TARGET_NAME}) - else() - # using VCPKG for opus - find_package(OPUS REQUIRED) - target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${OPUS_INCLUDE_DIRS}) - target_link_libraries(${TARGET_NAME} ${OPUS_LIBRARIES}) - endif() +macro(TARGET_opus) + find_library(OPUS_LIBRARY_RELEASE NAMES opus PATHS ${VCPKG_INSTALL_ROOT}/lib) + find_library(OPUS_LIBRARY_DEBUG NAMES opus PATHS ${VCPKG_INSTALL_ROOT}/debug/lib) + select_library_configurations(OPUS) + target_link_libraries(${TARGET_NAME} ${OPUS_LIBRARY}) endmacro() From 3a5059a613704d562f5563a07456fbef84fe00db Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Mon, 13 Jan 2020 13:35:36 -0600 Subject: [PATCH 20/45] Re-enable hifiAC and pcmCodec projects These has been disabled for testing purpose. They are now restored. --- plugins/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 126f8d34ec..e0e6046691 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -38,11 +38,11 @@ if (NOT SERVER_ONLY AND NOT ANDROID) add_subdirectory(${DIR}) endif() -# server-side plugins -# set(DIR "pcmCodec") -# add_subdirectory(${DIR}) -# set(DIR "hifiCodec") -# add_subdirectory(${DIR}) +server-side plugins +set(DIR "pcmCodec") +add_subdirectory(${DIR}) +set(DIR "hifiCodec") +add_subdirectory(${DIR}) set(DIR "opusCodec") add_subdirectory(${DIR}) From 6a294404c2fb3f8c574e22bf8661e4446b14322b Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Mon, 13 Jan 2020 15:44:46 -0600 Subject: [PATCH 21/45] fix marcus being very stoopid --- plugins/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index e0e6046691..c88bb8a00d 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -38,7 +38,7 @@ if (NOT SERVER_ONLY AND NOT ANDROID) add_subdirectory(${DIR}) endif() -server-side plugins +# server-side plugins set(DIR "pcmCodec") add_subdirectory(${DIR}) set(DIR "hifiCodec") From 391f3accc6a27572c3b913d1bc2ac7c05289d79b Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Tue, 14 Jan 2020 23:20:24 +0100 Subject: [PATCH 22/45] Instrument HifiAC code with performance timers --- plugins/hifiCodec/CMakeLists.txt | 2 +- plugins/hifiCodec/src/HiFiCodec.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt index 0d4f093fc2..5515602d2f 100644 --- a/plugins/hifiCodec/CMakeLists.txt +++ b/plugins/hifiCodec/CMakeLists.txt @@ -8,7 +8,7 @@ set(TARGET_NAME hifiCodec) setup_hifi_client_server_plugin() -link_hifi_libraries(audio plugins) +link_hifi_libraries(shared audio plugins) target_hifiAudioCodec() if (BUILD_SERVER) install_beside_console() diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 99bb411539..a93e6c47d0 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -13,6 +13,7 @@ #include #include +#include const char* HiFiCodec::NAME { "hifiAC" }; @@ -44,6 +45,8 @@ public: } virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + PerformanceTimer perfTimer("HiFiEncoder::encode"); + encodedBuffer.resize(_encodedSize); AudioEncoder::process((const int16_t*)decodedBuffer.constData(), (int16_t*)encodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } @@ -58,11 +61,15 @@ public: } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + PerformanceTimer perfTimer("HiFiEncoder::decode"); + decodedBuffer.resize(_decodedSize); AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); } virtual void lostFrame(QByteArray& decodedBuffer) override { + PerformanceTimer perfTimer("HiFiEncoder::lostFrame"); + decodedBuffer.resize(_decodedSize); // this performs packet loss interpolation AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); From 830f3ab337658fc0886381be69f4a2077ee89bde Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Tue, 14 Jan 2020 23:20:24 +0100 Subject: [PATCH 23/45] Instrument HifiAC code with performance timers Signed-off-by: Marcus Llewellyn --- plugins/hifiCodec/CMakeLists.txt | 2 +- plugins/hifiCodec/src/HiFiCodec.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/hifiCodec/CMakeLists.txt b/plugins/hifiCodec/CMakeLists.txt index 0d4f093fc2..5515602d2f 100644 --- a/plugins/hifiCodec/CMakeLists.txt +++ b/plugins/hifiCodec/CMakeLists.txt @@ -8,7 +8,7 @@ set(TARGET_NAME hifiCodec) setup_hifi_client_server_plugin() -link_hifi_libraries(audio plugins) +link_hifi_libraries(shared audio plugins) target_hifiAudioCodec() if (BUILD_SERVER) install_beside_console() diff --git a/plugins/hifiCodec/src/HiFiCodec.cpp b/plugins/hifiCodec/src/HiFiCodec.cpp index 99bb411539..a93e6c47d0 100644 --- a/plugins/hifiCodec/src/HiFiCodec.cpp +++ b/plugins/hifiCodec/src/HiFiCodec.cpp @@ -13,6 +13,7 @@ #include #include +#include const char* HiFiCodec::NAME { "hifiAC" }; @@ -44,6 +45,8 @@ public: } virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + PerformanceTimer perfTimer("HiFiEncoder::encode"); + encodedBuffer.resize(_encodedSize); AudioEncoder::process((const int16_t*)decodedBuffer.constData(), (int16_t*)encodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); } @@ -58,11 +61,15 @@ public: } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + PerformanceTimer perfTimer("HiFiEncoder::decode"); + decodedBuffer.resize(_decodedSize); AudioDecoder::process((const int16_t*)encodedBuffer.constData(), (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, true); } virtual void lostFrame(QByteArray& decodedBuffer) override { + PerformanceTimer perfTimer("HiFiEncoder::lostFrame"); + decodedBuffer.resize(_decodedSize); // this performs packet loss interpolation AudioDecoder::process(nullptr, (int16_t*)decodedBuffer.data(), AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL, false); From f23dfc7d0de0acd1f143eadff8f05f2ff9ec2a0e Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Tue, 14 Jan 2020 17:41:55 -0600 Subject: [PATCH 24/45] Cleanup some kruft. Removed some unecessary files, defines, and includes. --- plugins/opusCodec/src/OpusCodecManager.cpp | 16 -- plugins/opusCodec/src/OpusDecoder.cpp | 2 - plugins/opusCodec/src/OpusEncoder.cpp | 1 - plugins/opusCodec/src/OpusEncoder.h | 1 - plugins/opusCodec/src/OpusWrapper.cpp | 174 --------------------- plugins/opusCodec/src/OpusWrapper.h | 131 ---------------- 6 files changed, 325 deletions(-) delete mode 100644 plugins/opusCodec/src/OpusWrapper.cpp delete mode 100644 plugins/opusCodec/src/OpusWrapper.h diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp index d86208c6f2..1e3d73a229 100644 --- a/plugins/opusCodec/src/OpusCodecManager.cpp +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -15,25 +15,9 @@ #include -// Not sure how many of these are needed, but here they are. -#include -#include -#include -#include -#include - #include "OpusEncoder.h" #include "OpusDecoder.h" -#define FRAME_SIZE 960 -#define SAMPLE_RATE 48000 -#define CHANNELS 2 -#define APPLICATION OPUS_APPLICATION_AUDIO -#define BITRATE 64000 - -#define MAX_FRAME_SIZE 6*FRAME_SIZE -#define MAX_PACKET_SIZE 3*1276 - const char* AthenaOpusCodec::NAME { "opus" }; void AthenaOpusCodec::init() { diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index 265021c8fd..b410ed9bd5 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -2,8 +2,6 @@ #include #include - - #include "OpusDecoder.h" static QLoggingCategory decoder("AthenaOpusDecoder"); diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 593e74e921..04af3121db 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -3,7 +3,6 @@ #include #include "OpusEncoder.h" -#include "OpusWrapper.h" #include "opus/opus.h" static QLoggingCategory encoder("AthenaOpusEncoder"); diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index bae5a6402a..b456f76093 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -1,7 +1,6 @@ #ifndef OPUSENCODER_H #define OPUSENCODER_H #include -#include "OpusWrapper.h" #include "opus/opus.h" diff --git a/plugins/opusCodec/src/OpusWrapper.cpp b/plugins/opusCodec/src/OpusWrapper.cpp deleted file mode 100644 index 0d8ff87965..0000000000 --- a/plugins/opusCodec/src/OpusWrapper.cpp +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2018 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "OpusWrapper.h" - -std::string opus::ErrorToString(int error) { - switch (error) { - case OPUS_OK: - return "OK"; - case OPUS_BAD_ARG: - return "One or more invalid/out of range arguments."; - case OPUS_BUFFER_TOO_SMALL: - return "The mode struct passed is invalid."; - case OPUS_INTERNAL_ERROR: - return "An internal error was detected."; - case OPUS_INVALID_PACKET: - return "The compressed data passed is corrupted."; - case OPUS_UNIMPLEMENTED: - return "Invalid/unsupported request number."; - case OPUS_INVALID_STATE: - return "An encoder or decoder structure is invalid or already freed."; - default: - return "Unknown error code: " + std::to_string(error); - } -} - -void opus::internal::OpusDestroyer::operator()(OpusEncoder* encoder) const -noexcept { - opus_encoder_destroy(encoder); -} - -void opus::internal::OpusDestroyer::operator()(OpusDecoder* decoder) const -noexcept { - opus_decoder_destroy(decoder); -} - -opus::Encoder::Encoder(opus_int32 sample_rate, int num_channels, - int application, int expected_loss_percent) - : num_channels_{ num_channels } { - int error{}; - encoder_.reset( - opus_encoder_create(sample_rate, num_channels, application, &error)); - valid_ = error == OPUS_OK; - if (!valid()) { - // LOG(INFO) << "Could not construct encoder. Error: " << ErrorToString(error); - return; - } - if (expected_loss_percent > 0) { - // LOG(INFO) << "Enabling FEC in the encoder."; - Ctl(OPUS_SET_INBAND_FEC(1)); - Ctl(OPUS_SET_PACKET_LOSS_PERC(expected_loss_percent)); - } -} - -bool opus::Encoder::ResetState() { - valid_ = Ctl(OPUS_RESET_STATE) == OPUS_OK; - return valid_; -} - -bool opus::Encoder::SetBitrate(int bitrate) { - valid_ = Ctl(OPUS_SET_BITRATE(bitrate)) == OPUS_OK; - return valid_; -} - -bool opus::Encoder::SetVariableBitrate(int vbr) { - valid_ = Ctl(OPUS_SET_VBR(vbr)) == OPUS_OK; - return valid_; -} - -bool opus::Encoder::SetComplexity(int complexity) { - valid_ = Ctl(OPUS_SET_COMPLEXITY(complexity)) == OPUS_OK; - return valid_; -} - -int opus::Encoder::GetLookahead() { - opus_int32 skip{}; - valid_ = Ctl(OPUS_GET_LOOKAHEAD(&skip)) == OPUS_OK; - return skip; -} - -std::vector> opus::Encoder::Encode( - const std::vector& pcm, int frame_size) { - constexpr auto sample_size = sizeof(pcm[0]); - const auto frame_length = frame_size * num_channels_ * sample_size; - auto data_length = pcm.size() * sample_size; - if (data_length % frame_length != 0u) { - // LOG(WARNING) << "PCM samples contain an incomplete frame. Ignoring the " - // "incomplete frame."; - data_length -= (data_length % frame_length); - } - - std::vector> encoded; - for (std::size_t i{}; i < data_length; i += frame_length) { - encoded.push_back(EncodeFrame(pcm.begin() + (i / sample_size), frame_size)); - } - return encoded; -} - -std::vector opus::Encoder::EncodeFrame( - const std::vector::const_iterator& frame_start, - int frame_size) { - const auto frame_length = (frame_size * num_channels_ * sizeof(*frame_start)); - std::vector encoded(frame_length); - auto num_bytes = opus_encode(encoder_.get(), &*frame_start, frame_size, - encoded.data(), encoded.size()); - if (num_bytes < 0) { - // LOG(ERROR) << "Encode error: " << opus::ErrorToString(num_bytes); - return {}; - } - encoded.resize(num_bytes); - return encoded; -} - -opus::Decoder::Decoder(opus_uint32 sample_rate, int num_channels) - : num_channels_(num_channels) { - int error{}; - decoder_.reset(opus_decoder_create(sample_rate, num_channels, &error)); - valid_ = error == OPUS_OK; -} - -std::vector opus::Decoder::Decode( - const std::vector>& packets, int frame_size, - bool decode_fec) { - std::vector decoded; - for (const auto& enc : packets) { - auto just_decoded = Decode(enc, frame_size, decode_fec); - decoded.insert(std::end(decoded), std::begin(just_decoded), - std::end(just_decoded)); - } - return decoded; -} - -std::vector opus::Decoder::Decode( - const std::vector& packet, int frame_size, bool decode_fec) { - const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); - std::vector decoded(frame_length); - auto num_samples = opus_decode(decoder_.get(), packet.data(), packet.size(), - decoded.data(), frame_size, decode_fec); - if (num_samples < 0) { - // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); - return {}; - } - decoded.resize(num_samples * num_channels_); - return decoded; -} - -std::vector opus::Decoder::DecodeDummy(int frame_size) { - const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); - std::vector decoded(frame_length); - auto num_samples = - opus_decode(decoder_.get(), nullptr, 0, decoded.data(), frame_size, true); - if (num_samples < 0) { - // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); - return {}; - } - decoded.resize(num_samples * num_channels_); - return decoded; -} \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusWrapper.h b/plugins/opusCodec/src/OpusWrapper.h deleted file mode 100644 index 713dff87ff..0000000000 --- a/plugins/opusCodec/src/OpusWrapper.h +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2018 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPUSCPP_OPUS_WRAPPER_H_ -#define OPUSCPP_OPUS_WRAPPER_H_ - -#include -#include -#include - -#include "opus/opus.h" - -namespace opus { - - std::string ErrorToString(int error); - - namespace internal { - // Deleter for OpusEncoders and OpusDecoders - struct OpusDestroyer { - void operator()(OpusEncoder* encoder) const noexcept; - void operator()(OpusDecoder* decoder) const noexcept; - }; - template - using opus_uptr = std::unique_ptr; - } // namespace internal - - class Encoder { - public: - // see documentation at: - // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gaa89264fd93c9da70362a0c9b96b9ca88 - // Fs corresponds to sample_rate - // - // If expected_loss_percent is positive, FEC will be enabled - Encoder(opus_int32 sample_rate, int num_channels, int application, - int expected_loss_percent = 0); - - // Resets internal state of encoder. This should be called between encoding - // different streams so that back-to-back decoding and one-at-a-time decoding - // give the same result. Returns true on success. - bool ResetState(); - - // Sets the desired bitrate. Rates from 500 to 512000 are meaningful as well - // as the special values OPUS_AUTO and OPUS_BITRATE_MAX. If this method - // is not called, the default value of OPUS_AUTO is used. - // Returns true on success. - bool SetBitrate(int bitrate); - - // Enables or disables variable bitrate in the encoder. By default, variable - // bitrate is enabled. Returns true on success. - bool SetVariableBitrate(int vbr); - - // Sets the computational complexity of the encoder, in the range of 0 to 10, - // inclusive, with 10 being the highest complexity. Returns true on success. - bool SetComplexity(int complexity); - - // Gets the total samples of delay added by the entire codec. This value - // is the minimum amount of 'preskip' that has to be specified in an - // ogg-stream that encapsulates the encoded audio. - int GetLookahead(); - - // Takes audio data and encodes it. Returns a sequence of encoded packets. - // pcm.size() must be divisible by frame_size * (number of channels); - // pcm must not contain any incomplete packets. - // see documentation for pcm and frame_size at: - // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gad2d6bf6a9ffb6674879d7605ed073e25 - std::vector> Encode( - const std::vector& pcm, int frame_size); - - int valid() const { return valid_; } - - private: - std::vector EncodeFrame( - const std::vector::const_iterator& frame_start, - int frame_size); - - template - int Ctl(int request, Ts... args) const { - return opus_encoder_ctl(encoder_.get(), request, args...); - } - - int num_channels_{}; - bool valid_{}; - internal::opus_uptr encoder_; - }; - - class Decoder { - public: - // see documentation at: - // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga753f6fe0b699c81cfd47d70c8e15a0bd - // Fs corresponds to sample_rate - Decoder(opus_uint32 sample_rate, int num_channels); - - // Takes a sequence of encoded packets and decodes them. Returns the decoded - // audio. - // see documentation at: - // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 - std::vector Decode( - const std::vector>& packets, int frame_size, - bool decode_fec); - - int valid() const { return valid_; } - - // Takes an encoded packet and decodes it. Returns the decoded audio - // see documentation at: - // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 - std::vector Decode(const std::vector& packet, - int frame_size, bool decode_fec); - - // Generates a dummy frame by passing nullptr to the underlying opus decode. - std::vector DecodeDummy(int frame_size); - - private: - int num_channels_{}; - bool valid_{}; - internal::opus_uptr decoder_; - }; - -} // namespace opus - -#endif \ No newline at end of file From 24669c98f6d7c4870bc9a9d1614fbbfa01f74a9e Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Wed, 15 Jan 2020 12:57:18 -0600 Subject: [PATCH 25/45] Made changes requested by reviewer Fixed missing copyright headers. Fixed exceeding 120 columns. Fixed #include Fixed if conditional style. Fixed C++ reference symbol placement. --- plugins/opusCodec/src/OpusDecoder.cpp | 40 +++++++++++---- plugins/opusCodec/src/OpusDecoder.h | 10 ++++ plugins/opusCodec/src/OpusEncoder.cpp | 70 +++++++++++++++++++-------- plugins/opusCodec/src/OpusEncoder.h | 14 +++++- 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index b410ed9bd5..3e436c58e4 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -1,3 +1,13 @@ +// +// OpusCodecManager.h +// plugins/opusCodec/src +// +// Copyright 2020 Dale Glass +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + #include #include #include @@ -60,25 +70,30 @@ void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &deco PerformanceTimer perfTimer("AthenaOpusDecoder::decode"); // The audio system encodes and decodes always in fixed size chunks - int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opus_num_channels; + int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) + * _opus_num_channels; decodedBuffer.resize( buffer_size ); int buffer_frames = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); - qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " << decodedBuffer.size() << "; frameCount = " << buffer_frames; - int decoded_frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), buffer_frames, 0 ); + qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " + << decodedBuffer.size() << "; frameCount = " << buffer_frames; + int decoded_frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), + encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), buffer_frames, 0 ); if ( decoded_frames >= 0 ) { //qCDebug(decoder) << "Decoded " << decoded_frames << " Opus frames, " << buffer_frames << " expected"; if ( decoded_frames < buffer_frames ) { - qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames << " were expected!"; + qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames + << " were expected!"; int start = decoded_frames * static_cast(sizeof(int16_t)) * _opus_num_channels; memset( &decodedBuffer.data()[start], 0, static_cast(decodedBuffer.length() - start)); } else if ( decoded_frames > buffer_frames ) { // This should never happen - qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames << " were expected! Buffer overflow!?"; + qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames + << " were expected! Buffer overflow!?"; } } else { qCCritical(decoder) << "Failed to decode audio: " << error_to_string(decoded_frames); @@ -93,23 +108,28 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) PerformanceTimer perfTimer("AthenaOpusDecoder::lostFrame"); - int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opus_num_channels; + int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) + * _opus_num_channels; decodedBuffer.resize( buffer_size ); int buffer_frames = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); - int decoded_frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), buffer_frames, 1 ); + int decoded_frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), + buffer_frames, 1 ); if ( decoded_frames >= 0 ) { - //qCDebug(decoder) << "Produced " << decoded_frames << " opus frames from a lost frame, " << buffer_frames << " expected"; + //qCDebug(decoder) << "Produced " << decoded_frames << " opus frames from a lost frame, " << buffer_frames + // << " expected"; if ( decoded_frames < buffer_frames ) { - qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames << " were expected!"; + qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames + << " were expected!"; int start = decoded_frames * static_cast(sizeof(int16_t)) * _opus_num_channels; memset( &decodedBuffer.data()[start], 0, static_cast(decodedBuffer.length() - start)); } else if ( decoded_frames > buffer_frames ) { // This should never happen - qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames << " were expected! Buffer overflow!?"; + qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames + << " were expected! Buffer overflow!?"; } } else { qCCritical(decoder) << "Failed to decode lost frame: " << error_to_string(decoded_frames); diff --git a/plugins/opusCodec/src/OpusDecoder.h b/plugins/opusCodec/src/OpusDecoder.h index 0aaadedc7a..01722fcc6c 100644 --- a/plugins/opusCodec/src/OpusDecoder.h +++ b/plugins/opusCodec/src/OpusDecoder.h @@ -1,3 +1,13 @@ +// +// OpusCodecManager.h +// plugins/opusCodec/src +// +// Copyright 2020 Dale Glass +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + #ifndef OPUSDECODER_H #define OPUSDECODER_H diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 04af3121db..700164427e 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -1,9 +1,18 @@ +// +// OpusCodecManager.h +// plugins/opusCodec/src +// +// Copyright 2020 Dale Glass +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// #include #include +#include #include "OpusEncoder.h" -#include "opus/opus.h" static QLoggingCategory encoder("AthenaOpusEncoder"); @@ -68,7 +77,8 @@ void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& enco encodedBuffer.resize( decodedBuffer.size() ); int frame_size = decodedBuffer.length()/ _opus_channels / static_cast(sizeof(opus_int16)); - int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.constData()), frame_size, reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); + int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.constData()), frame_size, + reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); if ( bytes >= 0 ) { //qCDebug(encoder) << "Encoded " << decodedBuffer.length() << " bytes into " << bytes << " opus bytes"; @@ -76,7 +86,8 @@ void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& enco } else { encodedBuffer.resize(0); - qCWarning(encoder) << "Error when encoding " << decodedBuffer.length() << " bytes of audio: " << error_to_string(bytes); + qCWarning(encoder) << "Error when encoding " << decodedBuffer.length() << " bytes of audio: " + << error_to_string(bytes); } } @@ -94,8 +105,9 @@ void AthenaOpusEncoder::setComplexity(int complexity) assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_COMPLEXITY(complexity)); - if ( ret != OPUS_OK ) + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting complexity to " << complexity << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getBitrate() const @@ -110,8 +122,10 @@ void AthenaOpusEncoder::setBitrate(int bitrate) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_BITRATE(bitrate)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting bitrate to " << bitrate << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getVBR() const @@ -126,8 +140,10 @@ void AthenaOpusEncoder::setVBR(int vbr) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_VBR(vbr)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting VBR to " << vbr << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getVBRConstraint() const @@ -142,8 +158,10 @@ void AthenaOpusEncoder::setVBRConstraint(int vbr_const) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_VBR_CONSTRAINT(vbr_const)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting VBR constraint to " << vbr_const << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getMaxBandwidth() const @@ -158,8 +176,10 @@ void AthenaOpusEncoder::setMaxBandwidth(int maxbw) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxbw)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting max bandwidth to " << maxbw << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getBandwidth() const @@ -174,8 +194,10 @@ void AthenaOpusEncoder::setBandwidth(int bw) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_BANDWIDTH(bw)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting bandwidth to " << bw << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getSignal() const @@ -190,8 +212,10 @@ void AthenaOpusEncoder::setSignal(int signal) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_SIGNAL(signal)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting signal to " << signal << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getApplication() const @@ -202,12 +226,14 @@ int AthenaOpusEncoder::getApplication() const return ret; } -void AthenaOpusEncoder::setApplication(int app) +void AthenaOpusEncoder::setApplication(int application) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(app)); - if ( ret != OPUS_OK ) - qCWarning(encoder) << "Error when setting application to " << app << ": " << error_to_string(ret); + int ret = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(application)); + + if (ret != OPUS_OK) { + qCWarning(encoder) << "Error when setting application to " << application << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getLookahead() const @@ -230,8 +256,10 @@ void AthenaOpusEncoder::setInbandFEC(int fec) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(fec)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting inband FEC to " << fec << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getExpectedPacketLossPercent() const @@ -246,8 +274,10 @@ void AthenaOpusEncoder::setExpectedPacketLossPercent(int perc) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(perc)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting loss percent to " << perc << ": " << error_to_string(ret); + } } int AthenaOpusEncoder::getDTX() const @@ -262,8 +292,8 @@ void AthenaOpusEncoder::setDTX(int dtx) { assert(_encoder); int ret = opus_encoder_ctl(_encoder, OPUS_SET_DTX(dtx)); - if ( ret != OPUS_OK ) + + if (ret != OPUS_OK) { qCWarning(encoder) << "Error when setting DTX to " << dtx << ": " << error_to_string(ret); -} - - + } +} \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index b456f76093..920423a96d 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -1,7 +1,17 @@ +// +// OpusCodecManager.h +// plugins/opusCodec/src +// +// Copyright 2020 Dale Glass +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + #ifndef OPUSENCODER_H #define OPUSENCODER_H #include -#include "opus/opus.h" +#include class AthenaOpusEncoder : public Encoder { @@ -61,7 +71,7 @@ private: int _opus_expected_loss = 0; - OpusEncoder *_encoder = nullptr; + OpusEncoder* _encoder = nullptr; }; From 5ff8654363c4a4355222bbd4b6f727b7d0ffc9a4 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Wed, 15 Jan 2020 13:04:59 -0600 Subject: [PATCH 26/45] Update plugins/opusCodec/src/plugin.json Co-Authored-By: Thijs Wenker --- plugins/opusCodec/src/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/opusCodec/src/plugin.json b/plugins/opusCodec/src/plugin.json index bd7c707f7d..a7605ca322 100644 --- a/plugins/opusCodec/src/plugin.json +++ b/plugins/opusCodec/src/plugin.json @@ -1,4 +1,4 @@ { - "name":"Opus Codec", + "name": "Opus Codec", "version":1 } From 0a3dfdefcedcaee2dc1b81f99456ae6793ab6024 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Wed, 15 Jan 2020 13:05:43 -0600 Subject: [PATCH 27/45] Update plugins/opusCodec/src/plugin.json Co-Authored-By: Thijs Wenker --- plugins/opusCodec/src/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/opusCodec/src/plugin.json b/plugins/opusCodec/src/plugin.json index a7605ca322..17217d6017 100644 --- a/plugins/opusCodec/src/plugin.json +++ b/plugins/opusCodec/src/plugin.json @@ -1,4 +1,4 @@ { "name": "Opus Codec", - "version":1 + "version": 1 } From d96da21dd7491ef457a24c51448fd5a471fb21c1 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Wed, 15 Jan 2020 13:06:42 -0600 Subject: [PATCH 28/45] Update plugins/opusCodec/src/OpusDecoder.cpp Co-Authored-By: Thijs Wenker --- plugins/opusCodec/src/OpusDecoder.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index 3e436c58e4..b285b3bb7a 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -110,7 +110,7 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opus_num_channels; - decodedBuffer.resize( buffer_size ); + decodedBuffer.resize(buffer_size); int buffer_frames = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); int decoded_frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), @@ -138,4 +138,3 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) } - From 9a67ee19e030ded9e5fa5786e0835c5bb38e66a5 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Wed, 15 Jan 2020 13:49:25 -0600 Subject: [PATCH 29/45] More review fixes since I missed 66 of them. --- plugins/opusCodec/src/OpusDecoder.cpp | 108 +++++------ plugins/opusCodec/src/OpusDecoder.h | 10 +- plugins/opusCodec/src/OpusEncoder.cpp | 264 ++++++++++++-------------- plugins/opusCodec/src/OpusEncoder.h | 8 +- 4 files changed, 178 insertions(+), 212 deletions(-) diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index 3e436c58e4..e95d813fb4 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -18,36 +18,35 @@ static QLoggingCategory decoder("AthenaOpusDecoder"); static QString error_to_string(int error) { switch (error) { - case OPUS_OK: - return "OK"; - case OPUS_BAD_ARG: - return "One or more invalid/out of range arguments."; - case OPUS_BUFFER_TOO_SMALL: - return "The mode struct passed is invalid."; - case OPUS_INTERNAL_ERROR: - return "An internal error was detected."; - case OPUS_INVALID_PACKET: - return "The compressed data passed is corrupted."; - case OPUS_UNIMPLEMENTED: - return "Invalid/unsupported request number."; - case OPUS_INVALID_STATE: - return "An encoder or decoder structure is invalid or already freed."; - default: - return QString("Unknown error code: %i").arg(error); + case OPUS_OK: + return "OK"; + case OPUS_BAD_ARG: + return "One or more invalid/out of range arguments."; + case OPUS_BUFFER_TOO_SMALL: + return "The mode struct passed is invalid."; + case OPUS_INTERNAL_ERROR: + return "An internal error was detected."; + case OPUS_INVALID_PACKET: + return "The compressed data passed is corrupted."; + case OPUS_UNIMPLEMENTED: + return "Invalid/unsupported request number."; + case OPUS_INVALID_STATE: + return "An encoder or decoder structure is invalid or already freed."; + default: + return QString("Unknown error code: %i").arg(error); } } -AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) -{ +AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) { int error; - _opus_sample_rate = sampleRate; - _opus_num_channels = numChannels; + _opusSampleRate = sampleRate; + _opusNumChannels = numChannels; _decoder = opus_decoder_create(sampleRate, numChannels, &error); - if ( error != OPUS_OK ) { + if (error != OPUS_OK) { qCCritical(decoder) << "Failed to initialize Opus encoder: " << error_to_string(error); _decoder = nullptr; return; @@ -57,42 +56,39 @@ AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) qCDebug(decoder) << "Opus decoder initialized, sampleRate = " << sampleRate << "; numChannels = " << numChannels; } -AthenaOpusDecoder::~AthenaOpusDecoder() -{ - if ( _decoder ) +AthenaOpusDecoder::~AthenaOpusDecoder() { + if (_decoder) opus_decoder_destroy(_decoder); } -void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &decodedBuffer) -{ +void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &decodedBuffer) { assert(_decoder); PerformanceTimer perfTimer("AthenaOpusDecoder::decode"); // The audio system encodes and decodes always in fixed size chunks - int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) - * _opus_num_channels; + int bufferSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) + * _opusNumChannels; - decodedBuffer.resize( buffer_size ); - int buffer_frames = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); + decodedBuffer.resize(bufferSize); + int bufferFrames = decodedBuffer.size() / _opusNumChannels / static_cast(sizeof( opus_int16 )); qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " - << decodedBuffer.size() << "; frameCount = " << buffer_frames; - int decoded_frames = opus_decode( _decoder, reinterpret_cast(encodedBuffer.data()), - encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), buffer_frames, 0 ); + << decodedBuffer.size() << "; frameCount = " << bufferFrames; + int decoded_frames = opus_decode(_decoder, reinterpret_cast(encodedBuffer.data()), + encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), bufferFrames, 0); - if ( decoded_frames >= 0 ) { - //qCDebug(decoder) << "Decoded " << decoded_frames << " Opus frames, " << buffer_frames << " expected"; + if (decoded_frames >= 0) { - if ( decoded_frames < buffer_frames ) { - qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames + if (decoded_frames < bufferFrames) { + qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << bufferFrames << " were expected!"; - int start = decoded_frames * static_cast(sizeof(int16_t)) * _opus_num_channels; + int start = decoded_frames * static_cast(sizeof(int16_t)) * _opusNumChannels; memset( &decodedBuffer.data()[start], 0, static_cast(decodedBuffer.length() - start)); - } else if ( decoded_frames > buffer_frames ) { + } else if (decoded_frames > bufferFrames) { // This should never happen - qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames + qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << bufferFrames << " were expected! Buffer overflow!?"; } } else { @@ -102,40 +98,36 @@ void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &deco } -void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) -{ +void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) { assert(_decoder); PerformanceTimer perfTimer("AthenaOpusDecoder::lostFrame"); - int buffer_size = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) - * _opus_num_channels; - decodedBuffer.resize( buffer_size ); - int buffer_frames = decodedBuffer.size() / _opus_num_channels / static_cast(sizeof( opus_int16 )); + int bufferSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) + * _opusNumChannels; + decodedBuffer.resize(bufferSize); + int bufferFrames = decodedBuffer.size() / _opusNumChannels / static_cast(sizeof(opus_int16)); - int decoded_frames = opus_decode( _decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), - buffer_frames, 1 ); + int decoded_frames = opus_decode(_decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), + bufferFrames, 1 ); - if ( decoded_frames >= 0 ) { - //qCDebug(decoder) << "Produced " << decoded_frames << " opus frames from a lost frame, " << buffer_frames - // << " expected"; + if (decoded_frames >= 0) { - if ( decoded_frames < buffer_frames ) { - qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << buffer_frames + if ( decoded_frames < bufferFrames ) { + qCWarning(decoder) << "Opus decoder returned " << decoded_frames << ", but " << bufferFrames << " were expected!"; - int start = decoded_frames * static_cast(sizeof(int16_t)) * _opus_num_channels; + int start = decoded_frames * static_cast(sizeof(int16_t)) * _opusNumChannels; memset( &decodedBuffer.data()[start], 0, static_cast(decodedBuffer.length() - start)); - } else if ( decoded_frames > buffer_frames ) { + } else if (decoded_frames > bufferFrames) { // This should never happen - qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << buffer_frames + qCCritical(decoder) << "Opus decoder returned " << decoded_frames << ", but only " << bufferFrames << " were expected! Buffer overflow!?"; } + } else { qCCritical(decoder) << "Failed to decode lost frame: " << error_to_string(decoded_frames); decodedBuffer.fill('\0'); } } - - diff --git a/plugins/opusCodec/src/OpusDecoder.h b/plugins/opusCodec/src/OpusDecoder.h index 01722fcc6c..f7ae6eb6da 100644 --- a/plugins/opusCodec/src/OpusDecoder.h +++ b/plugins/opusCodec/src/OpusDecoder.h @@ -13,7 +13,7 @@ #include -#include "opus/opus.h" +#include class AthenaOpusDecoder : public Decoder { @@ -23,16 +23,16 @@ public: virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; - virtual void lostFrame( QByteArray &decodedBuffer) override; + virtual void lostFrame(QByteArray &decodedBuffer) override; private: int _encodedSize; OpusDecoder *_decoder = nullptr; - int _opus_sample_rate = 0; - int _opus_num_channels = 0; - int _decoded_size = 0; + int _opusSampleRate = 0; + int _opusNumChannels = 0; + int _decodedSize = 0; }; diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 700164427e..e0b20a6e65 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -16,40 +16,39 @@ static QLoggingCategory encoder("AthenaOpusEncoder"); -static QString error_to_string(int error) { +static QString errorToString(int error) { switch (error) { - case OPUS_OK: - return "OK"; - case OPUS_BAD_ARG: - return "One or more invalid/out of range arguments."; - case OPUS_BUFFER_TOO_SMALL: - return "The mode struct passed is invalid."; - case OPUS_INTERNAL_ERROR: - return "An internal error was detected."; - case OPUS_INVALID_PACKET: - return "The compressed data passed is corrupted."; - case OPUS_UNIMPLEMENTED: - return "Invalid/unsupported request number."; - case OPUS_INVALID_STATE: - return "An encoder or decoder structure is invalid or already freed."; - default: - return QString("Unknown error code: %i").arg(error); + case OPUS_OK: + return "OK"; + case OPUS_BAD_ARG: + return "One or more invalid/out of range arguments."; + case OPUS_BUFFER_TOO_SMALL: + return "The mode struct passed is invalid."; + case OPUS_INTERNAL_ERROR: + return "An internal error was detected."; + case OPUS_INVALID_PACKET: + return "The compressed data passed is corrupted."; + case OPUS_UNIMPLEMENTED: + return "Invalid/unsupported request number."; + case OPUS_INVALID_STATE: + return "An encoder or decoder structure is invalid or already freed."; + default: + return QString("Unknown error code: %i").arg(error); } } -AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) -{ - _opus_sample_rate = sampleRate; - _opus_channels = numChannels; +AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) { + _opusSampleRate = sampleRate; + _opusChannels = numChannels; int error; _encoder = opus_encoder_create(sampleRate, numChannels, DEFAULT_APPLICATION, &error); - if ( error != OPUS_OK ) { - qCCritical(encoder) << "Failed to initialize Opus encoder: " << error_to_string(error); + if (error != OPUS_OK) { + qCCritical(encoder) << "Failed to initialize Opus encoder: " << errorToString(error); _encoder = nullptr; return; } @@ -62,8 +61,7 @@ AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) qCDebug(encoder) << "Opus encoder initialized, sampleRate = " << sampleRate << "; numChannels = " << numChannels; } -AthenaOpusEncoder::~AthenaOpusEncoder() -{ +AthenaOpusEncoder::~AthenaOpusEncoder() { opus_encoder_destroy(_encoder); } @@ -75,225 +73,201 @@ void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& enco assert(_encoder); encodedBuffer.resize( decodedBuffer.size() ); - int frame_size = decodedBuffer.length()/ _opus_channels / static_cast(sizeof(opus_int16)); + int frameSize = decodedBuffer.length()/ _opusChannels / static_cast(sizeof(opus_int16)); - int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.constData()), frame_size, + int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.constData()), frameSize, reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); - if ( bytes >= 0 ) { - //qCDebug(encoder) << "Encoded " << decodedBuffer.length() << " bytes into " << bytes << " opus bytes"; + if (bytes >= 0) { encodedBuffer.resize(bytes); } else { encodedBuffer.resize(0); qCWarning(encoder) << "Error when encoding " << decodedBuffer.length() << " bytes of audio: " - << error_to_string(bytes); + << errorToString(bytes); } } -int AthenaOpusEncoder::getComplexity() const -{ +int AthenaOpusEncoder::getComplexity() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_COMPLEXITY(&ret)); - return ret; + int returnValue; + opus_encoder_ctl(_encoder, OPUS_GET_COMPLEXITY(&returnValue)); + return returnValue; } -void AthenaOpusEncoder::setComplexity(int complexity) -{ +void AthenaOpusEncoder::setComplexity(int complexity) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_COMPLEXITY(complexity)); + int returnValue = opus_encoder_ctl(_encoder, OPUS_SET_COMPLEXITY(complexity)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting complexity to " << complexity << ": " << error_to_string(ret); + if (returnValue != OPUS_OK) { + qCWarning(encoder) << "Error when setting complexity to " << complexity << ": " << errorToString(returnValue); } } -int AthenaOpusEncoder::getBitrate() const -{ +int AthenaOpusEncoder::getBitrate() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_BITRATE(&ret)); - return ret; + int returnValue; + opus_encoder_ctl(_encoder, OPUS_GET_BITRATE(&returnValue)); + return returnValue; } -void AthenaOpusEncoder::setBitrate(int bitrate) -{ +void AthenaOpusEncoder::setBitrate(int bitrate) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_BITRATE(bitrate)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_BITRATE(bitrate)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting bitrate to " << bitrate << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting bitrate to " << bitrate << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getVBR() const -{ +int AthenaOpusEncoder::getVBR() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_VBR(&ret)); - return ret; + int returnValue; + opus_encoder_ctl(_encoder, OPUS_GET_VBR(&returnValue)); + return returnValue; } -void AthenaOpusEncoder::setVBR(int vbr) -{ +void AthenaOpusEncoder::setVBR(int vbr) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_VBR(vbr)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_VBR(vbr)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting VBR to " << vbr << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting VBR to " << vbr << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getVBRConstraint() const -{ +int AthenaOpusEncoder::getVBRConstraint() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_VBR_CONSTRAINT(&ret)); - return ret; + int returnValue; + opus_encoder_ctl(_encoder, OPUS_GET_VBR_CONSTRAINT(&returnValue)); + return returnValue; } -void AthenaOpusEncoder::setVBRConstraint(int vbr_const) -{ +void AthenaOpusEncoder::setVBRConstraint(int vbr_const) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_VBR_CONSTRAINT(vbr_const)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_VBR_CONSTRAINT(vbr_const)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting VBR constraint to " << vbr_const << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting VBR constraint to " << vbr_const << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getMaxBandwidth() const -{ +int AthenaOpusEncoder::getMaxBandwidth() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_MAX_BANDWIDTH(&ret)); - return ret; + int returnValue; + opus_encoder_ctl(_encoder, OPUS_GET_MAX_BANDWIDTH(&returnValue)); + return returnValue; } -void AthenaOpusEncoder::setMaxBandwidth(int maxbw) -{ +void AthenaOpusEncoder::setMaxBandwidth(int maxbw) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxbw)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxbw)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting max bandwidth to " << maxbw << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting max bandwidth to " << maxbw << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getBandwidth() const -{ +int AthenaOpusEncoder::getBandwidth() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_BANDWIDTH(&ret)); - return ret; + int bandwidth; + opus_encoder_ctl(_encoder, OPUS_GET_BANDWIDTH(&bandwidth)); + return bandwidth; } -void AthenaOpusEncoder::setBandwidth(int bw) -{ +void AthenaOpusEncoder::setBandwidth(int bandwidth) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_BANDWIDTH(bw)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_BANDWIDTH(bandwidth)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting bandwidth to " << bw << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting bandwidth to " << bandwidth << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getSignal() const -{ +int AthenaOpusEncoder::getSignal() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_SIGNAL(&ret)); - return ret; + int signal; + opus_encoder_ctl(_encoder, OPUS_GET_SIGNAL(&signal)); + return signal; } -void AthenaOpusEncoder::setSignal(int signal) -{ +void AthenaOpusEncoder::setSignal(int signal) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_SIGNAL(signal)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_SIGNAL(signal)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting signal to " << signal << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting signal to " << signal << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getApplication() const -{ +int AthenaOpusEncoder::getApplication() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_APPLICATION(&ret)); - return ret; + int applicationValue; + opus_encoder_ctl(_encoder, OPUS_GET_APPLICATION(&applicationValue)); + return applicationValue; } -void AthenaOpusEncoder::setApplication(int application) -{ +void AthenaOpusEncoder::setApplication(int application) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(application)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(application)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting application to " << application << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting application to " << application << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getLookahead() const -{ +int AthenaOpusEncoder::getLookahead() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_LOOKAHEAD(&ret)); - return ret; + int lookAhead; + opus_encoder_ctl(_encoder, OPUS_GET_LOOKAHEAD(&lookAhead)); + return lookAhead; } -int AthenaOpusEncoder::getInbandFEC() const -{ +int AthenaOpusEncoder::getInbandFEC() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_INBAND_FEC(&ret)); - return ret; + int fec; + opus_encoder_ctl(_encoder, OPUS_GET_INBAND_FEC(&fec)); + return fec; } -void AthenaOpusEncoder::setInbandFEC(int fec) -{ +void AthenaOpusEncoder::setInbandFEC(int fec) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(fec)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(fec)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting inband FEC to " << fec << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting inband FEC to " << fec << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getExpectedPacketLossPercent() const -{ +int AthenaOpusEncoder::getExpectedPacketLossPercent() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_PACKET_LOSS_PERC(&ret)); - return ret; + int lossPercentage; + opus_encoder_ctl(_encoder, OPUS_GET_PACKET_LOSS_PERC(&lossPercentage)); + return lossPercentage; } -void AthenaOpusEncoder::setExpectedPacketLossPercent(int perc) -{ +void AthenaOpusEncoder::setExpectedPacketLossPercentage(int percent) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(perc)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(percent)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting loss percent to " << perc << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting loss percent to " << percent << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getDTX() const -{ +int AthenaOpusEncoder::getDTX() const { assert(_encoder); - int ret; - opus_encoder_ctl(_encoder, OPUS_GET_DTX(&ret)); - return ret; + int dtx; + opus_encoder_ctl(_encoder, OPUS_GET_DTX(&dtx)); + return dtx; } -void AthenaOpusEncoder::setDTX(int dtx) -{ +void AthenaOpusEncoder::setDTX(int dtx) { assert(_encoder); - int ret = opus_encoder_ctl(_encoder, OPUS_SET_DTX(dtx)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_DTX(dtx)); - if (ret != OPUS_OK) { - qCWarning(encoder) << "Error when setting DTX to " << dtx << ": " << error_to_string(ret); + if (errorCode != OPUS_OK) { + qCWarning(encoder) << "Error when setting DTX to " << dtx << ": " << errorToString(errorCode); } } \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index 920423a96d..fc1ca0b827 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -58,7 +58,7 @@ public: void setInbandFEC(int fec); int getExpectedPacketLossPercent() const; - void setExpectedPacketLossPercent(int perc); + void setExpectedPacketLossPercentage(int percent); int getDTX() const; void setDTX(int dtx); @@ -66,9 +66,9 @@ public: private: - int _opus_sample_rate = 0; - int _opus_channels = 0; - int _opus_expected_loss = 0; + int _opusSampleRate = 0; + int _opusChannels = 0; + int _opusExpectedLoss = 0; OpusEncoder* _encoder = nullptr; From fef469cca45c7a2bc14509a24019a1dd7671d09a Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Wed, 15 Jan 2020 18:00:27 -0600 Subject: [PATCH 30/45] More style fixes for the Great and Powerful Thoys --- plugins/opusCodec/CMakeLists.txt | 2 +- plugins/opusCodec/src/OpusDecoder.h | 2 +- plugins/opusCodec/src/OpusEncoder.cpp | 22 +++++++++++----------- plugins/opusCodec/src/OpusEncoder.h | 14 +++++++------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/plugins/opusCodec/CMakeLists.txt b/plugins/opusCodec/CMakeLists.txt index 06ed6eab95..583aff85d6 100644 --- a/plugins/opusCodec/CMakeLists.txt +++ b/plugins/opusCodec/CMakeLists.txt @@ -12,5 +12,5 @@ link_hifi_libraries(shared audio plugins) target_opus() if (BUILD_SERVER) - install_beside_console() + install_beside_console() endif () diff --git a/plugins/opusCodec/src/OpusDecoder.h b/plugins/opusCodec/src/OpusDecoder.h index f7ae6eb6da..095893856b 100644 --- a/plugins/opusCodec/src/OpusDecoder.h +++ b/plugins/opusCodec/src/OpusDecoder.h @@ -29,7 +29,7 @@ public: private: int _encodedSize; - OpusDecoder *_decoder = nullptr; + OpusDecoder* _decoder = nullptr; int _opusSampleRate = 0; int _opusNumChannels = 0; int _decodedSize = 0; diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index e0b20a6e65..124ca3e3e1 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -41,7 +41,7 @@ static QString errorToString(int error) { AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) { _opusSampleRate = sampleRate; - _opusChannels = numChannels; + _opusChannels = numChannels; int error; @@ -72,8 +72,8 @@ void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& enco PerformanceTimer perfTimer("AthenaOpusEncoder::encode"); assert(_encoder); - encodedBuffer.resize( decodedBuffer.size() ); - int frameSize = decodedBuffer.length()/ _opusChannels / static_cast(sizeof(opus_int16)); + encodedBuffer.resize(decodedBuffer.size()); + int frameSize = decodedBuffer.length() / _opusChannels / static_cast(sizeof(opus_int16)); int bytes = opus_encode(_encoder, reinterpret_cast(decodedBuffer.constData()), frameSize, reinterpret_cast(encodedBuffer.data()), encodedBuffer.size() ); @@ -231,28 +231,28 @@ int AthenaOpusEncoder::getInbandFEC() const { return fec; } -void AthenaOpusEncoder::setInbandFEC(int fec) { +void AthenaOpusEncoder::setInbandFEC(int inBandFEC) { assert(_encoder); - int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(fec)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(inBandFEC)); if (errorCode != OPUS_OK) { - qCWarning(encoder) << "Error when setting inband FEC to " << fec << ": " << errorToString(errorCode); + qCWarning(encoder) << "Error when setting inband FEC to " << inBandFEC << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getExpectedPacketLossPercent() const { +int AthenaOpusEncoder::getExpectedPacketLossPercentage() const { assert(_encoder); int lossPercentage; opus_encoder_ctl(_encoder, OPUS_GET_PACKET_LOSS_PERC(&lossPercentage)); return lossPercentage; } -void AthenaOpusEncoder::setExpectedPacketLossPercentage(int percent) { +void AthenaOpusEncoder::setExpectedPacketLossPercentage(int percentage) { assert(_encoder); - int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(percent)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(percentage)); if (errorCode != OPUS_OK) { - qCWarning(encoder) << "Error when setting loss percent to " << percent << ": " << errorToString(errorCode); + qCWarning(encoder) << "Error when setting loss percent to " << percentage << ": " << errorToString(errorCode); } } @@ -270,4 +270,4 @@ void AthenaOpusEncoder::setDTX(int dtx) { if (errorCode != OPUS_OK) { qCWarning(encoder) << "Error when setting DTX to " << dtx << ": " << errorToString(errorCode); } -} \ No newline at end of file +} diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index fc1ca0b827..107b0f82ca 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -38,27 +38,27 @@ public: void setVBR(int vbr); int getVBRConstraint() const; - void setVBRConstraint(int vbr_const); + void setVBRConstraint(int vbrConstraint); int getMaxBandwidth() const; - void setMaxBandwidth(int maxbw); + void setMaxBandwidth(int maxBandwidth); int getBandwidth() const; - void setBandwidth(int bw); + void setBandwidth(int bandwidth); int getSignal() const; void setSignal(int signal); int getApplication() const; - void setApplication(int app); + void setApplication(int application); int getLookahead() const; int getInbandFEC() const; - void setInbandFEC(int fec); + void setInbandFEC(int inBandFEC); - int getExpectedPacketLossPercent() const; - void setExpectedPacketLossPercentage(int percent); + int getExpectedPacketLossPercentage() const; + void setExpectedPacketLossPercentage(int percentage); int getDTX() const; void setDTX(int dtx); From 26d507c39a6bc3b8671b240b4f17c84e7a1e4cdd Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Thu, 16 Jan 2020 11:30:45 -0600 Subject: [PATCH 31/45] Yet more style fixes. --- plugins/opusCodec/src/OpusDecoder.cpp | 7 ++++--- plugins/opusCodec/src/OpusEncoder.cpp | 6 +++--- plugins/opusCodec/src/OpusEncoder.h | 10 +++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index e95d813fb4..40e4954cb3 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -57,8 +57,9 @@ AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) { } AthenaOpusDecoder::~AthenaOpusDecoder() { - if (_decoder) + if (_decoder) { opus_decoder_destroy(_decoder); + } } @@ -71,7 +72,7 @@ void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &deco * _opusNumChannels; decodedBuffer.resize(bufferSize); - int bufferFrames = decodedBuffer.size() / _opusNumChannels / static_cast(sizeof( opus_int16 )); + int bufferFrames = decodedBuffer.size() / _opusNumChannels / static_cast(sizeof(opus_int16)); qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " << decodedBuffer.size() << "; frameCount = " << bufferFrames; @@ -109,7 +110,7 @@ void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) { int bufferFrames = decodedBuffer.size() / _opusNumChannels / static_cast(sizeof(opus_int16)); int decoded_frames = opus_decode(_decoder, nullptr, 0, reinterpret_cast(decodedBuffer.data()), - bufferFrames, 1 ); + bufferFrames, 1); if (decoded_frames >= 0) { diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 124ca3e3e1..75b190b0dc 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -160,12 +160,12 @@ int AthenaOpusEncoder::getMaxBandwidth() const { return returnValue; } -void AthenaOpusEncoder::setMaxBandwidth(int maxbw) { +void AthenaOpusEncoder::setMaxBandwidth(int maxBandwidth) { assert(_encoder); - int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxbw)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxBandwidth)); if (errorCode != OPUS_OK) { - qCWarning(encoder) << "Error when setting max bandwidth to " << maxbw << ": " << errorToString(errorCode); + qCWarning(encoder) << "Error when setting max bandwidth to " << maxBandwidth << ": " << errorToString(errorCode); } } diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index 107b0f82ca..10640bf409 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -16,11 +16,6 @@ class AthenaOpusEncoder : public Encoder { public: - const int DEFAULT_BITRATE = 128000; - const int DEFAULT_COMPLEXITY = 10; - const int DEFAULT_APPLICATION = OPUS_APPLICATION_VOIP; - const int DEFAULT_SIGNAL = OPUS_AUTO; - AthenaOpusEncoder(int sampleRate, int numChannels); ~AthenaOpusEncoder() override; @@ -66,6 +61,11 @@ public: private: + const int DEFAULT_BITRATE = 128000; + const int DEFAULT_COMPLEXITY = 10; + const int DEFAULT_APPLICATION = OPUS_APPLICATION_VOIP; + const int DEFAULT_SIGNAL = OPUS_AUTO; + int _opusSampleRate = 0; int _opusChannels = 0; int _opusExpectedLoss = 0; From cb78daa0b193a62819a9207aab1282211b9a66ad Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Thu, 16 Jan 2020 14:34:08 -0600 Subject: [PATCH 32/45] Delete trailing whitespace. --- plugins/opusCodec/src/OpusEncoder.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 75b190b0dc..3408701633 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -147,7 +147,7 @@ int AthenaOpusEncoder::getVBRConstraint() const { void AthenaOpusEncoder::setVBRConstraint(int vbr_const) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_VBR_CONSTRAINT(vbr_const)); - + if (errorCode != OPUS_OK) { qCWarning(encoder) << "Error when setting VBR constraint to " << vbr_const << ": " << errorToString(errorCode); } @@ -163,7 +163,7 @@ int AthenaOpusEncoder::getMaxBandwidth() const { void AthenaOpusEncoder::setMaxBandwidth(int maxBandwidth) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxBandwidth)); - + if (errorCode != OPUS_OK) { qCWarning(encoder) << "Error when setting max bandwidth to " << maxBandwidth << ": " << errorToString(errorCode); } From 997abf654345261572acd5d071bddcfa914deda3 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sun, 19 Jan 2020 15:40:23 -0600 Subject: [PATCH 33/45] Remove logging statement. --- plugins/opusCodec/src/OpusDecoder.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/plugins/opusCodec/src/OpusDecoder.cpp index 40e4954cb3..e3e4e3645a 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/plugins/opusCodec/src/OpusDecoder.cpp @@ -73,9 +73,6 @@ void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &deco decodedBuffer.resize(bufferSize); int bufferFrames = decodedBuffer.size() / _opusNumChannels / static_cast(sizeof(opus_int16)); - - qCDebug(decoder) << "Opus decode: encodedBytes = " << encodedBuffer.length() << "; decodedBufferBytes = " - << decodedBuffer.size() << "; frameCount = " << bufferFrames; int decoded_frames = opus_decode(_decoder, reinterpret_cast(encodedBuffer.data()), encodedBuffer.length(), reinterpret_cast(decodedBuffer.data()), bufferFrames, 0); From b8e41fd244c68b2063193a488e7964ad5ffcb63b Mon Sep 17 00:00:00 2001 From: Fluffy Jenkins Date: Tue, 28 Jan 2020 22:06:27 +0000 Subject: [PATCH 34/45] Added basic beta default scripts feature --- scripts/defaultScripts.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/scripts/defaultScripts.js b/scripts/defaultScripts.js index 795458f05d..8ba12f8e8b 100644 --- a/scripts/defaultScripts.js +++ b/scripts/defaultScripts.js @@ -39,7 +39,7 @@ var DEFAULT_SCRIPTS_COMBINED = [ var DEFAULT_SCRIPTS_SEPARATE = [ "system/controllers/controllerScripts.js", "communityModules/notificationCore/notificationCore.js", - "communityModules/chat/FloofChat.js" + {"stable": "communityModules/chat/FloofChat.js", "beta": "https://content.fluffy.ws/scripts/chat/FloofChat.js"} //"system/chat.js" ]; @@ -53,7 +53,8 @@ var MENU_CATEGORY = "Developer > Scripting"; var MENU_ITEM = "Debug defaultScripts.js"; var SETTINGS_KEY = '_debugDefaultScriptsIsChecked'; -var previousSetting = Settings.getValue(SETTINGS_KEY); +var SETTINGS_KEY_BETA = '_betaDefaultScriptsIsChecked'; +var previousSetting = Settings.getValue(SETTINGS_KEY, false); if (previousSetting === '' || previousSetting === false || previousSetting === 'false') { previousSetting = false; @@ -71,23 +72,31 @@ if (Menu.menuExists(MENU_CATEGORY) && !Menu.menuItemExists(MENU_CATEGORY, MENU_I isChecked: previousSetting, }); } - function loadSeparateDefaults() { var currentlyRunningScripts = ScriptDiscoveryService.getRunning(); for (var i in DEFAULT_SCRIPTS_SEPARATE) { var shouldLoadCurrentDefaultScript = true; + var scriptItem = DEFAULT_SCRIPTS_SEPARATE[i]; + if (typeof scriptItem === "object") { + if (previousSettingBeta) { + console.info("Loading Beta item " + scriptItem.beta); + scriptItem = scriptItem.beta; + } else { + scriptItem = scriptItem.stable; + } + } for (var j = 0; j < currentlyRunningScripts.length; j++) { var currentRunningScriptObject = currentlyRunningScripts[j]; - var currentDefaultScriptName = DEFAULT_SCRIPTS_SEPARATE[i].substr((DEFAULT_SCRIPTS_SEPARATE[i].lastIndexOf("/") + 1), DEFAULT_SCRIPTS_SEPARATE[i].length); + var currentDefaultScriptName = scriptItem.substr((scriptItem.lastIndexOf("/") + 1), scriptItem.length); if (currentDefaultScriptName === currentRunningScriptObject.name) { shouldLoadCurrentDefaultScript = false; } } if (shouldLoadCurrentDefaultScript) { - Script.load(DEFAULT_SCRIPTS_SEPARATE[i]); + Script.load(scriptItem); } } } @@ -161,7 +170,7 @@ function removeMenuItem() { } } -Script.scriptEnding.connect(function() { +Script.scriptEnding.connect(function () { removeMenuItem(); }); From a66bbcb12564a6b6a7b53fffe06a07732467953b Mon Sep 17 00:00:00 2001 From: Fluffy Jenkins Date: Tue, 28 Jan 2020 22:23:01 +0000 Subject: [PATCH 35/45] Made fixes --- scripts/defaultScripts.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/defaultScripts.js b/scripts/defaultScripts.js index 8ba12f8e8b..af802ddc1e 100644 --- a/scripts/defaultScripts.js +++ b/scripts/defaultScripts.js @@ -54,7 +54,8 @@ var MENU_ITEM = "Debug defaultScripts.js"; var SETTINGS_KEY = '_debugDefaultScriptsIsChecked'; var SETTINGS_KEY_BETA = '_betaDefaultScriptsIsChecked'; -var previousSetting = Settings.getValue(SETTINGS_KEY, false); +var previousSetting = Settings.getValue(SETTINGS_KEY, false);; +var previousSettingBeta = Settings.getValue(SETTINGS_KEY_BETA, false); if (previousSetting === '' || previousSetting === false || previousSetting === 'false') { previousSetting = false; @@ -91,7 +92,7 @@ function loadSeparateDefaults() { var currentRunningScriptObject = currentlyRunningScripts[j]; var currentDefaultScriptName = scriptItem.substr((scriptItem.lastIndexOf("/") + 1), scriptItem.length); if (currentDefaultScriptName === currentRunningScriptObject.name) { - shouldLoadCurrentDefaultScript = false; + ScriptDiscoveryService.stopScript(currentRunningScriptObject.url); } } From 5492759a0c8223f9838b920d4f932fefbb35ebe2 Mon Sep 17 00:00:00 2001 From: Fluffy Jenkins Date: Wed, 29 Jan 2020 19:50:23 +0000 Subject: [PATCH 36/45] Made requested changes and changed how it loads --- scripts/defaultScripts.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/defaultScripts.js b/scripts/defaultScripts.js index af802ddc1e..7ea1632314 100644 --- a/scripts/defaultScripts.js +++ b/scripts/defaultScripts.js @@ -54,7 +54,7 @@ var MENU_ITEM = "Debug defaultScripts.js"; var SETTINGS_KEY = '_debugDefaultScriptsIsChecked'; var SETTINGS_KEY_BETA = '_betaDefaultScriptsIsChecked'; -var previousSetting = Settings.getValue(SETTINGS_KEY, false);; +var previousSetting = Settings.getValue(SETTINGS_KEY, false); var previousSettingBeta = Settings.getValue(SETTINGS_KEY_BETA, false); if (previousSetting === '' || previousSetting === false || previousSetting === 'false') { @@ -73,6 +73,7 @@ if (Menu.menuExists(MENU_CATEGORY) && !Menu.menuItemExists(MENU_CATEGORY, MENU_I isChecked: previousSetting, }); } + function loadSeparateDefaults() { var currentlyRunningScripts = ScriptDiscoveryService.getRunning(); @@ -81,7 +82,7 @@ function loadSeparateDefaults() { var scriptItem = DEFAULT_SCRIPTS_SEPARATE[i]; if (typeof scriptItem === "object") { if (previousSettingBeta) { - console.info("Loading Beta item " + scriptItem.beta); + console.log("Loading Beta item " + scriptItem.beta); scriptItem = scriptItem.beta; } else { scriptItem = scriptItem.stable; @@ -92,7 +93,11 @@ function loadSeparateDefaults() { var currentRunningScriptObject = currentlyRunningScripts[j]; var currentDefaultScriptName = scriptItem.substr((scriptItem.lastIndexOf("/") + 1), scriptItem.length); if (currentDefaultScriptName === currentRunningScriptObject.name) { - ScriptDiscoveryService.stopScript(currentRunningScriptObject.url); + if (currentRunningScriptObject.url !== scriptItem) { + ScriptDiscoveryService.stopScript(currentRunningScriptObject.url); + } else { + shouldLoadCurrentDefaultScript = false; + } } } From d3e2f7204b9141f0b98a990163d649132e59a84a Mon Sep 17 00:00:00 2001 From: Kasen IO Date: Thu, 30 Jan 2020 11:30:31 -0500 Subject: [PATCH 37/45] init --- .../static/styles/responsive.css | 0 .../hifi-jsdoc-template/tmpl/layout.tmpl | 43 ++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 tools/jsdoc/hifi-jsdoc-template/static/styles/responsive.css diff --git a/tools/jsdoc/hifi-jsdoc-template/static/styles/responsive.css b/tools/jsdoc/hifi-jsdoc-template/static/styles/responsive.css new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/jsdoc/hifi-jsdoc-template/tmpl/layout.tmpl b/tools/jsdoc/hifi-jsdoc-template/tmpl/layout.tmpl index 7093109ae1..0a06d07459 100644 --- a/tools/jsdoc/hifi-jsdoc-template/tmpl/layout.tmpl +++ b/tools/jsdoc/hifi-jsdoc-template/tmpl/layout.tmpl @@ -5,7 +5,7 @@ <?js= title ?> - + @@ -16,6 +16,7 @@ + @@ -39,27 +40,39 @@ var nightSheet = document.querySelector('[href="styles/night.css"]'); nightSheet.disabled = darkDisabled; } + + var responsiveDisabled = JSON.parse(localStorage.getItem('responsiveDisabled')); + if (responsiveDisabled == null) { + localStorage.setItem('responsiveDisabled', JSON.stringify(false)); + } else { + var responsiveSheet = document.querySelector('[href="styles/responsive.css"]'); + responsiveSheet.disabled = responsiveDisabled; + } - -
-

- - - -

Looking for Project Athena
Documentation?

-
- + +
+
+

+ + + +

Looking for Project Athena
Documentation?

Toggle mdi-theme-light-dark -
+ + + Toggle + mdi-monitor-cellphone +
+