From a39ad97a1f7aa64892a4d52b3cbe70459989fe73 Mon Sep 17 00:00:00 2001 From: Marcus Llewellyn Date: Sat, 21 Dec 2019 12:05:14 -0600 Subject: [PATCH] Opus cleanup and C++ wrapper Did a tiny amount of fleshing out and some cleanup. Also added a Google implemented C++ Opus wrapper in case it will be useful. --- plugins/opusCodec/src/OpusCodecManager.cpp | 29 +++- plugins/opusCodec/src/OpusCodecManager.h | 16 +- plugins/opusCodec/src/OpusWrapper.cpp | 174 +++++++++++++++++++++ plugins/opusCodec/src/OpusWrapper.h | 131 ++++++++++++++++ 4 files changed, 333 insertions(+), 17 deletions(-) create mode 100644 plugins/opusCodec/src/OpusWrapper.cpp create mode 100644 plugins/opusCodec/src/OpusWrapper.h diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp index 3c47ac15fb..8f7d4fe6e5 100644 --- a/plugins/opusCodec/src/OpusCodecManager.cpp +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -22,6 +22,15 @@ #include #include +#define FRAME_SIZE 960 +#define SAMPLE_RATE 48000 +#define CHANNELS 2 +#define APPLICATION OPUS_APPLICATION_AUDIO +#define BITRATE 64000 + +#define MAX_FRAME_SIZE 6*FRAME_SIZE +#define MAX_PACKET_SIZE 3*1276 + const char* OpusCodec::NAME { "opus" }; void OpusCodec::init() { @@ -45,9 +54,23 @@ bool OpusCodec::isSupported() const { } +class OpusEncoder : public Encoder { +public: + OpusEncoder(int sampleRate, int numChannels) { + + } + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer.resize(_encodedSize); + } + +private: + int _encodedSize; +}; + Encoder* OpusCodec::createEncoder(int sampleRate, int numChannels) { - return this; + return new OpusEncoder(sampleRate, numChannels); } Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { @@ -55,9 +78,9 @@ Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { } void OpusCodec::releaseEncoder(Encoder* encoder) { - // do nothing + delete encoder; } void OpusCodec::releaseDecoder(Decoder* decoder) { - // do nothing + delete decoder; } \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusCodecManager.h b/plugins/opusCodec/src/OpusCodecManager.h index b02a549e8c..b75e78b23c 100644 --- a/plugins/opusCodec/src/OpusCodecManager.h +++ b/plugins/opusCodec/src/OpusCodecManager.h @@ -9,8 +9,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef hifi__opusCodecManager_h -#define hifi__opusCodecManager_h +#ifndef hifi__OpusCodecManager_h +#define hifi__OpusCodecManager_h #include @@ -35,18 +35,6 @@ public: virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = decodedBuffer; - } - - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = encodedBuffer; - } - - virtual void lostFrame(QByteArray& decodedBuffer) override { - memset(decodedBuffer.data(), 0, decodedBuffer.size()); - } - private: static const char* NAME; }; diff --git a/plugins/opusCodec/src/OpusWrapper.cpp b/plugins/opusCodec/src/OpusWrapper.cpp new file mode 100644 index 0000000000..0d8ff87965 --- /dev/null +++ b/plugins/opusCodec/src/OpusWrapper.cpp @@ -0,0 +1,174 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "OpusWrapper.h" + +std::string opus::ErrorToString(int error) { + switch (error) { + case OPUS_OK: + return "OK"; + case OPUS_BAD_ARG: + return "One or more invalid/out of range arguments."; + case OPUS_BUFFER_TOO_SMALL: + return "The mode struct passed is invalid."; + case OPUS_INTERNAL_ERROR: + return "An internal error was detected."; + case OPUS_INVALID_PACKET: + return "The compressed data passed is corrupted."; + case OPUS_UNIMPLEMENTED: + return "Invalid/unsupported request number."; + case OPUS_INVALID_STATE: + return "An encoder or decoder structure is invalid or already freed."; + default: + return "Unknown error code: " + std::to_string(error); + } +} + +void opus::internal::OpusDestroyer::operator()(OpusEncoder* encoder) const +noexcept { + opus_encoder_destroy(encoder); +} + +void opus::internal::OpusDestroyer::operator()(OpusDecoder* decoder) const +noexcept { + opus_decoder_destroy(decoder); +} + +opus::Encoder::Encoder(opus_int32 sample_rate, int num_channels, + int application, int expected_loss_percent) + : num_channels_{ num_channels } { + int error{}; + encoder_.reset( + opus_encoder_create(sample_rate, num_channels, application, &error)); + valid_ = error == OPUS_OK; + if (!valid()) { + // LOG(INFO) << "Could not construct encoder. Error: " << ErrorToString(error); + return; + } + if (expected_loss_percent > 0) { + // LOG(INFO) << "Enabling FEC in the encoder."; + Ctl(OPUS_SET_INBAND_FEC(1)); + Ctl(OPUS_SET_PACKET_LOSS_PERC(expected_loss_percent)); + } +} + +bool opus::Encoder::ResetState() { + valid_ = Ctl(OPUS_RESET_STATE) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetBitrate(int bitrate) { + valid_ = Ctl(OPUS_SET_BITRATE(bitrate)) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetVariableBitrate(int vbr) { + valid_ = Ctl(OPUS_SET_VBR(vbr)) == OPUS_OK; + return valid_; +} + +bool opus::Encoder::SetComplexity(int complexity) { + valid_ = Ctl(OPUS_SET_COMPLEXITY(complexity)) == OPUS_OK; + return valid_; +} + +int opus::Encoder::GetLookahead() { + opus_int32 skip{}; + valid_ = Ctl(OPUS_GET_LOOKAHEAD(&skip)) == OPUS_OK; + return skip; +} + +std::vector> opus::Encoder::Encode( + const std::vector& pcm, int frame_size) { + constexpr auto sample_size = sizeof(pcm[0]); + const auto frame_length = frame_size * num_channels_ * sample_size; + auto data_length = pcm.size() * sample_size; + if (data_length % frame_length != 0u) { + // LOG(WARNING) << "PCM samples contain an incomplete frame. Ignoring the " + // "incomplete frame."; + data_length -= (data_length % frame_length); + } + + std::vector> encoded; + for (std::size_t i{}; i < data_length; i += frame_length) { + encoded.push_back(EncodeFrame(pcm.begin() + (i / sample_size), frame_size)); + } + return encoded; +} + +std::vector opus::Encoder::EncodeFrame( + const std::vector::const_iterator& frame_start, + int frame_size) { + const auto frame_length = (frame_size * num_channels_ * sizeof(*frame_start)); + std::vector encoded(frame_length); + auto num_bytes = opus_encode(encoder_.get(), &*frame_start, frame_size, + encoded.data(), encoded.size()); + if (num_bytes < 0) { + // LOG(ERROR) << "Encode error: " << opus::ErrorToString(num_bytes); + return {}; + } + encoded.resize(num_bytes); + return encoded; +} + +opus::Decoder::Decoder(opus_uint32 sample_rate, int num_channels) + : num_channels_(num_channels) { + int error{}; + decoder_.reset(opus_decoder_create(sample_rate, num_channels, &error)); + valid_ = error == OPUS_OK; +} + +std::vector opus::Decoder::Decode( + const std::vector>& packets, int frame_size, + bool decode_fec) { + std::vector decoded; + for (const auto& enc : packets) { + auto just_decoded = Decode(enc, frame_size, decode_fec); + decoded.insert(std::end(decoded), std::begin(just_decoded), + std::end(just_decoded)); + } + return decoded; +} + +std::vector opus::Decoder::Decode( + const std::vector& packet, int frame_size, bool decode_fec) { + const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); + std::vector decoded(frame_length); + auto num_samples = opus_decode(decoder_.get(), packet.data(), packet.size(), + decoded.data(), frame_size, decode_fec); + if (num_samples < 0) { + // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); + return {}; + } + decoded.resize(num_samples * num_channels_); + return decoded; +} + +std::vector opus::Decoder::DecodeDummy(int frame_size) { + const auto frame_length = (frame_size * num_channels_ * sizeof(opus_int16)); + std::vector decoded(frame_length); + auto num_samples = + opus_decode(decoder_.get(), nullptr, 0, decoded.data(), frame_size, true); + if (num_samples < 0) { + // LOG(ERROR) << "Decode error: " << opus::ErrorToString(num_samples); + return {}; + } + decoded.resize(num_samples * num_channels_); + return decoded; +} \ No newline at end of file diff --git a/plugins/opusCodec/src/OpusWrapper.h b/plugins/opusCodec/src/OpusWrapper.h new file mode 100644 index 0000000000..713dff87ff --- /dev/null +++ b/plugins/opusCodec/src/OpusWrapper.h @@ -0,0 +1,131 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPUSCPP_OPUS_WRAPPER_H_ +#define OPUSCPP_OPUS_WRAPPER_H_ + +#include +#include +#include + +#include "opus/opus.h" + +namespace opus { + + std::string ErrorToString(int error); + + namespace internal { + // Deleter for OpusEncoders and OpusDecoders + struct OpusDestroyer { + void operator()(OpusEncoder* encoder) const noexcept; + void operator()(OpusDecoder* decoder) const noexcept; + }; + template + using opus_uptr = std::unique_ptr; + } // namespace internal + + class Encoder { + public: + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gaa89264fd93c9da70362a0c9b96b9ca88 + // Fs corresponds to sample_rate + // + // If expected_loss_percent is positive, FEC will be enabled + Encoder(opus_int32 sample_rate, int num_channels, int application, + int expected_loss_percent = 0); + + // Resets internal state of encoder. This should be called between encoding + // different streams so that back-to-back decoding and one-at-a-time decoding + // give the same result. Returns true on success. + bool ResetState(); + + // Sets the desired bitrate. Rates from 500 to 512000 are meaningful as well + // as the special values OPUS_AUTO and OPUS_BITRATE_MAX. If this method + // is not called, the default value of OPUS_AUTO is used. + // Returns true on success. + bool SetBitrate(int bitrate); + + // Enables or disables variable bitrate in the encoder. By default, variable + // bitrate is enabled. Returns true on success. + bool SetVariableBitrate(int vbr); + + // Sets the computational complexity of the encoder, in the range of 0 to 10, + // inclusive, with 10 being the highest complexity. Returns true on success. + bool SetComplexity(int complexity); + + // Gets the total samples of delay added by the entire codec. This value + // is the minimum amount of 'preskip' that has to be specified in an + // ogg-stream that encapsulates the encoded audio. + int GetLookahead(); + + // Takes audio data and encodes it. Returns a sequence of encoded packets. + // pcm.size() must be divisible by frame_size * (number of channels); + // pcm must not contain any incomplete packets. + // see documentation for pcm and frame_size at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__encoder.html#gad2d6bf6a9ffb6674879d7605ed073e25 + std::vector> Encode( + const std::vector& pcm, int frame_size); + + int valid() const { return valid_; } + + private: + std::vector EncodeFrame( + const std::vector::const_iterator& frame_start, + int frame_size); + + template + int Ctl(int request, Ts... args) const { + return opus_encoder_ctl(encoder_.get(), request, args...); + } + + int num_channels_{}; + bool valid_{}; + internal::opus_uptr encoder_; + }; + + class Decoder { + public: + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga753f6fe0b699c81cfd47d70c8e15a0bd + // Fs corresponds to sample_rate + Decoder(opus_uint32 sample_rate, int num_channels); + + // Takes a sequence of encoded packets and decodes them. Returns the decoded + // audio. + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 + std::vector Decode( + const std::vector>& packets, int frame_size, + bool decode_fec); + + int valid() const { return valid_; } + + // Takes an encoded packet and decodes it. Returns the decoded audio + // see documentation at: + // https://mf4.xiph.org/jenkins/view/opus/job/opus/ws/doc/html/group__opus__decoder.html#ga7d1111f64c36027ddcb81799df9b3fc9 + std::vector Decode(const std::vector& packet, + int frame_size, bool decode_fec); + + // Generates a dummy frame by passing nullptr to the underlying opus decode. + std::vector DecodeDummy(int frame_size); + + private: + int num_channels_{}; + bool valid_{}; + internal::opus_uptr decoder_; + }; + +} // namespace opus + +#endif \ No newline at end of file