From 6a9344a3f6b5f966a3d9b8d5c64db8f4008fe6ea Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Wed, 8 Jul 2015 15:14:24 -0700 Subject: [PATCH 1/4] fix bug in old version of decode --- libraries/shared/src/ByteCountCoding.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libraries/shared/src/ByteCountCoding.h b/libraries/shared/src/ByteCountCoding.h index 2a39ee7a8c..efde19d45b 100644 --- a/libraries/shared/src/ByteCountCoding.h +++ b/libraries/shared/src/ByteCountCoding.h @@ -21,6 +21,8 @@ #include #include +#include + #include #include @@ -111,12 +113,12 @@ template inline QByteArray ByteCountCoded::encode() const { } template inline void ByteCountCoded::decode(const QByteArray& fromEncodedBytes) { - // first convert the ByteArray into a BitArray... QBitArray encodedBits; int bitCount = BITS_IN_BYTE * fromEncodedBytes.count(); encodedBits.resize(bitCount); + // copies the QByteArray into a QBitArray for(int byte = 0; byte < fromEncodedBytes.count(); byte++) { char originalByte = fromEncodedBytes.at(byte); for(int bit = 0; bit < BITS_IN_BYTE; bit++) { @@ -128,8 +130,8 @@ template inline void ByteCountCoded::decode(const QByteArray& fro } // next, read the leading bits to determine the correct number of bytes to decode (may not match the QByteArray) - int encodedByteCount = 0; - int leadBits = 1; + int encodedByteCount = 1; // there is at least 1 byte (after the leadBits) + int leadBits = 1; // there is always at least 1 lead bit int bitAt; for (bitAt = 0; bitAt < bitCount; bitAt++) { if (encodedBits.at(bitAt)) { @@ -139,16 +141,15 @@ template inline void ByteCountCoded::decode(const QByteArray& fro break; } } - encodedByteCount++; // always at least one byte - int expectedBitCount = encodedByteCount * BITS_IN_BYTE; + int expectedBitCount = (encodedByteCount * BITS_IN_BYTE) - leadBits; T value = 0; - + if (expectedBitCount <= (encodedBits.size() - leadBits)) { // Now, keep reading... int valueStartsAt = bitAt + 1; T bitValue = 1; - for (bitAt = valueStartsAt; bitAt < expectedBitCount; bitAt++) { + for (bitAt = valueStartsAt; bitAt < (expectedBitCount + leadBits); bitAt++) { if(encodedBits.at(bitAt)) { value += bitValue; } From e885ac1821c26c7e3ebed8e92db38239e1770255 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Wed, 8 Jul 2015 17:32:51 -0700 Subject: [PATCH 2/4] improved performance of ByteCountCoding<> decode --- interface/src/Application.cpp | 1 + interface/src/Util.cpp | 72 +++++++++++++++++++++ interface/src/Util.h | 1 + libraries/shared/src/ByteCountCoding.h | 88 ++++++++++++++------------ 4 files changed, 123 insertions(+), 39 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 48fa53777d..cd15f2f4c7 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1014,6 +1014,7 @@ void Application::paintGL() { void Application::runTests() { runTimingTests(); + //runUnitTests(); } void Application::audioMuteToggled() { diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index bf4df3f3d2..5f04010b47 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -21,6 +21,7 @@ #include +#include #include #include @@ -229,6 +230,43 @@ void runTimingTests() { elapsedUsecs = (float)startTime.nsecsElapsed() * NSEC_TO_USEC; qCDebug(interfaceapp, "vec3 assign and dot() usecs: %f, last result:%f", (double)(elapsedUsecs / numTests), (double)result); + + + quint64 BYTE_CODE_MAX_VALUE = 99999999; + quint64 BYTE_CODE_TESTS_SKIP = 999; + + QByteArray extraJunk; + const int EXTRA_JUNK_SIZE = 200; + extraJunk.append((unsigned char)255); + for (int i = 0; i < EXTRA_JUNK_SIZE; i++) { + extraJunk.append(QString("junk")); + } + + { + startTime.start(); + quint64 tests = 0; + quint64 failed = 0; + for (quint64 value = 0; value < BYTE_CODE_MAX_VALUE; value += BYTE_CODE_TESTS_SKIP) { + quint64 valueA = value; // usecTimestampNow(); + ByteCountCoded codedValueA = valueA; + QByteArray codedValueABuffer = codedValueA; + codedValueABuffer.append(extraJunk); + ByteCountCoded decodedValueA; + decodedValueA.decode(codedValueABuffer); + quint64 valueADecoded = decodedValueA; + tests++; + if (valueA != valueADecoded) { + qDebug() << "FAILED! value:" << valueA << "decoded:" << valueADecoded; + failed++; + } + + } + elapsedUsecs = (float)startTime.nsecsElapsed() * NSEC_TO_USEC; + qCDebug(interfaceapp) << "ByteCountCoded usecs: " << elapsedUsecs + << "per test:" << (double) (elapsedUsecs / tests) + << "tests:" << tests + << "failed:" << failed; + } } bool rayIntersectsSphere(const glm::vec3& rayStarting, const glm::vec3& rayNormalizedDirection, @@ -271,3 +309,37 @@ bool pointInSphere(glm::vec3& point, glm::vec3& sphereCenter, double sphereRadiu } return false; } + +void runUnitTests() { + + quint64 LAST_TEST = 10; + quint64 SKIP_BY = 1; + + for (quint64 value = 0; value <= LAST_TEST; value += SKIP_BY) { + qDebug() << "value:" << value; + + ByteCountCoded codedValue = value; + + QByteArray codedValueBuffer = codedValue; + + codedValueBuffer.append((unsigned char)255); + codedValueBuffer.append(QString("junk")); + + qDebug() << "codedValueBuffer:"; + outputBufferBits((const unsigned char*)codedValueBuffer.constData(), codedValueBuffer.size()); + + ByteCountCoded valueDecoder = codedValueBuffer; + quint64 valueDecoded = valueDecoder; + qDebug() << "valueDecoded:" << valueDecoded; + + + if (value == valueDecoded) { + qDebug() << "SUCCESS!"; + } else { + qDebug() << "FAILED!"; + } + + } +} + + diff --git a/interface/src/Util.h b/interface/src/Util.h index ed05209747..d252c26bef 100644 --- a/interface/src/Util.h +++ b/interface/src/Util.h @@ -30,6 +30,7 @@ void drawText(int x, int y, float scale, float radians, int mono, void renderCollisionOverlay(int width, int height, float magnitude, float red = 0, float blue = 0, float green = 0); void runTimingTests(); +void runUnitTests(); bool rayIntersectsSphere(const glm::vec3& rayStarting, const glm::vec3& rayNormalizedDirection, const glm::vec3& sphereCenter, float sphereRadius, float& distance); diff --git a/libraries/shared/src/ByteCountCoding.h b/libraries/shared/src/ByteCountCoding.h index efde19d45b..88f07230be 100644 --- a/libraries/shared/src/ByteCountCoding.h +++ b/libraries/shared/src/ByteCountCoding.h @@ -26,6 +26,8 @@ #include #include +#include "SharedUtil.h" + #include "NumericalConstants.h" template class ByteCountCoded { @@ -40,6 +42,7 @@ public: QByteArray encode() const; void decode(const QByteArray& fromEncoded); + void decode(const char* encodedBuffer, int encodedSize); bool operator==(const ByteCountCoded& other) const { return data == other.data; } bool operator!=(const ByteCountCoded& other) const { return data != other.data; } @@ -113,50 +116,57 @@ template inline QByteArray ByteCountCoded::encode() const { } template inline void ByteCountCoded::decode(const QByteArray& fromEncodedBytes) { - // first convert the ByteArray into a BitArray... - QBitArray encodedBits; - int bitCount = BITS_IN_BYTE * fromEncodedBytes.count(); - encodedBits.resize(bitCount); - - // copies the QByteArray into a QBitArray - for(int byte = 0; byte < fromEncodedBytes.count(); byte++) { - char originalByte = fromEncodedBytes.at(byte); - for(int bit = 0; bit < BITS_IN_BYTE; bit++) { - int shiftBy = BITS_IN_BYTE - (bit + 1); - char maskBit = ( 1 << shiftBy); - bool bitValue = originalByte & maskBit; - encodedBits.setBit(byte * BITS_IN_BYTE + bit, bitValue); - } - } - - // next, read the leading bits to determine the correct number of bytes to decode (may not match the QByteArray) + decode(fromEncodedBytes.constData(), fromEncodedBytes.size()); +} + +template inline void ByteCountCoded::decode(const char* encodedBuffer, int encodedSize) { + data = 0; // reset data + int bitCount = BITS_IN_BYTE * encodedSize; + int encodedByteCount = 1; // there is at least 1 byte (after the leadBits) int leadBits = 1; // there is always at least 1 lead bit - int bitAt; - for (bitAt = 0; bitAt < bitCount; bitAt++) { - if (encodedBits.at(bitAt)) { - encodedByteCount++; - leadBits++; - } else { - break; - } - } - int expectedBitCount = (encodedByteCount * BITS_IN_BYTE) - leadBits; - - T value = 0; - - if (expectedBitCount <= (encodedBits.size() - leadBits)) { - // Now, keep reading... - int valueStartsAt = bitAt + 1; - T bitValue = 1; - for (bitAt = valueStartsAt; bitAt < (expectedBitCount + leadBits); bitAt++) { - if(encodedBits.at(bitAt)) { - value += bitValue; + bool inLeadBits = true; + int bitAt = 0; + int expectedBitCount; // unknown at this point + int lastValueBit; + T bitValue = 1; + + for(int byte = 0; byte < encodedSize; byte++) { + char originalByte = encodedBuffer[byte]; + unsigned char maskBit = 128; + for(int bit = 0; bit < BITS_IN_BYTE; bit++) { + //int shiftBy = BITS_IN_BYTE - (bit + 1); + //char maskBit = (1 << shiftBy); + bool bitIsSet = originalByte & maskBit; + + // Processing of the lead bits + if (inLeadBits) { + if (bitIsSet) { + encodedByteCount++; + leadBits++; + } else { + inLeadBits = false; // once we hit our first 0, we know we're out of the lead bits + expectedBitCount = (encodedByteCount * BITS_IN_BYTE) - leadBits; + lastValueBit = expectedBitCount + bitAt; + // check to see if the remainder of our buffer is sufficient + if (expectedBitCount > (bitCount - leadBits)) { + break; + } + } + } else { + if (bitAt > lastValueBit) { + break; + } + + if(bitIsSet) { + data += bitValue; + } + bitValue *= 2; } - bitValue *= 2; + bitAt++; + maskBit = maskBit >> 1; } } - data = value; } #endif // hifi_ByteCountCoding_h From c0cdf3256cca585fdc6f5d72304f45faf22c143f Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Wed, 8 Jul 2015 17:34:36 -0700 Subject: [PATCH 3/4] comment and dead code --- libraries/shared/src/ByteCountCoding.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/shared/src/ByteCountCoding.h b/libraries/shared/src/ByteCountCoding.h index 88f07230be..735c51a783 100644 --- a/libraries/shared/src/ByteCountCoding.h +++ b/libraries/shared/src/ByteCountCoding.h @@ -133,10 +133,8 @@ template inline void ByteCountCoded::decode(const char* encodedBu for(int byte = 0; byte < encodedSize; byte++) { char originalByte = encodedBuffer[byte]; - unsigned char maskBit = 128; + unsigned char maskBit = 128; // LEFT MOST BIT set for(int bit = 0; bit < BITS_IN_BYTE; bit++) { - //int shiftBy = BITS_IN_BYTE - (bit + 1); - //char maskBit = (1 << shiftBy); bool bitIsSet = originalByte & maskBit; // Processing of the lead bits @@ -148,6 +146,7 @@ template inline void ByteCountCoded::decode(const char* encodedBu inLeadBits = false; // once we hit our first 0, we know we're out of the lead bits expectedBitCount = (encodedByteCount * BITS_IN_BYTE) - leadBits; lastValueBit = expectedBitCount + bitAt; + // check to see if the remainder of our buffer is sufficient if (expectedBitCount > (bitCount - leadBits)) { break; From 7790e35d2497ca6ff631413ba943f9bf8ea63b3d Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Wed, 8 Jul 2015 18:19:12 -0700 Subject: [PATCH 4/4] added bytes consumed result to decode, and exit early after bytes consumed --- interface/src/Application.cpp | 2 +- interface/src/Util.cpp | 8 +++++--- libraries/shared/src/ByteCountCoding.h | 16 +++++++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index cd15f2f4c7..a834ce1288 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1014,7 +1014,7 @@ void Application::paintGL() { void Application::runTests() { runTimingTests(); - //runUnitTests(); + runUnitTests(); } void Application::audioMuteToggled() { diff --git a/interface/src/Util.cpp b/interface/src/Util.cpp index 5f04010b47..82f7d55b5d 100644 --- a/interface/src/Util.cpp +++ b/interface/src/Util.cpp @@ -232,7 +232,7 @@ void runTimingTests() { (double)(elapsedUsecs / numTests), (double)result); - quint64 BYTE_CODE_MAX_VALUE = 99999999; + quint64 BYTE_CODE_MAX_TEST_VALUE = 99999999; quint64 BYTE_CODE_TESTS_SKIP = 999; QByteArray extraJunk; @@ -246,7 +246,7 @@ void runTimingTests() { startTime.start(); quint64 tests = 0; quint64 failed = 0; - for (quint64 value = 0; value < BYTE_CODE_MAX_VALUE; value += BYTE_CODE_TESTS_SKIP) { + for (quint64 value = 0; value < BYTE_CODE_MAX_TEST_VALUE; value += BYTE_CODE_TESTS_SKIP) { quint64 valueA = value; // usecTimestampNow(); ByteCountCoded codedValueA = valueA; QByteArray codedValueABuffer = codedValueA; @@ -328,9 +328,11 @@ void runUnitTests() { qDebug() << "codedValueBuffer:"; outputBufferBits((const unsigned char*)codedValueBuffer.constData(), codedValueBuffer.size()); - ByteCountCoded valueDecoder = codedValueBuffer; + ByteCountCoded valueDecoder; + size_t bytesConsumed = valueDecoder.decode(codedValueBuffer); quint64 valueDecoded = valueDecoder; qDebug() << "valueDecoded:" << valueDecoded; + qDebug() << "bytesConsumed:" << bytesConsumed; if (value == valueDecoded) { diff --git a/libraries/shared/src/ByteCountCoding.h b/libraries/shared/src/ByteCountCoding.h index 735c51a783..ce6f121ddb 100644 --- a/libraries/shared/src/ByteCountCoding.h +++ b/libraries/shared/src/ByteCountCoding.h @@ -41,8 +41,8 @@ public: ByteCountCoded(const QByteArray& fromEncoded) : data(0) { decode(fromEncoded); } QByteArray encode() const; - void decode(const QByteArray& fromEncoded); - void decode(const char* encodedBuffer, int encodedSize); + size_t decode(const QByteArray& fromEncoded); + size_t decode(const char* encodedBuffer, int encodedSize); bool operator==(const ByteCountCoded& other) const { return data == other.data; } bool operator!=(const ByteCountCoded& other) const { return data != other.data; } @@ -115,12 +115,13 @@ template inline QByteArray ByteCountCoded::encode() const { return output; } -template inline void ByteCountCoded::decode(const QByteArray& fromEncodedBytes) { - decode(fromEncodedBytes.constData(), fromEncodedBytes.size()); +template inline size_t ByteCountCoded::decode(const QByteArray& fromEncodedBytes) { + return decode(fromEncodedBytes.constData(), fromEncodedBytes.size()); } -template inline void ByteCountCoded::decode(const char* encodedBuffer, int encodedSize) { +template inline size_t ByteCountCoded::decode(const char* encodedBuffer, int encodedSize) { data = 0; // reset data + size_t bytesConsumed = 0; int bitCount = BITS_IN_BYTE * encodedSize; int encodedByteCount = 1; // there is at least 1 byte (after the leadBits) @@ -133,6 +134,7 @@ template inline void ByteCountCoded::decode(const char* encodedBu for(int byte = 0; byte < encodedSize; byte++) { char originalByte = encodedBuffer[byte]; + bytesConsumed++; unsigned char maskBit = 128; // LEFT MOST BIT set for(int bit = 0; bit < BITS_IN_BYTE; bit++) { bool bitIsSet = originalByte & maskBit; @@ -165,7 +167,11 @@ template inline void ByteCountCoded::decode(const char* encodedBu bitAt++; maskBit = maskBit >> 1; } + if (!inLeadBits && bitAt > lastValueBit) { + break; + } } + return bytesConsumed; } #endif // hifi_ByteCountCoding_h