Removing heap allocation from property flag parsing, adding some manual tests

libraries/shared/src/BufferParser.h

@@ -44,10 +44,7 @@ public:
 
     template <typename T>
     inline void readFlags(PropertyFlags<T>& result) {
-        // FIXME doing heap allocation
+        _offset += result.decode(_data + _offset, remaining());
-        QByteArray encoded((const char*)(_data + _offset), remaining());
-        result.decode(encoded);
-        _offset += result.getEncodedLength();
     }
 
     template<typename T>

libraries/shared/src/PropertyFlags.h

@@ -25,6 +25,7 @@
 #include <QBitArray>
 #include <QByteArray>
 
+#include "ByteCountCoding.h"
 #include <SharedUtil.h>
 
 template<typename Enum>class PropertyFlags {
@@ -51,7 +52,8 @@ public:
     void setHasProperty(Enum flag, bool value = true);
     bool getHasProperty(Enum flag) const;
     QByteArray encode();
-    void decode(const QByteArray& fromEncoded);
+    size_t decode(const uint8_t* data, size_t length);
+    size_t decode(const QByteArray& fromEncoded);
 
     operator QByteArray() { return encode(); };
 
@@ -193,51 +195,63 @@ template<typename Enum> inline QByteArray PropertyFlags<Enum>::encode() {
     return output;
 }
 
-template<typename Enum> inline void PropertyFlags<Enum>::decode(const QByteArray& fromEncodedBytes) {
+template<typename Enum> 
+inline size_t PropertyFlags<Enum>::decode(const uint8_t* data, size_t size) {
+    clear();
+    //clear(); // we are cleared out!
 
-    clear(); // we are cleared out!
+    size_t bytesConsumed = 0;
+    int bitCount = BITS_IN_BYTE * size;
 
-    // first convert the ByteArray into a BitArray...
+    int encodedByteCount = 1; // there is at least 1 byte (after the leadBits)
-    QBitArray encodedBits;
+    int leadBits = 1; // there is always at least 1 lead bit
-    int bitCount = BITS_PER_BYTE * fromEncodedBytes.count();
+    bool inLeadBits = true;
-    encodedBits.resize(bitCount);
+    int bitAt = 0;
-    
+    int expectedBitCount; // unknown at this point
-    for(int byte = 0; byte < fromEncodedBytes.count(); byte++) {
+    int lastValueBit;
-        char originalByte = fromEncodedBytes.at(byte);
+    for (int byte = 0; byte < size; byte++) {
-        for(int bit = 0; bit < BITS_PER_BYTE; bit++) {
+        char originalByte = data[byte];
-            int shiftBy = BITS_PER_BYTE - (bit + 1);
+        bytesConsumed++;
-            char maskBit = ( 1 << shiftBy);
+        unsigned char maskBit = 0x80; // LEFT MOST BIT set
-            bool bitValue = originalByte & maskBit;
+        for (int bit = 0; bit < BITS_IN_BYTE; bit++) {
-            encodedBits.setBit(byte * BITS_PER_BYTE + bit, bitValue);
+            bool bitIsSet = originalByte & maskBit;
+            // Processing of the lead bits
+            if (inLeadBits) {
+                if (bitIsSet) {
+                    encodedByteCount++;
+                    leadBits++;
+                } else {
+                    inLeadBits = false; // once we hit our first 0, we know we're out of the lead bits
+                    expectedBitCount = (encodedByteCount * BITS_IN_BYTE) - leadBits;
+                    lastValueBit = expectedBitCount + bitAt;
+
+                    // check to see if the remainder of our buffer is sufficient
+                    if (expectedBitCount > (bitCount - leadBits)) {
+                        break;
+                    }
+                }
+            } else {
+                if (bitAt > lastValueBit) {
+                    break;
+                }
+
+                if (bitIsSet) {
+                    setHasProperty(static_cast<Enum>(bitAt - leadBits), true);
+                }
+            }
+            bitAt++;
+            maskBit >>= 1;
         }
-    }
+        if (!inLeadBits && bitAt > lastValueBit) {
-    
-    // next, read the leading bits to determine the correct number of bytes to decode (may not match the QByteArray)
-    int encodedByteCount = 0;
-    int leadBits = 1;
-    int bitAt;
-    for (bitAt = 0; bitAt < bitCount; bitAt++) {
-        if (encodedBits.at(bitAt)) {
-            encodedByteCount++;
-            leadBits++;
-        } else {
             break;
         }
     }
-    encodedByteCount++; // always at least one byte
+    _encodedLength = bytesConsumed;
-    _encodedLength = encodedByteCount;
+    return bytesConsumed;
+}
 
-    int expectedBitCount = encodedByteCount * BITS_PER_BYTE;
+template<typename Enum> inline size_t PropertyFlags<Enum>::decode(const QByteArray& fromEncodedBytes) {
-    
+    return decode(reinterpret_cast<const uint8_t*>(fromEncodedBytes.data()), fromEncodedBytes.size());
-    // Now, keep reading...
-    if (expectedBitCount <= (encodedBits.size() - leadBits)) {
-        int flagsStartAt = bitAt + 1; 
-        for (bitAt = flagsStartAt; bitAt < expectedBitCount; bitAt++) {
-            if (encodedBits.at(bitAt)) {
-                setHasProperty((Enum)(bitAt - flagsStartAt));
-            }
-        }
-    }
 }
 
 template<typename Enum> inline void PropertyFlags<Enum>::debugDumpBits() {

tests/entities/CMakeLists.txt

@@ -0,0 +1,12 @@
+
+set(TARGET_NAME "entities-test")
+
+# This is not a testcase -- just set it up as a regular hifi project
+setup_hifi_project()
+
+set_target_properties(${TARGET_NAME} PROPERTIES FOLDER "Tests/manual-tests/")
+
+# link in the shared libraries
+link_hifi_libraries(entities avatars shared octree gpu model fbx networking animation environment)
+
+copy_dlls_beside_windows_executable()

tests/entities/src/main.cpp

@@ -0,0 +1,165 @@
+//
+//  main.cpp
+//  tests/render-utils/src
+//
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <QCoreApplication>
+#include <QFile>
+#include <QTimer>
+#include <QElapsedTimer>
+#include <QLoggingCategory>
+#include <QDir>
+#include <ByteCountCoding.h>
+
+#include <PathUtils.h>
+#include <BoxEntityItem.h>
+#include <Octree.h>
+
+const QString& getTestResourceDir() {
+    static QString dir;
+    if (dir.isEmpty()) {
+        QDir path(__FILE__);
+        path.cdUp();
+        dir = path.cleanPath(path.absoluteFilePath("../")) + "/";
+        qDebug() << "Qml Test Path: " << dir;
+    }
+    return dir;
+}
+
+class StopWatch {
+public:
+    void start() {
+        Q_ASSERT(_start == 0);
+        _start = usecTimestampNow();
+    }
+
+    void stop() {
+        Q_ASSERT(_start != 0);
+        _last = usecTimestampNow() - _start;
+        _start = 0;
+        _total += _last;
+        _count++;
+    }
+
+    quint64 getLast() {
+        return _last;
+    }
+
+    quint64 getTotal() {
+        return _total;
+    }
+
+    float getAverage() {
+        return (float)_total / (float)_count;
+    }
+
+    void reset() {
+        _last = _start = _total = _count = 0;
+    }
+
+private:
+    size_t _count{ 0 };
+    quint64 _total{ 0 };
+    quint64 _start{ 0 };
+    quint64 _last{ 0 };
+};
+
+template <typename T>
+void testByteCountCodedStable(const T& value) {
+    ByteCountCoded<T> coder((T)value);
+    auto encoded = coder.encode();
+    auto originalEncodedSize = encoded.size();
+    for (int i = 0; i < 10; ++i) {
+        encoded.append(qrand());
+    }
+    ByteCountCoded<T> decoder;
+    decoder.decode(encoded);
+    Q_ASSERT(decoder.data == coder.data);
+    auto consumed = decoder.decode(encoded.data(), encoded.size());
+    Q_ASSERT(consumed == originalEncodedSize);
+
+}
+
+template <typename T>
+void testByteCountCoded() {
+    testByteCountCodedStable<T>(0);
+    testByteCountCodedStable<T>(1);
+    testByteCountCodedStable<T>(1 << 16);
+    testByteCountCodedStable<T>(std::numeric_limits<T>::max() >> 16);
+    testByteCountCodedStable<T>(std::numeric_limits<T>::max() >> 8);
+    testByteCountCodedStable<T>(std::numeric_limits<T>::max() >> 1);
+    testByteCountCodedStable<T>(std::numeric_limits<T>::max());
+}
+
+void testPropertyFlags(uint32_t value) {
+    EntityPropertyFlags original;
+    original.clear();
+    auto enumSize = sizeof(EntityPropertyList);
+    for (size_t i = 0; i < sizeof(EntityPropertyList) * 8; ++i) {
+        original.setHasProperty((EntityPropertyList)i);
+    }
+    QByteArray encoded = original.encode();
+    auto originalSize = encoded.size();
+    for (size_t i = 0; i < sizeof(EntityPropertyList); ++i) {
+        encoded.append(qrand());
+    }
+
+    EntityPropertyFlags decodeOld, decodeNew;
+    {
+        decodeOld.decode(encoded);
+        Q_ASSERT(decodeOld == original);
+    }
+
+    {
+        auto decodeSize = decodeNew.decode((const uint8_t*)encoded.data(), encoded.size());
+        Q_ASSERT(originalSize == decodeSize);
+        Q_ASSERT(decodeNew == original);
+    }
+}
+
+void testPropertyFlags() {
+    testPropertyFlags(0);
+    testPropertyFlags(1);
+    testPropertyFlags(1 << 16);
+    testPropertyFlags(0xFFFF);
+}
+
+int main(int argc, char** argv) {    
+    QCoreApplication app(argc, argv);
+    {
+        auto start = usecTimestampNow();
+        for (int i = 0; i < 1000; ++i) {
+            testPropertyFlags();
+            testByteCountCoded<quint8>();
+            testByteCountCoded<quint16>();
+            testByteCountCoded<quint32>();
+            testByteCountCoded<quint64>();
+        }
+        auto duration = usecTimestampNow() - start;
+        qDebug() << duration;
+
+    }
+    DependencyManager::set<NodeList>(NodeType::Unassigned);
+
+    QFile file(getTestResourceDir() + "packet.bin");
+    if (!file.open(QIODevice::ReadOnly)) return -1;
+    QByteArray packet = file.readAll();
+    EntityItemPointer item = BoxEntityItem::factory(EntityItemID(), EntityItemProperties());
+    ReadBitstreamToTreeParams params;
+    params.bitstreamVersion = 33;
+
+    auto start = usecTimestampNow();
+    for (int i = 0; i < 1000; ++i) {
+        item->readEntityDataFromBuffer(reinterpret_cast<const unsigned char*>(packet.constData()), packet.size(), params);
+    }
+    float duration = (usecTimestampNow() - start);
+    qDebug() << (duration / 1000.0f);
+    return 0;
+}
+
+#include "main.moc"