From 89f3a091f11db26a475b81aca391fccf93b0c691 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 10 Mar 2015 09:02:35 -0700 Subject: [PATCH 1/6] remove debugging --- interface/src/ModelUploader.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/interface/src/ModelUploader.cpp b/interface/src/ModelUploader.cpp index 4093b3e610..aca02cb904 100644 --- a/interface/src/ModelUploader.cpp +++ b/interface/src/ModelUploader.cpp @@ -338,9 +338,6 @@ void ModelUploader::populateBasicMapping(QVariantHash& mapping, QString filename geometry.blendshapeChannelNames.contains("Blink_Right") && geometry.blendshapeChannelNames.contains("Squint_Right")); -qDebug() << "likelyMixamoFile:" << likelyMixamoFile; -qDebug() << "geometry.blendshapeChannelNames:" << geometry.blendshapeChannelNames; - if (!mapping.contains(BLENDSHAPE_FIELD) && likelyMixamoFile) { QVariantHash blendshapes; blendshapes.insertMulti("BrowsD_L", QVariantList() << "BrowsDown_Left" << 1.0); From c8298ca617e4e59cdf8b5a425e146553f6288c5c Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 10 Mar 2015 10:49:53 -0700 Subject: [PATCH 2/6] migrate SVO reading to use QDataStream in step toward reading from HTTP url --- libraries/octree/src/Octree.cpp | 239 +++++++++++++++++--------------- libraries/octree/src/Octree.h | 1 + 2 files changed, 129 insertions(+), 111 deletions(-) diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index 6b8f8c31e8..6c36b28c7a 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -18,7 +18,10 @@ #include #include // to load voxels from file +#include #include +#include +#include #include #include @@ -1836,141 +1839,155 @@ int Octree::encodeTreeBitstreamRecursion(OctreeElement* element, bool Octree::readFromSVOFile(const char* fileName) { bool fileOk = false; - PacketVersion gotVersion = 0; - std::ifstream file(fileName, std::ios::in|std::ios::binary|std::ios::ate); + QFile file(fileName); + QFileInfo fileInfo(fileName); + bool isOpen = file.open(QIODevice::ReadOnly); + QDataStream fileInputStream(&file); // read the data serialized from the file - if(file.is_open()) { + // get file length.... + unsigned long fileLength = fileInfo.size(); + + if(isOpen) { emit importSize(1.0f, 1.0f, 1.0f); emit importProgress(0); qDebug("Loading file %s...", fileName); + + fileOk = readFromStream(fileLength, fileInputStream); - // get file length.... - unsigned long fileLength = file.tellg(); - file.seekg( 0, std::ios::beg ); + emit importProgress(100); + file.close(); + } + + return fileOk; +} + +bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream) { + bool fileOk = false; + + PacketVersion gotVersion = 0; + + unsigned long headerLength = 0; // bytes in the header + + bool wantImportProgress = true; + + PacketType expectedType = expectedDataPacketType(); + PacketVersion expectedVersion = versionForPacketType(expectedType); + bool hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion); + + // before reading the file, check to see if this version of the Octree supports file versions + if (getWantSVOfileVersions()) { + + // read just enough of the file to parse the header... + const unsigned long HEADER_LENGTH = sizeof(PacketType) + sizeof(PacketVersion); + unsigned char fileHeader[HEADER_LENGTH]; + //file.read((char*)&fileHeader, HEADER_LENGTH); + int bytesRead = inputStream.readRawData((char*)&fileHeader, HEADER_LENGTH); + qDebug() << "HEADER_LENGTH... bytesRead:" << bytesRead; - unsigned long headerLength = 0; // bytes in the header + headerLength = HEADER_LENGTH; // we need this later to skip to the data + + unsigned char* dataAt = (unsigned char*)&fileHeader; + unsigned long dataLength = HEADER_LENGTH; + + // if so, read the first byte of the file and see if it matches the expected version code + PacketType gotType; + memcpy(&gotType, dataAt, sizeof(gotType)); + + dataAt += sizeof(expectedType); + dataLength -= sizeof(expectedType); + gotVersion = *dataAt; - bool wantImportProgress = true; + if (gotType == expectedType) { + if (canProcessVersion(gotVersion)) { + dataAt += sizeof(gotVersion); + dataLength -= sizeof(gotVersion); + fileOk = true; + qDebug("SVO file version match. Expected: %d Got: %d", + versionForPacketType(expectedDataPacketType()), gotVersion); - PacketType expectedType = expectedDataPacketType(); - PacketVersion expectedVersion = versionForPacketType(expectedType); - bool hasBufferBreaks = versionHasSVOfileBreaks(expectedVersion); - - // before reading the file, check to see if this version of the Octree supports file versions - if (getWantSVOfileVersions()) { - - // read just enough of the file to parse the header... - const unsigned long HEADER_LENGTH = sizeof(PacketType) + sizeof(PacketVersion); - unsigned char fileHeader[HEADER_LENGTH]; - file.read((char*)&fileHeader, HEADER_LENGTH); - - headerLength = HEADER_LENGTH; // we need this later to skip to the data - - unsigned char* dataAt = (unsigned char*)&fileHeader; - unsigned long dataLength = HEADER_LENGTH; - - // if so, read the first byte of the file and see if it matches the expected version code - PacketType gotType; - memcpy(&gotType, dataAt, sizeof(gotType)); - - dataAt += sizeof(expectedType); - dataLength -= sizeof(expectedType); - gotVersion = *dataAt; - - if (gotType == expectedType) { - if (canProcessVersion(gotVersion)) { - dataAt += sizeof(gotVersion); - dataLength -= sizeof(gotVersion); - fileOk = true; - qDebug("SVO file version match. Expected: %d Got: %d", - versionForPacketType(expectedDataPacketType()), gotVersion); - - hasBufferBreaks = versionHasSVOfileBreaks(gotVersion); - } else { - qDebug("SVO file version mismatch. Expected: %d Got: %d", - versionForPacketType(expectedDataPacketType()), gotVersion); - } + hasBufferBreaks = versionHasSVOfileBreaks(gotVersion); } else { - qDebug() << "SVO file type mismatch. Expected: " << nameForPacketType(expectedType) - << " Got: " << nameForPacketType(gotType); + qDebug("SVO file version mismatch. Expected: %d Got: %d", + versionForPacketType(expectedDataPacketType()), gotVersion); } - } else { - qDebug() << " NOTE: this file type does not include type and version information."; - fileOk = true; // assume the file is ok - } - - if (hasBufferBreaks) { - qDebug() << " this version includes buffer breaks"; - } else { - qDebug() << " this version does not include buffer breaks"; + qDebug() << "SVO file type mismatch. Expected: " << nameForPacketType(expectedType) + << " Got: " << nameForPacketType(gotType); } - if (fileOk) { + } else { + qDebug() << " NOTE: this file type does not include type and version information."; + fileOk = true; // assume the file is ok + } + + if (hasBufferBreaks) { + qDebug() << " this version includes buffer breaks"; + } else { + qDebug() << " this version does not include buffer breaks"; + } + + if (fileOk) { + + // if this version of the file does not include buffer breaks, then we need to load the entire file at once + if (!hasBufferBreaks) { - // if this version of the file does not include buffer breaks, then we need to load the entire file at once - if (!hasBufferBreaks) { + // read the entire file into a buffer, WHAT!? Why not. + unsigned long dataLength = streamLength - headerLength; + unsigned char* entireFileDataSection = new unsigned char[dataLength]; + inputStream.readRawData((char*)entireFileDataSection, dataLength); + + unsigned char* dataAt = entireFileDataSection; + + ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS, NULL, 0, + SharedNodePointer(), wantImportProgress, gotVersion); + + readBitstreamToTree(dataAt, dataLength, args); + delete[] entireFileDataSection; + + } else { + - // read the entire file into a buffer, WHAT!? Why not. - unsigned long dataLength = fileLength - headerLength; - unsigned char* entireFileDataSection = new unsigned char[dataLength]; - file.read((char*)entireFileDataSection, dataLength); - unsigned char* dataAt = entireFileDataSection; + unsigned long dataLength = streamLength - headerLength; + unsigned long remainingLength = dataLength; + const unsigned long MAX_CHUNK_LENGTH = MAX_OCTREE_PACKET_SIZE * 2; + unsigned char* fileChunk = new unsigned char[MAX_CHUNK_LENGTH]; + while (remainingLength > 0) { + quint16 chunkLength = 0; + + inputStream.readRawData((char*)&chunkLength, sizeof(chunkLength)); + remainingLength -= sizeof(chunkLength); + + if (chunkLength > remainingLength) { + qDebug() << "UNEXPECTED chunk size of:" << chunkLength + << "greater than remaining length:" << remainingLength; + break; + } + + if (chunkLength > MAX_CHUNK_LENGTH) { + qDebug() << "UNEXPECTED chunk size of:" << chunkLength + << "greater than MAX_CHUNK_LENGTH:" << MAX_CHUNK_LENGTH; + break; + } + + inputStream.readRawData((char*)fileChunk, chunkLength); + + remainingLength -= chunkLength; + + unsigned char* dataAt = fileChunk; + unsigned long dataLength = chunkLength; + ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS, NULL, 0, SharedNodePointer(), wantImportProgress, gotVersion); readBitstreamToTree(dataAt, dataLength, args); - delete[] entireFileDataSection; - - } else { - - - unsigned long dataLength = fileLength - headerLength; - unsigned long remainingLength = dataLength; - const unsigned long MAX_CHUNK_LENGTH = MAX_OCTREE_PACKET_SIZE * 2; - unsigned char* fileChunk = new unsigned char[MAX_CHUNK_LENGTH]; - - while (remainingLength > 0) { - quint16 chunkLength = 0; - - file.read((char*)&chunkLength, sizeof(chunkLength)); // read the chunk size from the file - remainingLength -= sizeof(chunkLength); - - if (chunkLength > remainingLength) { - qDebug() << "UNEXPECTED chunk size of:" << chunkLength - << "greater than remaining length:" << remainingLength; - break; - } - - if (chunkLength > MAX_CHUNK_LENGTH) { - qDebug() << "UNEXPECTED chunk size of:" << chunkLength - << "greater than MAX_CHUNK_LENGTH:" << MAX_CHUNK_LENGTH; - break; - } - - file.read((char*)fileChunk, chunkLength); // read in a section of the file larger than the chunk; - - remainingLength -= chunkLength; - - unsigned char* dataAt = fileChunk; - unsigned long dataLength = chunkLength; - - ReadBitstreamToTreeParams args(WANT_COLOR, NO_EXISTS_BITS, NULL, 0, - SharedNodePointer(), wantImportProgress, gotVersion); - - readBitstreamToTree(dataAt, dataLength, args); - } - - delete[] fileChunk; } + + delete[] fileChunk; } - - emit importProgress(100); - - file.close(); } + return fileOk; } diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h index a2265e38ed..c4e4e2205b 100644 --- a/libraries/octree/src/Octree.h +++ b/libraries/octree/src/Octree.h @@ -327,6 +327,7 @@ public: // these will read/write files that match the wireformat, excluding the 'V' leading void writeToSVOFile(const char* filename, OctreeElement* element = NULL); bool readFromSVOFile(const char* filename); + bool readFromStream(unsigned long streamLength, QDataStream& inputStream); unsigned long getOctreeElementsCount(); From 5facb39aa7b6cc0d5b0baa27a3643a72ff713ad3 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 10 Mar 2015 12:21:37 -0700 Subject: [PATCH 3/6] support for import entities from network URL --- examples/editEntities.js | 24 +++++++++++++++++- interface/src/Application.cpp | 2 +- libraries/octree/src/Octree.cpp | 43 ++++++++++++++++++++++++++++++--- libraries/octree/src/Octree.h | 2 ++ 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/examples/editEntities.js b/examples/editEntities.js index 689f73eed4..a2ecd7ff02 100644 --- a/examples/editEntities.js +++ b/examples/editEntities.js @@ -699,7 +699,7 @@ function setupModelMenus() { Menu.addMenuItem({ menuName: "File", menuItemName: "Models", isSeparator: true, beforeItem: "Settings" }); Menu.addMenuItem({ menuName: "File", menuItemName: "Export Entities", shortcutKey: "CTRL+META+E", afterItem: "Models" }); Menu.addMenuItem({ menuName: "File", menuItemName: "Import Entities", shortcutKey: "CTRL+META+I", afterItem: "Export Entities" }); - + Menu.addMenuItem({ menuName: "File", menuItemName: "Import Entities from URL", shortcutKey: "CTRL+META+U", afterItem: "Import Entities" }); Menu.addMenuItem({ menuName: "View", menuItemName: MENU_AUTO_FOCUS_ON_SELECT, afterItem: MENU_INSPECT_TOOL_ENABLED, isCheckable: true, isChecked: Settings.getValue(SETTING_AUTO_FOCUS_ON_SELECT) == "true" }); @@ -800,6 +800,28 @@ function handeMenuEvent(menuItem) { if (filename) { var success = Clipboard.importEntities(filename); + if (success) { + var distance = cameraManager.enabled ? cameraManager.zoomDistance : DEFAULT_ENTITY_DRAG_DROP_DISTANCE; + var direction = Quat.getFront(Camera.orientation); + var offset = Vec3.multiply(distance, direction); + var position = Vec3.sum(Camera.position, offset); + + position.x = Math.max(0, position.x); + position.y = Math.max(0, position.y); + position.z = Math.max(0, position.z); + + var pastedEntityIDs = Clipboard.pasteEntities(position); + + selectionManager.setSelections(pastedEntityIDs); + } else { + Window.alert("There was an error importing the entity file."); + } + } + } else if (menuItem == "Import Entities from URL") { + var url = Window.prompt("URL of SVO to import", ""); + if (url) { + var success = Clipboard.importEntities(url); + if (success) { var distance = cameraManager.enabled ? cameraManager.zoomDistance : DEFAULT_ENTITY_DRAG_DROP_DISTANCE; var direction = Quat.getFront(Camera.orientation); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 6945e88949..4662f92427 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1723,7 +1723,7 @@ void Application::saveSettings() { bool Application::importEntities(const QString& filename) { _entityClipboard.eraseAllOctreeElements(); - bool success = _entityClipboard.readFromSVOFile(filename.toLocal8Bit().constData()); + bool success = _entityClipboard.readFromSVOURL(filename); if (success) { _entityClipboard.reaverageOctreeElements(); } diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index 6c36b28c7a..fddbe0bef5 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -20,13 +20,18 @@ #include #include +#include #include #include +#include +#include +#include #include #include -#include #include +#include +#include #include #include #include @@ -1862,6 +1867,38 @@ bool Octree::readFromSVOFile(const char* fileName) { return fileOk; } +bool Octree::readFromSVOURL(const QString& urlString) { + bool readOk = false; + + // determine if this is a local file or a network resource + QUrl url(urlString); + + if (url.isLocalFile()) { + readOk = readFromSVOFile(qPrintable(url.toLocalFile())); + } else { + QNetworkRequest request; + request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); + request.setUrl(url); + + QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); + QNetworkReply* reply = networkAccessManager.get(request); + + qDebug() << "Downloading svo at" << qPrintable(urlString); + + QEventLoop loop; + QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); + loop.exec(); + + if (reply->error() == QNetworkReply::NoError) { + int resourceSize = reply->bytesAvailable(); + QDataStream inputStream(reply); + readOk = readFromStream(resourceSize, inputStream); + } + } + return readOk; +} + + bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream) { bool fileOk = false; @@ -1881,9 +1918,7 @@ bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream // read just enough of the file to parse the header... const unsigned long HEADER_LENGTH = sizeof(PacketType) + sizeof(PacketVersion); unsigned char fileHeader[HEADER_LENGTH]; - //file.read((char*)&fileHeader, HEADER_LENGTH); - int bytesRead = inputStream.readRawData((char*)&fileHeader, HEADER_LENGTH); - qDebug() << "HEADER_LENGTH... bytesRead:" << bytesRead; + inputStream.readRawData((char*)&fileHeader, HEADER_LENGTH); headerLength = HEADER_LENGTH; // we need this later to skip to the data diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h index c4e4e2205b..672a3b63d5 100644 --- a/libraries/octree/src/Octree.h +++ b/libraries/octree/src/Octree.h @@ -326,7 +326,9 @@ public: // these will read/write files that match the wireformat, excluding the 'V' leading void writeToSVOFile(const char* filename, OctreeElement* element = NULL); + bool readFromSVOFile(const char* filename); + bool readFromSVOURL(const QString& url); // will support file urls as well... bool readFromStream(unsigned long streamLength, QDataStream& inputStream); From feddea821ad4d144e5754dcd807114a090b3cc08 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 10 Mar 2015 12:35:12 -0700 Subject: [PATCH 4/6] cleanup import from file vs url --- examples/editEntities.js | 37 ++++++++++++----------------------- interface/src/Application.cpp | 12 ++++++++++-- interface/src/Application.h | 2 +- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/examples/editEntities.js b/examples/editEntities.js index a2ecd7ff02..e97fee583a 100644 --- a/examples/editEntities.js +++ b/examples/editEntities.js @@ -726,6 +726,7 @@ function cleanupModelMenus() { Menu.removeSeparator("File", "Models"); Menu.removeMenuItem("File", "Export Entities"); Menu.removeMenuItem("File", "Import Entities"); + Menu.removeMenuItem("File", "Import Entities from URL"); Menu.removeMenuItem("View", MENU_INSPECT_TOOL_ENABLED); Menu.removeMenuItem("View", MENU_AUTO_FOCUS_ON_SELECT); @@ -795,32 +796,18 @@ function handeMenuEvent(menuItem) { } } } - } else if (menuItem == "Import Entities") { - var filename = Window.browse("Select models to import", "", "*.svo") - if (filename) { - var success = Clipboard.importEntities(filename); - - if (success) { - var distance = cameraManager.enabled ? cameraManager.zoomDistance : DEFAULT_ENTITY_DRAG_DROP_DISTANCE; - var direction = Quat.getFront(Camera.orientation); - var offset = Vec3.multiply(distance, direction); - var position = Vec3.sum(Camera.position, offset); - - position.x = Math.max(0, position.x); - position.y = Math.max(0, position.y); - position.z = Math.max(0, position.z); - - var pastedEntityIDs = Clipboard.pasteEntities(position); - - selectionManager.setSelections(pastedEntityIDs); - } else { - Window.alert("There was an error importing the entity file."); - } + } else if (menuItem == "Import Entities" || menuItem == "Import Entities from URL") { + + var importURL; + if (menuItem == "Import Entities") { + importURL = Window.browse("Select models to import", "", "*.svo"); + //importURL = "file://" + filename; + } else { + importURL = Window.prompt("URL of SVO to import", ""); } - } else if (menuItem == "Import Entities from URL") { - var url = Window.prompt("URL of SVO to import", ""); - if (url) { - var success = Clipboard.importEntities(url); + + if (importURL) { + var success = Clipboard.importEntities(importURL); if (success) { var distance = cameraManager.enabled ? cameraManager.zoomDistance : DEFAULT_ENTITY_DRAG_DROP_DISTANCE; diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 4662f92427..fa511097cb 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -1721,9 +1721,17 @@ void Application::saveSettings() { _myAvatar->saveData(); } -bool Application::importEntities(const QString& filename) { +bool Application::importEntities(const QString& urlOrFilename) { _entityClipboard.eraseAllOctreeElements(); - bool success = _entityClipboard.readFromSVOURL(filename); + + QUrl url(urlOrFilename); + + // if the URL appears to be invalid or relative, then it is probably a local file + if (!url.isValid() || url.isRelative()) { + url = QUrl::fromLocalFile(urlOrFilename); + } + + bool success = _entityClipboard.readFromSVOURL(url.toString()); if (success) { _entityClipboard.reaverageOctreeElements(); } diff --git a/interface/src/Application.h b/interface/src/Application.h index 1da5f430b6..d409f6c6d1 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -325,7 +325,7 @@ public slots: QVector pasteEntities(float x, float y, float z); bool exportEntities(const QString& filename, const QVector& entityIDs); bool exportEntities(const QString& filename, float x, float y, float z, float scale); - bool importEntities(const QString& filename); + bool importEntities(const QString& url); void setLowVelocityFilter(bool lowVelocityFilter); void loadDialog(); From e16961c958244368a7dfe06a5f06565f2129bd83 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 10 Mar 2015 12:36:31 -0700 Subject: [PATCH 5/6] removed dead code --- examples/editEntities.js | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/editEntities.js b/examples/editEntities.js index e97fee583a..9568f4207f 100644 --- a/examples/editEntities.js +++ b/examples/editEntities.js @@ -801,7 +801,6 @@ function handeMenuEvent(menuItem) { var importURL; if (menuItem == "Import Entities") { importURL = Window.browse("Select models to import", "", "*.svo"); - //importURL = "file://" + filename; } else { importURL = Window.prompt("URL of SVO to import", ""); } From b9671f3f9b2afc8597f6e98f9f08c67bdff24559 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Tue, 10 Mar 2015 15:27:44 -0700 Subject: [PATCH 6/6] CR feedback --- libraries/octree/src/Octree.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index fddbe0bef5..2b29529f4e 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -1845,14 +1845,13 @@ bool Octree::readFromSVOFile(const char* fileName) { bool fileOk = false; QFile file(fileName); - QFileInfo fileInfo(fileName); - bool isOpen = file.open(QIODevice::ReadOnly); - QDataStream fileInputStream(&file); // read the data serialized from the file + fileOk = file.open(QIODevice::ReadOnly); - // get file length.... - unsigned long fileLength = fileInfo.size(); + if(fileOk) { + QDataStream fileInputStream(&file); + QFileInfo fileInfo(fileName); + unsigned long fileLength = fileInfo.size(); - if(isOpen) { emit importSize(1.0f, 1.0f, 1.0f); emit importProgress(0);