Mirrors/overte-JulianGro
3
0
Fork 0
mirror of https://github.com/JulianGro/overte.git synced 2025-04-25 16:55:07 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

resolve conflicts on merge with origin/protocol

Browse source
This commit is contained in:
Stephen Birarda 2015-08-27 14:45:22 -07:00
commit c80c4a9b45
53 changed files with 1861 additions and 527 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
assignment-client/src/AssignmentFactory.cpp
View file

@ -16,6 +16,7 @@
#include "audio/AudioMixer.h"
#include "avatars/AvatarMixer.h"
#include "entities/EntityServer.h"
#include "assets/AssetServer.h"
ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) {
@ -33,6 +34,8 @@ ThreadedAssignment* AssignmentFactory::unpackAssignment(NLPacket& packet) {
return new Agent(packet);
case Assignment::EntityServerType:
return new EntityServer(packet);
case Assignment::AssetServerType:
return new AssetServer(packet);
default:
return NULL;
}

202
assignment-client/src/assets/AssetServer.cpp Normal file
View file

@ -0,0 +1,202 @@
//
// AssetServer.cpp
// assignment-client/src/assets
//
// Created by Ryan Huffman on 2015/07/21
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "AssetServer.h"
#include <QBuffer>
#include <QCryptographicHash>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QCoreApplication>
#include <QEventLoop>
#include <QRunnable>
#include <QString>
#include "NetworkLogging.h"
#include "NodeType.h"
#include "SendAssetTask.h"
const QString ASSET_SERVER_LOGGING_TARGET_NAME = "asset-server";
AssetServer::AssetServer(NLPacket& packet) :
ThreadedAssignment(packet),
_taskPool(this)
{
// Most of the work will be I/O bound, reading from disk and constructing packet objects,
// so the ideal is greater than the number of cores on the system.
_taskPool.setMaxThreadCount(20);
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
packetReceiver.registerMessageListener(PacketType::AssetUpload, this, "handleAssetUpload");
}
void AssetServer::run() {
ThreadedAssignment::commonInit(ASSET_SERVER_LOGGING_TARGET_NAME, NodeType::AssetServer);
auto nodeList = DependencyManager::get<NodeList>();
nodeList->addNodeTypeToInterestSet(NodeType::Agent);
_resourcesDirectory = QDir(QCoreApplication::applicationDirPath()).filePath("resources/assets");
if (!_resourcesDirectory.exists()) {
qDebug() << "Creating resources directory";
_resourcesDirectory.mkpath(".");
}
qDebug() << "Serving files from: " << _resourcesDirectory.path();
// Scan for new files
qDebug() << "Looking for new files in asset directory";
auto files = _resourcesDirectory.entryInfoList(QDir::Files);
QRegExp filenameRegex { "^[a-f0-9]{" + QString::number(HASH_HEX_LENGTH) + "}(\\..+)?$" };
for (const auto& fileInfo : files) {
auto filename = fileInfo.fileName();
if (!filenameRegex.exactMatch(filename)) {
qDebug() << "Found file: " << filename;
if (!fileInfo.isReadable()) {
qDebug() << "\tCan't open file for reading: " << filename;
continue;
}
// Read file
QFile file { fileInfo.absoluteFilePath() };
file.open(QFile::ReadOnly);
QByteArray data = file.readAll();
auto hash = hashData(data);
qDebug() << "\tMoving " << filename << " to " << hash;
file.rename(_resourcesDirectory.absoluteFilePath(hash) + "." + fileInfo.suffix());
}
}
}
void AssetServer::handleAssetGetInfo(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode) {
QByteArray assetHash;
MessageID messageID;
uint8_t extensionLength;
if (packet->getPayloadSize() < qint64(HASH_HEX_LENGTH + sizeof(messageID) + sizeof(extensionLength))) {
qDebug() << "ERROR bad file request";
return;
}
packet->readPrimitive(&messageID);
assetHash = packet->readWithoutCopy(HASH_HEX_LENGTH);
packet->readPrimitive(&extensionLength);
QByteArray extension = packet->read(extensionLength);
auto replyPacket = NLPacket::create(PacketType::AssetGetInfoReply);
replyPacket->writePrimitive(messageID);
replyPacket->write(assetHash);
QString fileName = QString(assetHash) + "." + extension;
QFileInfo fileInfo { _resourcesDirectory.filePath(fileName) };
if (fileInfo.exists() && fileInfo.isReadable()) {
qDebug() << "Opening file: " << fileInfo.filePath();
replyPacket->writePrimitive(AssetServerError::NO_ERROR);
replyPacket->writePrimitive(fileInfo.size());
} else {
qDebug() << "Asset not found: " << assetHash;
replyPacket->writePrimitive(AssetServerError::ASSET_NOT_FOUND);
}
auto nodeList = DependencyManager::get<NodeList>();
nodeList->sendPacket(std::move(replyPacket), *senderNode);
}
void AssetServer::handleAssetGet(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode) {
MessageID messageID;
QByteArray assetHash;
uint8_t extensionLength;
DataOffset start;
DataOffset end;
auto minSize = qint64(sizeof(messageID) + HASH_HEX_LENGTH + sizeof(extensionLength) + sizeof(start) + sizeof(end));
if (packet->getPayloadSize() < minSize) {
qDebug() << "ERROR bad file request";
return;
}
packet->readPrimitive(&messageID);
assetHash = packet->read(HASH_HEX_LENGTH);
packet->readPrimitive(&extensionLength);
QByteArray extension = packet->read(extensionLength);
packet->readPrimitive(&start);
packet->readPrimitive(&end);
qDebug() << "Received a request for the file (" << messageID << "): " << assetHash << " from " << start << " to " << end;
// Queue task
QString filePath = _resourcesDirectory.filePath(QString(assetHash) + "." + QString(extension));
auto task = new SendAssetTask(messageID, assetHash, filePath, start, end, senderNode);
_taskPool.start(task);
}
void AssetServer::handleAssetUpload(QSharedPointer<NLPacketList> packetList, SharedNodePointer senderNode) {
auto data = packetList->getMessage();
QBuffer buffer { &data };
buffer.open(QIODevice::ReadOnly);
MessageID messageID;
buffer.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
uint8_t extensionLength;
buffer.read(reinterpret_cast<char*>(&extensionLength), sizeof(extensionLength));
QByteArray extension = buffer.read(extensionLength);
qDebug() << "Got extension: " << extension;
uint64_t fileSize;
buffer.read(reinterpret_cast<char*>(&fileSize), sizeof(fileSize));
qDebug() << "Receiving a file of size " << fileSize;
auto replyPacket = NLPacket::create(PacketType::AssetUploadReply);
replyPacket->writePrimitive(messageID);
if (fileSize > MAX_UPLOAD_SIZE) {
replyPacket->writePrimitive(AssetServerError::ASSET_TOO_LARGE);
} else {
QByteArray fileData = buffer.read(fileSize);
QString hash = hashData(fileData);
qDebug() << "Got data: (" << hash << ") ";
QFile file { _resourcesDirectory.filePath(QString(hash)) + "." + QString(extension) };
if (file.exists()) {
qDebug() << "[WARNING] This file already exists: " << hash;
} else {
file.open(QIODevice::WriteOnly);
file.write(fileData);
file.close();
}
replyPacket->writePrimitive(AssetServerError::NO_ERROR);
replyPacket->write(hash.toLatin1());
}
auto nodeList = DependencyManager::get<NodeList>();
nodeList->sendPacket(std::move(replyPacket), *senderNode);
}
QString AssetServer::hashData(const QByteArray& data) {
return QString(QCryptographicHash::hash(data, QCryptographicHash::Sha256).toHex());
}

47
assignment-client/src/assets/AssetServer.h Normal file
View file

@ -0,0 +1,47 @@
//
// AssetServer.h
// assignment-client/src/assets
//
// Created by Ryan Huffman on 2015/07/21
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AssetServer_h
#define hifi_AssetServer_h
#include <QDir>
#include <ThreadedAssignment.h>
#include <QThreadPool>
#include "AssetUtils.h"
class AssetServer : public ThreadedAssignment {
Q_OBJECT
public:
AssetServer(NLPacket& packet);
static QString hashData(const QByteArray& data);
public slots:
void run();
private slots:
void handleAssetGetInfo(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
void handleAssetGet(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
void handleAssetUpload(QSharedPointer<NLPacketList> packetList, SharedNodePointer senderNode);
private:
static void writeError(NLPacketList* packetList, AssetServerError error);
QDir _resourcesDirectory;
QThreadPool _taskPool;
};
inline void writeError(NLPacketList* packetList, AssetServerError error) {
packetList->writePrimitive(error);
}
#endif

70
assignment-client/src/assets/SendAssetTask.cpp Normal file
View file

@ -0,0 +1,70 @@
//
// SendAssetTask.cpp
// assignment-client/src/assets
//
// Created by Ryan Huffman on 2015/08/26
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "SendAssetTask.h"
#include <QFile>
#include <DependencyManager.h>
#include <NetworkLogging.h>
#include <NLPacket.h>
#include <NLPacketList.h>
#include <NodeList.h>
#include "AssetUtils.h"
SendAssetTask::SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end,
const SharedNodePointer& sendToNode) :
QRunnable(),
_messageID(messageID),
_assetHash(assetHash),
_filePath(filePath),
_start(start),
_end(end),
_sendToNode(sendToNode)
{
}
void SendAssetTask::run() {
qDebug() << "Starting task to send asset: " << _assetHash << " for messageID " << _messageID;
auto replyPacketList = std::unique_ptr<NLPacketList>(new NLPacketList(PacketType::AssetGetReply, QByteArray(), true, true));
replyPacketList->write(_assetHash);
replyPacketList->writePrimitive(_messageID);
if (_end <= _start) {
writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE);
} else {
QFile file { _filePath };
if (file.open(QIODevice::ReadOnly)) {
if (file.size() < _end) {
writeError(replyPacketList.get(), AssetServerError::INVALID_BYTE_RANGE);
qCDebug(networking) << "Bad byte range: " << _assetHash << " " << _start << ":" << _end;
} else {
auto size = _end - _start;
file.seek(_start);
replyPacketList->writePrimitive(AssetServerError::NO_ERROR);
replyPacketList->writePrimitive(size);
replyPacketList->write(file.read(size));
qCDebug(networking) << "Sending asset: " << _assetHash;
}
file.close();
} else {
qCDebug(networking) << "Asset not found: " << _filePath << "(" << _assetHash << ")";
writeError(replyPacketList.get(), AssetServerError::ASSET_NOT_FOUND);
}
}
auto nodeList = DependencyManager::get<NodeList>();
nodeList->sendPacketList(std::move(replyPacketList), *_sendToNode);
}

42
assignment-client/src/assets/SendAssetTask.h Normal file
View file

@ -0,0 +1,42 @@
//
// SendAssetTask.h
// assignment-client/src/assets
//
// Created by Ryan Huffman on 2015/08/26
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_SendAssetTask_h
#define hifi_SendAssetTask_h
#include <QByteArray>
#include <QString>
#include <QRunnable>
#include "AssetUtils.h"
#include "AssetServer.h"
#include "Node.h"
class SendAssetTask : public QRunnable {
public:
SendAssetTask(MessageID messageID, const QByteArray& assetHash, QString filePath, DataOffset start, DataOffset end,
const SharedNodePointer& sendToNode);
void run();
signals:
void finished();
private:
MessageID _messageID;
QByteArray _assetHash;
QString _filePath;
DataOffset _start;
DataOffset _end;
SharedNodePointer _sendToNode;
};
#endif

1
domain-server/src/DomainServer.cpp
View file

@ -573,7 +573,6 @@ void DomainServer::populateDefaultStaticAssignmentsExcludingTypes(const QSet<Ass
if (!excludedTypes.contains(defaultedType)
&& defaultedType != Assignment::UNUSED_0
&& defaultedType != Assignment::UNUSED_1
&& defaultedType != Assignment::UNUSED_2
&& defaultedType != Assignment::AgentType) {
// type has not been set from a command line or config file config, use the default
// by clearing whatever exists and writing a single default assignment with no payload

149
interface/resources/qml/Stats.qml
View file

@ -28,39 +28,39 @@ Item {
anchors.fill: parent
onClicked: { root.expanded = !root.expanded; }
}
Column {
id: generalCol
spacing: 4; x: 4; y: 4;
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Servers: " + root.serverCount
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Avatars: " + root.avatarCount
text: "Avatars: " + root.avatarCount
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Framerate: " + root.framerate
text: "Framerate: " + root.framerate
}
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Simrate: " + root.simrate
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Packets In/Out: " + root.packetInCount + "/" + root.packetOutCount
text: "Packets In/Out: " + root.packetInCount + "/" + root.packetOutCount
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Mbps In/Out: " + root.mbpsIn.toFixed(2) + "/" + root.mbpsOut.toFixed(2)
text: "Mbps In/Out: " + root.mbpsIn.toFixed(2) + "/" + root.mbpsOut.toFixed(2)
}
}
}
@ -77,30 +77,35 @@ Item {
Column {
id: pingCol
spacing: 4; x: 4; y: 4;
Text {
Text {
color: root.fontColor
font.pixelSize: root.fontSize
text: "Audio ping: " + root.audioPing
text: "Audio ping: " + root.audioPing
}
Text {
Text {
color: root.fontColor
font.pixelSize: root.fontSize
text: "Avatar ping: " + root.avatarPing
text: "Avatar ping: " + root.avatarPing
}
Text {
Text {
color: root.fontColor
font.pixelSize: root.fontSize
text: "Entities avg ping: " + root.entitiesPing
text: "Entities avg ping: " + root.entitiesPing
}
Text {
Text {
color: root.fontColor
font.pixelSize: root.fontSize
visible: root.expanded;
text: "Asset ping: " + root.assetPing
}
Text {
color: root.fontColor
font.pixelSize: root.fontSize
visible: root.expanded;
text: "Voxel max ping: " + 0
}
}
}
Rectangle {
width: geoCol.width + 8
height: geoCol.height + 8
@ -112,34 +117,34 @@ Item {
Column {
id: geoCol
spacing: 4; x: 4; y: 4;
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Position: " + root.position.x.toFixed(1) + ", " +
root.position.y.toFixed(1) + ", " + root.position.z.toFixed(1)
text: "Position: " + root.position.x.toFixed(1) + ", " +
root.position.y.toFixed(1) + ", " + root.position.z.toFixed(1)
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Velocity: " + root.velocity.toFixed(1)
text: "Velocity: " + root.velocity.toFixed(1)
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Yaw: " + root.yaw.toFixed(1)
text: "Yaw: " + root.yaw.toFixed(1)
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded;
text: "Avatar Mixer: " + root.avatarMixerKbps + " kbps, " +
root.avatarMixerPps + "pps";
visible: root.expanded;
text: "Avatar Mixer: " + root.avatarMixerKbps + " kbps, " +
root.avatarMixerPps + "pps";
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded;
text: "Downloads: ";
visible: root.expanded;
text: "Downloads: ";
}
}
}
@ -154,72 +159,72 @@ Item {
Column {
id: octreeCol
spacing: 4; x: 4; y: 4;
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
text: "Triangles: " + root.triangles +
" / Quads: " + root.quads + " / Material Switches: " + root.materialSwitches
text: "Triangles: " + root.triangles +
" / Quads: " + root.quads + " / Material Switches: " + root.materialSwitches
}
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded;
text: "\tMesh Parts Rendered Opaque: " + root.meshOpaque +
" / Translucent: " + root.meshTranslucent;
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded;
text: "\tOpaque considered: " + root.opaqueConsidered +
" / Out of view: " + root.opaqueOutOfView + " / Too small: " + root.opaqueTooSmall;
text: "\tMesh Parts Rendered Opaque: " + root.meshOpaque +
" / Translucent: " + root.meshTranslucent;
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: !root.expanded
text: "Octree Elements Server: " + root.serverElements +
" Local: " + root.localElements;
visible: root.expanded;
text: "\tOpaque considered: " + root.opaqueConsidered +
" / Out of view: " + root.opaqueOutOfView + " / Too small: " + root.opaqueTooSmall;
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded
text: "Octree Sending Mode: " + root.sendingMode;
visible: !root.expanded
text: "Octree Elements Server: " + root.serverElements +
" Local: " + root.localElements;
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded
text: "Octree Packets to Process: " + root.packetStats;
visible: root.expanded
text: "Octree Sending Mode: " + root.sendingMode;
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded
text: "Octree Elements - ";
visible: root.expanded
text: "Octree Packets to Process: " + root.packetStats;
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded
text: "\tServer: " + root.serverElements +
" Internal: " + root.serverInternal +
" Leaves: " + root.serverLeaves;
visible: root.expanded
text: "Octree Elements - ";
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded
text: "\tLocal: " + root.localElements +
" Internal: " + root.localInternal +
" Leaves: " + root.localLeaves;
visible: root.expanded
text: "\tServer: " + root.serverElements +
" Internal: " + root.serverInternal +
" Leaves: " + root.serverLeaves;
}
Text {
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded
text: "LOD: " + root.lodStatus;
visible: root.expanded
text: "\tLocal: " + root.localElements +
" Internal: " + root.localInternal +
" Leaves: " + root.localLeaves;
}
Text {
color: root.fontColor;
font.pixelSize: root.fontSize
visible: root.expanded
text: "LOD: " + root.lodStatus;
}
}
}

16
interface/src/Application.cpp
View file

@ -52,6 +52,7 @@
#include <AccountManager.h>
#include <AddressManager.h>
#include <AssetClient.h>
#include <ApplicationVersion.h>
#include <CursorManager.h>
#include <AudioInjector.h>
@ -297,6 +298,7 @@ bool setupEssentials(int& argc, char** argv) {
auto autoUpdater = DependencyManager::set<AutoUpdater>();
auto pathUtils = DependencyManager::set<PathUtils>();
auto actionFactory = DependencyManager::set<InterfaceActionFactory>();
auto assetClient = DependencyManager::set<AssetClient>();
auto userInputMapper = DependencyManager::set<UserInputMapper>();
return true;
@ -444,6 +446,17 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) :
audioThread->start();
QThread* assetThread = new QThread();
assetThread->setObjectName("Asset Thread");
auto assetClient = DependencyManager::get<AssetClient>();
assetClient->moveToThread(assetThread);
assetThread->start();
const DomainHandler& domainHandler = nodeList->getDomainHandler();
connect(&domainHandler, SIGNAL(hostnameChanged(const QString&)), SLOT(domainChanged(const QString&)));
@ -513,7 +526,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) :
// tell the NodeList instance who to tell the domain server we care about
nodeList->addSetOfNodeTypesToNodeInterestSet(NodeSet() << NodeType::AudioMixer << NodeType::AvatarMixer
<< NodeType::EntityServer);
<< NodeType::EntityServer << NodeType::AssetServer);
// connect to the packet sent signal of the _entityEditSender
connect(&_entityEditSender, &EntityEditPacketSender::packetSent, this, &Application::packetSent);
@ -2010,6 +2023,7 @@ void Application::sendPingPackets() {
case NodeType::AvatarMixer:
case NodeType::AudioMixer:
case NodeType::EntityServer:
case NodeType::AssetServer:
return true;
default:
return false;

31
interface/src/Menu.cpp
View file

@ -22,6 +22,7 @@
#include <UserActivityLogger.h>
#include <VrMenu.h>
#include <AssetClient.h>
#include "Application.h"
#include "AccountManager.h"
@ -89,6 +90,36 @@ Menu::Menu() {
addActionToQMenuAndActionHash(fileMenu, MenuOption::RunningScripts, Qt::CTRL | Qt::Key_J,
qApp, SLOT(toggleRunningScriptsWidget()));
// Asset uploading
{
auto action = new QAction("Upload File", fileMenu);
fileMenu->addAction(action);
action->setMenuRole(QAction::NoRole);
_actionHash.insert("Upload File", action);
connect(action, &QAction::triggered, [this](bool checked) {
qDebug() << "Clicked upload file";
auto filename = QFileDialog::getOpenFileUrl(nullptr, "Select a file to upload");
if (!filename.isEmpty()) {
qDebug() << "Selected: " << filename;
QFile file { filename.path() };
if (file.open(QIODevice::ReadOnly)) {
QFileInfo fileInfo { filename.path() };
auto extension = fileInfo.suffix();
auto data = file.readAll();
auto assetClient = DependencyManager::get<AssetClient>();
assetClient->uploadAsset(data, extension, [this, extension](bool result, QString hash) mutable {
if (result) {
QMessageBox::information(this, "Upload Successful", "URL: apt:/" + hash + "." + extension);
} else {
QMessageBox::warning(this, "Upload Failed", "There was an error uploading the file.");
}
});
}
}
});
}
auto addressManager = DependencyManager::get<AddressManager>();
addDisabledActionAndSeparator(fileMenu, "History");

2
interface/src/ui/Stats.cpp
View file

@ -126,8 +126,10 @@ void Stats::updateStats() {
if (Menu::getInstance()->isOptionChecked(MenuOption::TestPing)) {
SharedNodePointer audioMixerNode = nodeList->soloNodeOfType(NodeType::AudioMixer);
SharedNodePointer avatarMixerNode = nodeList->soloNodeOfType(NodeType::AvatarMixer);
SharedNodePointer assetServerNode = nodeList->soloNodeOfType(NodeType::AssetServer);
STAT_UPDATE(audioPing, audioMixerNode ? audioMixerNode->getPingMs() : -1);
STAT_UPDATE(avatarPing, avatarMixerNode ? avatarMixerNode->getPingMs() : -1);
STAT_UPDATE(assetPing, assetServerNode ? assetServerNode->getPingMs() : -1);
//// Now handle entity servers, since there could be more than one, we average their ping times
int totalPingOctree = 0;

2
interface/src/ui/Stats.h
View file

@ -39,6 +39,7 @@ class Stats : public QQuickItem {
STATS_PROPERTY(int, audioPing, 0)
STATS_PROPERTY(int, avatarPing, 0)
STATS_PROPERTY(int, entitiesPing, 0)
STATS_PROPERTY(int, assetPing, 0)
STATS_PROPERTY(QVector3D, position, QVector3D(0, 0, 0) )
STATS_PROPERTY(float, velocity, 0)
STATS_PROPERTY(float, yaw, 0)
@ -105,6 +106,7 @@ signals:
void audioPingChanged();
void avatarPingChanged();
void entitiesPingChanged();
void assetPingChanged();
void positionChanged();
void velocityChanged();
void yawChanged();

17
libraries/animation/src/AnimationCache.cpp
View file

@ -39,14 +39,16 @@ QSharedPointer<Resource> AnimationCache::createResource(const QUrl& url, const Q
return QSharedPointer<Resource>(new Animation(url), &Resource::allReferencesCleared);
}
AnimationReader::AnimationReader(const QUrl& url, QNetworkReply* reply) :
Animation::Animation(const QUrl& url) : Resource(url) {}
AnimationReader::AnimationReader(const QUrl& url, const QByteArray& data) :
_url(url),
_reply(reply) {
_data(data) {
}
void AnimationReader::run() {
try {
if (!_reply) {
if (_data.isEmpty()) {
throw QString("Reply is NULL ?!");
}
QString urlname = _url.path().toLower();
@ -58,7 +60,7 @@ void AnimationReader::run() {
// Parse the FBX directly from the QNetworkReply
FBXGeometry* fbxgeo = nullptr;
if (_url.path().toLower().endsWith(".fbx")) {
fbxgeo = readFBX(_reply, QVariantHash(), _url.path());
fbxgeo = readFBX(_data, QVariantHash(), _url.path());
} else {
QString errorStr("usupported format");
emit onError(299, errorStr);
@ -71,11 +73,8 @@ void AnimationReader::run() {
} catch (const QString& error) {
emit onError(299, error);
}
_reply->deleteLater();
}
Animation::Animation(const QUrl& url) : Resource(url) {}
bool Animation::isLoaded() const {
return _loaded && _geometry;
}
@ -108,9 +107,9 @@ const QVector<FBXAnimationFrame>& Animation::getFramesReference() const {
return _geometry->animationFrames;
}
void Animation::downloadFinished(QNetworkReply* reply) {
void Animation::downloadFinished(const QByteArray& data) {
// parse the animation/fbx file on a background thread.
AnimationReader* animationReader = new AnimationReader(reply->url(), reply);
AnimationReader* animationReader = new AnimationReader(_url, data);
connect(animationReader, SIGNAL(onSuccess(FBXGeometry*)), SLOT(animationParseSuccess(FBXGeometry*)));
connect(animationReader, SIGNAL(onError(int, QString)), SLOT(animationParseError(int, QString)));
QThreadPool::globalInstance()->start(animationReader);

6
libraries/animation/src/AnimationCache.h
View file

@ -65,7 +65,7 @@ public:
const QVector<FBXAnimationFrame>& getFramesReference() const;
protected:
virtual void downloadFinished(QNetworkReply* reply);
virtual void downloadFinished(const QByteArray& data) override;
protected slots:
void animationParseSuccess(FBXGeometry* geometry);
@ -81,7 +81,7 @@ class AnimationReader : public QObject, public QRunnable {
Q_OBJECT
public:
AnimationReader(const QUrl& url, QNetworkReply* reply);
AnimationReader(const QUrl& url, const QByteArray& data);
virtual void run();
signals:
@ -90,7 +90,7 @@ signals:
private:
QUrl _url;
QNetworkReply* _reply;
QByteArray _data;
};
class AnimationDetails {

16
libraries/audio/src/Sound.cpp
View file

@ -56,16 +56,17 @@ Sound::Sound(const QUrl& url, bool isStereo) :
}
void Sound::downloadFinished(QNetworkReply* reply) {
void Sound::downloadFinished(const QByteArray& data) {
// replace our byte array with the downloaded data
QByteArray rawAudioByteArray = reply->readAll();
QString fileName = reply->url().fileName();
QByteArray rawAudioByteArray = QByteArray(data);
QString fileName = getURL().fileName();
const QString WAV_EXTENSION = ".wav";
if (reply->hasRawHeader("Content-Type") || fileName.endsWith(WAV_EXTENSION)) {
if (fileName.endsWith(WAV_EXTENSION)) {
QByteArray headerContentType = reply->rawHeader("Content-Type");
QString headerContentType = "audio/x-wav";
//QByteArray headerContentType = reply->rawHeader("Content-Type");
// WAV audio file encountered
if (headerContentType == "audio/x-wav"
@ -80,9 +81,9 @@ void Sound::downloadFinished(QNetworkReply* reply) {
} else {
// check if this was a stereo raw file
// since it's raw the only way for us to know that is if the file was called .stereo.raw
if (reply->url().fileName().toLower().endsWith("stereo.raw")) {
if (fileName.toLower().endsWith("stereo.raw")) {
_isStereo = true;
qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << reply->url() << "as stereo audio file.";
qCDebug(audio) << "Processing sound of" << rawAudioByteArray.size() << "bytes from" << getURL() << "as stereo audio file.";
}
// Process as RAW file
@ -94,7 +95,6 @@ void Sound::downloadFinished(QNetworkReply* reply) {
}
_isReady = true;
reply->deleteLater();
}
void Sound::downSample(const QByteArray& rawAudioByteArray) {

2
libraries/audio/src/Sound.h
View file

@ -39,7 +39,7 @@ private:
void downSample(const QByteArray& rawAudioByteArray);
void interpretAsWav(const QByteArray& inputAudioByteArray, QByteArray& outputAudioByteArray);
virtual void downloadFinished(QNetworkReply* reply);
virtual void downloadFinished(const QByteArray& data) override;
};
typedef QSharedPointer<Sound> SharedSoundPointer;

217
libraries/networking/src/AssetClient.cpp Normal file
View file

@ -0,0 +1,217 @@
//
// AssetClient.cpp
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/21
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "AssetClient.h"
#include <QBuffer>
#include <QThread>
#include "AssetRequest.h"
#include "NodeList.h"
#include "PacketReceiver.h"
#include "AssetUtils.h"
MessageID AssetClient::_currentID = 0;
AssetClient::AssetClient() {
auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
packetReceiver.registerListener(PacketType::AssetGetInfoReply, this, "handleAssetGetInfoReply");
packetReceiver.registerMessageListener(PacketType::AssetGetReply, this, "handleAssetGetReply");
packetReceiver.registerListener(PacketType::AssetUploadReply, this, "handleAssetUploadReply");
}
AssetRequest* AssetClient::create(QString hash, QString extension) {
if (QThread::currentThread() != thread()) {
AssetRequest* req;
QMetaObject::invokeMethod(this, "create",
Qt::BlockingQueuedConnection,
Q_RETURN_ARG(AssetRequest*, req),
Q_ARG(QString, hash),
Q_ARG(QString, extension));
return req;
}
if (hash.length() != HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
return nullptr;
}
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (assetServer) {
auto assetClient = DependencyManager::get<AssetClient>();
auto request = new AssetRequest(assetClient.data(), hash, extension);
return request;
}
return nullptr;
}
bool AssetClient::getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback) {
if (hash.length() != HASH_HEX_LENGTH) {
qDebug() << "Invalid hash size";
return false;
}
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (assetServer) {
auto packet = NLPacket::create(PacketType::AssetGet);
auto messageID = ++_currentID;
packet->writePrimitive(messageID);
packet->write(hash.toLatin1());
packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1());
packet->writePrimitive(start);
packet->writePrimitive(end);
nodeList->sendPacket(std::move(packet), *assetServer);
_pendingRequests[messageID] = callback;
return true;
}
return false;
}
bool AssetClient::getAssetInfo(QString hash, QString extension, GetInfoCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (assetServer) {
auto packet = NLPacket::create(PacketType::AssetGetInfo);
auto messageID = ++_currentID;
packet->writePrimitive(messageID);
packet->write(hash.toLatin1().constData(), HASH_HEX_LENGTH);
packet->writePrimitive(uint8_t(extension.length()));
packet->write(extension.toLatin1());
nodeList->sendPacket(std::move(packet), *assetServer);
_pendingInfoRequests[messageID] = callback;
return true;
}
return false;
}
void AssetClient::handleAssetGetInfoReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode) {
MessageID messageID;
packet->readPrimitive(&messageID);
auto assetHash = QString(packet->read(HASH_HEX_LENGTH));
AssetServerError error;
packet->readPrimitive(&error);
AssetInfo info { assetHash, 0 };
if (error == NO_ERROR) {
packet->readPrimitive(&info.size);
}
if (_pendingInfoRequests.contains(messageID)) {
auto callback = _pendingInfoRequests.take(messageID);
callback(error == NO_ERROR, info);
}
}
void AssetClient::handleAssetGetReply(QSharedPointer<NLPacketList> packetList, SharedNodePointer senderNode) {
QByteArray data = packetList->getMessage();
QBuffer packet { &data };
packet.open(QIODevice::ReadOnly);
auto assetHash = packet.read(HASH_HEX_LENGTH);
qDebug() << "Got reply for asset: " << assetHash;
MessageID messageID;
packet.read(reinterpret_cast<char*>(&messageID), sizeof(messageID));
AssetServerError error;
packet.read(reinterpret_cast<char*>(&error), sizeof(AssetServerError));
QByteArray assetData;
if (!error) {
DataOffset length;
packet.read(reinterpret_cast<char*>(&length), sizeof(DataOffset));
data = packet.read(length);
} else {
qDebug() << "Failure getting asset: " << error;
}
if (_pendingRequests.contains(messageID)) {
auto callback = _pendingRequests.take(messageID);
callback(error == NO_ERROR, data);
}
}
bool AssetClient::uploadAsset(QByteArray data, QString extension, UploadResultCallback callback) {
auto nodeList = DependencyManager::get<NodeList>();
SharedNodePointer assetServer = nodeList->soloNodeOfType(NodeType::AssetServer);
if (assetServer) {
auto packetList = std::unique_ptr<NLPacketList>(new NLPacketList(PacketType::AssetUpload, QByteArray(), true, true));
auto messageID = ++_currentID;
packetList->writePrimitive(messageID);
packetList->writePrimitive(static_cast<uint8_t>(extension.length()));
packetList->write(extension.toLatin1().constData(), extension.length());
qDebug() << "Extension length: " << extension.length();
qDebug() << "Extension: " << extension;
uint64_t size = data.length();
packetList->writePrimitive(size);
packetList->write(data.constData(), size);
nodeList->sendPacketList(std::move(packetList), *assetServer);
_pendingUploads[messageID] = callback;
return true;
}
return false;
}
void AssetClient::handleAssetUploadReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode) {
MessageID messageID;
packet->readPrimitive(&messageID);
AssetServerError error;
packet->readPrimitive(&error);
QString hashString { "" };
if (error) {
qDebug() << "Error uploading file to asset server";
} else {
auto hashData = packet->read(HASH_HEX_LENGTH);
hashString = QString(hashData);
qDebug() << "Successfully uploaded asset to asset-server - SHA256 hash is " << hashString;
}
if (_pendingUploads.contains(messageID)) {
auto callback = _pendingUploads.take(messageID);
callback(error == NO_ERROR, hashString);
}
}

61
libraries/networking/src/AssetClient.h Normal file
View file

@ -0,0 +1,61 @@
//
// AssetClient.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/21
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AssetClient_h
#define hifi_AssetClient_h
#include <QString>
#include <DependencyManager.h>
#include "AssetUtils.h"
#include "LimitedNodeList.h"
#include "NLPacket.h"
class AssetRequest;
struct AssetInfo {
QString hash;
int64_t size;
};
using ReceivedAssetCallback = std::function<void(bool result, QByteArray data)>;
using GetInfoCallback = std::function<void(bool result, AssetInfo info)>;
using UploadResultCallback = std::function<void(bool result, QString hash)>;
class AssetClient : public QObject, public Dependency {
Q_OBJECT
public:
AssetClient();
Q_INVOKABLE AssetRequest* create(QString hash, QString extension);
private slots:
void handleAssetGetInfoReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
void handleAssetGetReply(QSharedPointer<NLPacketList> packetList, SharedNodePointer senderNode);
void handleAssetUploadReply(QSharedPointer<NLPacket> packet, SharedNodePointer senderNode);
private:
friend class AssetRequest;
friend class Menu;
bool getAssetInfo(QString hash, QString extension, GetInfoCallback callback);
bool getAsset(QString hash, QString extension, DataOffset start, DataOffset end, ReceivedAssetCallback callback);
bool uploadAsset(QByteArray data, QString extension, UploadResultCallback callback);
static MessageID _currentID;
QHash<MessageID, ReceivedAssetCallback> _pendingRequests;
QHash<MessageID, GetInfoCallback> _pendingInfoRequests;
QHash<MessageID, UploadResultCallback> _pendingUploads;
};
#endif

75
libraries/networking/src/AssetRequest.cpp Normal file
View file

@ -0,0 +1,75 @@
//
// AssetRequest.cpp
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/24
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "AssetRequest.h"
#include <algorithm>
#include <QThread>
#include "AssetClient.h"
#include "NodeList.h"
AssetRequest::AssetRequest(QObject* parent, QString hash, QString extension) :
QObject(parent),
_hash(hash),
_extension(extension)
{
}
void AssetRequest::start() {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "start", Qt::AutoConnection);
return;
}
if (_state == NOT_STARTED) {
_state = WAITING_FOR_INFO;
auto assetClient = DependencyManager::get<AssetClient>();
assetClient->getAssetInfo(_hash, _extension, [this](bool success, AssetInfo info) {
_info = info;
_data.resize(info.size);
const DataOffset CHUNK_SIZE = 1024000000;
qDebug() << "Got size of " << _hash << " : " << info.size << " bytes";
// Round up
int numChunks = (info.size + CHUNK_SIZE - 1) / CHUNK_SIZE;
auto assetClient = DependencyManager::get<AssetClient>();
for (int i = 0; i < numChunks; ++i) {
++_numPendingRequests;
auto start = i * CHUNK_SIZE;
auto end = std::min((i + 1) * CHUNK_SIZE, info.size);
assetClient->getAsset(_hash, _extension, start, end, [this, start, end](bool success, QByteArray data) {
Q_ASSERT(data.size() == (end - start));
if (success) {
_result = Success;
memcpy((_data.data() + start), data.constData(), end - start);
_totalReceived += data.size();
emit progress(_totalReceived, _info.size);
} else {
_result = Error;
qDebug() << "Got error retrieving asset";
}
--_numPendingRequests;
if (_numPendingRequests == 0) {
_state = FINISHED;
emit finished(this);
}
});
}
});
}
}

61
libraries/networking/src/AssetRequest.h Normal file
View file

@ -0,0 +1,61 @@
//
// AssetRequest.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/24
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AssetRequest_h
#define hifi_AssetRequest_h
#include <QByteArray>
#include <QObject>
#include <QString>
#include "AssetClient.h"
#include "AssetUtils.h"
class AssetRequest : public QObject {
Q_OBJECT
public:
enum State {
NOT_STARTED = 0,
WAITING_FOR_INFO,
WAITING_FOR_DATA,
FINISHED
};
enum Result {
Success = 0,
Timeout,
NotFound,
Error,
};
AssetRequest(QObject* parent, QString hash, QString extension);
Q_INVOKABLE void start();
const QByteArray& getData() { return _data; }
signals:
void finished(AssetRequest*);
void progress(qint64 totalReceived, qint64 total);
private:
State _state = NOT_STARTED;
Result _result;
AssetInfo _info;
uint64_t _totalReceived { 0 };
QString _hash;
QString _extension;
QByteArray _data;
int _numPendingRequests { 0 };
};
#endif

50
libraries/networking/src/AssetResourceRequest.cpp Normal file
View file

@ -0,0 +1,50 @@
//
// AssetResourceRequest.cpp
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "AssetResourceRequest.h"
#include "AssetClient.h"
#include "AssetRequest.h"
void AssetResourceRequest::doSend() {
// Make request to atp
auto assetClient = DependencyManager::get<AssetClient>();
auto parts = _url.path().split(".", QString::SkipEmptyParts);
auto hash = parts[0];
auto extension = parts.length() > 1 ? parts[1] : "";
auto request = assetClient->create(hash, extension);
if (!request) {
return;
}
connect(request, &AssetRequest::progress, this, &AssetResourceRequest::progress);
QObject::connect(request, &AssetRequest::finished, [this](AssetRequest* req) mutable {
if (_state != IN_PROGRESS) return;
_state = FINISHED;
if (true) {
_data = req->getData();
_result = ResourceRequest::SUCCESS;
emit finished();
} else {
_result = ResourceRequest::ERROR;
emit finished();
}
});
request->start();
}
void AssetResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) {
qDebug() << "Got asset data: " << bytesReceived << " / " << bytesTotal;
emit progress(bytesReceived, bytesTotal);
}

31
libraries/networking/src/AssetResourceRequest.h Normal file
View file

@ -0,0 +1,31 @@
//
// AssetResourceRequest.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AssetResourceRequest_h
#define hifi_AssetResourceRequest_h
#include <QUrl>
#include "ResourceRequest.h"
class AssetResourceRequest : public ResourceRequest {
Q_OBJECT
public:
AssetResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { }
protected:
virtual void doSend() override;
private slots:
void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal);
};
#endif

30
libraries/networking/src/AssetUtils.h Normal file
View file

@ -0,0 +1,30 @@
//
// AssetUtils.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/30
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_AssetUtils_h
#define hifi_AssetUtils_h
#include "NLPacketList.h"
using MessageID = uint32_t;
using DataOffset = int64_t;
const size_t HASH_HEX_LENGTH = 64;
const uint64_t MAX_UPLOAD_SIZE = 1000 * 1000 * 1000; // 1GB
enum AssetServerError : uint8_t {
NO_ERROR = 0,
ASSET_NOT_FOUND,
INVALID_BYTE_RANGE,
ASSET_TOO_LARGE,
};
#endif

2
libraries/networking/src/Assignment.cpp
View file

@ -28,6 +28,8 @@ Assignment::Type Assignment::typeForNodeType(NodeType_t nodeType) {
return Assignment::AgentType;
case NodeType::EntityServer:
return Assignment::EntityServerType;
case NodeType::AssetServer:
return Assignment::AssetServerType;
default:
return Assignment::AllTypes;
}

6
libraries/networking/src/Assignment.h
View file

@ -29,9 +29,9 @@ public:
AudioMixerType = 0,
AvatarMixerType = 1,
AgentType = 2,
UNUSED_0 = 3,
UNUSED_1 = 4,
UNUSED_2 = 5,
AssetServerType = 3,
UNUSED_0 = 4,
UNUSED_1 = 5,
EntityServerType = 6,
AllTypes = 7
};

28
libraries/networking/src/FileResourceRequest.cpp Normal file
View file

@ -0,0 +1,28 @@
//
// FileResourceRequest.cpp
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "FileResourceRequest.h"
#include <QFile>
void FileResourceRequest::doSend() {
QString filename = _url.toLocalFile();
QFile file(filename);
_state = FINISHED;
if (file.open(QFile::ReadOnly)) {
_data = file.readAll();
_result = ResourceRequest::SUCCESS;
emit finished();
} else {
_result = ResourceRequest::ERROR;
emit finished();
}
}

28
libraries/networking/src/FileResourceRequest.h Normal file
View file

@ -0,0 +1,28 @@
//
// FileResourceRequest.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_FileResourceRequest_h
#define hifi_FileResourceRequest_h
#include <QUrl>
#include "ResourceRequest.h"
class FileResourceRequest : public ResourceRequest {
Q_OBJECT
public:
FileResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { }
protected:
virtual void doSend() override;
};
#endif

96
libraries/networking/src/HTTPResourceRequest.cpp Normal file
View file

@ -0,0 +1,96 @@
//
// HTTPResourceRequest.cpp
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "HTTPResourceRequest.h"
#include <QFile>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <SharedUtil.h>
#include "NetworkAccessManager.h"
#include "NetworkLogging.h"
HTTPResourceRequest::~HTTPResourceRequest() {
if (_reply) {
_reply->disconnect(this);
_reply->deleteLater();
_reply = nullptr;
}
}
void HTTPResourceRequest::doSend() {
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
QNetworkRequest networkRequest = QNetworkRequest(_url);
networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
if (_cacheEnabled) {
networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache);
} else {
networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork);
}
_reply = networkAccessManager.get(networkRequest);
connect(_reply, &QNetworkReply::finished, this, &HTTPResourceRequest::onRequestFinished);
static const int TIMEOUT_MS = 10000;
connect(&_sendTimer, &QTimer::timeout, this, &HTTPResourceRequest::onTimeout);
_sendTimer.setSingleShot(true);
_sendTimer.start(TIMEOUT_MS);
}
void HTTPResourceRequest::onRequestFinished() {
Q_ASSERT(_state == IN_PROGRESS);
Q_ASSERT(_reply);
_state = FINISHED;
auto error = _reply->error();
if (error == QNetworkReply::NoError) {
_data = _reply->readAll();
_loadedFromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
_result = ResourceRequest::SUCCESS;
emit finished();
} else if (error == QNetworkReply::TimeoutError) {
_result = ResourceRequest::TIMEOUT;
emit finished();
} else {
_result = ResourceRequest::ERROR;
emit finished();
}
_reply->deleteLater();
_reply = nullptr;
}
void HTTPResourceRequest::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) {
if (_state == IN_PROGRESS) {
// We've received data, so reset the timer
_sendTimer.start();
}
emit progress(bytesReceived, bytesTotal);
}
void HTTPResourceRequest::onTimeout() {
Q_ASSERT(_state != UNSENT);
if (_state == IN_PROGRESS) {
qCDebug(networking) << "Timed out loading " << _url;
_reply->abort();
_state = FINISHED;
_result = TIMEOUT;
emit finished();
}
}

41
libraries/networking/src/HTTPResourceRequest.h Normal file
View file

@ -0,0 +1,41 @@
//
// HTTPResourceRequest.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_HTTPResourceRequest_h
#define hifi_HTTPResourceRequest_h
#include <QNetworkReply>
#include <QUrl>
#include <QTimer>
#include "ResourceRequest.h"
class HTTPResourceRequest : public ResourceRequest {
Q_OBJECT
public:
~HTTPResourceRequest();
HTTPResourceRequest(QObject* parent, const QUrl& url) : ResourceRequest(parent, url) { }
protected:
virtual void doSend() override;
private slots:
void onTimeout();
void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal);
void onRequestFinished();
private:
QTimer _sendTimer;
QNetworkReply* _reply { nullptr };
};
#endif

20
libraries/networking/src/LimitedNodeList.cpp
View file

@ -32,6 +32,7 @@
#include "HifiSockAddr.h"
#include "UUID.h"
#include "NetworkLogging.h"
#include "udt/Packet.h"
const char SOLO_NODE_TYPES[2] = {
NodeType::AvatarMixer,
@ -344,6 +345,19 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr<NLPacketList> packetList,
return _nodeSocket.writePacketList(std::move(packetList), sockAddr);
}
qint64 LimitedNodeList::sendPacketList(std::unique_ptr<NLPacketList> packetList, const Node& destinationNode) {
// close the last packet in the list
packetList->closeCurrentPacket();
for (std::unique_ptr<udt::Packet>& packet : packetList->_packets) {
NLPacket* nlPacket = static_cast<NLPacket*>(packet.get());
collectPacketStats(*nlPacket);
fillPacketHeader(*nlPacket, destinationNode.getConnectionSecret());
}
return _nodeSocket.writePacketList(std::move(packetList), *destinationNode.getActiveSocket());
}
qint64 LimitedNodeList::sendPacket(std::unique_ptr<NLPacket> packet, const Node& destinationNode,
const HifiSockAddr& overridenSockAddr) {
// use the node's active socket as the destination socket if there is no overriden socket address
@ -428,6 +442,10 @@ void LimitedNodeList::handleNodeKill(const SharedNodePointer& node) {
qCDebug(networking) << "Killed" << *node;
node->stopPingTimer();
emit nodeKilled(node);
if (auto activeSocket = node->getActiveSocket()) {
_nodeSocket.cleanupConnection(*activeSocket);
}
}
SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType,
@ -531,7 +549,7 @@ unsigned int LimitedNodeList::broadcastToNodes(std::unique_ptr<NLPacket> packet,
return n;
}
SharedNodePointer LimitedNodeList::soloNodeOfType(char nodeType) {
SharedNodePointer LimitedNodeList::soloNodeOfType(NodeType_t nodeType) {
return nodeMatchingPredicate([&](const SharedNodePointer& node){
return node->getType() == nodeType;
});

3
libraries/networking/src/LimitedNodeList.h
View file

@ -128,6 +128,7 @@ public:
qint64 sendPacketList(NLPacketList& packetList, const HifiSockAddr& sockAddr,
const QUuid& connectionSecret = QUuid());
qint64 sendPacketList(std::unique_ptr<NLPacketList> packetList, const HifiSockAddr& sockAddr);
qint64 sendPacketList(std::unique_ptr<NLPacketList> packetList, const Node& destinationNode);
void (*linkedDataCreateCallback)(Node *);
@ -150,7 +151,7 @@ public:
int updateNodeWithDataFromPacket(QSharedPointer<NLPacket> packet, SharedNodePointer matchingNode);
unsigned int broadcastToNodes(std::unique_ptr<NLPacket> packet, const NodeSet& destinationNodeTypes);
SharedNodePointer soloNodeOfType(char nodeType);
SharedNodePointer soloNodeOfType(NodeType_t nodeType);
void getPacketStats(float &packetsPerSecond, float &bytesPerSecond);
void resetPacketStats();

1
libraries/networking/src/NodeType.h
View file

@ -22,6 +22,7 @@ namespace NodeType {
const NodeType_t Agent = 'I';
const NodeType_t AudioMixer = 'M';
const NodeType_t AvatarMixer = 'W';
const NodeType_t AssetServer = 'A';
const NodeType_t Unassigned = 1;
void init();

19
libraries/networking/src/PacketReceiver.cpp
View file

@ -19,8 +19,10 @@
#include "NodeList.h"
#include "SharedUtil.h"
Q_DECLARE_METATYPE(QSharedPointer<NLPacketList>);
PacketReceiver::PacketReceiver(QObject* parent) : QObject(parent) {
qRegisterMetaType<QSharedPointer<NLPacket>>();
qRegisterMetaType<QSharedPointer<NLPacketList>>();
}
bool PacketReceiver::registerListenerForTypes(PacketTypeList types, QObject* listener, const char* slot) {
@ -206,12 +208,13 @@ void PacketReceiver::registerVerifiedListener(PacketType type, QObject* object,
void PacketReceiver::unregisterListener(QObject* listener) {
Q_ASSERT_X(listener, "PacketReceiver::unregisterListener", "No listener to unregister");
QMutexLocker packetListenerLocker(&_packetListenerLock);
std::remove_if(std::begin(_packetListenerMap), std::end(_packetListenerMap),
[&listener](const ObjectMethodPair& pair) {
return pair.first == listener;
});
packetListenerLocker.unlock();
{
QMutexLocker packetListenerLocker(&_packetListenerLock);
std::remove_if(std::begin(_packetListenerMap), std::end(_packetListenerMap),
[&listener](const ObjectMethodPair& pair) {
return pair.first == listener;
});
}
QMutexLocker directConnectSetLocker(&_directConnectSetMutex);
_directlyConnectedObjects.remove(listener);
@ -454,7 +457,6 @@ void PacketReceiver::handleVerifiedPacket(std::unique_ptr<udt::Packet> packet) {
// if it exists, remove the listener from _directlyConnectedObjects
QMutexLocker locker(&_directConnectSetMutex);
_directlyConnectedObjects.remove(listener.first);
locker.unlock();
}
} else if (it == _packetListenerMap.end()) {
@ -463,7 +465,4 @@ void PacketReceiver::handleVerifiedPacket(std::unique_ptr<udt::Packet> packet) {
// insert a dummy listener so we don't print this again
_packetListenerMap.insert(nlPacket->getType(), { nullptr, QMetaMethod() });
}
packetListenerLocker.unlock();
}

186
libraries/networking/src/ResourceCache.cpp
View file

@ -155,11 +155,17 @@ void ResourceCache::clearUnusedResource() {
void ResourceCache::attemptRequest(Resource* resource) {
auto sharedItems = DependencyManager::get<ResourceCacheSharedItems>();
if (_requestLimit <= 0) {
qDebug() << "REQUEST LIMIT REACHED (" << _requestLimit << "), queueing: " << resource->getURL();
// wait until a slot becomes available
sharedItems->_pendingRequests.append(resource);
return;
}
_requestLimit--;
qDebug() << "-- Decreasing limit for : " << resource->getURL();
// Disable request limiting for ATP
if (resource->getURL() != URL_SCHEME_ATP) {
_requestLimit--;
}
sharedItems->_loadingRequests.append(resource);
resource->makeRequest();
}
@ -168,7 +174,10 @@ void ResourceCache::requestCompleted(Resource* resource) {
auto sharedItems = DependencyManager::get<ResourceCacheSharedItems>();
sharedItems->_loadingRequests.removeOne(resource);
_requestLimit++;
qDebug() << "++ Increasing limit after finished: " << resource->getURL();
if (resource->getURL() != URL_SCHEME_ATP) {
_requestLimit++;
}
// look for the highest priority pending request
int highestIndex = -1;
@ -196,24 +205,22 @@ int ResourceCache::_requestLimit = DEFAULT_REQUEST_LIMIT;
Resource::Resource(const QUrl& url, bool delayLoad) :
_url(url),
_request(url) {
_activeUrl(url),
_request(nullptr) {
init();
_request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
_request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache);
// start loading immediately unless instructed otherwise
if (!(_startedLoading || delayLoad)) {
attemptRequest();
QTimer::singleShot(0, this, &Resource::ensureLoading);
}
}
Resource::~Resource() {
if (_reply) {
if (_request) {
ResourceCache::requestCompleted(this);
delete _reply;
_reply = nullptr;
_request->deleteLater();
_request = nullptr;
}
}
@ -259,21 +266,17 @@ float Resource::getLoadPriority() {
}
void Resource::refresh() {
if (_reply && !(_loaded || _failedToLoad)) {
if (_request && !(_loaded || _failedToLoad)) {
return;
}
if (_reply) {
if (_request) {
_request->disconnect(this);
_request->deleteLater();
_request = nullptr;
ResourceCache::requestCompleted(this);
_reply->disconnect(this);
_replyTimer->disconnect(this);
_reply->deleteLater();
_reply = nullptr;
_replyTimer->deleteLater();
_replyTimer = nullptr;
}
init();
_request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork);
ensureLoading();
emit onRefresh();
}
@ -303,6 +306,7 @@ void Resource::init() {
_failedToLoad = false;
_loaded = false;
_attempts = 0;
_activeUrl = _url;
if (_url.isEmpty()) {
_startedLoading = _loaded = true;
@ -318,6 +322,7 @@ void Resource::attemptRequest() {
}
void Resource::finishedLoading(bool success) {
qDebug() << "Finished loading: " << _url;
if (success) {
_loaded = true;
} else {
@ -330,94 +335,91 @@ void Resource::reinsert() {
_cache->_resources.insert(_url, _self);
}
static const int REPLY_TIMEOUT_MS = 5000;
void Resource::handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) {
_bytesReceived = bytesReceived;
_bytesTotal = bytesTotal;
_replyTimer->start(REPLY_TIMEOUT_MS);
}
void Resource::handleReplyError() {
handleReplyErrorInternal(_reply->error());
}
void Resource::handleReplyTimeout() {
handleReplyErrorInternal(QNetworkReply::TimeoutError);
}
void Resource::makeRequest() {
_reply = NetworkAccessManager::getInstance().get(_request);
Q_ASSERT(!_request);
connect(_reply, SIGNAL(downloadProgress(qint64,qint64)), SLOT(handleDownloadProgress(qint64,qint64)));
connect(_reply, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(handleReplyError()));
connect(_reply, SIGNAL(finished()), SLOT(handleReplyFinished()));
_request = ResourceManager::createResourceRequest(this, _activeUrl);
if (!_request) {
qDebug() << "Failed to get request for " << _url;
ResourceCache::requestCompleted(this);
finishedLoading(false);
return;
}
qDebug() << "Starting request for: " << _url;
connect(_request, &ResourceRequest::progress, this, &Resource::handleDownloadProgress);
connect(_request, &ResourceRequest::finished, this, &Resource::handleReplyFinished);
_replyTimer = new QTimer(this);
connect(_replyTimer, SIGNAL(timeout()), SLOT(handleReplyTimeout()));
_replyTimer->setSingleShot(true);
_replyTimer->start(REPLY_TIMEOUT_MS);
_bytesReceived = _bytesTotal = 0;
_request->send();
}
void Resource::handleReplyErrorInternal(QNetworkReply::NetworkError error) {
_reply->disconnect(this);
_replyTimer->disconnect(this);
_reply->deleteLater();
_reply = nullptr;
_replyTimer->deleteLater();
_replyTimer = nullptr;
ResourceCache::requestCompleted(this);
// retry for certain types of failures
switch (error) {
case QNetworkReply::RemoteHostClosedError:
case QNetworkReply::TimeoutError:
case QNetworkReply::TemporaryNetworkFailureError:
case QNetworkReply::ProxyConnectionClosedError:
case QNetworkReply::ProxyTimeoutError:
case QNetworkReply::UnknownNetworkError:
case QNetworkReply::UnknownProxyError:
case QNetworkReply::UnknownContentError:
case QNetworkReply::ProtocolFailure: {
// retry with increasing delays
const int MAX_ATTEMPTS = 8;
const int BASE_DELAY_MS = 1000;
if (++_attempts < MAX_ATTEMPTS) {
QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest()));
qCWarning(networking) << "error downloading url =" << _url.toDisplayString() << ", error =" << error << ", retrying (" << _attempts << "/" << MAX_ATTEMPTS << ")";
return;
}
// fall through to final failure
}
default:
qCCritical(networking) << "error downloading, url =" << _url.toDisplayString() << ", error =" << error;
emit failed(error);
finishedLoading(false);
break;
}
void Resource::handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal) {
_bytesReceived = bytesReceived;
_bytesTotal = bytesTotal;
}
void Resource::handleReplyFinished() {
Q_ASSERT(_request);
bool fromCache = _reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
qCDebug(networking) << "success downloading url =" << _url.toDisplayString() << (fromCache ? "from cache" : "");
auto result = _request->getResult();
if (result == ResourceRequest::SUCCESS) {
_data = _request->getData();
qDebug() << "Request finished for " << _url << ", " << _activeUrl;
_reply->disconnect(this);
_replyTimer->disconnect(this);
QNetworkReply* reply = _reply;
_reply = nullptr;
_replyTimer->deleteLater();
_replyTimer = nullptr;
ResourceCache::requestCompleted(this);
_request->disconnect(this);
_request->deleteLater();
_request = nullptr;
finishedLoading(true);
emit loaded(*reply);
downloadFinished(reply);
ResourceCache::requestCompleted(this);
emit loaded(_data);
downloadFinished(_data);
} else {
_request->disconnect(this);
_request->deleteLater();
_request = nullptr;
if (result == ResourceRequest::Result::TIMEOUT) {
qDebug() << "Timed out loading" << _url << "received" << _bytesReceived << "total" << _bytesTotal;
} else {
qDebug() << "Error loading " << _url;
}
bool retry = false;
switch (result) {
case ResourceRequest::Result::TIMEOUT:
case ResourceRequest::Result::ERROR: {
// retry with increasing delays
const int MAX_ATTEMPTS = 8;
const int BASE_DELAY_MS = 1000;
if (++_attempts < MAX_ATTEMPTS) {
QTimer::singleShot(BASE_DELAY_MS * (int)pow(2.0, _attempts), this, SLOT(attemptRequest()));
retry = true;
break;
}
// fall through to final failure
}
default:
finishedLoading(false);
break;
}
auto error = result == ResourceRequest::TIMEOUT ? QNetworkReply::TimeoutError : QNetworkReply::UnknownNetworkError;
emit failed(error);
if (!retry) {
ResourceCache::requestCompleted(this);
}
}
}
void Resource::downloadFinished(QNetworkReply* reply) {
void Resource::downloadFinished(const QByteArray& data) {
;
}

20
libraries/networking/src/ResourceCache.h
View file

@ -26,6 +26,8 @@
#include <DependencyManager.h>
#include "ResourceManager.h"
class QNetworkReply;
class QTimer;
@ -102,7 +104,7 @@ protected:
void reserveUnusedResource(qint64 resourceSize);
void clearUnusedResource();
static void attemptRequest(Resource* resource);
Q_INVOKABLE static void attemptRequest(Resource* resource);
static void requestCompleted(Resource* resource);
private:
@ -174,7 +176,7 @@ public:
signals:
/// Fired when the resource has been loaded.
void loaded(QNetworkReply& request);
void loaded(const QByteArray& request);
/// Fired when resource failed to load.
void failed(QNetworkReply::NetworkError error);
@ -189,7 +191,7 @@ protected:
virtual void init();
/// Called when the download has finished. The recipient should delete the reply when done with it.
virtual void downloadFinished(QNetworkReply* reply);
virtual void downloadFinished(const QByteArray& data);
/// Should be called by subclasses when all the loading that will be done has been done.
Q_INVOKABLE void finishedLoading(bool success);
@ -198,31 +200,29 @@ protected:
virtual void reinsert();
QUrl _url;
QNetworkRequest _request;
QUrl _activeUrl;
bool _startedLoading = false;
bool _failedToLoad = false;
bool _loaded = false;
QHash<QPointer<QObject>, float> _loadPriorities;
QWeakPointer<Resource> _self;
QPointer<ResourceCache> _cache;
QByteArray _data;
private slots:
void handleDownloadProgress(qint64 bytesReceived, qint64 bytesTotal);
void handleReplyError();
void handleDownloadProgress(uint64_t bytesReceived, uint64_t bytesTotal);
void handleReplyFinished();
void handleReplyTimeout();
private:
void setLRUKey(int lruKey) { _lruKey = lruKey; }
void makeRequest();
void handleReplyErrorInternal(QNetworkReply::NetworkError error);
void retry();
friend class ResourceCache;
ResourceRequest* _request = nullptr;
int _lruKey = 0;
QNetworkReply* _reply = nullptr;
QTimer* _replyTimer = nullptr;
qint64 _bytesReceived = 0;
qint64 _bytesTotal = 0;

33
libraries/networking/src/ResourceManager.cpp Normal file
View file

@ -0,0 +1,33 @@
//
// ResourceManager.cpp
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "ResourceManager.h"
#include "AssetResourceRequest.h"
#include "FileResourceRequest.h"
#include "HTTPResourceRequest.h"
#include <SharedUtil.h>
ResourceRequest* ResourceManager::createResourceRequest(QObject* parent, const QUrl& url) {
auto scheme = url.scheme();
if (scheme == URL_SCHEME_FILE) {
return new FileResourceRequest(parent, url);
} else if (scheme == URL_SCHEME_HTTP || scheme == URL_SCHEME_HTTPS || scheme == URL_SCHEME_FTP) {
return new HTTPResourceRequest(parent, url);
} else if (scheme == URL_SCHEME_ATP) {
return new AssetResourceRequest(parent, url);
}
qDebug() << "Failed to load: " << url.url();
return nullptr;
}

30
libraries/networking/src/ResourceManager.h Normal file
View file

@ -0,0 +1,30 @@
//
// ResourceManager.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_ResourceManager_h
#define hifi_ResourceManager_h
#include <functional>
#include "ResourceRequest.h"
const QString URL_SCHEME_FILE = "file";
const QString URL_SCHEME_HTTP = "http";
const QString URL_SCHEME_HTTPS = "https";
const QString URL_SCHEME_FTP = "ftp";
const QString URL_SCHEME_ATP = "atp";
class ResourceManager {
public:
static ResourceRequest* createResourceRequest(QObject* parent, const QUrl& url);
};
#endif

24
libraries/networking/src/ResourceRequest.cpp Normal file
View file

@ -0,0 +1,24 @@
//
// ResourceRequest.cpp
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#include "ResourceRequest.h"
ResourceRequest::ResourceRequest(QObject* parent, const QUrl& url) :
QObject(parent),
_url(url) {
}
void ResourceRequest::send() {
Q_ASSERT(_state == UNSENT);
_state = IN_PROGRESS;
doSend();
}

60
libraries/networking/src/ResourceRequest.h Normal file
View file

@ -0,0 +1,60 @@
//
// ResourceRequest.h
// libraries/networking/src
//
// Created by Ryan Huffman on 2015/07/23
// Copyright 2015 High Fidelity, Inc.
//
// Distributed under the Apache License, Version 2.0.
// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
//
#ifndef hifi_ResourceRequest_h
#define hifi_ResourceRequest_h
#include <QObject>
#include <QUrl>
class ResourceRequest : public QObject {
Q_OBJECT
public:
ResourceRequest(QObject* parent, const QUrl& url);
enum State {
UNSENT = 0,
IN_PROGRESS,
FINISHED
};
enum Result {
SUCCESS,
ERROR,
TIMEOUT,
NOT_FOUND
};
void send();
QByteArray getData() { return _data; }
State getState() const { return _state; }
Result getResult() const { return _result; }
QUrl getUrl() const { return _url; }
bool loadedFromCache() const { return _loadedFromCache; }
void setCacheEnabled(bool value) { _cacheEnabled = value; }
signals:
void progress(uint64_t bytesReceived, uint64_t bytesTotal);
void finished();
protected:
virtual void doSend() = 0;
QUrl _url;
State _state { UNSENT };
Result _result;
QByteArray _data;
bool _cacheEnabled { true };
bool _loadedFromCache { false };
};
#endif

5
libraries/networking/src/udt/Connection.cpp
View file

@ -76,6 +76,7 @@ SendQueue& Connection::getSendQueue() {
QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::packetSent);
QObject::connect(_sendQueue.get(), &SendQueue::packetSent, this, &Connection::recordSentPackets);
QObject::connect(_sendQueue.get(), &SendQueue::packetRetransmitted, this, &Connection::recordRetransmission);
QObject::connect(_sendQueue.get(), &SendQueue::queueInactive, this, &Connection::queueInactive);
// set defaults on the send queue from our congestion control object
_sendQueue->setPacketSendPeriod(_congestionControl->_packetSendPeriod);
@ -85,6 +86,10 @@ SendQueue& Connection::getSendQueue() {
return *_sendQueue;
}
void Connection::queueInactive() {
emit connectionInactive(_destination);
}
void Connection::sendReliablePacket(std::unique_ptr<Packet> packet) {
Q_ASSERT_X(packet->isReliable(), "Connection::send", "Trying to send an unreliable packet reliably.");
getSendQueue().queuePacket(std::move(packet));

2
libraries/networking/src/udt/Connection.h
View file

@ -73,10 +73,12 @@ public:
signals:
void packetSent();
void connectionInactive(HifiSockAddr sockAdrr);
private slots:
void recordSentPackets(int payload, int total);
void recordRetransmission();
void queueInactive();
private:
void sendACK(bool wasCausedBySyncTimeout = true);

4
libraries/networking/src/udt/Packet.cpp
View file

@ -93,6 +93,8 @@ Packet::Packet(Packet&& other) :
_isReliable = other._isReliable;
_isPartOfMessage = other._isPartOfMessage;
_sequenceNumber = other._sequenceNumber;
_packetPosition = other._packetPosition;
_messageNumber = other._messageNumber;
}
Packet& Packet::operator=(Packet&& other) {
@ -101,6 +103,8 @@ Packet& Packet::operator=(Packet&& other) {
_isReliable = other._isReliable;
_isPartOfMessage = other._isPartOfMessage;
_sequenceNumber = other._sequenceNumber;
_packetPosition = other._packetPosition;
_messageNumber = other._messageNumber;
return *this;
}

8
libraries/networking/src/udt/PacketHeaders.h
View file

@ -73,7 +73,13 @@ enum class PacketType : uint8_t {
EntityEdit,
DomainServerConnectionToken,
DomainSettingsRequest,
DomainSettings
DomainSettings,
AssetGet,
AssetGetReply,
AssetUpload,
AssetUploadReply,
AssetGetInfo,
AssetGetInfoReply
};
const int NUM_BYTES_MD5_HASH = 16;

148
libraries/networking/src/udt/PacketList.cpp
View file

@ -82,7 +82,7 @@ std::unique_ptr<Packet> PacketList::createPacket() {
std::unique_ptr<Packet> PacketList::createPacketWithExtendedHeader() {
// use the static create method to create a new packet
auto packet = createPacket();
if (!_extendedHeader.isEmpty()) {
// add the extended header to the front of the packet
if (packet->write(_extendedHeader) == -1) {
@ -90,7 +90,7 @@ std::unique_ptr<Packet> PacketList::createPacketWithExtendedHeader() {
<< "- make sure that _extendedHeader is not larger than the payload capacity.";
}
}
return packet;
}
@ -112,83 +112,91 @@ QByteArray PacketList::getMessage() {
}
qint64 PacketList::writeData(const char* data, qint64 maxSize) {
if (!_currentPacket) {
// we don't have a current packet, time to set one up
_currentPacket = createPacketWithExtendedHeader();
}
// check if this block of data can fit into the currentPacket
if (maxSize <= _currentPacket->bytesAvailableForWrite()) {
// it fits, just write it to the current packet
_currentPacket->write(data, maxSize);
// return the number of bytes written
return maxSize;
} else {
// it does not fit - this may need to be in the next packet
if (!_isOrdered) {
auto newPacket = createPacketWithExtendedHeader();
if (_segmentStartIndex >= 0) {
// We in the process of writing a segment for an unordered PacketList.
// We need to try and pull the first part of the segment out to our new packet
// check now to see if this is an unsupported write
int numBytesToEnd = _currentPacket->bytesAvailableForWrite();
if ((newPacket->size() - numBytesToEnd) < maxSize) {
// this is an unsupported case - the segment is bigger than the size of an individual packet
// but the PacketList is not going to be sent ordered
qDebug() << "Error in PacketList::writeData - attempted to write a segment to an unordered packet that is"
<< "larger than the payload size.";
Q_ASSERT(false);
}
int segmentSize = _currentPacket->pos() - _segmentStartIndex;
// copy from currentPacket where the segment started to the beginning of the newPacket
newPacket->write(_currentPacket->getPayload() + _segmentStartIndex, segmentSize);
// the current segment now starts at the beginning of the new packet
_segmentStartIndex = _extendedHeader.size();
// shrink the current payload to the actual size of the packet
_currentPacket->setPayloadSize(_segmentStartIndex);
}
// move the current packet to our list of packets
_packets.push_back(std::move(_currentPacket));
// write the data to the newPacket
newPacket->write(data, maxSize);
// swap our current packet with the new packet
_currentPacket.swap(newPacket);
// return the number of bytes written to the new packet
return maxSize;
auto sizeRemaining = maxSize;
while (sizeRemaining > 0) {
if (!_currentPacket) {
// we don't have a current packet, time to set one up
_currentPacket = createPacketWithExtendedHeader();
}
// check if this block of data can fit into the currentPacket
if (sizeRemaining <= _currentPacket->bytesAvailableForWrite()) {
// it fits, just write it to the current packet
_currentPacket->write(data, sizeRemaining);
sizeRemaining = 0;
} else {
// we're an ordered PacketList - let's fit what we can into the current packet and then put the leftover
// into a new packet
int numBytesToEnd = _currentPacket->bytesAvailableForWrite();
_currentPacket->write(data, numBytesToEnd);
// move the current packet to our list of packets
_packets.push_back(std::move(_currentPacket));
// recursively call our writeData method for the remaining data to write to a new packet
return numBytesToEnd + writeData(data + numBytesToEnd, maxSize - numBytesToEnd);
// it does not fit - this may need to be in the next packet
if (!_isOrdered) {
auto newPacket = createPacketWithExtendedHeader();
if (_segmentStartIndex >= 0) {
// We in the process of writing a segment for an unordered PacketList.
// We need to try and pull the first part of the segment out to our new packet
// check now to see if this is an unsupported write
int numBytesToEnd = _currentPacket->bytesAvailableForWrite();
if ((newPacket->size() - numBytesToEnd) < sizeRemaining) {
// this is an unsupported case - the segment is bigger than the size of an individual packet
// but the PacketList is not going to be sent ordered
qDebug() << "Error in PacketList::writeData - attempted to write a segment to an unordered packet that is"
<< "larger than the payload size.";
Q_ASSERT(false);
}
int segmentSize = _currentPacket->pos() - _segmentStartIndex;
// copy from currentPacket where the segment started to the beginning of the newPacket
newPacket->write(_currentPacket->getPayload() + _segmentStartIndex, segmentSize);
// the current segment now starts at the beginning of the new packet
_segmentStartIndex = _extendedHeader.size();
// shrink the current payload to the actual size of the packet
_currentPacket->setPayloadSize(_segmentStartIndex);
}
// move the current packet to our list of packets
_packets.push_back(std::move(_currentPacket));
// write the data to the newPacket
newPacket->write(data, sizeRemaining);
// swap our current packet with the new packet
_currentPacket.swap(newPacket);
// We've written all of the data, so set sizeRemaining to 0
sizeRemaining = 0;
} else {
// we're an ordered PacketList - let's fit what we can into the current packet and then put the leftover
// into a new packet
int numBytesToEnd = _currentPacket->bytesAvailableForWrite();
_currentPacket->write(data, numBytesToEnd);
// Remove number of bytes written from sizeRemaining
sizeRemaining -= numBytesToEnd;
// Move the data pointer forward
data += numBytesToEnd;
// move the current packet to our list of packets
_packets.push_back(std::move(_currentPacket));
}
}
}
return maxSize;
}
void PacketList::closeCurrentPacket(bool shouldSendEmpty) {
if (shouldSendEmpty && !_currentPacket) {
_currentPacket = createPacketWithExtendedHeader();
}
if (_currentPacket) {
// move the current packet to our list of packets
_packets.push_back(std::move(_currentPacket));

290
libraries/networking/src/udt/SendQueue.cpp
View file

@ -12,6 +12,7 @@
#include "SendQueue.h"
#include <algorithm>
#include <thread>
#include <QtCore/QCoreApplication>
#include <QtCore/QThread>
@ -24,7 +25,27 @@
#include "Socket.h"
using namespace udt;
using namespace std::chrono;
class DoubleLock {
public:
DoubleLock(std::mutex& mutex1, std::mutex& mutex2) : _mutex1(mutex1), _mutex2(mutex2) { }
DoubleLock(const DoubleLock&) = delete;
DoubleLock& operator=(const DoubleLock&) = delete;
// Either locks all the mutexes or none of them
bool try_lock() { return (std::try_lock(_mutex1, _mutex2) == -1); }
// Locks all the mutexes
void lock() { std::lock(_mutex1, _mutex2); }
// Undefined behavior if not locked
void unlock() { _mutex1.unlock(); _mutex2.unlock(); }
private:
std::mutex& _mutex1;
std::mutex& _mutex2;
};
std::unique_ptr<SendQueue> SendQueue::create(Socket* socket, HifiSockAddr destination) {
auto queue = std::unique_ptr<SendQueue>(new SendQueue(socket, destination));
@ -200,8 +221,8 @@ void SendQueue::sendNewPacketAndAddToSentList(std::unique_ptr<Packet> newPacket,
// Insert the packet we have just sent in the sent list
QWriteLocker locker(&_sentLock);
_sentPackets[newPacket->getSequenceNumber()].swap(newPacket);
Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list");
}
Q_ASSERT_X(!newPacket, "SendQueue::sendNewPacketAndAddToSentList()", "Overriden packet in sent list");
emit packetSent(packetSize, payloadSize);
}
@ -210,8 +231,8 @@ void SendQueue::run() {
_isRunning = true;
while (_isRunning) {
// Record timing
_lastSendTimestamp = high_resolution_clock::now();
// Record how long the loop takes to execute
auto loopStartTimestamp = high_resolution_clock::now();
std::unique_lock<std::mutex> handshakeLock { _handshakeMutex };
@ -222,12 +243,13 @@ void SendQueue::run() {
// hold the time of last send in a static
static auto lastSendHandshake = high_resolution_clock::time_point();
static const int HANDSHAKE_RESEND_INTERVAL_MS = 100;
static const auto HANDSHAKE_RESEND_INTERVAL_MS = std::chrono::milliseconds(100);
// calculation the duration since the last handshake send
auto sinceLastHandshake = duration_cast<milliseconds>(high_resolution_clock::now() - lastSendHandshake);
auto sinceLastHandshake = std::chrono::duration_cast<std::chrono::milliseconds>(high_resolution_clock::now()
- lastSendHandshake);
if (sinceLastHandshake.count() >= HANDSHAKE_RESEND_INTERVAL_MS) {
if (sinceLastHandshake >= HANDSHAKE_RESEND_INTERVAL_MS) {
// it has been long enough since last handshake, send another
static auto handshakePacket = ControlPacket::create(ControlPacket::Handshake, 0);
@ -238,7 +260,8 @@ void SendQueue::run() {
// we wait for the ACK or the re-send interval to expire
_handshakeACKCondition.wait_until(handshakeLock,
high_resolution_clock::now() + milliseconds(HANDSHAKE_RESEND_INTERVAL_MS));
high_resolution_clock::now()
+ HANDSHAKE_RESEND_INTERVAL_MS);
// Once we're here we've either received the handshake ACK or it's going to be time to re-send a handshake.
// Either way let's continue processing - no packets will be sent if no handshake ACK has been received.
@ -246,109 +269,23 @@ void SendQueue::run() {
handshakeLock.unlock();
bool naksEmpty = true; // used at the end of processing to see if we should wait for NAKs
bool resentPacket = false;
bool sentAPacket = maybeResendPacket();
bool flowWindowFull = false;
// the following while makes sure that we find a packet to re-send, if there is one
while (!resentPacket) {
std::unique_lock<std::mutex> nakLocker(_naksLock);
if (_naks.getLength() > 0) {
naksEmpty = _naks.getLength() > 1;
// pull the sequence number we need to re-send
SequenceNumber resendNumber = _naks.popFirstSequenceNumber();
nakLocker.unlock();
// pull the packet to re-send from the sent packets list
QReadLocker sentLocker(&_sentLock);
// see if we can find the packet to re-send
auto it = _sentPackets.find(resendNumber);
if (it != _sentPackets.end()) {
// we found the packet - grab it
auto& resendPacket = *(it->second);
// unlock the sent packets
sentLocker.unlock();
// send it off
sendPacket(resendPacket);
emit packetRetransmitted();
// mark that we did resend a packet
resentPacket = true;
// break out of our while now that we have re-sent a packet
break;
} else {
// we didn't find this packet in the sentPackets queue - assume this means it was ACKed
// we'll fire the loop again to see if there is another to re-send
continue;
}
} else {
naksEmpty = true;
}
// break from the while, we didn't resend a packet
break;
}
bool packetsEmpty = false; // used after processing to check if we should wait for packets
bool sentPacket = false;
// if we didn't find a packet to re-send AND we think we can fit a new packet on the wire
// (this is according to the current flow window size) then we send out a new packet
if (_hasReceivedHandshakeACK
&& !resentPacket
&& seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber) <= _flowWindowSize) {
// we didn't re-send a packet, so time to send a new one
std::unique_lock<std::mutex> locker(_packetsLock);
if (_packets.size() > 0) {
SequenceNumber nextNumber = getNextSequenceNumber();
// grab the first packet we will send
std::unique_ptr<Packet> firstPacket;
firstPacket.swap(_packets.front());
_packets.pop_front();
std::unique_ptr<Packet> secondPacket;
if (((uint32_t) nextNumber & 0xF) == 0) {
// the first packet is the first in a probe pair - every 16 (rightmost 16 bits = 0) packets
// pull off a second packet if we can before we unlock
if (_packets.size() > 0) {
secondPacket.swap(_packets.front());
_packets.pop_front();
}
}
packetsEmpty = _packets.size() == 0;
// unlock the packets, we're done pulling
locker.unlock();
sentPacket = true;
// definitely send the first packet
sendNewPacketAndAddToSentList(move(firstPacket), nextNumber);
// do we have a second in a pair to send as well?
if (secondPacket) {
nextNumber = getNextSequenceNumber();
sendNewPacketAndAddToSentList(move(secondPacket), nextNumber);
}
} else {
packetsEmpty = true;
locker.unlock();
}
if (!sentAPacket) {
flowWindowFull = (seqlen(SequenceNumber { (uint32_t) _lastACKSequenceNumber }, _currentSequenceNumber) >
_flowWindowSize);
sentAPacket = maybeSendNewPacket();
}
// Keep track of how long the flow window has been full for
if (flowWindowFull && !_flowWindowWasFull) {
_flowWindowFullSince = loopStartTimestamp;
}
_flowWindowWasFull = flowWindowFull;
// since we're a while loop, give the thread a chance to process events
QCoreApplication::processEvents();
@ -357,30 +294,133 @@ void SendQueue::run() {
break;
}
if (packetsEmpty && naksEmpty) {
// During our processing above the loss list and packet list were both empty.
if (_hasReceivedHandshakeACK && !sentAPacket) {
static const std::chrono::seconds CONSIDER_INACTIVE_AFTER { 5 };
// If that is still the case we should use a condition_variable_any to sleep until we have data to handle.
// To confirm that the queue of packets and the NAKs list are still both empty we'll need to use the DoubleLock
DoubleLock doubleLock(_packetsLock, _naksLock);
// The packets queue and loss list mutexes are now both locked - check if they're still both empty
if (_packets.empty() && _naks.getLength() == 0) {
// both are empty - let's use a condition_variable_any to wait
_emptyCondition.wait(doubleLock);
if (flowWindowFull && (high_resolution_clock::now() - _flowWindowFullSince) > CONSIDER_INACTIVE_AFTER) {
// If the flow window has been full for over CONSIDER_INACTIVE_AFTER,
// then signal the queue is inactive
emit queueInactive();
} else {
// During our processing above we didn't send any packets and the flow window is not full.
// we have the double lock again - it'll be unlocked once it goes out of scope
// skip to the next iteration
// If that is still the case we should use a condition_variable_any to sleep until we have data to handle.
// To confirm that the queue of packets and the NAKs list are still both empty we'll need to use the DoubleLock
DoubleLock doubleLock(_packetsLock, _naksLock);
// The packets queue and loss list mutexes are now both locked - check if they're still both empty
if (doubleLock.try_lock() && _packets.empty() && _naks.getLength() == 0) {
// both are empty - let's use a condition_variable_any to wait
auto cvStatus = _emptyCondition.wait_for(doubleLock, CONSIDER_INACTIVE_AFTER);
// we have the double lock again - Make sure to unlock it
doubleLock.unlock();
// Check if we've been inactive for too long
if (cvStatus == std::cv_status::timeout) {
emit queueInactive();
}
// skip to the next iteration
continue;
}
}
}
auto loopEndTimestamp = high_resolution_clock::now();
// sleep as long as we need until next packet send, if we can
auto timeToSleep = (loopStartTimestamp + std::chrono::microseconds(_packetSendPeriod)) - loopEndTimestamp;
if (timeToSleep > timeToSleep.zero()) {
std::this_thread::sleep_for(timeToSleep);
}
}
}
bool SendQueue::maybeSendNewPacket() {
// we didn't re-send a packet, so time to send a new one
std::unique_lock<std::mutex> locker(_packetsLock);
if (_packets.size() > 0) {
SequenceNumber nextNumber = getNextSequenceNumber();
// grab the first packet we will send
std::unique_ptr<Packet> firstPacket;
firstPacket.swap(_packets.front());
_packets.pop_front();
std::unique_ptr<Packet> secondPacket;
if (((uint32_t) nextNumber & 0xF) == 0) {
// the first packet is the first in a probe pair - every 16 (rightmost 16 bits = 0) packets
// pull off a second packet if we can before we unlock
if (_packets.size() > 0) {
secondPacket.swap(_packets.front());
_packets.pop_front();
}
}
// unlock the packets, we're done pulling
locker.unlock();
// definitely send the first packet
sendNewPacketAndAddToSentList(move(firstPacket), nextNumber);
// do we have a second in a pair to send as well?
if (secondPacket) {
sendNewPacketAndAddToSentList(move(secondPacket), getNextSequenceNumber());
}
// We sent our packet(s), return here
return true;
}
// No packets were sent
return false;
}
bool SendQueue::maybeResendPacket() {
// the following while makes sure that we find a packet to re-send, if there is one
while (true) {
std::unique_lock<std::mutex> naksLocker(_naksLock);
if (_naks.getLength() > 0) {
// pull the sequence number we need to re-send
SequenceNumber resendNumber = _naks.popFirstSequenceNumber();
naksLocker.unlock();
// pull the packet to re-send from the sent packets list
QReadLocker sentLocker(&_sentLock);
// see if we can find the packet to re-send
auto it = _sentPackets.find(resendNumber);
if (it != _sentPackets.end()) {
// we found the packet - grab it
auto& resendPacket = *(it->second);
// unlock the sent packets
sentLocker.unlock();
// send it off
sendPacket(resendPacket);
emit packetRetransmitted();
// Signal that we did resend a packet
return true;
} else {
// we didn't find this packet in the sentPackets queue - assume this means it was ACKed
// we'll fire the loop again to see if there is another to re-send
continue;
}
}
// sleep as long as we need until next packet send, if we can
auto now = high_resolution_clock::now();
auto microsecondDuration = duration_cast<microseconds>((_lastSendTimestamp + microseconds(_packetSendPeriod)) - now);
if (microsecondDuration.count() > 0) {
usleep(microsecondDuration.count());
}
// break from the while, we didn't resend a packet
break;
}
// No packet was resent
return false;
}

27
libraries/networking/src/udt/SendQueue.h
View file

@ -42,21 +42,8 @@ class SendQueue : public QObject {
Q_OBJECT
public:
class DoubleLock {
public:
DoubleLock(std::mutex& mutex1, std::mutex& mutex2) : _mutex1(mutex1), _mutex2(mutex2) { lock(); }
~DoubleLock() { unlock(); }
DoubleLock(const DoubleLock&) = delete;
DoubleLock& operator=(const DoubleLock&) = delete;
void lock() { std::lock(_mutex1, _mutex2); }
void unlock() { _mutex1.unlock(); _mutex2.unlock(); }
private:
std::mutex& _mutex1;
std::mutex& _mutex2;
};
using high_resolution_clock = std::chrono::high_resolution_clock;
using time_point = high_resolution_clock::time_point;
static std::unique_ptr<SendQueue> create(Socket* socket, HifiSockAddr destination);
@ -82,6 +69,8 @@ signals:
void packetSent(int dataSize, int payloadSize);
void packetRetransmitted();
void queueInactive();
private slots:
void run();
@ -93,6 +82,9 @@ private:
void sendPacket(const Packet& packet);
void sendNewPacketAndAddToSentList(std::unique_ptr<Packet> newPacket, SequenceNumber sequenceNumber);
bool maybeSendNewPacket(); // Figures out what packet to send next
bool maybeResendPacket(); // Determines whether to resend a packet and wich one
// Increments current sequence number and return it
SequenceNumber getNextSequenceNumber();
MessageNumber getNextMessageNumber();
@ -110,11 +102,14 @@ private:
std::atomic<uint32_t> _atomicCurrentSequenceNumber { 0 };// Atomic for last sequence number sent out
std::atomic<int> _packetSendPeriod { 0 }; // Interval between two packet send event in microseconds, set from CC
std::chrono::high_resolution_clock::time_point _lastSendTimestamp; // Record last time of packet departure
std::atomic<bool> _isRunning { false };
std::atomic<int> _flowWindowSize { 0 }; // Flow control window size (number of packets that can be on wire) - set from CC
// Used to detect when the connection becomes inactive for too long
bool _flowWindowWasFull = false;
time_point _flowWindowFullSince;
mutable std::mutex _naksLock; // Protects the naks list.
LossList _naks; // Sequence numbers of packets to resend

10
libraries/networking/src/udt/Socket.cpp
View file

@ -147,12 +147,18 @@ Connection& Socket::findOrCreateConnection(const HifiSockAddr& sockAddr) {
if (it == _connectionsHash.end()) {
auto connection = std::unique_ptr<Connection>(new Connection(this, sockAddr, _ccFactory->create()));
QObject::connect(connection.get(), &Connection::connectionInactive, this, &Socket::cleanupConnection);
it = _connectionsHash.insert(it, std::make_pair(sockAddr, std::move(connection)));
}
return *it->second;
}
void Socket::cleanupConnection(HifiSockAddr sockAddr) {
qCDebug(networking) << "Socket::cleanupConnection called for connection to" << sockAddr;
_connectionsHash.erase(sockAddr);
}
void Socket::messageReceived(std::unique_ptr<PacketList> packetList) {
if (_packetListHandler) {
_packetListHandler(std::move(packetList));
@ -205,8 +211,8 @@ void Socket::readPendingDatagrams() {
// if this was a reliable packet then signal the matching connection with the sequence number
auto& connection = findOrCreateConnection(senderSockAddr);
connection.processReceivedSequenceNumber(packet->getSequenceNumber(),
packet->getDataSize(),
packet->getPayloadSize());
packet->getDataSize(),
packet->getPayloadSize());
}
if (packet->isPartOfMessage()) {

3
libraries/networking/src/udt/Socket.h
View file

@ -73,6 +73,9 @@ public:
ConnectionStats::Stats sampleStatsForConnection(const HifiSockAddr& destination);
std::vector<HifiSockAddr> getConnectionSockAddrs();
public slots:
void cleanupConnection(HifiSockAddr sockAddr);
private slots:
void readPendingDatagrams();
void rateControlSync();

36
libraries/render-utils/src/GeometryCache.cpp
View file

@ -1678,15 +1678,15 @@ void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend) {
}
}
GeometryReader::GeometryReader(const QUrl& url, QNetworkReply* reply, const QVariantHash& mapping) :
GeometryReader::GeometryReader(const QUrl& url, const QByteArray& data, const QVariantHash& mapping) :
_url(url),
_reply(reply),
_data(data),
_mapping(mapping) {
}
void GeometryReader::run() {
try {
if (!_reply) {
if (_data.isEmpty()) {
throw QString("Reply is NULL ?!");
}
QString urlname = _url.path().toLower();
@ -1701,9 +1701,9 @@ void GeometryReader::run() {
if (_url.path().toLower().endsWith(".fbx")) {
const bool grabLightmaps = true;
const float lightmapLevel = 1.0f;
fbxgeo = readFBX(_reply, _mapping, _url.path(), grabLightmaps, lightmapLevel);
fbxgeo = readFBX(_data, _mapping, _url.path(), grabLightmaps, lightmapLevel);
} else if (_url.path().toLower().endsWith(".obj")) {
fbxgeo = OBJReader().readOBJ(_reply, _mapping, &_url);
fbxgeo = OBJReader().readOBJ(_data, _mapping);
} else {
QString errorStr("usupported format");
emit onError(NetworkGeometry::ModelParseError, errorStr);
@ -1717,7 +1717,6 @@ void GeometryReader::run() {
qCDebug(renderutils) << "Error reading " << _url << ": " << error;
emit onError(NetworkGeometry::ModelParseError, error);
}
_reply->deleteLater();
}
NetworkGeometry::NetworkGeometry(const QUrl& url, bool delayLoad, const QVariantHash& mapping, const QUrl& textureBaseUrl) :
@ -1746,8 +1745,10 @@ void NetworkGeometry::attemptRequest() {
void NetworkGeometry::attemptRequestInternal() {
if (_url.path().toLower().endsWith(".fst")) {
_mappingUrl = _url;
requestMapping(_url);
} else {
_modelUrl = _url;
requestModel(_url);
}
}
@ -1838,8 +1839,8 @@ void NetworkGeometry::requestMapping(const QUrl& url) {
_resource->deleteLater();
}
_resource = new Resource(url, false);
connect(_resource, SIGNAL(loaded(QNetworkReply&)), SLOT(mappingRequestDone(QNetworkReply&)));
connect(_resource, SIGNAL(failed(QNetworkReply::NetworkError)), SLOT(mappingRequestError(QNetworkReply::NetworkError)));
connect(_resource, &Resource::loaded, this, &NetworkGeometry::mappingRequestDone);
connect(_resource, &Resource::failed, this, &NetworkGeometry::mappingRequestError);
}
void NetworkGeometry::requestModel(const QUrl& url) {
@ -1847,18 +1848,19 @@ void NetworkGeometry::requestModel(const QUrl& url) {
if (_resource) {
_resource->deleteLater();
}
_modelUrl = url;
_resource = new Resource(url, false);
connect(_resource, SIGNAL(loaded(QNetworkReply&)), SLOT(modelRequestDone(QNetworkReply&)));
connect(_resource, SIGNAL(failed(QNetworkReply::NetworkError)), SLOT(modelRequestError(QNetworkReply::NetworkError)));
connect(_resource, &Resource::loaded, this, &NetworkGeometry::modelRequestDone);
connect(_resource, &Resource::failed, this, &NetworkGeometry::modelRequestError);
}
void NetworkGeometry::mappingRequestDone(QNetworkReply& reply) {
void NetworkGeometry::mappingRequestDone(const QByteArray& data) {
assert(_state == RequestMappingState);
// parse the mapping file
_mapping = FSTReader::readMapping(reply.readAll());
_mapping = FSTReader::readMapping(data);
QUrl replyUrl = reply.url();
QUrl replyUrl = _mappingUrl;
QString modelUrlStr = _mapping.value("filename").toString();
if (modelUrlStr.isNull()) {
qCDebug(renderutils) << "Mapping file " << _url << "has no \"filename\" entry";
@ -1873,8 +1875,8 @@ void NetworkGeometry::mappingRequestDone(QNetworkReply& reply) {
_textureBaseUrl = replyUrl.resolved(texdir);
}
QUrl modelUrl = replyUrl.resolved(modelUrlStr);
requestModel(modelUrl);
_modelUrl = replyUrl.resolved(modelUrlStr);
requestModel(_modelUrl);
}
}
@ -1884,13 +1886,13 @@ void NetworkGeometry::mappingRequestError(QNetworkReply::NetworkError error) {
emit onFailure(*this, MappingRequestError);
}
void NetworkGeometry::modelRequestDone(QNetworkReply& reply) {
void NetworkGeometry::modelRequestDone(const QByteArray& data) {
assert(_state == RequestModelState);
_state = ParsingModelState;
// asynchronously parse the model file.
GeometryReader* geometryReader = new GeometryReader(reply.url(), &reply, _mapping);
GeometryReader* geometryReader = new GeometryReader(_modelUrl, data, _mapping);
connect(geometryReader, SIGNAL(onSuccess(FBXGeometry*)), SLOT(modelParseSuccess(FBXGeometry*)));
connect(geometryReader, SIGNAL(onError(int, QString)), SLOT(modelParseError(int, QString)));

10
libraries/render-utils/src/GeometryCache.h
View file

@ -348,10 +348,10 @@ signals:
void onFailure(NetworkGeometry& networkGeometry, Error error);
protected slots:
void mappingRequestDone(QNetworkReply& reply);
void mappingRequestDone(const QByteArray& data);
void mappingRequestError(QNetworkReply::NetworkError error);
void modelRequestDone(QNetworkReply& reply);
void modelRequestDone(const QByteArray& data);
void modelRequestError(QNetworkReply::NetworkError error);
void modelParseSuccess(FBXGeometry* geometry);
@ -371,6 +371,8 @@ protected:
State _state;
QUrl _url;
QUrl _mappingUrl;
QUrl _modelUrl;
QVariantHash _mapping;
QUrl _textureBaseUrl;
@ -386,14 +388,14 @@ protected:
class GeometryReader : public QObject, public QRunnable {
Q_OBJECT
public:
GeometryReader(const QUrl& url, QNetworkReply* reply, const QVariantHash& mapping);
GeometryReader(const QUrl& url, const QByteArray& data, const QVariantHash& mapping);
virtual void run();
signals:
void onSuccess(FBXGeometry* geometry);
void onError(int error, QString str);
private:
QUrl _url;
QNetworkReply* _reply;
QByteArray _data;
QVariantHash _mapping;
};

25
libraries/render-utils/src/TextureCache.cpp
View file

@ -215,8 +215,7 @@ NetworkTexture::NetworkTexture(const QUrl& url, TextureType type, const QByteArr
class ImageReader : public QRunnable {
public:
ImageReader(const QWeakPointer<Resource>& texture, TextureType type, QNetworkReply* reply, const QUrl& url = QUrl(),
const QByteArray& content = QByteArray());
ImageReader(const QWeakPointer<Resource>& texture, TextureType type, const QByteArray& data, const QUrl& url = QUrl());
virtual void run();
@ -224,27 +223,25 @@ private:
QWeakPointer<Resource> _texture;
TextureType _type;
QNetworkReply* _reply;
QUrl _url;
QByteArray _content;
};
void NetworkTexture::downloadFinished(QNetworkReply* reply) {
void NetworkTexture::downloadFinished(const QByteArray& data) {
// send the reader off to the thread pool
QThreadPool::globalInstance()->start(new ImageReader(_self, _type, reply));
QThreadPool::globalInstance()->start(new ImageReader(_self, _type, data, _url));
}
void NetworkTexture::loadContent(const QByteArray& content) {
QThreadPool::globalInstance()->start(new ImageReader(_self, _type, NULL, _url, content));
QThreadPool::globalInstance()->start(new ImageReader(_self, _type, content, _url));
}
ImageReader::ImageReader(const QWeakPointer<Resource>& texture, TextureType type, QNetworkReply* reply,
const QUrl& url, const QByteArray& content) :
ImageReader::ImageReader(const QWeakPointer<Resource>& texture, TextureType type, const QByteArray& data,
const QUrl& url) :
_texture(texture),
_type(type),
_reply(reply),
_url(url),
_content(content) {
_content(data) {
}
std::once_flag onceListSupportedFormatsflag;
@ -297,16 +294,8 @@ public:
void ImageReader::run() {
QSharedPointer<Resource> texture = _texture.toStrongRef();
if (texture.isNull()) {
if (_reply) {
_reply->deleteLater();
}
return;
}
if (_reply) {
_url = _reply->url();
_content = _reply->readAll();
_reply->deleteLater();
}
listSupportedImageFormats();

2
libraries/render-utils/src/TextureCache.h
View file

@ -123,7 +123,7 @@ public:
TextureType getType() const { return _type; }
protected:
virtual void downloadFinished(QNetworkReply* reply);
virtual void downloadFinished(const QByteArray& data) override;
Q_INVOKABLE void loadContent(const QByteArray& content);
// FIXME: This void* should be a gpu::Texture* but i cannot get it to work for now, moving on...

57
libraries/script-engine/src/BatchLoader.cpp
View file

@ -17,9 +17,7 @@
#include "BatchLoader.h"
#include <NetworkAccessManager.h>
#include <SharedUtil.h>
#include "ResourceManager.h"
BatchLoader::BatchLoader(const QList<QUrl>& urls)
: QObject(),
@ -27,6 +25,7 @@ BatchLoader::BatchLoader(const QList<QUrl>& urls)
_finished(false),
_urls(urls.toSet()),
_data() {
qRegisterMetaType<QMap<QUrl, QString>>("QMap<QUrl, QString>");
}
void BatchLoader::start() {
@ -35,45 +34,27 @@ void BatchLoader::start() {
}
_started = true;
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
for (QUrl url : _urls) {
if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "ftp") {
QNetworkRequest request = QNetworkRequest(url);
request.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
QNetworkReply* reply = networkAccessManager.get(request);
qCDebug(scriptengine) << "Downloading file at" << url;
connect(reply, &QNetworkReply::finished, [=]() {
if (reply->error()) {
_data.insert(url, QString());
} else {
_data.insert(url, reply->readAll());
}
reply->deleteLater();
checkFinished();
});
// If we end up being destroyed before the reply finishes, clean it up
connect(this, &QObject::destroyed, reply, &QObject::deleteLater);
} else {
#ifdef _WIN32
QString fileName = url.toString();
#else
QString fileName = url.toLocalFile();
#endif
qCDebug(scriptengine) << "Reading file at " << fileName;
QFile scriptFile(fileName);
if (scriptFile.open(QFile::ReadOnly | QFile::Text)) {
QTextStream in(&scriptFile);
_data.insert(url, in.readAll());
auto request = ResourceManager::createResourceRequest(this, url);
if (!request) {
continue;
}
connect(request, &ResourceRequest::finished, this, [=]() {
if (request->getResult() == ResourceRequest::SUCCESS) {
_data.insert(url, request->getData());
} else {
_data.insert(url, QString());
}
}
request->deleteLater();
checkFinished();
});
// If we end up being destroyed before the reply finishes, clean it up
connect(this, &QObject::destroyed, request, &QObject::deleteLater);
qCDebug(scriptengine) << "Loading script at " << url;
request->send();
}
checkFinished();
}

33
libraries/script-engine/src/ScriptCache.cpp
View file

@ -17,7 +17,6 @@
#include <QObject>
#include <assert.h>
#include <NetworkAccessManager.h>
#include <SharedUtil.h>
#include "ScriptCache.h"
@ -28,8 +27,6 @@ ScriptCache::ScriptCache(QObject* parent) {
}
QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& isPending, bool reload) {
assert(!_scriptCache.contains(url) || !reload);
QString scriptContents;
if (_scriptCache.contains(url) && !reload) {
qCDebug(scriptengine) << "Found script in cache:" << url.toString();
@ -44,18 +41,10 @@ QString ScriptCache::getScript(const QUrl& url, ScriptUser* scriptUser, bool& is
if (alreadyWaiting) {
qCDebug(scriptengine) << "Already downloading script at:" << url.toString();
} else {
QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
QNetworkRequest networkRequest = QNetworkRequest(url);
networkRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
if (reload) {
networkRequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork);
qCDebug(scriptengine) << "Redownloading script at:" << url.toString();
} else {
qCDebug(scriptengine) << "Downloading script at:" << url.toString();
}
QNetworkReply* reply = networkAccessManager.get(networkRequest);
connect(reply, &QNetworkReply::finished, this, &ScriptCache::scriptDownloaded);
auto request = ResourceManager::createResourceRequest(this, url);
request->setCacheEnabled(!reload);
connect(request, &ResourceRequest::finished, this, &ScriptCache::scriptDownloaded);
request->send();
}
}
return scriptContents;
@ -69,27 +58,25 @@ void ScriptCache::deleteScript(const QUrl& url) {
}
void ScriptCache::scriptDownloaded() {
QNetworkReply* reply = qobject_cast<QNetworkReply*>(sender());
QUrl url = reply->url();
ResourceRequest* req = qobject_cast<ResourceRequest*>(sender());
QUrl url = req->getUrl();
QList<ScriptUser*> scriptUsers = _scriptUsers.values(url);
_scriptUsers.remove(url);
if (reply->error() == QNetworkReply::NoError && reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) == 200) {
_scriptCache[url] = reply->readAll();
if (req->getResult() == ResourceRequest::SUCCESS) {
_scriptCache[url] = req->getData();
qCDebug(scriptengine) << "Done downloading script at:" << url.toString();
foreach(ScriptUser* user, scriptUsers) {
user->scriptContentsAvailable(url, _scriptCache[url]);
}
} else {
qCWarning(scriptengine) << "Error loading script from URL " << reply->url().toString()
<< "- HTTP status code is" << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()
<< "and error from QNetworkReply is" << reply->errorString();
qCWarning(scriptengine) << "Error loading script from URL " << url;
foreach(ScriptUser* user, scriptUsers) {
user->errorInLoadingScript(url);
}
}
reply->deleteLater();
req->deleteLater();
}