Mirrors/overte-lubosz
3
0
Fork 0
mirror of https://github.com/lubosz/overte.git synced 2025-04-10 18:36:32 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #12427 from huffman/fix/http-connection-crash

Browse source 
Fix rolling backups, delete backup, and recovery id
This commit is contained in:
Stephen Birarda 2018-02-16 16:34:40 -07:00 committed by GitHub
commit 97f7b71db2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 146 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
domain-server/src/AssetsBackupHandler.cpp
View file

@ -306,7 +306,7 @@ void AssetsBackupHandler::recoverBackup(QuaZip& zip) {
restoreAllAssets();
}
void AssetsBackupHandler::deleteBackup(QuaZip& zip) {
void AssetsBackupHandler::deleteBackup(const QString& absoluteFilePath) {
Q_ASSERT(QThread::currentThread() == thread());
if (operationInProgress()) {
@ -315,10 +315,10 @@ void AssetsBackupHandler::deleteBackup(QuaZip& zip) {
}
auto it = find_if(begin(_backups), end(_backups), [&](const std::vector<AssetServerBackup>::value_type& value) {
return value.filePath == zip.getZipName();
return value.filePath == absoluteFilePath;
});
if (it == end(_backups)) {
qCDebug(asset_backup) << "Could not find backup" << zip.getZipName() << "to delete.";
qCDebug(asset_backup) << "Could not find backup" << absoluteFilePath << "to delete.";
return;
}

2
domain-server/src/AssetsBackupHandler.h
View file

@ -37,7 +37,7 @@ public:
void loadBackup(QuaZip& zip) override;
void createBackup(QuaZip& zip) override;
void recoverBackup(QuaZip& zip) override;
void deleteBackup(QuaZip& zip) override;
void deleteBackup(const QString& absoluteFilePath) override;
void consolidateBackup(QuaZip& zip) override;
bool operationInProgress() { return getRecoveryStatus().first; }

2
domain-server/src/BackupHandler.h
View file

@ -30,7 +30,7 @@ public:
virtual void loadBackup(QuaZip& zip) = 0;
virtual void createBackup(QuaZip& zip) = 0;
virtual void recoverBackup(QuaZip& zip) = 0;
virtual void deleteBackup(QuaZip& zip) = 0;
virtual void deleteBackup(const QString& absoluteFilePath) = 0;
virtual void consolidateBackup(QuaZip& zip) = 0;
};
using BackupHandlerPointer = std::unique_ptr<BackupHandlerInterface>;

2
domain-server/src/ContentSettingsBackupHandler.h
View file

@ -28,7 +28,7 @@ public:
void recoverBackup(QuaZip& zip) override;
void deleteBackup(QuaZip& zip) override {}
void deleteBackup(const QString& absoluteFilePath) override {}
void consolidateBackup(QuaZip& zip) override {}
private:

129
domain-server/src/DomainContentBackupManager.cpp
View file

@ -89,8 +89,7 @@ void DomainContentBackupManager::parseSettings(const QJsonObject& settings) {
}
auto name = obj["Name"].toString();
auto format = obj["format"].toString();
format = name.replace(" ", "_").toLower();
auto format = name.replace(" ", "_").toLower();
qCDebug(domain_server) << " Name:" << name;
qCDebug(domain_server) << " format:" << format;
@ -116,6 +115,12 @@ void DomainContentBackupManager::parseSettings(const QJsonObject& settings) {
}
}
void DomainContentBackupManager::refreshBackupRules() {
for (auto& backup : _backupRules) {
backup.lastBackupSeconds = getMostRecentBackupTimeInSecs(backup.extensionFormat);
}
}
int64_t DomainContentBackupManager::getMostRecentBackupTimeInSecs(const QString& format) {
int64_t mostRecentBackupInSecs = 0;
@ -155,6 +160,7 @@ bool DomainContentBackupManager::process() {
if (!isStillRecovering) {
_isRecovering = false;
_recoveryFilename = "";
emit recoveryCompleted();
}
}
@ -235,8 +241,16 @@ void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, cons
}
QDir backupDir { _backupDirectory };
QFile backupFile { backupDir.filePath(backupName) };
auto absoluteFilePath { backupDir.filePath(backupName) };
QFile backupFile { absoluteFilePath };
auto success = backupFile.remove();
refreshBackupRules();
for (auto& handler : _backupHandlers) {
handler->deleteBackup(absoluteFilePath);
}
promise->resolve({
{ "success", success }
});
@ -268,6 +282,7 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise,
success = false;
} else {
_isRecovering = true;
_recoveryFilename = backupName;
for (auto& handler : _backupHandlers) {
handler->recoverBackup(zip);
}
@ -286,12 +301,7 @@ void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise,
});
}
void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise promise) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "getAllBackupInformation", Q_ARG(MiniPromise::Promise, promise));
return;
}
std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
QDir backupDir { _backupDirectory };
auto matchingFiles =
backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + "*.zip", MANUAL_BACKUP_PREFIX + "*.zip" },
@ -301,7 +311,7 @@ void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise pr
QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
QVariantList backups;
std::vector<BackupItemInfo> backups;
for (const auto& fileInfo : matchingFiles) {
auto fileName = fileInfo.fileName();
@ -324,17 +334,53 @@ void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise pr
availabilityProgress += progress / _backupHandlers.size();
}
backups.push_back(QVariantMap({
{ "id", fileInfo.fileName() },
{ "name", name },
{ "createdAtMillis", createdAt.toMSecsSinceEpoch() },
{ "isAvailable", isAvailable },
{ "availabilityProgress", availabilityProgress },
{ "isManualBackup", type == MANUAL_BACKUP_PREFIX }
}));
backups.emplace_back(fileInfo.fileName(), name, fileInfo.absoluteFilePath(), createdAt,
type == MANUAL_BACKUP_PREFIX);
}
}
return backups;
}
void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise promise) {
if (QThread::currentThread() != thread()) {
QMetaObject::invokeMethod(this, "getAllBackupsAndStatus", Q_ARG(MiniPromise::Promise, promise));
return;
}
QDir backupDir { _backupDirectory };
auto matchingFiles =
backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + "*.zip", MANUAL_BACKUP_PREFIX + "*.zip" },
QDir::Files | QDir::NoSymLinks, QDir::Name);
QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
QString nameFormat = "(.+)";
QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
auto backups = getAllBackups();
QVariantList variantBackups;
for (auto& backup : backups) {
bool isAvailable { true };
float availabilityProgress { 0.0f };
for (auto& handler : _backupHandlers) {
bool handlerIsAvailable { true };
float progress { 0.0f };
std::tie(handlerIsAvailable, progress) = handler->isAvailable(backup.absolutePath);
isAvailable &= handlerIsAvailable;
availabilityProgress += progress / _backupHandlers.size();
}
variantBackups.push_back(QVariantMap({
{ "id", backup.id },
{ "name", backup.name },
{ "createdAtMillis", backup.createdAt.toMSecsSinceEpoch() },
{ "isAvailable", isAvailable },
{ "availabilityProgress", availabilityProgress },
{ "isManualBackup", backup.isManualBackup }
}));
}
float recoveryProgress = 0.0f;
bool isRecovering = _isRecovering.load();
if (_isRecovering) {
@ -351,7 +397,7 @@ void DomainContentBackupManager::getAllBackupInformation(MiniPromise::Promise pr
};
QVariantMap info {
{ "backups", backups },
{ "backups", variantBackups },
{ "status", status }
};
@ -390,32 +436,27 @@ void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule)
}
void DomainContentBackupManager::load() {
QDir backupDir { _backupDirectory };
if (backupDir.exists()) {
auto matchingFiles = backupDir.entryInfoList({ "backup-*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
for (const auto& file : matchingFiles) {
QFile backupFile { file.absoluteFilePath() };
if (!backupFile.open(QIODevice::ReadOnly)) {
qCritical() << "Could not open file:" << file.absoluteFilePath();
qCritical() << " ERROR:" << backupFile.errorString();
continue;
}
QuaZip zip { &backupFile };
if (!zip.open(QuaZip::mdUnzip)) {
qCritical() << "Could not open backup archive:" << file.absoluteFilePath();
qCritical() << " ERROR:" << zip.getZipError();
continue;
}
for (auto& handler : _backupHandlers) {
handler->loadBackup(zip);
}
zip.close();
auto backups = getAllBackups();
for (auto& backup : backups) {
QFile backupFile { backup.absolutePath };
if (!backupFile.open(QIODevice::ReadOnly)) {
qCritical() << "Could not open file:" << backup.absolutePath;
qCritical() << " ERROR:" << backupFile.errorString();
continue;
}
QuaZip zip { &backupFile };
if (!zip.open(QuaZip::mdUnzip)) {
qCritical() << "Could not open backup archive:" << backup.absolutePath;
qCritical() << " ERROR:" << zip.getZipError();
continue;
}
for (auto& handler : _backupHandlers) {
handler->loadBackup(zip);
}
zip.close();
}
}

17
domain-server/src/DomainContentBackupManager.h
View file

@ -24,6 +24,17 @@
#include <shared/MiniPromises.h>
struct BackupItemInfo {
BackupItemInfo(QString pId, QString pName, QString pAbsolutePath, QDateTime pCreatedAt, bool pIsManualBackup) :
id(pId), name(pName), absolutePath(pAbsolutePath), createdAt(pCreatedAt), isManualBackup(pIsManualBackup) { };
QString id;
QString name;
QString absolutePath;
QDateTime createdAt;
bool isManualBackup;
};
class DomainContentBackupManager : public GenericThread {
Q_OBJECT
public:
@ -43,14 +54,13 @@ public:
int persistInterval = DEFAULT_PERSIST_INTERVAL,
bool debugTimestampNow = false);
std::vector<BackupItemInfo> getAllBackups();
void addBackupHandler(BackupHandlerPointer handler);
void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
void replaceData(QByteArray data);
public slots:
void getAllBackupInformation(MiniPromise::Promise promise);
void getAllBackupsAndStatus(MiniPromise::Promise promise);
void createManualBackup(MiniPromise::Promise promise, const QString& name);
void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
@ -68,6 +78,7 @@ protected:
void load();
void backup();
void removeOldBackupVersions(const BackupRule& rule);
void refreshBackupRules();
bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
int64_t getMostRecentBackupTimeInSecs(const QString& format);
void parseSettings(const QJsonObject& settings);

4
domain-server/src/DomainServer.cpp
View file

@ -2128,13 +2128,13 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
return true;
} else if (url.path() == URI_API_BACKUPS) {
auto deferred = makePromise("getAllBackupInformation");
auto deferred = makePromise("getAllBackupsAndStatus");
deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
});
_contentManager->getAllBackupInformation(deferred);
_contentManager->getAllBackupsAndStatus(deferred);
return true;
} else if (url.path().startsWith(URI_API_BACKUPS_ID)) {
auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());

2
domain-server/src/EntitiesBackupHandler.h
View file

@ -30,7 +30,7 @@ public:
void recoverBackup(QuaZip& zip) override;
// Delete a skeleton backup
void deleteBackup(QuaZip& zip) override {}
void deleteBackup(const QString& absoluteFilePath) override {}
// Create a full backup
void consolidateBackup(QuaZip& zip) override {}

72
libraries/embedded-webserver/src/HTTPConnection.cpp
View file

@ -133,57 +133,33 @@ QList<FormData> HTTPConnection::parseFormData() const {
}
void HTTPConnection::respond(const char* code, const QByteArray& content, const char* contentType, const Headers& headers) {
QByteArray data(content);
auto device { std::unique_ptr<QBuffer>(new QBuffer()) };
device->setBuffer(new QByteArray(content));
if (device->open(QIODevice::ReadOnly)) {
respond(code, std::move(device), contentType, headers);
} else {
qCritical() << "Error opening QBuffer to respond to " << _requestUrl.path();
}
respondWithStatusAndHeaders(code, contentType, headers, content.size());
_socket->write(content);
_socket->disconnectFromHost();
// make sure we receive no further read notifications
disconnect(_socket, &QTcpSocket::readyRead, this, nullptr);
}
void HTTPConnection::respond(const char* code, std::unique_ptr<QIODevice> device, const char* contentType, const Headers& headers) {
_responseDevice = std::move(device);
_socket->write("HTTP/1.1 ");
if (_responseDevice->isSequential()) {
qWarning() << "Error responding to HTTPConnection: sequential IO devices not supported";
_socket->write(StatusCode500);
_socket->write("\r\n");
respondWithStatusAndHeaders(StatusCode500, contentType, headers, 0);
_socket->disconnect(SIGNAL(readyRead()), this);
_socket->disconnectFromHost();
return;
}
_socket->write(code);
_socket->write("\r\n");
for (Headers::const_iterator it = headers.constBegin(), end = headers.constEnd();
it != end; it++) {
_socket->write(it.key());
_socket->write(": ");
_socket->write(it.value());
_socket->write("\r\n");
}
int csize = _responseDevice->size();
if (csize > 0) {
_socket->write("Content-Length: ");
_socket->write(QByteArray::number(csize));
_socket->write("\r\n");
_socket->write("Content-Type: ");
_socket->write(contentType);
_socket->write("\r\n");
}
_socket->write("Connection: close\r\n\r\n");
int totalToBeWritten = _responseDevice->size();
respondWithStatusAndHeaders(code, contentType, headers, totalToBeWritten);
if (_responseDevice->atEnd()) {
_socket->disconnectFromHost();
} else {
int totalToBeWritten = csize;
connect(_socket, &QTcpSocket::bytesWritten, this, [this, totalToBeWritten](size_t bytes) mutable {
constexpr size_t HTTP_RESPONSE_CHUNK_SIZE = 1024 * 10;
if (!_responseDevice->atEnd()) {
@ -201,6 +177,32 @@ void HTTPConnection::respond(const char* code, std::unique_ptr<QIODevice> device
disconnect(_socket, &QTcpSocket::readyRead, this, nullptr);
}
void HTTPConnection::respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 contentLength) {
_socket->write("HTTP/1.1 ");
_socket->write(code);
_socket->write("\r\n");
for (Headers::const_iterator it = headers.constBegin(), end = headers.constEnd();
it != end; it++) {
_socket->write(it.key());
_socket->write(": ");
_socket->write(it.value());
_socket->write("\r\n");
}
if (contentLength > 0) {
_socket->write("Content-Length: ");
_socket->write(QByteArray::number(contentLength));
_socket->write("\r\n");
_socket->write("Content-Type: ");
_socket->write(contentType);
_socket->write("\r\n");
}
_socket->write("Connection: close\r\n\r\n");
}
void HTTPConnection::readRequest() {
if (!_socket->canReadLine()) {
return;

1
libraries/embedded-webserver/src/HTTPConnection.h
View file

@ -105,6 +105,7 @@ protected slots:
void readContent ();
protected:
void respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 size);
/// The parent HTTP manager
HTTPManager* _parentManager;