From c002888808765940fc0b4dd42d444f6be25645d5 Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 6 Nov 2015 14:52:12 -0800 Subject: [PATCH 1/4] Added cache extractor to the tools directory It should find the local High Fidelity/Interface qt cache, iterate over each file in the cache and output each corresponding file into the High Fidelity/Interface/extracted dir. The file path will be determined from the source url. Untested on windows. --- tools/CMakeLists.txt | 3 + tools/cache-extract/CMakeLists.txt | 6 + tools/cache-extract/src/CacheExtractApp.cpp | 125 ++++++++++++++++++++ tools/cache-extract/src/CacheExtractApp.h | 47 ++++++++ tools/cache-extract/src/main.cpp | 17 +++ 5 files changed, 198 insertions(+) create mode 100644 tools/cache-extract/CMakeLists.txt create mode 100644 tools/cache-extract/src/CacheExtractApp.cpp create mode 100644 tools/cache-extract/src/CacheExtractApp.h create mode 100644 tools/cache-extract/src/main.cpp diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 2056044a4b..9bc7031720 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -10,3 +10,6 @@ set_target_properties(udt-test PROPERTIES FOLDER "Tools") add_subdirectory(vhacd-util) set_target_properties(vhacd-util PROPERTIES FOLDER "Tools") + +add_subdirectory(cache-extract) +set_target_properties(cache-extract PROPERTIES FOLDER "Tools") diff --git a/tools/cache-extract/CMakeLists.txt b/tools/cache-extract/CMakeLists.txt new file mode 100644 index 0000000000..045fa996d2 --- /dev/null +++ b/tools/cache-extract/CMakeLists.txt @@ -0,0 +1,6 @@ +set(TARGET_NAME cache-extract) + +setup_hifi_project() + +link_hifi_libraries() + diff --git a/tools/cache-extract/src/CacheExtractApp.cpp b/tools/cache-extract/src/CacheExtractApp.cpp new file mode 100644 index 0000000000..e7e60973af --- /dev/null +++ b/tools/cache-extract/src/CacheExtractApp.cpp @@ -0,0 +1,125 @@ +// +// CacheExtractApp.cpp +// tools/cache-extract/src +// +// Created by Anthony Thibault on 11/6/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "CacheExtractApp.h" +#include +#include +#include +#include + +// extracted from qnetworkdiskcache.cpp +#define CACHE_VERSION 8 +enum { + CacheMagic = 0xe8, + CurrentCacheVersion = CACHE_VERSION +}; +#define DATA_DIR QLatin1String("data") + +CacheExtractApp::CacheExtractApp(int& argc, char** argv) : + QCoreApplication(argc, argv) +{ + QString myDataLoc = QStandardPaths::writableLocation(QStandardPaths::DataLocation); + int lastSlash = myDataLoc.lastIndexOf(QDir::separator()); + QString cachePath = myDataLoc.leftRef(lastSlash).toString() + QDir::separator() + + "High Fidelity" + QDir::separator() + "Interface" + QDir::separator() + + DATA_DIR + QString::number(CACHE_VERSION) + QLatin1Char('/'); + + QString outputPath = myDataLoc.leftRef(lastSlash).toString() + QDir::separator() + + "High Fidelity" + QDir::separator() + "Interface" + QDir::separator() + "extracted"; + + qDebug() << "Searching cachePath = " << cachePath << "..."; + + // build list of files + QList fileList; + QDir dir(cachePath); + dir.setFilter(QDir::AllDirs | QDir::NoDotAndDotDot); + QFileInfoList list = dir.entryInfoList(); + for (int i = 0; i < list.size(); ++i) { + QFileInfo fileInfo = list.at(i); + if (fileInfo.isDir()) { + QDir subDir(fileInfo.filePath()); + subDir.setFilter(QDir::Files); + QFileInfoList subList = subDir.entryInfoList(); + for (int j = 0; j < subList.size(); ++j) { + fileList << subList.at(j).filePath(); + } + } + } + + // dump each cache file into the outputPath + for (int i = 0; i < fileList.size(); ++i) { + QByteArray contents; + MyMetaData metaData; + if (extractFile(fileList.at(i), metaData, contents)) { + QString outFileName = outputPath + metaData.url.path(); + int lastSlash = outFileName.lastIndexOf(QDir::separator()); + QString outDirName = outFileName.leftRef(lastSlash).toString(); + QDir dir(outputPath); + dir.mkpath(outDirName); + QFile out(outFileName); + if (out.open(QIODevice::WriteOnly)) { + out.write(contents); + out.close(); + } + } else { + qCritical() << "Error extracting = " << fileList.at(i); + } + } + + QMetaObject::invokeMethod(this, "quit", Qt::QueuedConnection); +} + +bool CacheExtractApp::extractFile(const QString& filePath, MyMetaData& metaData, QByteArray& data) const { + QFile f(filePath); + if (!f.open(QIODevice::ReadOnly)) { + qDebug() << "error opening " << filePath; + return false; + } + QDataStream in(&f); + // from qnetworkdiskcache.cpp QCacheItem::read() + qint32 marker, version, streamVersion; + in >> marker; + if (marker != CacheMagic) { + return false; + } + in >> version; + if (version != CurrentCacheVersion) { + return false; + } + in >> streamVersion; + if (streamVersion > in.version()) + return false; + in.setVersion(streamVersion); + + bool compressed; + in >> metaData; + in >> compressed; + if (compressed) { + QByteArray compressedData; + in >> compressedData; + QBuffer buffer; + buffer.setData(qUncompress(compressedData)); + buffer.open(QBuffer::ReadOnly); + data = buffer.readAll(); + } else { + data = f.readAll(); + } + return true; +} + +QDataStream &operator>>(QDataStream& in, MyMetaData& metaData) { + in >> metaData.url; + in >> metaData.expirationDate; + in >> metaData.lastModified; + in >> metaData.saveToDisk; + in >> metaData.attributes; + in >> metaData.rawHeaders; +} diff --git a/tools/cache-extract/src/CacheExtractApp.h b/tools/cache-extract/src/CacheExtractApp.h new file mode 100644 index 0000000000..3dde1f684d --- /dev/null +++ b/tools/cache-extract/src/CacheExtractApp.h @@ -0,0 +1,47 @@ +// +// CacheExtractApp.h +// tools/cache-extract/src +// +// Created by Anthony Thibault on 11/6/2015 +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_CacheExtractApp_h +#define hifi_CacheExtractApp_h + +#include +#include +#include +#include +#include +#include + +// copy of QNetworkCacheMetaData +class MyMetaData { +public: + typedef QPair RawHeader; + typedef QList RawHeaderList; + typedef QHash AttributesMap; + + QUrl url; + QDateTime expirationDate; + QDateTime lastModified; + bool saveToDisk; + AttributesMap attributes; + RawHeaderList rawHeaders; +}; + +QDataStream &operator>>(QDataStream &, MyMetaData &); + +class CacheExtractApp : public QCoreApplication { + Q_OBJECT +public: + CacheExtractApp(int& argc, char** argv); + + bool extractFile(const QString& filePath, MyMetaData& metaData, QByteArray& data) const; +}; + +#endif // hifi_CacheExtractApp_h diff --git a/tools/cache-extract/src/main.cpp b/tools/cache-extract/src/main.cpp new file mode 100644 index 0000000000..71a364ed3e --- /dev/null +++ b/tools/cache-extract/src/main.cpp @@ -0,0 +1,17 @@ +// +// main.cpp +// tools/cache-extract/src +// +// Created by Anthony Thibault on 11/6/2015. +// Copyright 2015 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include +#include "CacheExtractApp.h" + +int main (int argc, char** argv) { + CacheExtractApp app(argc, argv); + return app.exec(); +} From 086b0645273f33c9fc00cfd99cb82b82fef0725e Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 6 Nov 2015 15:07:38 -0800 Subject: [PATCH 2/4] Dumps all urls extracted to stdout. --- tools/cache-extract/src/CacheExtractApp.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/cache-extract/src/CacheExtractApp.cpp b/tools/cache-extract/src/CacheExtractApp.cpp index e7e60973af..a0d0bf54ec 100644 --- a/tools/cache-extract/src/CacheExtractApp.cpp +++ b/tools/cache-extract/src/CacheExtractApp.cpp @@ -68,6 +68,7 @@ CacheExtractApp::CacheExtractApp(int& argc, char** argv) : if (out.open(QIODevice::WriteOnly)) { out.write(contents); out.close(); + qDebug().noquote() << metaData.url.toDisplayString(); } } else { qCritical() << "Error extracting = " << fileList.at(i); From 9a484ff00dd71b94c34712759fc4a863a99c910a Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Fri, 6 Nov 2015 15:33:06 -0800 Subject: [PATCH 3/4] Fixes for windows --- tools/cache-extract/CMakeLists.txt | 1 + tools/cache-extract/src/CacheExtractApp.cpp | 22 ++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tools/cache-extract/CMakeLists.txt b/tools/cache-extract/CMakeLists.txt index 045fa996d2..1aaa4d9d04 100644 --- a/tools/cache-extract/CMakeLists.txt +++ b/tools/cache-extract/CMakeLists.txt @@ -3,4 +3,5 @@ set(TARGET_NAME cache-extract) setup_hifi_project() link_hifi_libraries() +copy_dlls_beside_windows_executable() diff --git a/tools/cache-extract/src/CacheExtractApp.cpp b/tools/cache-extract/src/CacheExtractApp.cpp index a0d0bf54ec..0cfaef3f83 100644 --- a/tools/cache-extract/src/CacheExtractApp.cpp +++ b/tools/cache-extract/src/CacheExtractApp.cpp @@ -14,6 +14,7 @@ #include #include #include +#include // extracted from qnetworkdiskcache.cpp #define CACHE_VERSION 8 @@ -21,19 +22,18 @@ enum { CacheMagic = 0xe8, CurrentCacheVersion = CACHE_VERSION }; -#define DATA_DIR QLatin1String("data") CacheExtractApp::CacheExtractApp(int& argc, char** argv) : QCoreApplication(argc, argv) { QString myDataLoc = QStandardPaths::writableLocation(QStandardPaths::DataLocation); - int lastSlash = myDataLoc.lastIndexOf(QDir::separator()); - QString cachePath = myDataLoc.leftRef(lastSlash).toString() + QDir::separator() + - "High Fidelity" + QDir::separator() + "Interface" + QDir::separator() + - DATA_DIR + QString::number(CACHE_VERSION) + QLatin1Char('/'); + int lastSlash = myDataLoc.lastIndexOf("/"); + QString cachePath = myDataLoc.leftRef(lastSlash).toString() + "/" + + "High Fidelity" + "/" + "Interface" + "/" + + "data" + QString::number(CACHE_VERSION) + "/"; - QString outputPath = myDataLoc.leftRef(lastSlash).toString() + QDir::separator() + - "High Fidelity" + QDir::separator() + "Interface" + QDir::separator() + "extracted"; + QString outputPath = myDataLoc.leftRef(lastSlash).toString() + "/" + + "High Fidelity" + "/" + "Interface" + "/" + "extracted"; qDebug() << "Searching cachePath = " << cachePath << "..."; @@ -60,9 +60,9 @@ CacheExtractApp::CacheExtractApp(int& argc, char** argv) : MyMetaData metaData; if (extractFile(fileList.at(i), metaData, contents)) { QString outFileName = outputPath + metaData.url.path(); - int lastSlash = outFileName.lastIndexOf(QDir::separator()); + int lastSlash = outFileName.lastIndexOf("/"); QString outDirName = outFileName.leftRef(lastSlash).toString(); - QDir dir(outputPath); + QDir dir; dir.mkpath(outDirName); QFile out(outFileName); if (out.open(QIODevice::WriteOnly)) { @@ -70,6 +70,9 @@ CacheExtractApp::CacheExtractApp(int& argc, char** argv) : out.close(); qDebug().noquote() << metaData.url.toDisplayString(); } + else { + qCritical() << "Error opening outputFile = " << outFileName; + } } else { qCritical() << "Error extracting = " << fileList.at(i); } @@ -123,4 +126,5 @@ QDataStream &operator>>(QDataStream& in, MyMetaData& metaData) { in >> metaData.saveToDisk; in >> metaData.attributes; in >> metaData.rawHeaders; + return in; } From 82b26b75f4024099e569ef6703594adf3321759b Mon Sep 17 00:00:00 2001 From: "Anthony J. Thibault" Date: Mon, 9 Nov 2015 08:14:27 -0800 Subject: [PATCH 4/4] Code convention fixes --- tools/cache-extract/src/CacheExtractApp.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/cache-extract/src/CacheExtractApp.h b/tools/cache-extract/src/CacheExtractApp.h index 3dde1f684d..3b34ff891d 100644 --- a/tools/cache-extract/src/CacheExtractApp.h +++ b/tools/cache-extract/src/CacheExtractApp.h @@ -22,9 +22,9 @@ // copy of QNetworkCacheMetaData class MyMetaData { public: - typedef QPair RawHeader; - typedef QList RawHeaderList; - typedef QHash AttributesMap; + using RawHeader = QPair; + using RawHeaderList = QList; + using AttributesMap = QHash; QUrl url; QDateTime expirationDate; @@ -34,7 +34,7 @@ public: RawHeaderList rawHeaders; }; -QDataStream &operator>>(QDataStream &, MyMetaData &); +QDataStream &operator>>(QDataStream& in, MyMetaData& metaData); class CacheExtractApp : public QCoreApplication { Q_OBJECT