From faafd26d2a77228c4872f885d15df6769f11f2a1 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 31 May 2018 16:50:32 -0700 Subject: [PATCH 1/9] Add uncompressed KTX files to baked texture output --- assignment-client/src/assets/AssetServer.cpp | 18 ---- assignment-client/src/assets/AssetServer.h | 5 - libraries/baking/src/TextureBaker.cpp | 98 +++++++++++++------ libraries/image/src/image/Image.cpp | 88 ++++++++--------- libraries/image/src/image/Image.h | 56 +++++------ libraries/ktx/src/TextureMeta.cpp | 4 + libraries/ktx/src/TextureMeta.h | 1 + .../src/model-networking/TextureCache.cpp | 23 ++++- libraries/shared/src/OwningBuffer.h | 29 ++++++ tools/oven/src/Oven.cpp | 6 -- 10 files changed, 190 insertions(+), 138 deletions(-) create mode 100644 libraries/shared/src/OwningBuffer.h diff --git a/assignment-client/src/assets/AssetServer.cpp b/assignment-client/src/assets/AssetServer.cpp index 22ed01fd00..e0c35b7148 100644 --- a/assignment-client/src/assets/AssetServer.cpp +++ b/assignment-client/src/assets/AssetServer.cpp @@ -291,18 +291,6 @@ AssetServer::AssetServer(ReceivedMessage& message) : _bakingTaskPool(this), _filesizeLimit(AssetUtils::MAX_UPLOAD_SIZE) { - // store the current state of image compression so we can reset it when this assignment is complete - _wasColorTextureCompressionEnabled = image::isColorTexturesCompressionEnabled(); - _wasGrayscaleTextureCompressionEnabled = image::isGrayscaleTexturesCompressionEnabled(); - _wasNormalTextureCompressionEnabled = image::isNormalTexturesCompressionEnabled(); - _wasCubeTextureCompressionEnabled = image::isCubeTexturesCompressionEnabled(); - - // enable compression in image library - image::setColorTexturesCompressionEnabled(true); - image::setGrayscaleTexturesCompressionEnabled(true); - image::setNormalTexturesCompressionEnabled(true); - image::setCubeTexturesCompressionEnabled(true); - BAKEABLE_TEXTURE_EXTENSIONS = image::getSupportedFormats(); qDebug() << "Supported baking texture formats:" << BAKEABLE_MODEL_EXTENSIONS; @@ -354,12 +342,6 @@ void AssetServer::aboutToFinish() { while (_pendingBakes.size() > 0) { QCoreApplication::processEvents(); } - - // re-set defaults in image library - image::setColorTexturesCompressionEnabled(_wasCubeTextureCompressionEnabled); - image::setGrayscaleTexturesCompressionEnabled(_wasGrayscaleTextureCompressionEnabled); - image::setNormalTexturesCompressionEnabled(_wasNormalTextureCompressionEnabled); - image::setCubeTexturesCompressionEnabled(_wasCubeTextureCompressionEnabled); } void AssetServer::run() { diff --git a/assignment-client/src/assets/AssetServer.h b/assignment-client/src/assets/AssetServer.h index 96a220d64d..b3d0f18a8f 100644 --- a/assignment-client/src/assets/AssetServer.h +++ b/assignment-client/src/assets/AssetServer.h @@ -167,11 +167,6 @@ private: using RequestQueue = QVector, SharedNodePointer>>; RequestQueue _queuedRequests; - bool _wasColorTextureCompressionEnabled { false }; - bool _wasGrayscaleTextureCompressionEnabled { false }; - bool _wasNormalTextureCompressionEnabled { false }; - bool _wasCubeTextureCompressionEnabled { false }; - uint64_t _filesizeLimit; }; diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp index b6957a2712..cadc1a2d7e 100644 --- a/libraries/baking/src/TextureBaker.cpp +++ b/libraries/baking/src/TextureBaker.cpp @@ -22,6 +22,8 @@ #include #include +#include + #include "ModelBakingLoggingCategory.h" const QString BAKED_TEXTURE_KTX_EXT = ".ktx"; @@ -124,47 +126,51 @@ void TextureBaker::processTexture() { TextureMeta meta; + auto originalCopyFilePath = _outputDirectory.absoluteFilePath(_textureURL.fileName()); { - auto filePath = _outputDirectory.absoluteFilePath(_textureURL.fileName()); - QFile file { filePath }; + QFile file { originalCopyFilePath }; if (!file.open(QIODevice::WriteOnly) || file.write(_originalTexture) == -1) { handleError("Could not write original texture for " + _textureURL.toString()); return; } - _outputFiles.push_back(filePath); + _originalTexture.clear(); + _outputFiles.push_back(originalCopyFilePath); meta.original = _metaTexturePathPrefix +_textureURL.fileName(); } - // IMPORTANT: _originalTexture is empty past this point - auto processedTexture = image::processImage(std::move(_originalTexture), _textureURL.toString().toStdString(), - ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType, _abortProcessing); - processedTexture->setSourceHash(hash); - - if (shouldStop()) { + auto buffer = std::shared_ptr((QIODevice*)new QFile(originalCopyFilePath)); + if (!buffer->open(QIODevice::ReadOnly)) { + handleError("Could not open original file at " + originalCopyFilePath); return; } - if (!processedTexture) { - handleError("Could not process texture " + _textureURL.toString()); - return; - } - - - auto memKTX = gpu::Texture::serialize(*processedTexture); - - if (!memKTX) { - handleError("Could not serialize " + _textureURL.toString() + " to KTX"); - return; - } - - const char* name = khronos::gl::texture::toString(memKTX->_header.getGLInternaFormat()); - if (name == nullptr) { - handleError("Could not determine internal format for compressed KTX: " + _textureURL.toString()); - return; - } - - // attempt to write the baked texture to the destination file path + // Compressed KTX { + // IMPORTANT: _originalTexture is empty past this point + auto processedTexture = image::processImage(buffer, _textureURL.toString().toStdString(), + ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType, true, _abortProcessing); + if (!processedTexture) { + handleError("Could not process texture " + _textureURL.toString()); + return; + } + processedTexture->setSourceHash(hash); + + if (shouldStop()) { + return; + } + + auto memKTX = gpu::Texture::serialize(*processedTexture); + if (!memKTX) { + handleError("Could not serialize " + _textureURL.toString() + " to KTX"); + return; + } + + const char* name = khronos::gl::texture::toString(memKTX->_header.getGLInternaFormat()); + if (name == nullptr) { + handleError("Could not determine internal format for compressed KTX: " + _textureURL.toString()); + return; + } + const char* data = reinterpret_cast(memKTX->_storage->data()); const size_t length = memKTX->_storage->size(); @@ -179,6 +185,40 @@ void TextureBaker::processTexture() { meta.availableTextureTypes[memKTX->_header.getGLInternaFormat()] = _metaTexturePathPrefix + fileName; } + // Uncompressed KTX + { + buffer->reset(); + auto processedTexture = image::processImage(std::move(buffer), _textureURL.toString().toStdString(), + ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType, false, _abortProcessing); + if (!processedTexture) { + handleError("Could not process texture " + _textureURL.toString()); + return; + } + processedTexture->setSourceHash(hash); + + if (shouldStop()) { + return; + } + + auto memKTX = gpu::Texture::serialize(*processedTexture); + if (!memKTX) { + handleError("Could not serialize " + _textureURL.toString() + " to KTX"); + return; + } + + const char* data = reinterpret_cast(memKTX->_storage->data()); + const size_t length = memKTX->_storage->size(); + + auto fileName = _baseFilename + ".ktx"; + auto filePath = _outputDirectory.absoluteFilePath(fileName); + QFile bakedTextureFile { filePath }; + if (!bakedTextureFile.open(QIODevice::WriteOnly) || bakedTextureFile.write(data, length) == -1) { + handleError("Could not write baked texture for " + _textureURL.toString()); + return; + } + _outputFiles.push_back(filePath); + meta.uncompressed = _metaTexturePathPrefix + fileName; + } { auto data = meta.serialize(); diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index 63a4725f64..7faf811dec 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -126,63 +126,63 @@ TextureUsage::TextureLoader TextureUsage::getTextureLoaderForType(Type type, con } gpu::TexturePointer TextureUsage::createStrict2DTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureColorFromImage(std::move(srcImage), srcImageName, true, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(std::move(srcImage), srcImageName, compress, true, abortProcessing); } gpu::TexturePointer TextureUsage::create2DTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureColorFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } gpu::TexturePointer TextureUsage::createAlbedoTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureColorFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } gpu::TexturePointer TextureUsage::createEmissiveTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureColorFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } gpu::TexturePointer TextureUsage::createLightmapTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureColorFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureColorFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } gpu::TexturePointer TextureUsage::createNormalTextureFromNormalImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureNormalMapFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureNormalMapFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } gpu::TexturePointer TextureUsage::createNormalTextureFromBumpImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureNormalMapFromImage(std::move(srcImage), srcImageName, true, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureNormalMapFromImage(std::move(srcImage), srcImageName, compress, true, abortProcessing); } gpu::TexturePointer TextureUsage::createRoughnessTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } gpu::TexturePointer TextureUsage::createRoughnessTextureFromGlossImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, true, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, compress, true, abortProcessing); } gpu::TexturePointer TextureUsage::createMetallicTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return process2DTextureGrayscaleFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } gpu::TexturePointer TextureUsage::createCubeTextureFromImage(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, true, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, true, abortProcessing); } gpu::TexturePointer TextureUsage::createCubeTextureFromImageWithoutIrradiance(QImage&& srcImage, const std::string& srcImageName, - const std::atomic& abortProcessing) { - return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, false, abortProcessing); + bool compress, const std::atomic& abortProcessing) { + return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } @@ -253,17 +253,11 @@ uint32 packR11G11B10F(const glm::vec3& color) { return glm::packF2x11_1x10(ucolor); } -QImage processRawImageData(QByteArray&& content, const std::string& filename) { - // Take a local copy to force move construction - // https://github.com/isocpp/CppCoreGuidelines/blob/master/CppCoreGuidelines.md#f18-for-consume-parameters-pass-by-x-and-stdmove-the-parameter - QByteArray localCopy = std::move(content); - +QImage processRawImageData(QIODevice& content, const std::string& filename) { // Help the QImage loader by extracting the image file format from the url filename ext. // Some tga are not created properly without it. auto filenameExtension = filename.substr(filename.find_last_of('.') + 1); - QBuffer buffer; - buffer.setData(localCopy); - QImageReader imageReader(&buffer, filenameExtension.c_str()); + QImageReader imageReader(&content, filenameExtension.c_str()); if (imageReader.canRead()) { return imageReader.read(); @@ -271,8 +265,8 @@ QImage processRawImageData(QByteArray&& content, const std::string& filename) { // Extension could be incorrect, try to detect the format from the content QImageReader newImageReader; newImageReader.setDecideFormatFromContent(true); - buffer.setData(localCopy); - newImageReader.setDevice(&buffer); + content.reset(); + newImageReader.setDevice(&content); if (newImageReader.canRead()) { qCWarning(imagelogging) << "Image file" << filename.c_str() << "has extension" << filenameExtension.c_str() @@ -284,11 +278,14 @@ QImage processRawImageData(QByteArray&& content, const std::string& filename) { return QImage(); } -gpu::TexturePointer processImage(QByteArray&& content, const std::string& filename, +gpu::TexturePointer processImage(std::shared_ptr content, const std::string& filename, int maxNumPixels, TextureUsage::Type textureType, - const std::atomic& abortProcessing) { + bool compress, const std::atomic& abortProcessing) { - QImage image = processRawImageData(std::move(content), filename); + QImage image = processRawImageData(*content.get(), filename); + // Texture content can take up a lot of memory. Here we release our ownership of that content + // in case it can be released. + content.reset(); int imageWidth = image.width(); int imageHeight = image.height(); @@ -314,7 +311,7 @@ gpu::TexturePointer processImage(QByteArray&& content, const std::string& filena } auto loader = TextureUsage::getTextureLoaderForType(textureType); - auto texture = loader(std::move(image), filename, abortProcessing); + auto texture = loader(std::move(image), filename, compress, abortProcessing); return texture; } @@ -804,7 +801,7 @@ void processTextureAlpha(const QImage& srcImage, bool& validAlpha, bool& alphaAs validAlpha = (numOpaques != NUM_PIXELS); } -gpu::TexturePointer TextureUsage::process2DTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, +gpu::TexturePointer TextureUsage::process2DTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, bool isStrict, const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "process2DTextureColorFromImage"); QImage image = processSourceImage(std::move(srcImage), false); @@ -825,7 +822,7 @@ gpu::TexturePointer TextureUsage::process2DTextureColorFromImage(QImage&& srcIma if ((image.width() > 0) && (image.height() > 0)) { gpu::Element formatMip; gpu::Element formatGPU; - if (isColorTexturesCompressionEnabled()) { + if (compress) { if (validAlpha) { // NOTE: This disables BC1a compression because it was producing odd artifacts on text textures // for the tutorial. Instead we use BC3 (which is larger) but doesn't produce the same artifacts). @@ -941,7 +938,8 @@ QImage processBumpMap(QImage&& image) { return result; } gpu::TexturePointer TextureUsage::process2DTextureNormalMapFromImage(QImage&& srcImage, const std::string& srcImageName, - bool isBumpMap, const std::atomic& abortProcessing) { + bool compress, bool isBumpMap, + const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "process2DTextureNormalMapFromImage"); QImage image = processSourceImage(std::move(srcImage), false); @@ -959,6 +957,7 @@ gpu::TexturePointer TextureUsage::process2DTextureNormalMapFromImage(QImage&& sr gpu::Element formatMip; gpu::Element formatGPU; if (isNormalTexturesCompressionEnabled()) { + if (compress) { formatGPU = gpu::Element::COLOR_COMPRESSED_BCX_XY; } else { #ifdef USE_GLES @@ -980,7 +979,7 @@ gpu::TexturePointer TextureUsage::process2DTextureNormalMapFromImage(QImage&& sr } gpu::TexturePointer TextureUsage::process2DTextureGrayscaleFromImage(QImage&& srcImage, const std::string& srcImageName, - bool isInvertedPixels, + bool compress, bool isInvertedPixels, const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "process2DTextureGrayscaleFromImage"); QImage image = processSourceImage(std::move(srcImage), false); @@ -999,6 +998,7 @@ gpu::TexturePointer TextureUsage::process2DTextureGrayscaleFromImage(QImage&& sr gpu::Element formatMip; gpu::Element formatGPU; if (isGrayscaleTexturesCompressionEnabled()) { + if (compress) { formatGPU = gpu::Element::COLOR_COMPRESSED_BCX_RED; } else { #ifdef USE_GLES @@ -1345,7 +1345,7 @@ QImage convertToHDRFormat(QImage&& srcImage, gpu::Element format) { } gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, - bool generateIrradiance, + bool compress, bool generateIrradiance, const std::atomic& abortProcessing) { PROFILE_RANGE(resource_parse, "processCubeTextureColorFromImage"); @@ -1373,7 +1373,7 @@ gpu::TexturePointer TextureUsage::processCubeTextureColorFromImage(QImage&& srcI gpu::Element formatMip; gpu::Element formatGPU; - if (isCubeTexturesCompressionEnabled()) { + if (compress) { formatGPU = gpu::Element::COLOR_COMPRESSED_BCX_HDR_RGB; } else { #ifdef USE_GLES diff --git a/libraries/image/src/image/Image.h b/libraries/image/src/image/Image.h index 39f5ea3bab..3dca8f5586 100644 --- a/libraries/image/src/image/Image.h +++ b/libraries/image/src/image/Image.h @@ -41,60 +41,50 @@ enum Type { UNUSED_TEXTURE }; -using TextureLoader = std::function&)>; +using TextureLoader = std::function&)>; TextureLoader getTextureLoaderForType(Type type, const QVariantMap& options = QVariantMap()); gpu::TexturePointer create2DTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createStrict2DTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createAlbedoTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createEmissiveTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createNormalTextureFromNormalImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createNormalTextureFromBumpImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createRoughnessTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createRoughnessTextureFromGlossImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createMetallicTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createCubeTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createCubeTextureFromImageWithoutIrradiance(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); gpu::TexturePointer createLightmapTextureFromImage(QImage&& image, const std::string& srcImageName, - const std::atomic& abortProcessing); + bool compress, const std::atomic& abortProcessing); -gpu::TexturePointer process2DTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, bool isStrict, - const std::atomic& abortProcessing); -gpu::TexturePointer process2DTextureNormalMapFromImage(QImage&& srcImage, const std::string& srcImageName, bool isBumpMap, - const std::atomic& abortProcessing); -gpu::TexturePointer process2DTextureGrayscaleFromImage(QImage&& srcImage, const std::string& srcImageName, bool isInvertedPixels, - const std::atomic& abortProcessing); -gpu::TexturePointer processCubeTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, bool generateIrradiance, - const std::atomic& abortProcessing); +gpu::TexturePointer process2DTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, + bool isStrict, const std::atomic& abortProcessing); +gpu::TexturePointer process2DTextureNormalMapFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, + bool isBumpMap, const std::atomic& abortProcessing); +gpu::TexturePointer process2DTextureGrayscaleFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, + bool isInvertedPixels, const std::atomic& abortProcessing); +gpu::TexturePointer processCubeTextureColorFromImage(QImage&& srcImage, const std::string& srcImageName, bool compress, + bool generateIrradiance, const std::atomic& abortProcessing); } // namespace TextureUsage const QStringList getSupportedFormats(); -bool isColorTexturesCompressionEnabled(); -bool isNormalTexturesCompressionEnabled(); -bool isGrayscaleTexturesCompressionEnabled(); -bool isCubeTexturesCompressionEnabled(); - -void setColorTexturesCompressionEnabled(bool enabled); -void setNormalTexturesCompressionEnabled(bool enabled); -void setGrayscaleTexturesCompressionEnabled(bool enabled); -void setCubeTexturesCompressionEnabled(bool enabled); - -gpu::TexturePointer processImage(QByteArray&& content, const std::string& url, +gpu::TexturePointer processImage(std::shared_ptr content, const std::string& url, int maxNumPixels, TextureUsage::Type textureType, - const std::atomic& abortProcessing = false); + bool compress = true, const std::atomic& abortProcessing = false); } // namespace image diff --git a/libraries/ktx/src/TextureMeta.cpp b/libraries/ktx/src/TextureMeta.cpp index 3a2abf24c4..c8427c1f60 100644 --- a/libraries/ktx/src/TextureMeta.cpp +++ b/libraries/ktx/src/TextureMeta.cpp @@ -33,6 +33,9 @@ bool TextureMeta::deserialize(const QByteArray& data, TextureMeta* meta) { if (root.contains("original")) { meta->original = root["original"].toString(); } + if (root.contains("uncompressed")) { + meta->uncompressed = root["uncompressed"].toString(); + } if (root.contains("compressed")) { auto compressed = root["compressed"].toObject(); for (auto it = compressed.constBegin(); it != compressed.constEnd(); it++) { @@ -57,6 +60,7 @@ QByteArray TextureMeta::serialize() { compressed[name] = kv.second.toString(); } root["original"] = original.toString(); + root["uncompressed"] = uncompressed.toString(); root["compressed"] = compressed; doc.setObject(root); diff --git a/libraries/ktx/src/TextureMeta.h b/libraries/ktx/src/TextureMeta.h index 6582c29e70..5450fee110 100644 --- a/libraries/ktx/src/TextureMeta.h +++ b/libraries/ktx/src/TextureMeta.h @@ -35,6 +35,7 @@ struct TextureMeta { QByteArray serialize(); QUrl original; + QUrl uncompressed; std::unordered_map availableTextureTypes; }; diff --git a/libraries/model-networking/src/model-networking/TextureCache.cpp b/libraries/model-networking/src/model-networking/TextureCache.cpp index ed21fd35bc..40b31cac53 100644 --- a/libraries/model-networking/src/model-networking/TextureCache.cpp +++ b/libraries/model-networking/src/model-networking/TextureCache.cpp @@ -50,6 +50,8 @@ #include +#include + Q_LOGGING_CATEGORY(trace_resource_parse_image, "trace.resource.parse.image") Q_LOGGING_CATEGORY(trace_resource_parse_image_raw, "trace.resource.parse.image.raw") Q_LOGGING_CATEGORY(trace_resource_parse_image_ktx, "trace.resource.parse.image.ktx") @@ -277,7 +279,7 @@ gpu::TexturePointer TextureCache::getImageTexture(const QString& path, image::Te return nullptr; } auto loader = image::TextureUsage::getTextureLoaderForType(type, options); - return gpu::TexturePointer(loader(std::move(image), path.toStdString(), false)); + return gpu::TexturePointer(loader(std::move(image), path.toStdString(), false, false)); } QSharedPointer TextureCache::createResource(const QUrl& url, const QSharedPointer& fallback, @@ -964,7 +966,6 @@ void NetworkTexture::loadMetaContent(const QByteArray& content) { return; } - auto& backend = DependencyManager::get()->getGPUContext()->getBackend(); for (auto pair : meta.availableTextureTypes) { gpu::Element elFormat; @@ -990,6 +991,21 @@ void NetworkTexture::loadMetaContent(const QByteArray& content) { } } +#ifndef Q_OS_ANDROID + if (!meta.uncompressed.isEmpty()) { + _currentlyLoadingResourceType = ResourceType::KTX; + _activeUrl = _activeUrl.resolved(meta.uncompressed); + + auto textureCache = DependencyManager::get(); + auto self = _self.lock(); + if (!self) { + return; + } + QMetaObject::invokeMethod(this, "attemptRequest", Qt::QueuedConnection); + return; + } +#endif + if (!meta.original.isEmpty()) { _currentlyLoadingResourceType = ResourceType::ORIGINAL; _activeUrl = _activeUrl.resolved(meta.original); @@ -1143,7 +1159,8 @@ void ImageReader::read() { PROFILE_RANGE_EX(resource_parse_image_raw, __FUNCTION__, 0xffff0000, 0); // IMPORTANT: _content is empty past this point - texture = image::processImage(std::move(_content), _url.toString().toStdString(), _maxNumPixels, networkTexture->getTextureType()); + auto buffer = std::shared_ptr((QIODevice*)new OwningBuffer(std::move(_content))); + texture = image::processImage(std::move(buffer), _url.toString().toStdString(), _maxNumPixels, networkTexture->getTextureType()); if (!texture) { qCWarning(modelnetworking) << "Could not process:" << _url; diff --git a/libraries/shared/src/OwningBuffer.h b/libraries/shared/src/OwningBuffer.h new file mode 100644 index 0000000000..80184286bc --- /dev/null +++ b/libraries/shared/src/OwningBuffer.h @@ -0,0 +1,29 @@ +// +// OwningBuffer.h +// shared/src +// +// Created by Ryan Huffman on 5/31/2018. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_OwningBuffer_h +#define hifi_OwningBuffer_h + +#include +class OwningBuffer : public QBuffer { +public: + OwningBuffer(const QByteArray& content) : _content(content) { + setData(_content); + } + OwningBuffer(QByteArray&& content) : _content(std::move(content)) { + setData(_content); + } + +private: + QByteArray _content; +}; + +#endif // hifi_OwningBuffer_h diff --git a/tools/oven/src/Oven.cpp b/tools/oven/src/Oven.cpp index c3fec2d15e..52b6db1aa5 100644 --- a/tools/oven/src/Oven.cpp +++ b/tools/oven/src/Oven.cpp @@ -25,12 +25,6 @@ Oven* Oven::_staticInstance { nullptr }; Oven::Oven() { _staticInstance = this; - // enable compression in image library - image::setColorTexturesCompressionEnabled(true); - image::setGrayscaleTexturesCompressionEnabled(true); - image::setNormalTexturesCompressionEnabled(true); - image::setCubeTexturesCompressionEnabled(true); - // setup our worker threads setupWorkerThreads(QThread::idealThreadCount()); From 24182d3451b828e659b55dd7199c807dc5a33a99 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 7 Jun 2018 15:27:50 -0700 Subject: [PATCH 2/9] Remove compression variables & functions in image.cpp --- libraries/image/src/image/Image.cpp | 56 ----------------------------- 1 file changed, 56 deletions(-) diff --git a/libraries/image/src/image/Image.cpp b/libraries/image/src/image/Image.cpp index 7faf811dec..f812e515de 100644 --- a/libraries/image/src/image/Image.cpp +++ b/libraries/image/src/image/Image.cpp @@ -48,11 +48,6 @@ std::atomic RECTIFIED_TEXTURE_COUNT{ 0 }; static const auto HDR_FORMAT = gpu::Element::COLOR_R11G11B10; -static std::atomic compressColorTextures { false }; -static std::atomic compressNormalTextures { false }; -static std::atomic compressGrayscaleTextures { false }; -static std::atomic compressCubeTextures { false }; - uint rectifyDimension(const uint& dimension) { if (dimension == 0) { return 0; @@ -185,55 +180,6 @@ gpu::TexturePointer TextureUsage::createCubeTextureFromImageWithoutIrradiance(QI return processCubeTextureColorFromImage(std::move(srcImage), srcImageName, compress, false, abortProcessing); } - -bool isColorTexturesCompressionEnabled() { -#if CPU_MIPMAPS - return compressColorTextures.load(); -#else - return false; -#endif -} - -bool isNormalTexturesCompressionEnabled() { -#if CPU_MIPMAPS - return compressNormalTextures.load(); -#else - return false; -#endif -} - -bool isGrayscaleTexturesCompressionEnabled() { -#if CPU_MIPMAPS - return compressGrayscaleTextures.load(); -#else - return false; -#endif -} - -bool isCubeTexturesCompressionEnabled() { -#if CPU_MIPMAPS - return compressCubeTextures.load(); -#else - return false; -#endif -} - -void setColorTexturesCompressionEnabled(bool enabled) { - compressColorTextures.store(enabled); -} - -void setNormalTexturesCompressionEnabled(bool enabled) { - compressNormalTextures.store(enabled); -} - -void setGrayscaleTexturesCompressionEnabled(bool enabled) { - compressGrayscaleTextures.store(enabled); -} - -void setCubeTexturesCompressionEnabled(bool enabled) { - compressCubeTextures.store(enabled); -} - static float denormalize(float value, const float minValue) { return value < minValue ? 0.0f : value; } @@ -956,7 +902,6 @@ gpu::TexturePointer TextureUsage::process2DTextureNormalMapFromImage(QImage&& sr if ((image.width() > 0) && (image.height() > 0)) { gpu::Element formatMip; gpu::Element formatGPU; - if (isNormalTexturesCompressionEnabled()) { if (compress) { formatGPU = gpu::Element::COLOR_COMPRESSED_BCX_XY; } else { @@ -997,7 +942,6 @@ gpu::TexturePointer TextureUsage::process2DTextureGrayscaleFromImage(QImage&& sr if ((image.width() > 0) && (image.height() > 0)) { gpu::Element formatMip; gpu::Element formatGPU; - if (isGrayscaleTexturesCompressionEnabled()) { if (compress) { formatGPU = gpu::Element::COLOR_COMPRESSED_BCX_RED; } else { From 3e65e0b0cd914b43d1aa640a6b7a91d0f52d663b Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Fri, 8 Jun 2018 13:21:47 -0700 Subject: [PATCH 3/9] Update processImage to not compress by default --- libraries/image/src/image/Image.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/image/src/image/Image.h b/libraries/image/src/image/Image.h index 3dca8f5586..ccf4845fca 100644 --- a/libraries/image/src/image/Image.h +++ b/libraries/image/src/image/Image.h @@ -84,7 +84,7 @@ const QStringList getSupportedFormats(); gpu::TexturePointer processImage(std::shared_ptr content, const std::string& url, int maxNumPixels, TextureUsage::Type textureType, - bool compress = true, const std::atomic& abortProcessing = false); + bool compress = false, const std::atomic& abortProcessing = false); } // namespace image From 9506e7edbe75d7a33805767ac554286ab8957ec0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 8 Jun 2018 14:47:20 -0700 Subject: [PATCH 4/9] check for update for dev-builds, handle semantic version --- interface/src/Application.cpp | 7 +- interface/src/ui/UpdateDialog.cpp | 36 +++++--- libraries/auto-updater/src/AutoUpdater.cpp | 68 ++++++++++----- libraries/auto-updater/src/AutoUpdater.h | 17 ++-- libraries/shared/src/ApplicationVersion.cpp | 94 +++++++++++++++++++++ libraries/shared/src/ApplicationVersion.h | 41 +++++++++ 6 files changed, 220 insertions(+), 43 deletions(-) create mode 100644 libraries/shared/src/ApplicationVersion.cpp create mode 100644 libraries/shared/src/ApplicationVersion.h diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index c5857dac53..4a785db2c2 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -905,7 +905,6 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) { DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); - DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); DependencyManager::set(); @@ -1784,10 +1783,12 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo // If launched from Steam, let it handle updates const QString HIFI_NO_UPDATER_COMMAND_LINE_KEY = "--no-updater"; bool noUpdater = arguments().indexOf(HIFI_NO_UPDATER_COMMAND_LINE_KEY) != -1; - if (!noUpdater) { + bool buildCanUpdate = BuildInfo::BUILD_TYPE == BuildInfo::BuildType::Stable + || BuildInfo::BUILD_TYPE == BuildInfo::BuildType::Master; + if (!noUpdater && buildCanUpdate) { constexpr auto INSTALLER_TYPE_CLIENT_ONLY = "client_only"; - auto applicationUpdater = DependencyManager::get(); + auto applicationUpdater = DependencyManager::set(); AutoUpdater::InstallerType type = installerType == INSTALLER_TYPE_CLIENT_ONLY ? AutoUpdater::InstallerType::CLIENT_ONLY : AutoUpdater::InstallerType::FULL; diff --git a/interface/src/ui/UpdateDialog.cpp b/interface/src/ui/UpdateDialog.cpp index 2dcc0c07eb..7ff2132ab9 100644 --- a/interface/src/ui/UpdateDialog.cpp +++ b/interface/src/ui/UpdateDialog.cpp @@ -21,19 +21,31 @@ UpdateDialog::UpdateDialog(QQuickItem* parent) : OffscreenQmlDialog(parent) { auto applicationUpdater = DependencyManager::get(); - int currentVersion = QCoreApplication::applicationVersion().toInt(); - int latestVersion = applicationUpdater.data()->getBuildData().lastKey(); - _updateAvailableDetails = "v" + QString::number(latestVersion) + " released on " - + QString(applicationUpdater.data()->getBuildData()[latestVersion]["releaseTime"]).replace(" ", " "); + if (applicationUpdater) { - _releaseNotes = ""; - for (int i = latestVersion; i > currentVersion; i--) { - if (applicationUpdater.data()->getBuildData().contains(i)) { - QString releaseNotes = applicationUpdater.data()->getBuildData()[i]["releaseNotes"]; - releaseNotes.remove("
"); - releaseNotes.remove(QRegExp("^\n+")); - _releaseNotes += "\n" + QString().sprintf("%d", i) + "\n" + releaseNotes + "\n"; + auto buildData = applicationUpdater.data()->getBuildData(); + ApplicationVersion latestVersion = buildData.lastKey(); + _updateAvailableDetails = "v" + latestVersion.versionString + " released on " + + QString(buildData[latestVersion]["releaseTime"]).replace(" ", " "); + + _releaseNotes = ""; + + auto it = buildData.end(); + while (it != buildData.begin()) { + --it; + + if (applicationUpdater->getCurrentVersion() < it.key()) { + // grab the release notes for this later version + QString releaseNotes = it.value()["releaseNotes"]; + releaseNotes.remove("
"); + releaseNotes.remove(QRegExp("^\n+")); + _releaseNotes += "\n" + it.key().versionString + "\n" + releaseNotes + "\n"; + } else { + break; + } } + + } } @@ -47,5 +59,5 @@ const QString& UpdateDialog::releaseNotes() const { void UpdateDialog::triggerUpgrade() { auto applicationUpdater = DependencyManager::get(); - applicationUpdater.data()->performAutoUpdate(applicationUpdater.data()->getBuildData().lastKey()); + applicationUpdater.data()->openLatestUpdateURL(); } diff --git a/libraries/auto-updater/src/AutoUpdater.cpp b/libraries/auto-updater/src/AutoUpdater.cpp index 6749cd9e10..300a22983a 100644 --- a/libraries/auto-updater/src/AutoUpdater.cpp +++ b/libraries/auto-updater/src/AutoUpdater.cpp @@ -11,13 +11,16 @@ #include "AutoUpdater.h" -#include - -#include -#include #include -AutoUpdater::AutoUpdater() { +#include +#include +#include +#include + +AutoUpdater::AutoUpdater() : + _currentVersion(BuildInfo::BUILD_TYPE == BuildInfo::BuildType::Stable ? BuildInfo::VERSION : BuildInfo::BUILD_NUMBER) +{ #if defined Q_OS_WIN32 _operatingSystem = "windows"; #elif defined Q_OS_MAC @@ -33,9 +36,22 @@ void AutoUpdater::checkForUpdate() { this->getLatestVersionData(); } +const QUrl BUILDS_XML_URL("https://highfidelity.com/builds.xml"); +const QUrl MASTER_BUILDS_XML_URL("https://highfidelity.com/dev-builds.xml"); + void AutoUpdater::getLatestVersionData() { QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance(); - QNetworkRequest latestVersionRequest(BUILDS_XML_URL); + + QUrl buildsURL; + + if (BuildInfo::BUILD_TYPE == BuildInfo::BuildType::Stable) { + buildsURL = BUILDS_XML_URL; + } else if (BuildInfo::BUILD_TYPE == BuildInfo::BuildType::Master) { + buildsURL = MASTER_BUILDS_XML_URL; + } + + QNetworkRequest latestVersionRequest(buildsURL); + latestVersionRequest.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true); latestVersionRequest.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT); QNetworkReply* reply = networkAccessManager.get(latestVersionRequest); @@ -52,12 +68,22 @@ void AutoUpdater::parseLatestVersionData() { QString clientOnly; }; - int version { 0 }; + QString version; QString downloadUrl; QString releaseTime; QString releaseNotes; QString commitSha; QString pullRequestNumber; + + QString versionKey; + + // stable builds look at the stable_version node (semantic version) + // master builds look at the version node (build number) + if (BuildInfo::BUILD_TYPE == BuildInfo::BuildType::Stable) { + versionKey = "stable_version"; + } else if (BuildInfo::BUILD_TYPE == BuildInfo::BuildType::Master) { + versionKey = "version"; + } while (xml.readNextStartElement()) { if (xml.name() == "projects") { @@ -77,8 +103,8 @@ void AutoUpdater::parseLatestVersionData() { QHash campaignInstallers; while (xml.readNextStartElement()) { - if (xml.name() == "version") { - version = xml.readElementText().toInt(); + if (xml.name() == versionKey) { + version = xml.readElementText(); } else if (xml.name() == "url") { downloadUrl = xml.readElementText(); } else if (xml.name() == "installers") { @@ -159,31 +185,31 @@ void AutoUpdater::parseLatestVersionData() { } void AutoUpdater::checkVersionAndNotify() { - if (BuildInfo::BUILD_TYPE != BuildInfo::BuildType::Stable || _builds.empty()) { - // No version checking is required in nightly/PR/dev builds or when no build - // data was found for the platform + if (_builds.empty()) { + // no build data was found for this platform return; } - int latestVersionAvailable = _builds.lastKey(); - if (QCoreApplication::applicationVersion().toInt() < latestVersionAvailable) { + + qDebug() << "Checking if update version" << _builds.lastKey().versionString + << "is newer than current version" << _currentVersion.versionString; + + if (_builds.lastKey() > _currentVersion) { emit newVersionIsAvailable(); } } -void AutoUpdater::performAutoUpdate(int version) { - // NOTE: This is not yet auto updating - however this is a checkpoint towards that end - // Next PR will handle the automatic download, upgrading and application restart - const QMap& chosenVersion = _builds.value(version); +void AutoUpdater::openLatestUpdateURL() { + const QMap& chosenVersion = _builds.last(); const QUrl& downloadUrl = chosenVersion.value("downloadUrl"); QDesktopServices::openUrl(downloadUrl); QCoreApplication::quit(); } -void AutoUpdater::downloadUpdateVersion(int version) { +void AutoUpdater::downloadUpdateVersion(const QString& version) { emit newVersionIsDownloaded(); } -void AutoUpdater::appendBuildData(int versionNumber, +void AutoUpdater::appendBuildData(const QString& versionNumber, const QString& downloadURL, const QString& releaseTime, const QString& releaseNotes, @@ -194,6 +220,6 @@ void AutoUpdater::appendBuildData(int versionNumber, thisBuildDetails.insert("releaseTime", releaseTime); thisBuildDetails.insert("releaseNotes", releaseNotes); thisBuildDetails.insert("pullRequestNumber", pullRequestNumber); - _builds.insert(versionNumber, thisBuildDetails); + _builds.insert(ApplicationVersion(versionNumber), thisBuildDetails); } diff --git a/libraries/auto-updater/src/AutoUpdater.h b/libraries/auto-updater/src/AutoUpdater.h index f56d7993e9..c788ac31d1 100644 --- a/libraries/auto-updater/src/AutoUpdater.h +++ b/libraries/auto-updater/src/AutoUpdater.h @@ -26,10 +26,9 @@ #include #include +#include #include -const QUrl BUILDS_XML_URL("https://highfidelity.com/builds.xml"); - class AutoUpdater : public QObject, public Dependency { Q_OBJECT SINGLETON_DEPENDENCY @@ -43,25 +42,29 @@ public: }; void checkForUpdate(); - const QMap>& getBuildData() { return _builds; } - void performAutoUpdate(int version); + const QMap>& getBuildData() { return _builds; } + void openLatestUpdateURL(); void setInstallerType(InstallerType type) { _installerType = type; } void setInstallerCampaign(QString campaign) { _installerCampaign = campaign; } + const ApplicationVersion& getCurrentVersion() const { return _currentVersion; } + signals: void latestVersionDataParsed(); void newVersionIsAvailable(); void newVersionIsDownloaded(); private: - QMap> _builds; + QMap> _builds; QString _operatingSystem; InstallerType _installerType { InstallerType::FULL }; QString _installerCampaign { "" }; + + ApplicationVersion _currentVersion; void getLatestVersionData(); - void downloadUpdateVersion(int version); - void appendBuildData(int versionNumber, + void downloadUpdateVersion(const QString& version); + void appendBuildData(const QString& versionNumber, const QString& downloadURL, const QString& releaseTime, const QString& releaseNotes, diff --git a/libraries/shared/src/ApplicationVersion.cpp b/libraries/shared/src/ApplicationVersion.cpp new file mode 100644 index 0000000000..5c2d5ad11c --- /dev/null +++ b/libraries/shared/src/ApplicationVersion.cpp @@ -0,0 +1,94 @@ +// +// ApplicationVersion.cpp +// libraries/shared/src +// +// Created by Stephen Birarda on 6/8/18. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "ApplicationVersion.h" + +#include + +#include +#include +#include + +ApplicationVersion::ApplicationVersion(const QString& versionString) : + versionString(versionString) +{ + // attempt to regex out a semantic version from the string + // handling both x.y.z and x.y formats + QRegExp semanticRegex("([\\d]+)\\.([\\d]+)(?:\\.([\\d]+))?"); + + int pos = semanticRegex.indexIn(versionString); + if (pos != -1) { + isSemantic = true; + auto captures = semanticRegex.capturedTexts(); + + major = captures[1].toInt(); + minor = captures[2].toInt(); + + if (captures.length() > 3) { + patch = captures[3].toInt(); + } else { + // the patch is implictly 0 if it was not included + patch = 0; + } + } else { + // if we didn't have a sematic style, we assume that we just have a build number + build = versionString.toInt(); + } +} + +bool ApplicationVersion::operator==(const ApplicationVersion& other) const { + if (isSemantic && other.isSemantic) { + return major == other.major && minor == other.minor && patch == other.patch; + } else if (!isSemantic && !other.isSemantic) { + return build == other.build; + } else { + assert(isSemantic == other.isSemantic); + return false; + } +} + +bool ApplicationVersion::operator<(const ApplicationVersion& other) const { + if (isSemantic && other.isSemantic) { + if (major == other.major) { + if (minor == other.minor) { + return patch < other.patch; + } else { + return minor < other.minor; + } + } else { + return major < other.major; + } + } else if (!isSemantic && !other.isSemantic) { + return build < other.build; + } else { + assert(isSemantic == other.isSemantic); + return false; + } +} + +bool ApplicationVersion::operator>(const ApplicationVersion& other) const { + if (isSemantic && other.isSemantic) { + if (major == other.major) { + if (minor == other.minor) { + return patch > other.patch; + } else { + return minor > other.minor; + } + } else { + return major > other.major; + } + } else if (!isSemantic && !other.isSemantic) { + return build > other.build; + } else { + assert(isSemantic == other.isSemantic); + return false; + } +} diff --git a/libraries/shared/src/ApplicationVersion.h b/libraries/shared/src/ApplicationVersion.h new file mode 100644 index 0000000000..5cb0a09a8d --- /dev/null +++ b/libraries/shared/src/ApplicationVersion.h @@ -0,0 +1,41 @@ +// +// ApplicationVersion.h +// libraries/shared/src +// +// Created by Stephen Birarda on 6/8/18. +// Copyright 2018 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_ApplicationVersion_h +#define hifi_ApplicationVersion_h + +#include + +class ApplicationVersion { +public: + ApplicationVersion(const QString& versionString); + + bool operator==(const ApplicationVersion& other) const; + bool operator!=(const ApplicationVersion& other) const { return !(*this == other); } + + bool operator <(const ApplicationVersion& other) const; + bool operator >(const ApplicationVersion& other) const; + + bool operator >=(const ApplicationVersion& other) const { return (*this == other) || (*this > other); } + bool operator <=(const ApplicationVersion& other) const { return (*this == other) || (*this < other); } + + int major = -1; + int minor = -1; + int patch = -1; + + int build = -1; + + bool isSemantic { false }; + + QString versionString; +}; + +#endif // hifi_ApplicationVersion_h From 8dd268addbf4777a53f1d5a525d0437d9c1cfff3 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 8 Jun 2018 17:13:04 -0700 Subject: [PATCH 5/9] handle semantic version updates in sandbox --- cmake/templates/console-build-info.json.in | 4 +- server-console/src/main.js | 55 +++++++++-------- server-console/src/modules/hf-updater.js | 70 ++++++++++++++++++++-- 3 files changed, 94 insertions(+), 35 deletions(-) diff --git a/cmake/templates/console-build-info.json.in b/cmake/templates/console-build-info.json.in index 6b4ee99292..e8cd8eee22 100644 --- a/cmake/templates/console-build-info.json.in +++ b/cmake/templates/console-build-info.json.in @@ -1,5 +1,7 @@ { "releaseType": "@RELEASE_TYPE@", + "buildNumber": "@BUILD_NUMBER@", + "stableBuild": "@STABLE_BUILD@", "buildIdentifier": "@BUILD_VERSION@", - "organization": "@BUILD_ORGANIZATION@" + "organization": "@BUILD_ORGANIZATION@" } diff --git a/server-console/src/main.js b/server-console/src/main.js index 8a92fc8a5d..f9c72ddf8a 100644 --- a/server-console/src/main.js +++ b/server-console/src/main.js @@ -60,7 +60,14 @@ function getBuildInfo() { } } - const DEFAULT_BUILD_INFO = { releaseType: "", buildIdentifier: "dev" }; + const DEFAULT_BUILD_INFO = { + releaseType: "", + buildIdentifier: "dev", + buildNumber: "0", + stableBuild: "0", + organization: "High Fidelity - dev" + }; + var buildInfo = DEFAULT_BUILD_INFO; if (buildInfoPath) { @@ -858,33 +865,25 @@ function onContentLoaded() { // maybeShowSplash(); if (buildInfo.releaseType == 'PRODUCTION' && !argv.noUpdater) { - var currentVersion = null; - try { - currentVersion = parseInt(buildInfo.buildIdentifier); - } catch (e) { - } - - if (currentVersion !== null) { - const CHECK_FOR_UPDATES_INTERVAL_SECONDS = 60 * 30; - var hasShownUpdateNotification = false; - const updateChecker = new updater.UpdateChecker(currentVersion, CHECK_FOR_UPDATES_INTERVAL_SECONDS); - updateChecker.on('update-available', function(latestVersion, url) { - if (!hasShownUpdateNotification) { - notifier.notify({ - icon: notificationIcon, - title: 'An update is available!', - message: 'High Fidelity version ' + latestVersion + ' is available', - wait: true, - url: url - }); - hasShownUpdateNotification = true; - } - }); - notifier.on('click', function(notifierObject, options) { - log.debug("Got click", options.url); - shell.openExternal(options.url); - }); - } + const CHECK_FOR_UPDATES_INTERVAL_SECONDS = 60 * 30; + var hasShownUpdateNotification = false; + const updateChecker = new updater.UpdateChecker(buildInfo, CHECK_FOR_UPDATES_INTERVAL_SECONDS); + updateChecker.on('update-available', function(latestVersion, url) { + if (!hasShownUpdateNotification) { + notifier.notify({ + icon: notificationIcon, + title: 'An update is available!', + message: 'High Fidelity version ' + latestVersion + ' is available', + wait: true, + url: url + }); + hasShownUpdateNotification = true; + } + }); + notifier.on('click', function(notifierObject, options) { + log.debug("Got click", options.url); + shell.openExternal(options.url); + }); } deleteOldFiles(logPath, DELETE_LOG_FILES_OLDER_THAN_X_SECONDS, LOG_FILE_REGEX); diff --git a/server-console/src/modules/hf-updater.js b/server-console/src/modules/hf-updater.js index 489364f655..d20da3c663 100644 --- a/server-console/src/modules/hf-updater.js +++ b/server-console/src/modules/hf-updater.js @@ -8,10 +8,48 @@ const os = require('os'); const platform = os.type() == 'Windows_NT' ? 'windows' : 'mac'; const BUILDS_URL = 'https://highfidelity.com/builds.xml'; +const DEV_BUILDS_URL = 'https://highfidelity.com/dev-builds.xml'; -function UpdateChecker(currentVersion, checkForUpdatesEveryXSeconds) { - this.currentVersion = currentVersion; - log.debug('cur', currentVersion); +// returns 1 if A is greater, 0 if equal, -1 if A is lesser +function semanticVersionCompare(versionA, versionB) { + var versionAParts = versionA.split('.'); + var versionBParts = versionB.split('.'); + + // make sure each version has 3 parts + var partsLength = versionAParts.length; + while (partsLength < 3) { + partsLength = versionAParts.push(0); + } + + partsLength = versionBParts.length; + while (partsLength < 3) { + partsLength = versionBParts.push(0); + } + + // map all of the parts to numbers + versionAParts = versionAParts.map(Number); + versionBParts = versionBParts.map(Number); + + for (var i = 0; i < 3; ++i) { + if (versionAParts[i] == versionBParts[i]) { + continue; + } else if (versionAParts[i] > versionBParts[i]) { + return 1; + } else { + return -1; + } + } + + return 0; +} + +function UpdateChecker(buildInfo, checkForUpdatesEveryXSeconds) { + this.stableBuild = (buildInfo.stableBuild == "1"); + + this.buildsURL = this.stableBuild ? BUILDS_URL : DEV_BUILDS_URL; + this.currentVersion = this.stableBuild ? buildInfo.buildIdentifier : parseInt(buildInfo.buildNumber); + + log.debug('Current version is', this.currentVersion); setInterval(this.checkForUpdates.bind(this), checkForUpdatesEveryXSeconds * 1000); this.checkForUpdates(); @@ -29,12 +67,32 @@ UpdateChecker.prototype = extend(UpdateChecker.prototype, { try { var $ = cheerio.load(body, { xmlMode: true }); const latestBuild = $('project[name="interface"] platform[name="' + platform + '"]').children().first(); - const latestVersion = parseInt(latestBuild.find('version').text()); - log.debug("Latest version is:", latestVersion, this.currentVersion); - if (latestVersion > this.currentVersion) { + + var latestVersion = 0; + + if (this.stableBuild) { + latestVersion = latestBuild.find('stable_version').text(); + } else { + latestVersion = parseInt(latestBuild.find('version').text()); + } + + log.debug("Latest available update version is:", latestVersion); + + updateAvailable = false; + + if (this.stableBuild) { + // compare the semantic versions to see if the update is newer + updateAvailable = (semanticVersionCompare(latestVersion, this.currentVersion) == 1); + } else { + // for master builds we just compare the versions as integers + updateAvailable = latestVersion > this.currentVersion; + } + + if (updateAvailable) { const url = latestBuild.find('url').text(); this.emit('update-available', latestVersion, url); } + } catch (e) { log.warn("Error when checking for updates", e); } From 7a9d77d0d9fa032c20c62d5b647418ff9278a7a2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 8 Jun 2018 17:15:27 -0700 Subject: [PATCH 6/9] force git abbreviated SHA to have length of 7 --- cmake/macros/SetPackagingParameters.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/macros/SetPackagingParameters.cmake b/cmake/macros/SetPackagingParameters.cmake index 029c829022..ef96e989d8 100644 --- a/cmake/macros/SetPackagingParameters.cmake +++ b/cmake/macros/SetPackagingParameters.cmake @@ -95,7 +95,7 @@ macro(SET_PACKAGING_PARAMETERS) endif () execute_process( - COMMAND git log -1 --format=${_GIT_LOG_FORMAT} + COMMAND git log -1 --abbrev=7 --format=${_GIT_LOG_FORMAT} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE _GIT_LOG_OUTPUT ERROR_VARIABLE _GIT_LOG_ERROR From 015c092e1c08d7d15a342bae3fb6855e17a9cc7e Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 11 Jun 2018 16:37:40 -0700 Subject: [PATCH 7/9] Update oven to only produce uncompressed ktx for cubemaps --- libraries/baking/src/TextureBaker.cpp | 8 ++++++-- libraries/baking/src/TextureBaker.h | 4 ++++ tools/oven/src/OvenCLIApplication.cpp | 6 ++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/libraries/baking/src/TextureBaker.cpp b/libraries/baking/src/TextureBaker.cpp index 4f42ad627c..ecfe724441 100644 --- a/libraries/baking/src/TextureBaker.cpp +++ b/libraries/baking/src/TextureBaker.cpp @@ -30,6 +30,8 @@ const QString BAKED_TEXTURE_KTX_EXT = ".ktx"; const QString BAKED_TEXTURE_BCN_SUFFIX = "_bcn.ktx"; const QString BAKED_META_TEXTURE_SUFFIX = ".texmeta.json"; +bool TextureBaker::_compressionEnabled = true; + TextureBaker::TextureBaker(const QUrl& textureURL, image::TextureUsage::Type textureType, const QDir& outputDirectory, const QString& metaTexturePathPrefix, const QString& baseFilename, const QByteArray& textureContent) : @@ -146,7 +148,7 @@ void TextureBaker::processTexture() { } // Compressed KTX - { + if (_compressionEnabled) { auto processedTexture = image::processImage(buffer, _textureURL.toString().toStdString(), ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType, true, _abortProcessing); if (!processedTexture) { @@ -186,7 +188,7 @@ void TextureBaker::processTexture() { } // Uncompressed KTX - { + if (_textureType == image::TextureUsage::Type::CUBE_TEXTURE) { buffer->reset(); auto processedTexture = image::processImage(std::move(buffer), _textureURL.toString().toStdString(), ABSOLUTE_MAX_TEXTURE_NUM_PIXELS, _textureType, false, _abortProcessing); @@ -218,6 +220,8 @@ void TextureBaker::processTexture() { } _outputFiles.push_back(filePath); meta.uncompressed = _metaTexturePathPrefix + fileName; + } else { + buffer.reset(); } { diff --git a/libraries/baking/src/TextureBaker.h b/libraries/baking/src/TextureBaker.h index 54839c001a..c8c4fb73b8 100644 --- a/libraries/baking/src/TextureBaker.h +++ b/libraries/baking/src/TextureBaker.h @@ -41,6 +41,8 @@ public: virtual void setWasAborted(bool wasAborted) override; + static void setCompressionEnabled(bool enabled) { _compressionEnabled = enabled; } + public slots: virtual void bake() override; virtual void abort() override; @@ -65,6 +67,8 @@ private: QString _metaTexturePathPrefix; std::atomic _abortProcessing { false }; + + static bool _compressionEnabled; }; #endif // hifi_TextureBaker_h diff --git a/tools/oven/src/OvenCLIApplication.cpp b/tools/oven/src/OvenCLIApplication.cpp index 6f87359134..c405c5f4a0 100644 --- a/tools/oven/src/OvenCLIApplication.cpp +++ b/tools/oven/src/OvenCLIApplication.cpp @@ -15,6 +15,7 @@ #include #include +#include #include "BakerCLI.h" @@ -47,10 +48,7 @@ OvenCLIApplication::OvenCLIApplication(int argc, char* argv[]) : if (parser.isSet(CLI_DISABLE_TEXTURE_COMPRESSION_PARAMETER)) { qDebug() << "Disabling texture compression"; - image::setColorTexturesCompressionEnabled(false); - image::setGrayscaleTexturesCompressionEnabled(false); - image::setNormalTexturesCompressionEnabled(false); - image::setCubeTexturesCompressionEnabled(false); + TextureBaker::setCompressionEnabled(false); } QMetaObject::invokeMethod(cli, "bakeFile", Qt::QueuedConnection, Q_ARG(QUrl, inputUrl), From 4713d8d44ca7deb534eea0c59fa6e15ab681946d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 12 Jun 2018 16:06:17 -0700 Subject: [PATCH 8/9] fix reference to builds URL for sandbox check --- server-console/src/modules/hf-updater.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-console/src/modules/hf-updater.js b/server-console/src/modules/hf-updater.js index d20da3c663..8362174c5d 100644 --- a/server-console/src/modules/hf-updater.js +++ b/server-console/src/modules/hf-updater.js @@ -58,7 +58,7 @@ util.inherits(UpdateChecker, events.EventEmitter); UpdateChecker.prototype = extend(UpdateChecker.prototype, { checkForUpdates: function() { log.debug("Checking for updates"); - request(BUILDS_URL, (error, response, body) => { + request(this.buildsURL, (error, response, body) => { if (error) { log.debug("Error", error); return; From 395767ed80ae15ce4bc27080b356d4e3da42f254 Mon Sep 17 00:00:00 2001 From: Gabriel Calero Date: Tue, 12 Jun 2018 20:11:22 -0300 Subject: [PATCH 9/9] Set android versionCode and versionName through gradle parameter RELEASE_NUMBER --- android/app/build.gradle | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index 699008092c..f780abdea0 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -4,12 +4,15 @@ android { compileSdkVersion 26 //buildToolsVersion '27.0.3' + def appVersionCode = Integer.valueOf(RELEASE_NUMBER ?: 1) + def appVersionName = RELEASE_NUMBER ?: "1.0" + defaultConfig { applicationId "io.highfidelity.hifiinterface" minSdkVersion 24 targetSdkVersion 26 - versionCode 1 - versionName "1.0" + versionCode appVersionCode + versionName appVersionName ndk { abiFilters 'arm64-v8a' } externalNativeBuild { cmake {