Adding a validation step at runtime for the cached KTX file in order to regenerate them if anything seems wrong

This commit is contained in:
samcake 2017-05-12 18:04:22 -07:00
parent 9d4411e3f0
commit bdb0414add
10 changed files with 56 additions and 9 deletions

View file

@ -441,7 +441,10 @@ void Texture::assignStoredMip(uint16 level, storage::StoragePointer& storage) {
// THen check that the mem texture passed make sense with its format // THen check that the mem texture passed make sense with its format
Size expectedSize = evalStoredMipSize(level, getStoredMipFormat()); Size expectedSize = evalStoredMipSize(level, getStoredMipFormat());
auto size = storage->size(); auto size = storage->size();
if (storage->size() <= expectedSize) { if (storage->size() < expectedSize) {
_storage->assignMipData(level, storage);
_stamp++;
} else if (size == expectedSize) {
_storage->assignMipData(level, storage); _storage->assignMipData(level, storage);
_stamp++; _stamp++;
} else if (size > expectedSize) { } else if (size > expectedSize) {
@ -469,6 +472,8 @@ void Texture::assignStoredMipFace(uint16 level, uint8 face, storage::StoragePoin
Size expectedSize = evalStoredMipFaceSize(level, getStoredMipFormat()); Size expectedSize = evalStoredMipFaceSize(level, getStoredMipFormat());
auto size = storage->size(); auto size = storage->size();
if (size <= expectedSize) { if (size <= expectedSize) {
_stamp++;
} else if (size == expectedSize) {
_storage->assignMipFaceData(level, face, storage); _storage->assignMipFaceData(level, face, storage);
_stamp++; _stamp++;
} else if (size > expectedSize) { } else if (size > expectedSize) {

View file

@ -542,6 +542,13 @@ bool Texture::evalTextureFormat(const ktx::Header& header, Element& mipFormat, E
} else { } else {
return false; return false;
} }
} else if (header.getGLFormat() == ktx::GLFormat::RG && header.getGLType() == ktx::GLType::UNSIGNED_BYTE && header.getTypeSize() == 1) {
mipFormat = Format::VEC2NU8_XY;
if (header.getGLInternaFormat_Uncompressed() == ktx::GLInternalFormat_Uncompressed::RG8) {
texelFormat = Format::VEC2NU8_XY;
} else {
return false;
}
} else if (header.getGLFormat() == ktx::GLFormat::COMPRESSED_FORMAT && header.getGLType() == ktx::GLType::COMPRESSED_TYPE) { } else if (header.getGLFormat() == ktx::GLFormat::COMPRESSED_FORMAT && header.getGLType() == ktx::GLType::COMPRESSED_TYPE) {
if (header.getGLInternaFormat_Compressed() == ktx::GLInternalFormat_Compressed::COMPRESSED_SRGB_S3TC_DXT1_EXT) { if (header.getGLInternaFormat_Compressed() == ktx::GLInternalFormat_Compressed::COMPRESSED_SRGB_S3TC_DXT1_EXT) {
mipFormat = Format::COLOR_COMPRESSED_SRGB; mipFormat = Format::COLOR_COMPRESSED_SRGB;

View file

@ -114,6 +114,9 @@ size_t Header::evalFaceSize(uint32_t level) const {
} }
size_t Header::evalImageSize(uint32_t level) const { size_t Header::evalImageSize(uint32_t level) const {
auto faceSize = evalFaceSize(level); auto faceSize = evalFaceSize(level);
if ((faceSize < 4) || ((faceSize & 0x3) != 0)) {
return 0;
}
if (numberOfFaces == NUM_CUBEMAPFACES && numberOfArrayElements == 0) { if (numberOfFaces == NUM_CUBEMAPFACES && numberOfArrayElements == 0) {
return faceSize; return faceSize;
} else { } else {
@ -139,6 +142,9 @@ ImageDescriptors Header::generateImageDescriptors() const {
size_t imageOffset = 0; size_t imageOffset = 0;
for (uint32_t level = 0; level < numberOfMipmapLevels; ++level) { for (uint32_t level = 0; level < numberOfMipmapLevels; ++level) {
auto imageSize = static_cast<uint32_t>(evalImageSize(level)); auto imageSize = static_cast<uint32_t>(evalImageSize(level));
if ((imageSize < 4) || ((imageSize & 0x3) != 0)) {
return ImageDescriptors();
}
if (imageSize == 0) { if (imageSize == 0) {
return ImageDescriptors(); return ImageDescriptors();
} }

View file

@ -148,12 +148,24 @@ namespace ktx {
size_t imageSize = *reinterpret_cast<const uint32_t*>(currentPtr); size_t imageSize = *reinterpret_cast<const uint32_t*>(currentPtr);
currentPtr += sizeof(uint32_t); currentPtr += sizeof(uint32_t);
auto expectedImageSize = header.evalImageSize(images.size());
if (imageSize != expectedImageSize) {
break;
} else if ((imageSize < 4) || (imageSize & 0x3)) {
break;
}
// The image size is the face size, beware!
size_t faceSize = imageSize;
if (numFaces == NUM_CUBEMAPFACES) {
imageSize = NUM_CUBEMAPFACES * faceSize;
}
// If enough data ahead then capture the pointer // If enough data ahead then capture the pointer
if ((currentPtr - srcBytes) + imageSize <= (srcSize)) { if ((currentPtr - srcBytes) + imageSize <= (srcSize)) {
auto padding = Header::evalPadding(imageSize); auto padding = Header::evalPadding(imageSize);
if (numFaces == NUM_CUBEMAPFACES) { if (numFaces == NUM_CUBEMAPFACES) {
size_t faceSize = imageSize / NUM_CUBEMAPFACES;
Image::FaceBytes faces(NUM_CUBEMAPFACES); Image::FaceBytes faces(NUM_CUBEMAPFACES);
for (uint32_t face = 0; face < NUM_CUBEMAPFACES; face++) { for (uint32_t face = 0; face < NUM_CUBEMAPFACES; face++) {
faces[face] = currentPtr; faces[face] = currentPtr;
@ -166,6 +178,7 @@ namespace ktx {
currentPtr += imageSize + padding; currentPtr += imageSize + padding;
} }
} else { } else {
// Stop here
break; break;
} }
} }
@ -190,6 +203,10 @@ namespace ktx {
// populate image table // populate image table
result->_images = parseImages(result->getHeader(), result->getTexelsDataSize(), result->getTexelsData()); result->_images = parseImages(result->getHeader(), result->getTexelsDataSize(), result->getTexelsData());
if (result->_images.size() != result->getHeader().getNumberOfLevels()) {
// Fail if the number of images produced doesn't match the header number of levels
return nullptr;
}
return result; return result;
} }

View file

@ -210,7 +210,8 @@ namespace ktx {
if (currentDataSize + sizeof(uint32_t) < allocatedImagesDataSize) { if (currentDataSize + sizeof(uint32_t) < allocatedImagesDataSize) {
uint32_t imageOffset = currentPtr - destBytes; uint32_t imageOffset = currentPtr - destBytes;
size_t imageSize = srcImages[l]._imageSize; size_t imageSize = srcImages[l]._imageSize;
*(reinterpret_cast<uint32_t*> (currentPtr)) = (uint32_t) imageSize; size_t imageFaceSize = srcImages[l]._faceSize;
*(reinterpret_cast<uint32_t*> (currentPtr)) = (uint32_t)imageFaceSize; // the imageSize written in the ktx is the FACE size
currentPtr += sizeof(uint32_t); currentPtr += sizeof(uint32_t);
currentDataSize += sizeof(uint32_t); currentDataSize += sizeof(uint32_t);

View file

@ -22,7 +22,7 @@ KTXCache::KTXCache(const std::string& dir, const std::string& ext) :
} }
KTXFilePointer KTXCache::writeFile(const char* data, Metadata&& metadata) { KTXFilePointer KTXCache::writeFile(const char* data, Metadata&& metadata) {
FilePointer file = FileCache::writeFile(data, std::move(metadata)); FilePointer file = FileCache::writeFile(data, std::move(metadata), true);
return std::static_pointer_cast<KTXFile>(file); return std::static_pointer_cast<KTXFile>(file);
} }

View file

@ -792,6 +792,8 @@ void ImageReader::read() {
texture = gpu::Texture::unserialize(ktxFile->getFilepath()); texture = gpu::Texture::unserialize(ktxFile->getFilepath());
if (texture) { if (texture) {
texture = textureCache->cacheTextureByHash(hash, texture); texture = textureCache->cacheTextureByHash(hash, texture);
} else {
qCWarning(modelnetworking) << "Invalid cached KTX " << _url << " under hash " << hash.c_str() << ", recreating...";
} }
} }
} }
@ -835,7 +837,7 @@ void ImageReader::read() {
const char* data = reinterpret_cast<const char*>(memKtx->_storage->data()); const char* data = reinterpret_cast<const char*>(memKtx->_storage->data());
size_t length = memKtx->_storage->size(); size_t length = memKtx->_storage->size();
auto& ktxCache = textureCache->_ktxCache; auto& ktxCache = textureCache->_ktxCache;
networkTexture->_file = ktxCache.writeFile(data, KTXCache::Metadata(hash, length)); networkTexture->_file = ktxCache.writeFile(data, KTXCache::Metadata(hash, length)); //
if (!networkTexture->_file) { if (!networkTexture->_file) {
qCWarning(modelnetworking) << _url << "file cache failed"; qCWarning(modelnetworking) << _url << "file cache failed";
} else { } else {

View file

@ -97,7 +97,7 @@ FilePointer FileCache::addFile(Metadata&& metadata, const std::string& filepath)
return file; return file;
} }
FilePointer FileCache::writeFile(const char* data, File::Metadata&& metadata) { FilePointer FileCache::writeFile(const char* data, File::Metadata&& metadata, bool overwrite) {
assert(_initialized); assert(_initialized);
std::string filepath = getFilepath(metadata.key); std::string filepath = getFilepath(metadata.key);
@ -107,8 +107,13 @@ FilePointer FileCache::writeFile(const char* data, File::Metadata&& metadata) {
// if file already exists, return it // if file already exists, return it
FilePointer file = getFile(metadata.key); FilePointer file = getFile(metadata.key);
if (file) { if (file) {
qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), metadata.key.c_str()); if (!overwrite) {
return file; qCWarning(file_cache, "[%s] Attempted to overwrite %s", _dirname.c_str(), metadata.key.c_str());
return file;
} else {
qCWarning(file_cache, "[%s] Overwriting %s", _dirname.c_str(), metadata.key.c_str());
file.reset();
}
} }
QSaveFile saveFile(QString::fromStdString(filepath)); QSaveFile saveFile(QString::fromStdString(filepath));

View file

@ -80,7 +80,7 @@ protected:
/// must be called after construction to create the cache on the fs and restore persisted files /// must be called after construction to create the cache on the fs and restore persisted files
void initialize(); void initialize();
FilePointer writeFile(const char* data, Metadata&& metadata); FilePointer writeFile(const char* data, Metadata&& metadata, bool overwrite = false);
FilePointer getFile(const Key& key); FilePointer getFile(const Key& key);
/// create a file /// create a file

View file

@ -21,6 +21,10 @@ ViewStorage::ViewStorage(const storage::StoragePointer& owner, size_t size, cons
StoragePointer Storage::createView(size_t viewSize, size_t offset) const { StoragePointer Storage::createView(size_t viewSize, size_t offset) const {
auto selfSize = size(); auto selfSize = size();
if ((viewSize < 4) || ((viewSize & 0x3) != 0)) {
throw std::runtime_error("Invalid mapping range");
}
if (0 == viewSize) { if (0 == viewSize) {
viewSize = selfSize; viewSize = selfSize;
} }