From 1c3259c8cf8a8de8421a3f2e8ce92f8b80bf3d5c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 10 Jan 2014 15:33:43 -0800 Subject: [PATCH] remove the starfield URL loading code, closes #1497 --- assignment-client/CMakeLists.txt | 5 - interface/src/starfield/Config.h | 1 - interface/src/starfield/Loader.h | 183 ----------------- libraries/shared/CMakeLists.txt | 4 - libraries/shared/src/UrlReader.cpp | 172 ---------------- libraries/shared/src/UrlReader.h | 305 ----------------------------- 6 files changed, 670 deletions(-) delete mode 100644 interface/src/starfield/Loader.h delete mode 100644 libraries/shared/src/UrlReader.cpp delete mode 100644 libraries/shared/src/UrlReader.h diff --git a/assignment-client/CMakeLists.txt b/assignment-client/CMakeLists.txt index c7b8ab5732..418523993b 100644 --- a/assignment-client/CMakeLists.txt +++ b/assignment-client/CMakeLists.txt @@ -39,8 +39,3 @@ include_directories(${ROOT_DIR}/externals/civetweb/include) if (UNIX) target_link_libraries(${TARGET_NAME} ${CMAKE_DL_LIBS}) endif (UNIX) - -# link curl for synchronous script downloads -find_package(CURL REQUIRED) -include_directories(${CURL_INCLUDE_DIRS}) -target_link_libraries(${TARGET_NAME} ${CURL_LIBRARY}) diff --git a/interface/src/starfield/Config.h b/interface/src/starfield/Config.h index d2d93be407..564f6f3a1a 100755 --- a/interface/src/starfield/Config.h +++ b/interface/src/starfield/Config.h @@ -36,7 +36,6 @@ #include #include -#include "UrlReader.h" #include "AngleUtil.h" #include "Radix2InplaceSort.h" #include "Radix2IntegerScanner.h" diff --git a/interface/src/starfield/Loader.h b/interface/src/starfield/Loader.h deleted file mode 100644 index e2f6105f33..0000000000 --- a/interface/src/starfield/Loader.h +++ /dev/null @@ -1,183 +0,0 @@ -// -// starfield/Loader.h -// interface -// -// Created by Tobias Schwinger on 3/29/13. -// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. -// - -#ifndef __interface__starfield__Loader__ -#define __interface__starfield__Loader__ - -#ifndef __interface__Starfield_impl__ -#error "This is an implementation file - not intended for direct inclusion." -#endif - -#include - -#include "Config.h" - -#include "starfield/data/InputVertex.h" -#include "starfield/data/BrightnessLevel.h" - -namespace starfield { - - class Loader : UrlReader { - public: - - bool loadVertices( - InputVertices& destination, char const* url, char const* cacheFile, unsigned limit) - { - _vertices = & destination; - _limit = limit; -#if STARFIELD_SAVE_MEMORY - if (_limit == 0 || _limit > 60000u) - _limit = 60000u; -#endif - _urlStr = url; // in case we fail early - - if (! UrlReader::readUrl(url, *this, cacheFile)) - { - qDebug("%s:%d: %s\n", - _urlStr, _lineNo, getError()); - - return false; - } - qDebug("Loaded %u stars.\n", _recordsRead); - - return true; - } - - protected: - friend class UrlReader; - - void begin(char const* url, - char const* type, - int64_t size, - int64_t stardate) { - - _lineNo = 0u; - _urlStr = url; // new value in http redirect - - _recordsRead = 0u; - - _vertices->clear(); - _vertices->reserve(_limit); -// qDebug("Stars.cpp: loader begin %s\n", url); - } - - size_t transfer(char* input, size_t bytes) { - - size_t consumed = 0u; - char const* end = input + bytes; - char* line, * next = input; - - for (;;) { - - // advance to next line - for (; next != end && isspace(*next); ++next); - consumed = next - input; - line = next; - ++_lineNo; - for (; next != end && *next != '\n' && *next != '\r'; ++next); - if (next == end) - return consumed; - *next++ = '\0'; - - // skip comments - if (*line == '\\' || *line == '/' || *line == ';') - continue; - - // parse - float azi, alt; - unsigned c; - setlocale(LC_NUMERIC, "C"); - if (sscanf(line, " %f %f #%x", & azi, & alt, & c) == 3) { - - if (spaceFor( getBrightness(c) )) { - - storeVertex(azi, alt, c); - } - - ++_recordsRead; - - } else { - - qDebug("Stars.cpp:%d: Bad input from %s\n", - _lineNo, _urlStr); - } - - } - return consumed; - } - - void end(bool ok) - { } - - private: - - bool atLimit() { return _limit > 0u && _recordsRead >= _limit; } - - bool spaceFor(BrightnessLevel b) { - - if (! atLimit()) { - return true; - } - - // just reached the limit? -> establish a minimum heap and - // remember the brightness at its top - if (_recordsRead == _limit) { - -// qDebug("Stars.cpp: vertex limit reached -> heap mode\n"); - - make_heap( - _vertices->begin(), _vertices->end(), - GreaterBrightness() ); - - _minBrightness = getBrightness( - _vertices->begin()->getColor() ); - } - - // not interested? say so - if (_minBrightness >= b) - return false; - - // otherwise free up space for the new vertex - pop_heap( - _vertices->begin(), _vertices->end(), - GreaterBrightness() ); - _vertices->pop_back(); - return true; - } - - void storeVertex(float azi, float alt, unsigned color) { - - _vertices->push_back(InputVertex(azi, alt, color)); - - if (atLimit()) { - - push_heap( - _vertices->begin(), _vertices->end(), - GreaterBrightness() ); - - _minBrightness = getBrightness( - _vertices->begin()->getColor() ); - } - } - - // variables - - InputVertices* _vertices; - unsigned _limit; - - unsigned _lineNo; - char const* _urlStr; - - unsigned _recordsRead; - BrightnessLevel _minBrightness; - }; - -} // anonymous namespace - -#endif - diff --git a/libraries/shared/CMakeLists.txt b/libraries/shared/CMakeLists.txt index 1923d906bb..c1fcf2e553 100644 --- a/libraries/shared/CMakeLists.txt +++ b/libraries/shared/CMakeLists.txt @@ -24,10 +24,6 @@ set(EXTERNAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external) if (WIN32) # include headers for external libraries and InterfaceConfig. include_directories(${EXTERNAL_ROOT_DIR}) -else (WIN32) - find_package(CURL REQUIRED) - include_directories(${CURL_INCLUDE_DIRS}) - target_link_libraries(${TARGET_NAME} ${CURL_LIBRARY}) endif (WIN32) # link required libraries on UNIX diff --git a/libraries/shared/src/UrlReader.cpp b/libraries/shared/src/UrlReader.cpp deleted file mode 100644 index 3f98326726..0000000000 --- a/libraries/shared/src/UrlReader.cpp +++ /dev/null @@ -1,172 +0,0 @@ -// -// UrlReader.cpp -// hifi -// -// Created by Tobias Schwinger on 3/21/13. -// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. -// - -#include - -#include -#include - -#ifndef _WIN32 -// (Windows port is incomplete and the build files do not support CURL, yet) - -#include - -#include "UrlReader.h" - -// -// ATTENTION: A certain part of the implementation lives in inlined code -// (see the bottom of the header file). -// -// Why? Because it allows stream parsing without having to call around a -// lot (one static and one dynamic call per character if the parser just -// reads one character at a time). -// -// Here is an overview of the code structure: -// -// readUrl -// -> transferBegin (sets up state) -// -> perform (starts CURL transfer) -// -> (specialized, type-erased) callback_template -// -> getInfo (fetches HTTP header, eventually initiates caching) -// -> stream.begin (client code - called once) -// -> feedBuffered (the buffering logic) -// -> stream.transfer (client code - called repeatedly) -// -> stream.end (client code - called when the transfer is done) -// -> transferEnd (closes cache file, if used) -// -// "->" means "calls or inlines", here -// - -size_t const UrlReader::max_read_ahead = CURL_MAX_WRITE_SIZE; - -char const* const UrlReader::success = "UrlReader: Success!"; -char const* const UrlReader::success_cached = "UrlReader: Using local file."; -char const* const UrlReader::error_init_failed = "UrlReader: Initialization failed."; -char const* const UrlReader::error_aborted = "UrlReader: Processing error."; -char const* const UrlReader::error_buffer_overflow = "UrlReader: Buffer overflow."; -char const* const UrlReader::error_leftover_input = "UrlReader: Incomplete processing."; - -#define _curlPtr static_cast(_curlHandle) - -UrlReader::UrlReader() - : _curlHandle(0l), _xtraBuffer(0l), _errorStr(0l), _cacheReadBuffer(0l) { - - _xtraBuffer = new(std::nothrow) char[max_read_ahead]; - if (! _xtraBuffer) { _errorStr = error_init_failed; return; } - _curlHandle = curl_easy_init(); - if (! _curlHandle) { _errorStr = error_init_failed; return; } - curl_easy_setopt(_curlPtr, CURLOPT_NOSIGNAL, 1l); - curl_easy_setopt(_curlPtr, CURLOPT_FAILONERROR, 1l); - curl_easy_setopt(_curlPtr, CURLOPT_FILETIME, 1l); - curl_easy_setopt(_curlPtr, CURLOPT_ENCODING, ""); -} - -UrlReader::~UrlReader() { - - delete[] _xtraBuffer; - delete[] _cacheReadBuffer; - if (! _curlHandle) { - return; - } - curl_easy_cleanup(_curlPtr); -} - -void UrlReader::perform(char const* url, transfer_callback* cb) { - - curl_easy_setopt(_curlPtr, CURLOPT_URL, url); - curl_easy_setopt(_curlPtr, CURLOPT_WRITEFUNCTION, cb); - curl_easy_setopt(_curlPtr, CURLOPT_WRITEDATA, this); - - CURLcode rc = curl_easy_perform(_curlPtr); - - if (rc == CURLE_OK) - { - while (_xtraSize > 0 && _errorStr == success) - cb(0l, 0, 0, this); - } - else if (_errorStr == success) - _errorStr = curl_easy_strerror(rc); -} - -void UrlReader::transferBegin(void* stream, char const* cacheFile) { - - _errorStr = success; - _streamPtr = stream; - _cacheFileName = cacheFile; - _cacheFile = 0l; - _cacheMode = no_cache; - _xtraSize = ~size_t(0); -} - -void UrlReader::getInfo(char const*& url, - char const*& type, int64_t& length, int64_t& stardate) { - - // fetch information from HTTP header - double clen; - long time; - curl_easy_getinfo(_curlPtr, CURLINFO_FILETIME, & time); - curl_easy_getinfo(_curlPtr, CURLINFO_EFFECTIVE_URL, & url); - curl_easy_getinfo(_curlPtr, CURLINFO_CONTENT_TYPE, & type); - curl_easy_getinfo(_curlPtr, CURLINFO_CONTENT_LENGTH_DOWNLOAD, & clen); - length = static_cast(clen); - curl_easy_getinfo(_curlPtr, CURLINFO_FILETIME, & time); - stardate = time; - -// printLog("UrlReader: Ready to transfer from URL '%s'\n", url); - - // check caching file time whether we actually want to download anything - if (_cacheFileName != 0l) { - struct stat s; - stat(_cacheFileName, & s); - if (time > s.st_mtime) { - // file on server is newer -> update cache file - _cacheFile = fopen(_cacheFileName, "wb"); -// printLog("UrlReader: Also writing content to cache file '%s'\n", _cacheFileName); - if (_cacheFile != 0l) { - _cacheMode = cache_write; - } - } else { - // file on server is older -> use cache file - if (! _cacheReadBuffer) { - _cacheReadBuffer = new (std::nothrow) char[max_read_ahead]; - if (! _cacheReadBuffer) { - // out of memory, no caching, have CURL catch it - return; - } - } - _cacheFile = fopen(_cacheFileName, "rb"); -// printLog("UrlReader: Delivering cached content from file '%s'\n", _cacheFileName); - if (_cacheFile != 0l) { - _cacheMode = cache_read; - } - // override error code returned by CURL when we abort the download - _errorStr = success_cached; - } - } -} - -void UrlReader::transferEnd() { - - if (_cacheFile != 0l) { - fclose(_cacheFile); - } -} - -#else // no-op version for incomplete Windows build: - -UrlReader::UrlReader() : _curlHandle(0l) { } -UrlReader::~UrlReader() { } -void UrlReader::perform(char const* url, transfer_callback* cb) { } -void UrlReader::transferBegin(void* stream, char const* cacheFile) { } -void UrlReader::getInfo(char const*& url, char const*& type, - int64_t& length, int64_t& stardate) { } -void UrlReader::transferEnd() { } - -#endif - - diff --git a/libraries/shared/src/UrlReader.h b/libraries/shared/src/UrlReader.h deleted file mode 100644 index 8a9084ce32..0000000000 --- a/libraries/shared/src/UrlReader.h +++ /dev/null @@ -1,305 +0,0 @@ -// -// UrlReader.h -// hifi -// -// Created by Tobias Schwinger on 3/21/13. -// Copyright (c) 2013 High Fidelity, Inc. All rights reserved. -// - -#ifndef __hifi__UrlReader__ -#define __hifi__UrlReader__ - -#include -#include -#include -#include - -// -// UrlReader class that encapsulates a context for sequential data retrieval -// via URLs. Use one per thread. -// -class UrlReader { -public: - - // - // Constructor - performs initialization, never throws. - // - UrlReader(); - - // - // Destructor - frees resources, never throws. - // - ~UrlReader(); - - // - // Reads data from an URL and forwards it to the instance of a class - // fulfilling the ContentStream concept. - // - // The call protocol on the ContentStream is detailed as follows: - // - // 1. begin(char const* url, - // char const* content_type, uint64_t bytes, uint64_t stardate) - // - // All information except 'url' is optional; 'content_type' can - // be a null pointer - 'bytes' and 'stardate' can be equal to - // to 'unavailable'. - // - // 2. transfer(char* buffer, size_t bytes) - // - // Called until all data has been received. The number of bytes - // actually processed should be returned. - // Unprocessed data is stored in an extra buffer whose size is - // given by the constant UrlReader::max_read_ahead - it can be - // assumed to be reasonably large for on-the-fly parsing. - // - // 3. end(bool ok) - // - // Called at the end of the transfer. - // - // Returns the same success code - // - template< class ContentStream > - bool readUrl(char const* url, ContentStream& s, char const* cacheFile = 0l); - - // - // Returns a pointer to a static C-string that describes the error - // condition. - // - inline char const* getError() const; - - // - // Can be called by the stream to set a user-defined error string. - // - inline void setError(char const* static_c_string); - - // - // Pointer to the C-string returned by a call to 'readUrl' when no - // error occurred. - // - static char const* const success; - - // - // Pointer to the C-string returned by a call to 'readUrl' when no - // error occurred and a local file has been read instead of the - // network stream. - // - static char const* const success_cached; - - // - // Pointer to the C-string returned by a call to 'readUrl' when the - // initialization has failed. - // - static char const* const error_init_failed; - - // - // Pointer to the C-string returned by a call to 'readUrl' when the - // transfer has been aborted by the client. - // - static char const* const error_aborted; - - // - // Pointer to the C-string returned by a call to 'readUrl' when - // leftover input from incomplete processing caused a buffer - // overflow. - // - static char const* const error_buffer_overflow; - - // - // Pointer to the C-string return by a call to 'readUrl' when the - // input provided was not completely consumed. - // - static char const* const error_leftover_input; - - // - // Constant of the maximum number of bytes that are buffered - // between invocations of 'transfer'. - // - static size_t const max_read_ahead; - - // - // Constant representing absent information in the call to the - // 'begin' member function of the target stream. - // - static int const unavailable = -1; - - // - // Constant for requesting to abort the current transfer when - // returned by the 'transfer' member function of the target stream. - // - static size_t const abort = ~0u; - -private: - // instances of this class shall not be copied - UrlReader(UrlReader const&); // = delete; - UrlReader& operator=(UrlReader const&); // = delete; - - inline bool isSuccess(); - - // entrypoints to compiled code - - typedef size_t transfer_callback(char*, size_t, size_t, void*); - - enum CacheMode { no_cache, cache_write, cache_read }; - - void transferBegin(void* stream, char const* cacheFile); - void transferEnd(); - - void perform(char const* url, transfer_callback* transfer); - - void getInfo(char const*& url, - char const*& type, int64_t& length, int64_t& stardate); - - // synthesized callback - - template< class Stream > static size_t callback_template(char *input, size_t size, - size_t nmemb, void* thiz); - - template< class Stream > size_t feedBuffered(Stream* stream, - char* input, size_t size); - - // state - - void* _curlHandle; - char* _xtraBuffer; - char const* _errorStr; - void* _streamPtr; - char const* _cacheFileName; - FILE* _cacheFile; - char* _cacheReadBuffer; - CacheMode _cacheMode; - size_t _xtraSize; -}; - -// inline functions - -inline char const* UrlReader::getError() const { - - return _errorStr; -} - -bool UrlReader::isSuccess() { - - return _errorStr == success || _errorStr == success_cached; -} - -template< class ContentStream > -bool UrlReader::readUrl(char const* url, ContentStream& s, char const* cacheFile) { - if (! _curlHandle) return false; - - this->transferBegin(& s, cacheFile); - this->perform(url, & callback_template); - this->transferEnd(); - bool ok = isSuccess(); - s.end(ok); - return ok; -} - -inline void UrlReader::setError(char const* staticCstring) { - - if (this->isSuccess()) - this->_errorStr = staticCstring; -} - -template< class Stream > -size_t UrlReader::feedBuffered(Stream* stream, char* input, size_t size) { - - size_t inputOffset = 0u; - - while (true) { - - char* buffer = input + inputOffset; - size_t bytes = size - inputOffset; - - // data in extra buffer? - if (_xtraSize > 0) { - - // fill extra buffer with beginning of input - size_t fill = max_read_ahead - _xtraSize; - if (bytes < fill) fill = bytes; - memcpy(_xtraBuffer + _xtraSize, buffer, fill); - // use extra buffer for next transfer - buffer = _xtraBuffer; - bytes = _xtraSize + fill; - inputOffset += fill; - } - - // call 'transfer' - size_t processed = stream->transfer(buffer, bytes); - if (processed == abort) { - - setError(error_aborted); - return 0u; - - } else if (! processed && ! input) { - - setError(error_leftover_input); - return 0u; - } - size_t unprocessed = bytes - processed; - - // can switch to input buffer, now? - if (buffer == _xtraBuffer && unprocessed <= inputOffset) { - - _xtraSize = 0u; - inputOffset -= unprocessed; - - } else { // no? unprocessed data -> extra buffer - - if (unprocessed > max_read_ahead) { - - setError(error_buffer_overflow); - return 0; - } - _xtraSize = unprocessed; - memmove(_xtraBuffer, buffer + processed, unprocessed); - - if (inputOffset == size || buffer != _xtraBuffer) { - - return size; - } - } - } // while -} - -template< class Stream > -size_t UrlReader::callback_template(char *input, size_t size, size_t nmemb, void* thiz) { - - size_t result = 0u; - UrlReader* me = static_cast(thiz); - Stream* stream = static_cast(me->_streamPtr); - size *= nmemb; - - // first call? - if (me->_xtraSize == ~size_t(0)) { - - me->_xtraSize = 0u; - // extract meta information and call 'begin' - char const* url, * type; - int64_t length, stardate; - me->getInfo(url, type, length, stardate); - if (me->_cacheMode != cache_read) { - stream->begin(url, type, length, stardate); - } - } - do { - // will have to repeat from here when reading a local file - - // read from cache file? - if (me->_cacheMode == cache_read) { - // change input buffer and start - input = me->_cacheReadBuffer; - size = fread(input, 1, max_read_ahead, me->_cacheFile); - nmemb = 1; - } else if (me->_cacheMode == cache_write) { - fwrite(input, 1, size, me->_cacheFile); - } - - result = me->feedBuffered(stream, input, size); - - } while (me->_cacheMode == cache_read && result != 0 && ! feof(me->_cacheFile)); - - return me->_cacheMode != cache_read ? result : 0; -} - -#endif /* defined(__hifi__UrlReader__) */ -