mirror of
https://github.com/overte-org/overte.git
synced 2025-07-24 01:03:58 +02:00
remove the starfield URL loading code, closes #1497
This commit is contained in:
Stephen Birarda
parent
f0460bac5f
commit
1c3259c8cf
6 changed files with 0 additions and 670 deletions
|
|
@ -39,8 +39,3 @@ include_directories(${ROOT_DIR}/externals/civetweb/include)
|
||||||
if (UNIX)
|
if (UNIX)
|
||||||
target_link_libraries(${TARGET_NAME} ${CMAKE_DL_LIBS})
|
target_link_libraries(${TARGET_NAME} ${CMAKE_DL_LIBS})
|
||||||
endif (UNIX)
|
endif (UNIX)
|
||||||
|
|
||||||
# link curl for synchronous script downloads
|
|
||||||
find_package(CURL REQUIRED)
|
|
||||||
include_directories(${CURL_INCLUDE_DIRS})
|
|
||||||
target_link_libraries(${TARGET_NAME} ${CURL_LIBRARY})
|
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,6 @@
|
||||||
#include <glm/gtc/matrix_transform.hpp>
|
#include <glm/gtc/matrix_transform.hpp>
|
||||||
#include <glm/gtc/matrix_access.hpp>
|
#include <glm/gtc/matrix_access.hpp>
|
||||||
|
|
||||||
#include "UrlReader.h"
|
|
||||||
#include "AngleUtil.h"
|
#include "AngleUtil.h"
|
||||||
#include "Radix2InplaceSort.h"
|
#include "Radix2InplaceSort.h"
|
||||||
#include "Radix2IntegerScanner.h"
|
#include "Radix2IntegerScanner.h"
|
||||||
|
|
|
||||||
|
|
@ -1,183 +0,0 @@
|
||||||
//
|
|
||||||
// starfield/Loader.h
|
|
||||||
// interface
|
|
||||||
//
|
|
||||||
// Created by Tobias Schwinger on 3/29/13.
|
|
||||||
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
|
|
||||||
//
|
|
||||||
|
|
||||||
#ifndef __interface__starfield__Loader__
|
|
||||||
#define __interface__starfield__Loader__
|
|
||||||
|
|
||||||
#ifndef __interface__Starfield_impl__
|
|
||||||
#error "This is an implementation file - not intended for direct inclusion."
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#include <locale.h>
|
|
||||||
|
|
||||||
#include "Config.h"
|
|
||||||
|
|
||||||
#include "starfield/data/InputVertex.h"
|
|
||||||
#include "starfield/data/BrightnessLevel.h"
|
|
||||||
|
|
||||||
namespace starfield {
|
|
||||||
|
|
||||||
class Loader : UrlReader {
|
|
||||||
public:
|
|
||||||
|
|
||||||
bool loadVertices(
|
|
||||||
InputVertices& destination, char const* url, char const* cacheFile, unsigned limit)
|
|
||||||
{
|
|
||||||
_vertices = & destination;
|
|
||||||
_limit = limit;
|
|
||||||
#if STARFIELD_SAVE_MEMORY
|
|
||||||
if (_limit == 0 || _limit > 60000u)
|
|
||||||
_limit = 60000u;
|
|
||||||
#endif
|
|
||||||
_urlStr = url; // in case we fail early
|
|
||||||
|
|
||||||
if (! UrlReader::readUrl(url, *this, cacheFile))
|
|
||||||
{
|
|
||||||
qDebug("%s:%d: %s\n",
|
|
||||||
_urlStr, _lineNo, getError());
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
qDebug("Loaded %u stars.\n", _recordsRead);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected:
|
|
||||||
friend class UrlReader;
|
|
||||||
|
|
||||||
void begin(char const* url,
|
|
||||||
char const* type,
|
|
||||||
int64_t size,
|
|
||||||
int64_t stardate) {
|
|
||||||
|
|
||||||
_lineNo = 0u;
|
|
||||||
_urlStr = url; // new value in http redirect
|
|
||||||
|
|
||||||
_recordsRead = 0u;
|
|
||||||
|
|
||||||
_vertices->clear();
|
|
||||||
_vertices->reserve(_limit);
|
|
||||||
// qDebug("Stars.cpp: loader begin %s\n", url);
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t transfer(char* input, size_t bytes) {
|
|
||||||
|
|
||||||
size_t consumed = 0u;
|
|
||||||
char const* end = input + bytes;
|
|
||||||
char* line, * next = input;
|
|
||||||
|
|
||||||
for (;;) {
|
|
||||||
|
|
||||||
// advance to next line
|
|
||||||
for (; next != end && isspace(*next); ++next);
|
|
||||||
consumed = next - input;
|
|
||||||
line = next;
|
|
||||||
++_lineNo;
|
|
||||||
for (; next != end && *next != '\n' && *next != '\r'; ++next);
|
|
||||||
if (next == end)
|
|
||||||
return consumed;
|
|
||||||
*next++ = '\0';
|
|
||||||
|
|
||||||
// skip comments
|
|
||||||
if (*line == '\\' || *line == '/' || *line == ';')
|
|
||||||
continue;
|
|
||||||
|
|
||||||
// parse
|
|
||||||
float azi, alt;
|
|
||||||
unsigned c;
|
|
||||||
setlocale(LC_NUMERIC, "C");
|
|
||||||
if (sscanf(line, " %f %f #%x", & azi, & alt, & c) == 3) {
|
|
||||||
|
|
||||||
if (spaceFor( getBrightness(c) )) {
|
|
||||||
|
|
||||||
storeVertex(azi, alt, c);
|
|
||||||
}
|
|
||||||
|
|
||||||
++_recordsRead;
|
|
||||||
|
|
||||||
} else {
|
|
||||||
|
|
||||||
qDebug("Stars.cpp:%d: Bad input from %s\n",
|
|
||||||
_lineNo, _urlStr);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
return consumed;
|
|
||||||
}
|
|
||||||
|
|
||||||
void end(bool ok)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
private:
|
|
||||||
|
|
||||||
bool atLimit() { return _limit > 0u && _recordsRead >= _limit; }
|
|
||||||
|
|
||||||
bool spaceFor(BrightnessLevel b) {
|
|
||||||
|
|
||||||
if (! atLimit()) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// just reached the limit? -> establish a minimum heap and
|
|
||||||
// remember the brightness at its top
|
|
||||||
if (_recordsRead == _limit) {
|
|
||||||
|
|
||||||
// qDebug("Stars.cpp: vertex limit reached -> heap mode\n");
|
|
||||||
|
|
||||||
make_heap(
|
|
||||||
_vertices->begin(), _vertices->end(),
|
|
||||||
GreaterBrightness() );
|
|
||||||
|
|
||||||
_minBrightness = getBrightness(
|
|
||||||
_vertices->begin()->getColor() );
|
|
||||||
}
|
|
||||||
|
|
||||||
// not interested? say so
|
|
||||||
if (_minBrightness >= b)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
// otherwise free up space for the new vertex
|
|
||||||
pop_heap(
|
|
||||||
_vertices->begin(), _vertices->end(),
|
|
||||||
GreaterBrightness() );
|
|
||||||
_vertices->pop_back();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
void storeVertex(float azi, float alt, unsigned color) {
|
|
||||||
|
|
||||||
_vertices->push_back(InputVertex(azi, alt, color));
|
|
||||||
|
|
||||||
if (atLimit()) {
|
|
||||||
|
|
||||||
push_heap(
|
|
||||||
_vertices->begin(), _vertices->end(),
|
|
||||||
GreaterBrightness() );
|
|
||||||
|
|
||||||
_minBrightness = getBrightness(
|
|
||||||
_vertices->begin()->getColor() );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// variables
|
|
||||||
|
|
||||||
InputVertices* _vertices;
|
|
||||||
unsigned _limit;
|
|
||||||
|
|
||||||
unsigned _lineNo;
|
|
||||||
char const* _urlStr;
|
|
||||||
|
|
||||||
unsigned _recordsRead;
|
|
||||||
BrightnessLevel _minBrightness;
|
|
||||||
};
|
|
||||||
|
|
||||||
} // anonymous namespace
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
@ -24,10 +24,6 @@ set(EXTERNAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external)
|
||||||
if (WIN32)
|
if (WIN32)
|
||||||
# include headers for external libraries and InterfaceConfig.
|
# include headers for external libraries and InterfaceConfig.
|
||||||
include_directories(${EXTERNAL_ROOT_DIR})
|
include_directories(${EXTERNAL_ROOT_DIR})
|
||||||
else (WIN32)
|
|
||||||
find_package(CURL REQUIRED)
|
|
||||||
include_directories(${CURL_INCLUDE_DIRS})
|
|
||||||
target_link_libraries(${TARGET_NAME} ${CURL_LIBRARY})
|
|
||||||
endif (WIN32)
|
endif (WIN32)
|
||||||
|
|
||||||
# link required libraries on UNIX
|
# link required libraries on UNIX
|
||||||
|
|
|
||||||
|
|
@ -1,172 +0,0 @@
|
||||||
//
|
|
||||||
// UrlReader.cpp
|
|
||||||
// hifi
|
|
||||||
//
|
|
||||||
// Created by Tobias Schwinger on 3/21/13.
|
|
||||||
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
|
|
||||||
//
|
|
||||||
|
|
||||||
#include <new>
|
|
||||||
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
|
|
||||||
#ifndef _WIN32
|
|
||||||
// (Windows port is incomplete and the build files do not support CURL, yet)
|
|
||||||
|
|
||||||
#include <curl/curl.h>
|
|
||||||
|
|
||||||
#include "UrlReader.h"
|
|
||||||
|
|
||||||
//
|
|
||||||
// ATTENTION: A certain part of the implementation lives in inlined code
|
|
||||||
// (see the bottom of the header file).
|
|
||||||
//
|
|
||||||
// Why? Because it allows stream parsing without having to call around a
|
|
||||||
// lot (one static and one dynamic call per character if the parser just
|
|
||||||
// reads one character at a time).
|
|
||||||
//
|
|
||||||
// Here is an overview of the code structure:
|
|
||||||
//
|
|
||||||
// readUrl
|
|
||||||
// -> transferBegin (sets up state)
|
|
||||||
// -> perform (starts CURL transfer)
|
|
||||||
// -> (specialized, type-erased) callback_template
|
|
||||||
// -> getInfo (fetches HTTP header, eventually initiates caching)
|
|
||||||
// -> stream.begin (client code - called once)
|
|
||||||
// -> feedBuffered (the buffering logic)
|
|
||||||
// -> stream.transfer (client code - called repeatedly)
|
|
||||||
// -> stream.end (client code - called when the transfer is done)
|
|
||||||
// -> transferEnd (closes cache file, if used)
|
|
||||||
//
|
|
||||||
// "->" means "calls or inlines", here
|
|
||||||
//
|
|
||||||
|
|
||||||
size_t const UrlReader::max_read_ahead = CURL_MAX_WRITE_SIZE;
|
|
||||||
|
|
||||||
char const* const UrlReader::success = "UrlReader: Success!";
|
|
||||||
char const* const UrlReader::success_cached = "UrlReader: Using local file.";
|
|
||||||
char const* const UrlReader::error_init_failed = "UrlReader: Initialization failed.";
|
|
||||||
char const* const UrlReader::error_aborted = "UrlReader: Processing error.";
|
|
||||||
char const* const UrlReader::error_buffer_overflow = "UrlReader: Buffer overflow.";
|
|
||||||
char const* const UrlReader::error_leftover_input = "UrlReader: Incomplete processing.";
|
|
||||||
|
|
||||||
#define _curlPtr static_cast<CURL*>(_curlHandle)
|
|
||||||
|
|
||||||
UrlReader::UrlReader()
|
|
||||||
: _curlHandle(0l), _xtraBuffer(0l), _errorStr(0l), _cacheReadBuffer(0l) {
|
|
||||||
|
|
||||||
_xtraBuffer = new(std::nothrow) char[max_read_ahead];
|
|
||||||
if (! _xtraBuffer) { _errorStr = error_init_failed; return; }
|
|
||||||
_curlHandle = curl_easy_init();
|
|
||||||
if (! _curlHandle) { _errorStr = error_init_failed; return; }
|
|
||||||
curl_easy_setopt(_curlPtr, CURLOPT_NOSIGNAL, 1l);
|
|
||||||
curl_easy_setopt(_curlPtr, CURLOPT_FAILONERROR, 1l);
|
|
||||||
curl_easy_setopt(_curlPtr, CURLOPT_FILETIME, 1l);
|
|
||||||
curl_easy_setopt(_curlPtr, CURLOPT_ENCODING, "");
|
|
||||||
}
|
|
||||||
|
|
||||||
UrlReader::~UrlReader() {
|
|
||||||
|
|
||||||
delete[] _xtraBuffer;
|
|
||||||
delete[] _cacheReadBuffer;
|
|
||||||
if (! _curlHandle) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
curl_easy_cleanup(_curlPtr);
|
|
||||||
}
|
|
||||||
|
|
||||||
void UrlReader::perform(char const* url, transfer_callback* cb) {
|
|
||||||
|
|
||||||
curl_easy_setopt(_curlPtr, CURLOPT_URL, url);
|
|
||||||
curl_easy_setopt(_curlPtr, CURLOPT_WRITEFUNCTION, cb);
|
|
||||||
curl_easy_setopt(_curlPtr, CURLOPT_WRITEDATA, this);
|
|
||||||
|
|
||||||
CURLcode rc = curl_easy_perform(_curlPtr);
|
|
||||||
|
|
||||||
if (rc == CURLE_OK)
|
|
||||||
{
|
|
||||||
while (_xtraSize > 0 && _errorStr == success)
|
|
||||||
cb(0l, 0, 0, this);
|
|
||||||
}
|
|
||||||
else if (_errorStr == success)
|
|
||||||
_errorStr = curl_easy_strerror(rc);
|
|
||||||
}
|
|
||||||
|
|
||||||
void UrlReader::transferBegin(void* stream, char const* cacheFile) {
|
|
||||||
|
|
||||||
_errorStr = success;
|
|
||||||
_streamPtr = stream;
|
|
||||||
_cacheFileName = cacheFile;
|
|
||||||
_cacheFile = 0l;
|
|
||||||
_cacheMode = no_cache;
|
|
||||||
_xtraSize = ~size_t(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
void UrlReader::getInfo(char const*& url,
|
|
||||||
char const*& type, int64_t& length, int64_t& stardate) {
|
|
||||||
|
|
||||||
// fetch information from HTTP header
|
|
||||||
double clen;
|
|
||||||
long time;
|
|
||||||
curl_easy_getinfo(_curlPtr, CURLINFO_FILETIME, & time);
|
|
||||||
curl_easy_getinfo(_curlPtr, CURLINFO_EFFECTIVE_URL, & url);
|
|
||||||
curl_easy_getinfo(_curlPtr, CURLINFO_CONTENT_TYPE, & type);
|
|
||||||
curl_easy_getinfo(_curlPtr, CURLINFO_CONTENT_LENGTH_DOWNLOAD, & clen);
|
|
||||||
length = static_cast<int64_t>(clen);
|
|
||||||
curl_easy_getinfo(_curlPtr, CURLINFO_FILETIME, & time);
|
|
||||||
stardate = time;
|
|
||||||
|
|
||||||
// printLog("UrlReader: Ready to transfer from URL '%s'\n", url);
|
|
||||||
|
|
||||||
// check caching file time whether we actually want to download anything
|
|
||||||
if (_cacheFileName != 0l) {
|
|
||||||
struct stat s;
|
|
||||||
stat(_cacheFileName, & s);
|
|
||||||
if (time > s.st_mtime) {
|
|
||||||
// file on server is newer -> update cache file
|
|
||||||
_cacheFile = fopen(_cacheFileName, "wb");
|
|
||||||
// printLog("UrlReader: Also writing content to cache file '%s'\n", _cacheFileName);
|
|
||||||
if (_cacheFile != 0l) {
|
|
||||||
_cacheMode = cache_write;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// file on server is older -> use cache file
|
|
||||||
if (! _cacheReadBuffer) {
|
|
||||||
_cacheReadBuffer = new (std::nothrow) char[max_read_ahead];
|
|
||||||
if (! _cacheReadBuffer) {
|
|
||||||
// out of memory, no caching, have CURL catch it
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_cacheFile = fopen(_cacheFileName, "rb");
|
|
||||||
// printLog("UrlReader: Delivering cached content from file '%s'\n", _cacheFileName);
|
|
||||||
if (_cacheFile != 0l) {
|
|
||||||
_cacheMode = cache_read;
|
|
||||||
}
|
|
||||||
// override error code returned by CURL when we abort the download
|
|
||||||
_errorStr = success_cached;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void UrlReader::transferEnd() {
|
|
||||||
|
|
||||||
if (_cacheFile != 0l) {
|
|
||||||
fclose(_cacheFile);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#else // no-op version for incomplete Windows build:
|
|
||||||
|
|
||||||
UrlReader::UrlReader() : _curlHandle(0l) { }
|
|
||||||
UrlReader::~UrlReader() { }
|
|
||||||
void UrlReader::perform(char const* url, transfer_callback* cb) { }
|
|
||||||
void UrlReader::transferBegin(void* stream, char const* cacheFile) { }
|
|
||||||
void UrlReader::getInfo(char const*& url, char const*& type,
|
|
||||||
int64_t& length, int64_t& stardate) { }
|
|
||||||
void UrlReader::transferEnd() { }
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,305 +0,0 @@
|
||||||
//
|
|
||||||
// UrlReader.h
|
|
||||||
// hifi
|
|
||||||
//
|
|
||||||
// Created by Tobias Schwinger on 3/21/13.
|
|
||||||
// Copyright (c) 2013 High Fidelity, Inc. All rights reserved.
|
|
||||||
//
|
|
||||||
|
|
||||||
#ifndef __hifi__UrlReader__
|
|
||||||
#define __hifi__UrlReader__
|
|
||||||
|
|
||||||
#include <stddef.h>
|
|
||||||
#include <stdint.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
|
|
||||||
//
|
|
||||||
// UrlReader class that encapsulates a context for sequential data retrieval
|
|
||||||
// via URLs. Use one per thread.
|
|
||||||
//
|
|
||||||
class UrlReader {
|
|
||||||
public:
|
|
||||||
|
|
||||||
//
|
|
||||||
// Constructor - performs initialization, never throws.
|
|
||||||
//
|
|
||||||
UrlReader();
|
|
||||||
|
|
||||||
//
|
|
||||||
// Destructor - frees resources, never throws.
|
|
||||||
//
|
|
||||||
~UrlReader();
|
|
||||||
|
|
||||||
//
|
|
||||||
// Reads data from an URL and forwards it to the instance of a class
|
|
||||||
// fulfilling the ContentStream concept.
|
|
||||||
//
|
|
||||||
// The call protocol on the ContentStream is detailed as follows:
|
|
||||||
//
|
|
||||||
// 1. begin(char const* url,
|
|
||||||
// char const* content_type, uint64_t bytes, uint64_t stardate)
|
|
||||||
//
|
|
||||||
// All information except 'url' is optional; 'content_type' can
|
|
||||||
// be a null pointer - 'bytes' and 'stardate' can be equal to
|
|
||||||
// to 'unavailable'.
|
|
||||||
//
|
|
||||||
// 2. transfer(char* buffer, size_t bytes)
|
|
||||||
//
|
|
||||||
// Called until all data has been received. The number of bytes
|
|
||||||
// actually processed should be returned.
|
|
||||||
// Unprocessed data is stored in an extra buffer whose size is
|
|
||||||
// given by the constant UrlReader::max_read_ahead - it can be
|
|
||||||
// assumed to be reasonably large for on-the-fly parsing.
|
|
||||||
//
|
|
||||||
// 3. end(bool ok)
|
|
||||||
//
|
|
||||||
// Called at the end of the transfer.
|
|
||||||
//
|
|
||||||
// Returns the same success code
|
|
||||||
//
|
|
||||||
template< class ContentStream >
|
|
||||||
bool readUrl(char const* url, ContentStream& s, char const* cacheFile = 0l);
|
|
||||||
|
|
||||||
//
|
|
||||||
// Returns a pointer to a static C-string that describes the error
|
|
||||||
// condition.
|
|
||||||
//
|
|
||||||
inline char const* getError() const;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Can be called by the stream to set a user-defined error string.
|
|
||||||
//
|
|
||||||
inline void setError(char const* static_c_string);
|
|
||||||
|
|
||||||
//
|
|
||||||
// Pointer to the C-string returned by a call to 'readUrl' when no
|
|
||||||
// error occurred.
|
|
||||||
//
|
|
||||||
static char const* const success;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Pointer to the C-string returned by a call to 'readUrl' when no
|
|
||||||
// error occurred and a local file has been read instead of the
|
|
||||||
// network stream.
|
|
||||||
//
|
|
||||||
static char const* const success_cached;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Pointer to the C-string returned by a call to 'readUrl' when the
|
|
||||||
// initialization has failed.
|
|
||||||
//
|
|
||||||
static char const* const error_init_failed;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Pointer to the C-string returned by a call to 'readUrl' when the
|
|
||||||
// transfer has been aborted by the client.
|
|
||||||
//
|
|
||||||
static char const* const error_aborted;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Pointer to the C-string returned by a call to 'readUrl' when
|
|
||||||
// leftover input from incomplete processing caused a buffer
|
|
||||||
// overflow.
|
|
||||||
//
|
|
||||||
static char const* const error_buffer_overflow;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Pointer to the C-string return by a call to 'readUrl' when the
|
|
||||||
// input provided was not completely consumed.
|
|
||||||
//
|
|
||||||
static char const* const error_leftover_input;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Constant of the maximum number of bytes that are buffered
|
|
||||||
// between invocations of 'transfer'.
|
|
||||||
//
|
|
||||||
static size_t const max_read_ahead;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Constant representing absent information in the call to the
|
|
||||||
// 'begin' member function of the target stream.
|
|
||||||
//
|
|
||||||
static int const unavailable = -1;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Constant for requesting to abort the current transfer when
|
|
||||||
// returned by the 'transfer' member function of the target stream.
|
|
||||||
//
|
|
||||||
static size_t const abort = ~0u;
|
|
||||||
|
|
||||||
private:
|
|
||||||
// instances of this class shall not be copied
|
|
||||||
UrlReader(UrlReader const&); // = delete;
|
|
||||||
UrlReader& operator=(UrlReader const&); // = delete;
|
|
||||||
|
|
||||||
inline bool isSuccess();
|
|
||||||
|
|
||||||
// entrypoints to compiled code
|
|
||||||
|
|
||||||
typedef size_t transfer_callback(char*, size_t, size_t, void*);
|
|
||||||
|
|
||||||
enum CacheMode { no_cache, cache_write, cache_read };
|
|
||||||
|
|
||||||
void transferBegin(void* stream, char const* cacheFile);
|
|
||||||
void transferEnd();
|
|
||||||
|
|
||||||
void perform(char const* url, transfer_callback* transfer);
|
|
||||||
|
|
||||||
void getInfo(char const*& url,
|
|
||||||
char const*& type, int64_t& length, int64_t& stardate);
|
|
||||||
|
|
||||||
// synthesized callback
|
|
||||||
|
|
||||||
template< class Stream > static size_t callback_template(char *input, size_t size,
|
|
||||||
size_t nmemb, void* thiz);
|
|
||||||
|
|
||||||
template< class Stream > size_t feedBuffered(Stream* stream,
|
|
||||||
char* input, size_t size);
|
|
||||||
|
|
||||||
// state
|
|
||||||
|
|
||||||
void* _curlHandle;
|
|
||||||
char* _xtraBuffer;
|
|
||||||
char const* _errorStr;
|
|
||||||
void* _streamPtr;
|
|
||||||
char const* _cacheFileName;
|
|
||||||
FILE* _cacheFile;
|
|
||||||
char* _cacheReadBuffer;
|
|
||||||
CacheMode _cacheMode;
|
|
||||||
size_t _xtraSize;
|
|
||||||
};
|
|
||||||
|
|
||||||
// inline functions
|
|
||||||
|
|
||||||
inline char const* UrlReader::getError() const {
|
|
||||||
|
|
||||||
return _errorStr;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool UrlReader::isSuccess() {
|
|
||||||
|
|
||||||
return _errorStr == success || _errorStr == success_cached;
|
|
||||||
}
|
|
||||||
|
|
||||||
template< class ContentStream >
|
|
||||||
bool UrlReader::readUrl(char const* url, ContentStream& s, char const* cacheFile) {
|
|
||||||
if (! _curlHandle) return false;
|
|
||||||
|
|
||||||
this->transferBegin(& s, cacheFile);
|
|
||||||
this->perform(url, & callback_template<ContentStream>);
|
|
||||||
this->transferEnd();
|
|
||||||
bool ok = isSuccess();
|
|
||||||
s.end(ok);
|
|
||||||
return ok;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline void UrlReader::setError(char const* staticCstring) {
|
|
||||||
|
|
||||||
if (this->isSuccess())
|
|
||||||
this->_errorStr = staticCstring;
|
|
||||||
}
|
|
||||||
|
|
||||||
template< class Stream >
|
|
||||||
size_t UrlReader::feedBuffered(Stream* stream, char* input, size_t size) {
|
|
||||||
|
|
||||||
size_t inputOffset = 0u;
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
|
|
||||||
char* buffer = input + inputOffset;
|
|
||||||
size_t bytes = size - inputOffset;
|
|
||||||
|
|
||||||
// data in extra buffer?
|
|
||||||
if (_xtraSize > 0) {
|
|
||||||
|
|
||||||
// fill extra buffer with beginning of input
|
|
||||||
size_t fill = max_read_ahead - _xtraSize;
|
|
||||||
if (bytes < fill) fill = bytes;
|
|
||||||
memcpy(_xtraBuffer + _xtraSize, buffer, fill);
|
|
||||||
// use extra buffer for next transfer
|
|
||||||
buffer = _xtraBuffer;
|
|
||||||
bytes = _xtraSize + fill;
|
|
||||||
inputOffset += fill;
|
|
||||||
}
|
|
||||||
|
|
||||||
// call 'transfer'
|
|
||||||
size_t processed = stream->transfer(buffer, bytes);
|
|
||||||
if (processed == abort) {
|
|
||||||
|
|
||||||
setError(error_aborted);
|
|
||||||
return 0u;
|
|
||||||
|
|
||||||
} else if (! processed && ! input) {
|
|
||||||
|
|
||||||
setError(error_leftover_input);
|
|
||||||
return 0u;
|
|
||||||
}
|
|
||||||
size_t unprocessed = bytes - processed;
|
|
||||||
|
|
||||||
// can switch to input buffer, now?
|
|
||||||
if (buffer == _xtraBuffer && unprocessed <= inputOffset) {
|
|
||||||
|
|
||||||
_xtraSize = 0u;
|
|
||||||
inputOffset -= unprocessed;
|
|
||||||
|
|
||||||
} else { // no? unprocessed data -> extra buffer
|
|
||||||
|
|
||||||
if (unprocessed > max_read_ahead) {
|
|
||||||
|
|
||||||
setError(error_buffer_overflow);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
_xtraSize = unprocessed;
|
|
||||||
memmove(_xtraBuffer, buffer + processed, unprocessed);
|
|
||||||
|
|
||||||
if (inputOffset == size || buffer != _xtraBuffer) {
|
|
||||||
|
|
||||||
return size;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // while
|
|
||||||
}
|
|
||||||
|
|
||||||
template< class Stream >
|
|
||||||
size_t UrlReader::callback_template(char *input, size_t size, size_t nmemb, void* thiz) {
|
|
||||||
|
|
||||||
size_t result = 0u;
|
|
||||||
UrlReader* me = static_cast<UrlReader*>(thiz);
|
|
||||||
Stream* stream = static_cast<Stream*>(me->_streamPtr);
|
|
||||||
size *= nmemb;
|
|
||||||
|
|
||||||
// first call?
|
|
||||||
if (me->_xtraSize == ~size_t(0)) {
|
|
||||||
|
|
||||||
me->_xtraSize = 0u;
|
|
||||||
// extract meta information and call 'begin'
|
|
||||||
char const* url, * type;
|
|
||||||
int64_t length, stardate;
|
|
||||||
me->getInfo(url, type, length, stardate);
|
|
||||||
if (me->_cacheMode != cache_read) {
|
|
||||||
stream->begin(url, type, length, stardate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
do {
|
|
||||||
// will have to repeat from here when reading a local file
|
|
||||||
|
|
||||||
// read from cache file?
|
|
||||||
if (me->_cacheMode == cache_read) {
|
|
||||||
// change input buffer and start
|
|
||||||
input = me->_cacheReadBuffer;
|
|
||||||
size = fread(input, 1, max_read_ahead, me->_cacheFile);
|
|
||||||
nmemb = 1;
|
|
||||||
} else if (me->_cacheMode == cache_write) {
|
|
||||||
fwrite(input, 1, size, me->_cacheFile);
|
|
||||||
}
|
|
||||||
|
|
||||||
result = me->feedBuffered(stream, input, size);
|
|
||||||
|
|
||||||
} while (me->_cacheMode == cache_read && result != 0 && ! feof(me->_cacheFile));
|
|
||||||
|
|
||||||
return me->_cacheMode != cache_read ? result : 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif /* defined(__hifi__UrlReader__) */
|
|
||||||
|
|
||||||
Loading…
Reference in a new issue