diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index cdf92918c6..284dd344e7 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -57,6 +57,29 @@ } ] }, + { + "label": "Monitoring", + "name": "monitoring", + "restart": false, + "settings": [ + { + "name": "enable_prometheus_exporter", + "label": "Enable Prometheus Exporter", + "help": "Enable a Prometheus exporter to make it possible to gather the stats that are available at Nodes tab with a Prometheus server. This makes it possible to keep track of long-term domain statistics for graphing, troubleshooting, and performance monitoring.", + "default": false, + "type": "checkbox", + "advanced": true + }, + { + "name": "prometheus_exporter_port", + "label": "Prometheus TCP Port", + "help": "This is the port where the Prometheus exporter accepts connections. It listens both on IPv4 and IPv6 and can be accessed remotely, so you should make sure to restrict access with a firewall as needed.", + "default": "9703", + "type": "int", + "advanced": true + } + ] + }, { "label": "Paths", "html_id": "paths", diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 9c6361faef..284281a64f 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -163,8 +163,7 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection, DomainServer::DomainServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _gatekeeper(this), - _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this), - _httpExporterManager(QHostAddress::Any, DOMAIN_SERVER_EXPORTER_PORT, QString("%1/resources/prometheus_exporter/").arg(QCoreApplication::applicationDirPath()), &_exporter) + _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) { if (_parentPID != -1) { watchParentProcess(_parentPID); @@ -230,7 +229,6 @@ DomainServer::DomainServer(int argc, char* argv[]) : this, &DomainServer::updateDownstreamNodes); connect(&_settingsManager, &DomainServerSettingsManager::settingsUpdated, this, &DomainServer::updateUpstreamNodes); - setupGroupCacheRefresh(); optionallySetupOAuth(); @@ -331,6 +329,8 @@ DomainServer::DomainServer(int argc, char* argv[]) : _nodePingMonitorTimer = new QTimer{ this }; connect(_nodePingMonitorTimer, &QTimer::timeout, this, &DomainServer::nodePingMonitor); _nodePingMonitorTimer->start(NODE_PING_MONITOR_INTERVAL_MSECS); + + initializeExporter(); } void DomainServer::parseCommandLine(int argc, char* argv[]) { @@ -425,6 +425,11 @@ DomainServer::~DomainServer() { _contentManager->terminate(); } + if ( _httpExporterManager ) { + _httpExporterManager->close(); + delete _httpExporterManager; + } + DependencyManager::destroy(); // cleanup the AssetClient thread @@ -3039,6 +3044,27 @@ void DomainServer::updateUpstreamNodes() { updateReplicationNodes(Upstream); } +void DomainServer::initializeExporter() +{ + static const QString ENABLE_EXPORTER = "monitoring.enable_prometheus_exporter"; + static const QString EXPORTER_PORT = "monitoring.prometheus_exporter_port"; + + bool isExporterEnabled = _settingsManager.valueOrDefaultValueForKeyPath(ENABLE_EXPORTER).toBool(); + int exporterPort = _settingsManager.valueOrDefaultValueForKeyPath(EXPORTER_PORT).toInt(); + + if ( exporterPort < 1 || exporterPort > 65535 ) { + qCWarning(domain_server) << "Prometheus exporter port " << exporterPort << " is out of range."; + isExporterEnabled = false; + } + + qCDebug(domain_server) << "Setting up Prometheus exporter"; + + if ( isExporterEnabled && !_httpExporterManager) { + qCInfo(domain_server) << "Starting Prometheus exporter on port " << exporterPort; + _httpExporterManager = new HTTPManager(QHostAddress::Any, static_cast(exporterPort), QString("%1/resources/prometheus_exporter/").arg(QCoreApplication::applicationDirPath()), &_exporter); + } +} + void DomainServer::updateReplicatedNodes() { // Make sure we have downstream nodes in our list static const QString REPLICATED_USERS_KEY = "users"; diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index f6bb9bc7ae..46968c7a0c 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -139,6 +139,7 @@ private slots: void updateReplicatedNodes(); void updateDownstreamNodes(); void updateUpstreamNodes(); + void initializeExporter(); void tokenGrantFinished(); void profileRequestFinished(); @@ -238,7 +239,7 @@ private: DomainServerExporter _exporter; HTTPManager _httpManager; - HTTPManager _httpExporterManager; + HTTPManager *_httpExporterManager { nullptr }; std::unique_ptr _httpsManager; QHash _allAssignments; diff --git a/libraries/embedded-webserver/src/HTTPManager.h b/libraries/embedded-webserver/src/HTTPManager.h index 597f6921cc..d25cde413e 100644 --- a/libraries/embedded-webserver/src/HTTPManager.h +++ b/libraries/embedded-webserver/src/HTTPManager.h @@ -34,7 +34,7 @@ class HTTPManager : public QTcpServer, public HTTPRequestHandler { public: /// Initializes the manager. HTTPManager(const QHostAddress& listenAddress, quint16 port, const QString& documentRoot, HTTPRequestHandler* requestHandler = nullptr); - + bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override; private slots: