diff --git a/resources/desktop/com.github.rssguard.appdata.xml b/resources/desktop/com.github.rssguard.appdata.xml index 652be1387..27d94511c 100644 --- a/resources/desktop/com.github.rssguard.appdata.xml +++ b/resources/desktop/com.github.rssguard.appdata.xml @@ -30,7 +30,7 @@ https://martinrotter.github.io/donate/ - + none diff --git a/resources/rssguard.qrc b/resources/rssguard.qrc index 664f365ca..12f0ff705 100755 --- a/resources/rssguard.qrc +++ b/resources/rssguard.qrc @@ -6,6 +6,7 @@ text/COPYING_GNU_GPL text/COPYING_GNU_GPL_HTML + scripts/adblock/adblock-server.js scripts/public_suffix_list.dat graphics/rssguard.ico diff --git a/resources/scripts/adblock/adblock-server.js b/resources/scripts/adblock/adblock-server.js index 98eaa3852..cc6ec728e 100755 --- a/resources/scripts/adblock/adblock-server.js +++ b/resources/scripts/adblock/adblock-server.js @@ -3,7 +3,7 @@ // How to install: // npm i -g @cliqz/adblocker // npm i -g concat-stream -// npm i -g psl +// npm i -g tldts-experimental // npm i -g node-fetch // // How to run: @@ -19,84 +19,100 @@ // }' 'http://localhost:' const fs = require('fs'); -const psl = require('psl'); +const tldts = require('tldts-experimental'); const adblock = require('@cliqz/adblocker') const http = require('http'); const concat = require('concat-stream'); const constants = require('node:http2'); const fetch = require("node-fetch"); +const cluster = require('cluster'); +const numCPUs = require('os').cpus().length; const port = process.argv[2]; const filtersFile = process.argv[3]; const engine = adblock.FiltersEngine.parse(fs.readFileSync(filtersFile, 'utf-8')); const hostname = '127.0.0.1'; -const server = http.createServer((req, res) => { - try { - console.log(new Date()); +if (cluster.isPrimary) { + console.log(`Primary ${process.pid} is running`); - const chunks = []; - req.on('data', chunk => chunks.push(chunk)); - req.on('end', () => { + // Fork workers. + for (let i = 0; i < numCPUs; i++) { + cluster.fork(); + } + + cluster.on('exit', (worker, code, signal) => { + console.log(`worker ${worker.process.pid} died`); + }); +} +else { + const server = http.createServer((req, res) => { + try { console.log(new Date()); - try { - const jsonData = Buffer.concat(chunks); - const jsonStruct = JSON.parse(jsonData.toString()); - - const askUrl = jsonStruct['url']; - const askFilter = jsonStruct['filter']; - const askCosmetic = jsonStruct['cosmetic']; - const askUrlType = jsonStruct['url_type']; - const fullUrl = new URL(askUrl); - - resultJson = {}; - - if (askFilter) { - const adblockMatch = engine.match(adblock.Request.fromRawDetails({ - type: askUrlType, - url: askUrl, - })); - - resultJson["filter"] = adblockMatch; - console.log(`adblocker: Filter is:\n${JSON.stringify(adblockMatch)}.`) - } - - if (askCosmetic) { - const adblockCosmetic = engine.getCosmeticsFilters({ - url: askUrl, - hostname: fullUrl.hostname, - domain: psl.parse(fullUrl.hostname).domain - }); - - resultJson["cosmetic"] = adblockCosmetic; - console.log(`adblocker: Cosmetic is:\n${JSON.stringify(adblockCosmetic)}.`) - } - - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify(resultJson)); - + const chunks = []; + req.on('data', chunk => chunks.push(chunk)); + req.on('end', () => { console.log(new Date()); - } - catch (inner_error) { - console.error(`adblocker: ${inner_error}.`); - res.statusCode = 500; - res.setHeader('Content-Type', 'text/plain'); - res.end(String(inner_error)); - } - }) - } - catch (error) { - console.error(`adblocker: ${inner_error}.`); + try { + const jsonData = Buffer.concat(chunks); + const jsonStruct = JSON.parse(jsonData.toString()); - res.statusCode = 500; - res.setHeader('Content-Type', 'text/plain'); - res.end(String(error)); - } -}); + const askUrl = jsonStruct['url']; + const askFilter = jsonStruct['filter']; + const askCosmetic = jsonStruct['cosmetic']; + const askUrlType = jsonStruct['url_type']; + const fullUrl = new URL(askUrl); -server.listen(port, hostname, () => { - console.log(`adblocker: Server started at local port ${port}.`); -}); \ No newline at end of file + resultJson = {}; + + if (askFilter) { + const adblockMatch = engine.match(adblock.Request.fromRawDetails({ + type: askUrlType, + url: askUrl, + })); + + resultJson["filter"] = adblockMatch; + console.log(`adblocker: Filter is:\n${JSON.stringify(adblockMatch)}.`) + } + + if (askCosmetic) { + const adblockCosmetic = engine.getCosmeticsFilters({ + url: askUrl, + hostname: fullUrl.hostname, + domain: tldts.parse(fullUrl).domain + }); + + resultJson["cosmetic"] = adblockCosmetic; + console.log(`adblocker: Cosmetic is:\n${JSON.stringify(adblockCosmetic)}.`) + } + + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(resultJson)); + + console.log(new Date()); + } + catch (inner_error) { + console.error(`adblocker: ${inner_error}.`); + + res.statusCode = 500; + res.setHeader('Content-Type', 'text/plain'); + res.end(String(inner_error)); + } + }) + } + catch (error) { + console.error(`adblocker: ${inner_error}.`); + + res.statusCode = 500; + res.setHeader('Content-Type', 'text/plain'); + res.end(String(error)); + } + }); + + server.listen(port, hostname, () => { + console.log(`adblocker: Server started at local port ${port}.`); + }); +} \ No newline at end of file diff --git a/src/librssguard/definitions/definitions.h b/src/librssguard/definitions/definitions.h index 540ab4613..46eea00c9 100755 --- a/src/librssguard/definitions/definitions.h +++ b/src/librssguard/definitions/definitions.h @@ -14,13 +14,10 @@ #define SERVICE_CODE_INOREADER "inoreader" #define SERVICE_CODE_GMAIL "gmail" +#define ADBLOCK_SERVER_PORT "48484" #define ADBLOCK_HOWTO "https://github.com/martinrotter/rssguard/blob/master/resources/docs/Documentation.md#adblock" -#define ADBLOCK_UPDATE_DAYS_INTERVAL 14 #define ADBLOCK_ICON_ACTIVE "adblock" #define ADBLOCK_ICON_DISABLED "adblock-disabled" -#define ADBLOCK_CUSTOMLIST_NAME "customlist.txt" -#define ADBLOCK_LISTS_SUBDIRECTORY "adblock" -#define ADBLOCK_EASYLIST_URL "https://easylist.to/easylist/easylist.txt" #define OAUTH_DECRYPTION_KEY 11451167756100761335ul #define OAUTH_REDIRECT_URI "http://localhost" diff --git a/src/librssguard/miscellaneous/application.cpp b/src/librssguard/miscellaneous/application.cpp index 77b77da81..48831ff5a 100755 --- a/src/librssguard/miscellaneous/application.cpp +++ b/src/librssguard/miscellaneous/application.cpp @@ -78,7 +78,10 @@ Application::Application(const QString& id, int& argc, char** argv) #if defined(USE_WEBENGINE) m_webFactory->urlIinterceptor()->load(); - m_webFactory->adBlock()->load(true); + + QTimer::singleShot(3000, this, [=]() { + m_webFactory->adBlock()->load(true); + }); #endif qDebugNN << LOGSEC_CORE diff --git a/src/librssguard/miscellaneous/skinfactory.cpp b/src/librssguard/miscellaneous/skinfactory.cpp index c43de3de9..4e69412db 100644 --- a/src/librssguard/miscellaneous/skinfactory.cpp +++ b/src/librssguard/miscellaneous/skinfactory.cpp @@ -60,9 +60,10 @@ QString SkinFactory::selectedSkinName() const { return qApp->settings()->value(GROUP(GUI), SETTING(GUI::Skin)).toString(); } -QString SkinFactory::adBlockedPage(const QString& url) { +QString SkinFactory::adBlockedPage(const QString& url, const QString& filter) { const QString& adblocked = currentSkin().m_adblocked.arg(tr("This page was blocked by AdBlock"), - tr(R"(Blocked URL: "%1")").arg(url)); + tr(R"(Blocked URL: "%1"
Used filter: "%2")").arg(url, + filter)); return currentSkin().m_layoutMarkupWrapper.arg(tr("This page was blocked by AdBlock"), adblocked); } diff --git a/src/librssguard/miscellaneous/skinfactory.h b/src/librssguard/miscellaneous/skinfactory.h index 9471ddcd9..404d5a5d3 100644 --- a/src/librssguard/miscellaneous/skinfactory.h +++ b/src/librssguard/miscellaneous/skinfactory.h @@ -50,7 +50,7 @@ class RSSGUARD_DLLSPEC SkinFactory : public QObject { // after application restart. QString selectedSkinName() const; - QString adBlockedPage(const QString& url); + QString adBlockedPage(const QString& url, const QString& filter); // Gets skin about a particular skin. Skin skinInfo(const QString& skin_name, bool* ok = nullptr) const; diff --git a/src/librssguard/network-web/adblock/adblockmanager.cpp b/src/librssguard/network-web/adblock/adblockmanager.cpp index 8fbf1ab3b..29619761c 100644 --- a/src/librssguard/network-web/adblock/adblockmanager.cpp +++ b/src/librssguard/network-web/adblock/adblockmanager.cpp @@ -3,6 +3,7 @@ #include "network-web/adblock/adblockmanager.h" #include "exceptions/applicationexception.h" +#include "exceptions/networkexception.h" #include "miscellaneous/application.h" #include "miscellaneous/settings.h" #include "network-web/adblock/adblockdialog.h" @@ -15,7 +16,10 @@ #include #include +#include +#include #include +#include #include #include #include @@ -24,24 +28,43 @@ AdBlockManager::AdBlockManager(QObject* parent) : QObject(parent), m_loaded(false), m_enabled(false), m_interceptor(new AdBlockUrlInterceptor(this)) { m_adblockIcon = new AdBlockIcon(this); m_adblockIcon->setObjectName(QSL("m_adblockIconAction")); - m_unifiedFiltersFile = qApp->userDataFolder() + QDir::separator() + QSL("adblock-unified-filters.txt"); + m_serverProcess = new QProcess(this); } -bool AdBlockManager::block(const AdblockRequestInfo& request) const { +AdBlockManager::~AdBlockManager() { + if (m_serverProcess->state() == QProcess::ProcessState::Running) { + m_serverProcess->kill(); + } +} + +BlockingResult AdBlockManager::block(const AdblockRequestInfo& request) const { if (!isEnabled()) { - return false; + return { false }; } const QString url_string = request.requestUrl().toEncoded().toLower(); const QString url_scheme = request.requestUrl().scheme().toLower(); if (!canRunOnScheme(url_scheme)) { - return false; + return { false }; } else { - // TODO: start server if needed, call it. - return false; + if (m_serverProcess->state() == QProcess::ProcessState::Running) { + try { + auto result = askServerIfBlocked(url_string); + + return result; + } + catch (const ApplicationException& ex) { + qCriticalNN << LOGSEC_ADBLOCK + << "HTTP error when calling server:" + << QUOTE_W_SPACE_DOT(ex.message()); + } + } + else { + return { false }; + } } } @@ -123,8 +146,106 @@ void AdBlockManager::showDialog() { AdBlockDialog(qApp->mainFormWidget()).exec(); } +BlockingResult AdBlockManager::askServerIfBlocked(const QString& url) const { + QJsonObject req_obj; + QByteArray out; + QElapsedTimer tmr; + + req_obj["url"] = url; + req_obj["filter"] = true; + + tmr.start(); + + auto network_res = NetworkFactory::performNetworkOperation(QSL("http://%1:%2").arg(QHostAddress(QHostAddress::SpecialAddress::LocalHost).toString(), + ADBLOCK_SERVER_PORT), + 500, + QJsonDocument(req_obj).toJson(), + out, + QNetworkAccessManager::Operation::PostOperation, + { { + QSL(HTTP_HEADERS_CONTENT_TYPE).toLocal8Bit(), + QSL("application/json").toLocal8Bit() } }); + + if (network_res.first == QNetworkReply::NetworkError::NoError) { + qDebugNN << LOGSEC_ADBLOCK + << "Query to server took " + << tmr.elapsed() + << " ms."; + + QJsonObject out_obj = QJsonDocument::fromJson(out).object(); + bool blocking = out_obj["filter"].toObject()["match"].toBool(); + + return { + blocking, + blocking + ? out_obj["filter"].toObject()["filter"].toObject()["filter"].toString() + : QString() + }; + } + else { + throw NetworkException(network_res.first); + } +} + void AdBlockManager::restartServer() { - // TODO: + if (m_serverProcess->state() == QProcess::ProcessState::Running) { + m_serverProcess->kill(); + + if (!m_serverProcess->waitForFinished(1000)) { + m_serverProcess->deleteLater(); + m_serverProcess = new QProcess(this); + } + } + + QString temp_server = QDir::toNativeSeparators(IOFactory::getSystemFolder(QStandardPaths::StandardLocation::TempLocation)) + + QDir::separator() + + QSL("adblock-server.js"); + + if (!IOFactory::copyFile(QSL(":/scripts/adblock/adblock-server.js"), temp_server)) { + qCriticalNN << LOGSEC_ADBLOCK << "Failed to copy server file to TEMP."; + } + +#if defined(Q_OS_WIN) + m_serverProcess->setProgram(QSL("node.exe")); +#else + m_serverProcess->setProgram(QSL("node")); +#endif + + m_serverProcess->setArguments({ + QDir::toNativeSeparators(temp_server), + ADBLOCK_SERVER_PORT, + QDir::toNativeSeparators(m_unifiedFiltersFile) + }); + + m_serverProcess->setProcessEnvironment(QProcessEnvironment::systemEnvironment()); + + auto pe = m_serverProcess->processEnvironment(); + QString node_path = +#if defined(Q_OS_WIN) + pe.value(QSL("APPDATA")) + +#elif defined(Q_OS_LINUX) + QSL("/usr/local/lib/node_modules") + +#else + QDir::toNativeSeparators(IOFactory::getSystemFolder(QStandardPaths::StandardLocation::GenericDataLocation)) + +#endif + QDir::separator() + + QSL("npm") + + QDir::separator() + + QSL("node_modules"); + + if (!pe.contains(QSL("NODE_PATH"))) { + pe.insert(QSL("NODE_PATH"), node_path); + } + + m_serverProcess->setProcessEnvironment(pe); + m_serverProcess->setProcessChannelMode(QProcess::ProcessChannelMode::ForwardedErrorChannel); + + if (!m_serverProcess->open()) { + qWarningNN << LOGSEC_ADBLOCK << "Failed to start server."; + } + else { + qDebugNN << LOGSEC_ADBLOCK << "Started server."; + } } void AdBlockManager::updateUnifiedFiltersFile() { @@ -132,8 +253,7 @@ void AdBlockManager::updateUnifiedFiltersFile() { QFile::remove(m_unifiedFiltersFile); } - // TODO: generate file - QByteArray unified_contents; + QString unified_contents; auto filter_lists = filterLists(); // Download filters one by one and append. @@ -142,10 +262,16 @@ void AdBlockManager::updateUnifiedFiltersFile() { auto res = NetworkFactory::performNetworkOperation(filter_list_url, 2000, {}, - out, QNetworkAccessManager::Operation::GetOperation); + out, + QNetworkAccessManager::Operation::GetOperation); if (res.first == QNetworkReply::NetworkError::NoError) { - unified_contents += out; + unified_contents = unified_contents.append(QString::fromUtf8(out)); + unified_contents = unified_contents.append('\n'); + + qDebugNN << LOGSEC_ADBLOCK + << "Downloaded filter list from" + << QUOTE_W_SPACE_DOT(filter_list_url); } else { qWarningNN << LOGSEC_ADBLOCK @@ -156,7 +282,7 @@ void AdBlockManager::updateUnifiedFiltersFile() { } } - unified_contents += customFilters().join(QSL("\n")).toUtf8(); + unified_contents = unified_contents.append(customFilters().join(QSL("\n"))); // Save. m_unifiedFiltersFile = IOFactory::getSystemFolder(QStandardPaths::StandardLocation::TempLocation) + @@ -164,10 +290,9 @@ void AdBlockManager::updateUnifiedFiltersFile() { QSL("adblock.filters"); try { - IOFactory::writeFile(m_unifiedFiltersFile, unified_contents); + IOFactory::writeFile(m_unifiedFiltersFile, unified_contents.toUtf8()); if (m_enabled) { - // TODO: re-start nodejs adblock server. restartServer(); } } diff --git a/src/librssguard/network-web/adblock/adblockmanager.h b/src/librssguard/network-web/adblock/adblockmanager.h index 2e856642c..161fd8a41 100644 --- a/src/librssguard/network-web/adblock/adblockmanager.h +++ b/src/librssguard/network-web/adblock/adblockmanager.h @@ -6,15 +6,26 @@ #include class QUrl; +class QProcess; class AdblockRequestInfo; class AdBlockUrlInterceptor; class AdBlockIcon; +struct BlockingResult { + bool m_blocked; + QString m_blockedByFilter; + + BlockingResult(bool blocked, QString blocked_by_filter = {}) + : m_blocked(blocked), m_blockedByFilter(std::move(blocked_by_filter)) {} + +}; + class AdBlockManager : public QObject { Q_OBJECT public: explicit AdBlockManager(QObject* parent = nullptr); + virtual ~AdBlockManager(); // If "initial_load" is false, then we want to explicitly turn off // Adblock if it is running or turn on when not running. @@ -27,7 +38,7 @@ class AdBlockManager : public QObject { AdBlockIcon* adBlockIcon() const; // General methods for adblocking. - bool block(const AdblockRequestInfo& request) const; + BlockingResult block(const AdblockRequestInfo& request) const; QString elementHidingRulesForDomain(const QUrl& url) const; QStringList filterLists() const; @@ -47,6 +58,8 @@ class AdBlockManager : public QObject { void enabledChanged(bool enabled); private: + BlockingResult askServerIfBlocked(const QString& url) const; + void restartServer(); private: @@ -55,6 +68,7 @@ class AdBlockManager : public QObject { AdBlockIcon* m_adblockIcon; AdBlockUrlInterceptor* m_interceptor; QString m_unifiedFiltersFile; + QProcess* m_serverProcess; }; inline AdBlockIcon* AdBlockManager::adBlockIcon() const { diff --git a/src/librssguard/network-web/adblock/adblockurlinterceptor.cpp b/src/librssguard/network-web/adblock/adblockurlinterceptor.cpp index ee26e0a00..a33df98ba 100644 --- a/src/librssguard/network-web/adblock/adblockurlinterceptor.cpp +++ b/src/librssguard/network-web/adblock/adblockurlinterceptor.cpp @@ -27,7 +27,7 @@ AdBlockUrlInterceptor::AdBlockUrlInterceptor(AdBlockManager* manager) : UrlInterceptor(manager), m_manager(manager) {} void AdBlockUrlInterceptor::interceptRequest(QWebEngineUrlRequestInfo& info) { - if (m_manager->block(AdblockRequestInfo(info))) { + if (m_manager->block(AdblockRequestInfo(info)).m_blocked) { info.block(true); qWarningNN << LOGSEC_ADBLOCK << "Blocked request:" << QUOTE_W_SPACE_DOT(info.requestUrl().toString()); diff --git a/src/librssguard/network-web/webpage.cpp b/src/librssguard/network-web/webpage.cpp index 0d6cf8f42..059695512 100644 --- a/src/librssguard/network-web/webpage.cpp +++ b/src/librssguard/network-web/webpage.cpp @@ -48,9 +48,9 @@ bool WebPage::acceptNavigationRequest(const QUrl& url, NavigationType type, bool if (is_main_frame) { auto blocked = qApp->web()->adBlock()->block(AdblockRequestInfo(url)); - if (blocked) { + if (blocked.m_blocked) { // This website is entirely blocked. - setHtml(qApp->skins()->adBlockedPage(url.toString()), + setHtml(qApp->skins()->adBlockedPage(url.toString(), blocked.m_blockedByFilter), QUrl::fromUserInput(INTERNAL_URL_ADBLOCKED)); return false; }