pogmommy/rssguard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

work on throttling and 429s

Browse source
This commit is contained in:
Martin Rotter 2025-02-04 15:17:30 +01:00
parent 6be83ed393
commit 2e0288d3d4
9 changed files with 66 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/librssguard-standard/src/definitions.h
View file

@ -8,10 +8,6 @@
#define ADVANCED_FEED_ADD_DIALOG_CODE 64
#define HTTP_CODE_NOT_MODIFIED 304
#define HTTP_CODE_TOO_MANY_REQUESTS 429
#define HTTP_CODE_UNAVAILABLE 503
#define RSS_REGEX_MATCHER "<link[^>]+type=\"application\\/(?:rss\\+xml)\"[^>]*>"
#define RSS_HREF_REGEX_MATCHER "href=\"([^\"]+)\""

47
src/librssguard-standard/src/standardserviceroot.cpp
View file

@ -195,10 +195,6 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const {
return ServiceRoot::additionalFlags() | Qt::ItemFlag::ItemIsDragEnabled | Qt::ItemFlag::ItemIsDropEnabled;
}
void StandardServiceRoot::clearFeedOverload(StandardFeed* feed) {
m_overloadedHosts.remove(QUrl(feed->source()).host());
}
QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>&
stated_messages,
@ -207,13 +203,6 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
Q_UNUSED(tagged_messages)
StandardFeed* f = static_cast<StandardFeed*>(feed);
if (checkIfFeedOverloaded(f)) {
qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(f->source())
<< "was signalled temporarily being down. Returning no articles for now.";
return {};
}
QByteArray feed_contents;
QString formatted_feed_contents;
int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
@ -244,27 +233,12 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
f->http2Status());
if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) {
if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS ||
network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) {
QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after")));
m_overloadedHosts.insert(QUrl(f->source()).host(), safe_dt);
qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source())
<< "indicates that there is too many requests right now on the same host.";
return {};
}
else {
qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError)
<< "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
throw FeedFetchException(Feed::Status::NetworkError,
NetworkFactory::networkErrorText(network_result.m_networkError));
}
qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError)
<< "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
throw FeedFetchException(Feed::Status::NetworkError,
NetworkFactory::networkErrorText(network_result.m_networkError));
}
else {
clearFeedOverload(f);
f->setLastEtag(network_result.m_headers.value(QSL("etag")));
if (network_result.m_httpCode == HTTP_CODE_NOT_MODIFIED && feed_contents.trimmed().isEmpty()) {
@ -592,19 +566,6 @@ void StandardServiceRoot::exportFeeds() {
form.data()->exec();
}
bool StandardServiceRoot::checkIfFeedOverloaded(StandardFeed* feed) const {
if (feed->sourceType() == StandardFeed::SourceType::Url ||
feed->sourceType() == StandardFeed::SourceType::EmbeddedBrowser) {
QString hostname = QUrl(feed->source()).host();
QDateTime retry_after = m_overloadedHosts.value(hostname);
return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc();
}
else {
return false;
}
}
QList<QAction*> StandardServiceRoot::serviceMenu() {
if (m_serviceMenu.isEmpty()) {
ServiceRoot::serviceMenu();

5
src/librssguard-standard/src/standardserviceroot.h
View file

@ -54,9 +54,6 @@ class StandardServiceRoot : public ServiceRoot {
void exportFeeds();
private:
void clearFeedOverload(StandardFeed* feed);
bool checkIfFeedOverloaded(StandardFeed* feed) const;
// Takes structure residing under given root item and adds feeds/categories from
// it to active structure.
// NOTE: This is used for import/export of the model.
@ -64,8 +61,6 @@ class StandardServiceRoot : public ServiceRoot {
QPointer<StandardFeed> m_feedForMetadata = {};
QList<QAction*> m_feedContextMenu = {};
QHash<QString, QDateTime> m_overloadedHosts;
};
#endif // STANDARDSERVICEROOT_H

43
src/librssguard/core/feeddownloader.cpp
View file

@ -170,6 +170,17 @@ void FeedDownloader::updateFeeds(const QList<Feed*>& feeds) {
}
}
void FeedDownloader::clearFeedOverload(Feed* feed) {
m_overloadedHosts.remove(QUrl(feed->source()).host());
}
bool FeedDownloader::checkIfFeedOverloaded(Feed* feed) const {
QString hostname = QUrl(feed->source()).host();
QDateTime retry_after = m_overloadedHosts.value(hostname);
return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc();
}
FeedUpdateResult FeedDownloader::updateThreadedFeed(const FeedUpdateRequest& fd) {
if (m_erroredAccounts.contains(fd.account)) {
// This feed is errored because its account errored when preparing feed update.
@ -214,11 +225,24 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
feed->setStatus(Feed::Status::Fetching);
const bool update_feed_list =
qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateFeedListDuringFetching)).toBool();
if (checkIfFeedOverloaded(feed)) {
qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(feed->source())
<< "was signalled temporarily being down. Returning no articles for now.";
feed->setStatus(Feed::Status::NetworkError, tr("feed is in network cooldown mode"));
if (update_feed_list) {
acc->itemChanged({feed});
}
return;
}
feed->setStatus(Feed::Status::Fetching);
if (update_feed_list) {
acc->itemChanged({feed});
}
@ -241,6 +265,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
<< QUOTE_W_SPACE_COMMA(feed->customId()) << "operation took" << NONQUOTE_W_SPACE(tmr.nsecsElapsed() / 1000)
<< "microseconds.";
clearFeedOverload(feed);
bool fix_future_datetimes =
qApp->settings()->value(GROUP(Messages), SETTING(Messages::FixupFutureArticleDateTimes)).toBool();
@ -420,6 +446,19 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
<< "message:" << QUOTE_W_SPACE_DOT(feed_ex.message());
feed->setStatus(feed_ex.feedStatus(), feed_ex.message());
if (feed_ex.feedStatus() == Feed::Status::NetworkError && !feed_ex.data().isNull()) {
NetworkResult network_result = feed_ex.data().value<NetworkResult>();
if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS ||
network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) {
QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after")));
m_overloadedHosts.insert(QUrl(feed->source()).host(), safe_dt);
qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source())
<< "indicates that there is too many requests right now on the same host.";
}
}
}
catch (const ApplicationException& app_ex) {
qCriticalNN << LOGSEC_NETWORK << "Unknown error when fetching feed:"

4
src/librssguard/core/feeddownloader.h
View file

@ -63,6 +63,9 @@ class FeedDownloader : public QObject {
void updateProgress(const Feed* feed, int current, int total);
private:
void clearFeedOverload(Feed* feed);
bool checkIfFeedOverloaded(Feed* feed) const;
void skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex);
void updateOneFeed(ServiceRoot* acc,
Feed* feed,
@ -82,6 +85,7 @@ class FeedDownloader : public QObject {
QList<FeedUpdateRequest> m_feeds = {};
QFutureWatcher<FeedUpdateResult> m_watcherLookup;
FeedDownloadResults m_results;
QHash<QString, QDateTime> m_overloadedHosts;
};
#endif // FEEDDOWNLOADER_H

4
src/librssguard/definitions/definitions.h
View file

@ -154,6 +154,10 @@
#define CLI_THREADS "threads"
#define HTTP_CODE_NOT_MODIFIED 304
#define HTTP_CODE_TOO_MANY_REQUESTS 429
#define HTTP_CODE_UNAVAILABLE 503
#define HTTP_HEADERS_ACCEPT "Accept"
#define HTTP_HEADERS_CONTENT_TYPE "Content-Type"
#define HTTP_HEADERS_CONTENT_LENGTH "Content-Length"

8
src/librssguard/exceptions/feedfetchexception.cpp
View file

@ -2,9 +2,13 @@
#include "exceptions/feedfetchexception.h"
FeedFetchException::FeedFetchException(Feed::Status feed_status, const QString& message)
: ApplicationException(message), m_feedStatus(feed_status) {}
FeedFetchException::FeedFetchException(Feed::Status feed_status, const QString& message, const QVariant& data)
: ApplicationException(message), m_data(data), m_feedStatus(feed_status) {}
Feed::Status FeedFetchException::feedStatus() const {
return m_feedStatus;
}
QVariant FeedFetchException::data() const {
return m_data;
}

7
src/librssguard/exceptions/feedfetchexception.h
View file

@ -8,11 +8,16 @@
class RSSGUARD_DLLSPEC FeedFetchException : public ApplicationException {
public:
explicit FeedFetchException(Feed::Status feed_status, const QString& message = {});
// "data" parameter should contain this, depending on "feed_status":
// - Feed::Status::NetworkError -> NetworkResult instance
// - other feed status values -> arbitrary data
explicit FeedFetchException(Feed::Status feed_status, const QString& message = {}, const QVariant& data = {});
Feed::Status feedStatus() const;
QVariant data() const;
private:
QVariant m_data;
Feed::Status m_feedStatus;
};

1
src/librssguard/network-web/networkfactory.h
View file

@ -96,6 +96,7 @@ class RSSGUARD_DLLSPEC NetworkFactory {
Http2Status http2_status = Http2Status::DontSet);
};
Q_DECLARE_METATYPE(NetworkResult)
Q_DECLARE_METATYPE(NetworkFactory::NetworkAuthentication)
#endif // NETWORKFACTORY_H