diff --git a/src/librssguard-standard/src/definitions.h b/src/librssguard-standard/src/definitions.h index 4e3dcc525..484c58977 100644 --- a/src/librssguard-standard/src/definitions.h +++ b/src/librssguard-standard/src/definitions.h @@ -8,10 +8,6 @@ #define ADVANCED_FEED_ADD_DIALOG_CODE 64 -#define HTTP_CODE_NOT_MODIFIED 304 -#define HTTP_CODE_TOO_MANY_REQUESTS 429 -#define HTTP_CODE_UNAVAILABLE 503 - #define RSS_REGEX_MATCHER "]+type=\"application\\/(?:rss\\+xml)\"[^>]*>" #define RSS_HREF_REGEX_MATCHER "href=\"([^\"]+)\"" diff --git a/src/librssguard-standard/src/standardserviceroot.cpp b/src/librssguard-standard/src/standardserviceroot.cpp index 43b1da640..55a54cbb5 100644 --- a/src/librssguard-standard/src/standardserviceroot.cpp +++ b/src/librssguard-standard/src/standardserviceroot.cpp @@ -195,10 +195,6 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const { return ServiceRoot::additionalFlags() | Qt::ItemFlag::ItemIsDragEnabled | Qt::ItemFlag::ItemIsDropEnabled; } -void StandardServiceRoot::clearFeedOverload(StandardFeed* feed) { - m_overloadedHosts.remove(QUrl(feed->source()).host()); -} - QList StandardServiceRoot::obtainNewMessages(Feed* feed, const QHash& stated_messages, @@ -207,13 +203,6 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, Q_UNUSED(tagged_messages) StandardFeed* f = static_cast(feed); - - if (checkIfFeedOverloaded(f)) { - qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(f->source()) - << "was signalled temporarily being down. Returning no articles for now."; - return {}; - } - QByteArray feed_contents; QString formatted_feed_contents; int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); @@ -244,27 +233,12 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, f->http2Status()); if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) { - if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS || - network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) { - - QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after"))); - - m_overloadedHosts.insert(QUrl(f->source()).host(), safe_dt); - - qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source()) - << "indicates that there is too many requests right now on the same host."; - return {}; - } - else { - qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError) - << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source()); - throw FeedFetchException(Feed::Status::NetworkError, - NetworkFactory::networkErrorText(network_result.m_networkError)); - } + qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError) + << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source()); + throw FeedFetchException(Feed::Status::NetworkError, + NetworkFactory::networkErrorText(network_result.m_networkError)); } else { - clearFeedOverload(f); - f->setLastEtag(network_result.m_headers.value(QSL("etag"))); if (network_result.m_httpCode == HTTP_CODE_NOT_MODIFIED && feed_contents.trimmed().isEmpty()) { @@ -592,19 +566,6 @@ void StandardServiceRoot::exportFeeds() { form.data()->exec(); } -bool StandardServiceRoot::checkIfFeedOverloaded(StandardFeed* feed) const { - if (feed->sourceType() == StandardFeed::SourceType::Url || - feed->sourceType() == StandardFeed::SourceType::EmbeddedBrowser) { - QString hostname = QUrl(feed->source()).host(); - QDateTime retry_after = m_overloadedHosts.value(hostname); - - return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc(); - } - else { - return false; - } -} - QList StandardServiceRoot::serviceMenu() { if (m_serviceMenu.isEmpty()) { ServiceRoot::serviceMenu(); diff --git a/src/librssguard-standard/src/standardserviceroot.h b/src/librssguard-standard/src/standardserviceroot.h index 482e309c9..5133d4e0c 100644 --- a/src/librssguard-standard/src/standardserviceroot.h +++ b/src/librssguard-standard/src/standardserviceroot.h @@ -54,9 +54,6 @@ class StandardServiceRoot : public ServiceRoot { void exportFeeds(); private: - void clearFeedOverload(StandardFeed* feed); - bool checkIfFeedOverloaded(StandardFeed* feed) const; - // Takes structure residing under given root item and adds feeds/categories from // it to active structure. // NOTE: This is used for import/export of the model. @@ -64,8 +61,6 @@ class StandardServiceRoot : public ServiceRoot { QPointer m_feedForMetadata = {}; QList m_feedContextMenu = {}; - - QHash m_overloadedHosts; }; #endif // STANDARDSERVICEROOT_H diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 8eaca567e..ce2969d27 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -170,6 +170,17 @@ void FeedDownloader::updateFeeds(const QList& feeds) { } } +void FeedDownloader::clearFeedOverload(Feed* feed) { + m_overloadedHosts.remove(QUrl(feed->source()).host()); +} + +bool FeedDownloader::checkIfFeedOverloaded(Feed* feed) const { + QString hostname = QUrl(feed->source()).host(); + QDateTime retry_after = m_overloadedHosts.value(hostname); + + return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc(); +} + FeedUpdateResult FeedDownloader::updateThreadedFeed(const FeedUpdateRequest& fd) { if (m_erroredAccounts.contains(fd.account)) { // This feed is errored because its account errored when preparing feed update. @@ -214,11 +225,24 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, Feed* feed, const QHash& stated_messages, const QHash& tagged_messages) { - feed->setStatus(Feed::Status::Fetching); - const bool update_feed_list = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateFeedListDuringFetching)).toBool(); + if (checkIfFeedOverloaded(feed)) { + qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(feed->source()) + << "was signalled temporarily being down. Returning no articles for now."; + + feed->setStatus(Feed::Status::NetworkError, tr("feed is in network cooldown mode")); + + if (update_feed_list) { + acc->itemChanged({feed}); + } + + return; + } + + feed->setStatus(Feed::Status::Fetching); + if (update_feed_list) { acc->itemChanged({feed}); } @@ -241,6 +265,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, << QUOTE_W_SPACE_COMMA(feed->customId()) << "operation took" << NONQUOTE_W_SPACE(tmr.nsecsElapsed() / 1000) << "microseconds."; + clearFeedOverload(feed); + bool fix_future_datetimes = qApp->settings()->value(GROUP(Messages), SETTING(Messages::FixupFutureArticleDateTimes)).toBool(); @@ -420,6 +446,19 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, << "message:" << QUOTE_W_SPACE_DOT(feed_ex.message()); feed->setStatus(feed_ex.feedStatus(), feed_ex.message()); + + if (feed_ex.feedStatus() == Feed::Status::NetworkError && !feed_ex.data().isNull()) { + NetworkResult network_result = feed_ex.data().value(); + + if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS || + network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) { + QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after"))); + m_overloadedHosts.insert(QUrl(feed->source()).host(), safe_dt); + + qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source()) + << "indicates that there is too many requests right now on the same host."; + } + } } catch (const ApplicationException& app_ex) { qCriticalNN << LOGSEC_NETWORK << "Unknown error when fetching feed:" diff --git a/src/librssguard/core/feeddownloader.h b/src/librssguard/core/feeddownloader.h index bbe64695f..bc235367d 100644 --- a/src/librssguard/core/feeddownloader.h +++ b/src/librssguard/core/feeddownloader.h @@ -63,6 +63,9 @@ class FeedDownloader : public QObject { void updateProgress(const Feed* feed, int current, int total); private: + void clearFeedOverload(Feed* feed); + bool checkIfFeedOverloaded(Feed* feed) const; + void skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex); void updateOneFeed(ServiceRoot* acc, Feed* feed, @@ -82,6 +85,7 @@ class FeedDownloader : public QObject { QList m_feeds = {}; QFutureWatcher m_watcherLookup; FeedDownloadResults m_results; + QHash m_overloadedHosts; }; #endif // FEEDDOWNLOADER_H diff --git a/src/librssguard/definitions/definitions.h b/src/librssguard/definitions/definitions.h index e454c14c8..73b017eec 100644 --- a/src/librssguard/definitions/definitions.h +++ b/src/librssguard/definitions/definitions.h @@ -154,6 +154,10 @@ #define CLI_THREADS "threads" +#define HTTP_CODE_NOT_MODIFIED 304 +#define HTTP_CODE_TOO_MANY_REQUESTS 429 +#define HTTP_CODE_UNAVAILABLE 503 + #define HTTP_HEADERS_ACCEPT "Accept" #define HTTP_HEADERS_CONTENT_TYPE "Content-Type" #define HTTP_HEADERS_CONTENT_LENGTH "Content-Length" diff --git a/src/librssguard/exceptions/feedfetchexception.cpp b/src/librssguard/exceptions/feedfetchexception.cpp index 36fd1b87f..503322660 100644 --- a/src/librssguard/exceptions/feedfetchexception.cpp +++ b/src/librssguard/exceptions/feedfetchexception.cpp @@ -2,9 +2,13 @@ #include "exceptions/feedfetchexception.h" -FeedFetchException::FeedFetchException(Feed::Status feed_status, const QString& message) - : ApplicationException(message), m_feedStatus(feed_status) {} +FeedFetchException::FeedFetchException(Feed::Status feed_status, const QString& message, const QVariant& data) + : ApplicationException(message), m_data(data), m_feedStatus(feed_status) {} Feed::Status FeedFetchException::feedStatus() const { return m_feedStatus; } + +QVariant FeedFetchException::data() const { + return m_data; +} diff --git a/src/librssguard/exceptions/feedfetchexception.h b/src/librssguard/exceptions/feedfetchexception.h index aef78197b..ce374a4a0 100644 --- a/src/librssguard/exceptions/feedfetchexception.h +++ b/src/librssguard/exceptions/feedfetchexception.h @@ -8,11 +8,16 @@ class RSSGUARD_DLLSPEC FeedFetchException : public ApplicationException { public: - explicit FeedFetchException(Feed::Status feed_status, const QString& message = {}); + // "data" parameter should contain this, depending on "feed_status": + // - Feed::Status::NetworkError -> NetworkResult instance + // - other feed status values -> arbitrary data + explicit FeedFetchException(Feed::Status feed_status, const QString& message = {}, const QVariant& data = {}); Feed::Status feedStatus() const; + QVariant data() const; private: + QVariant m_data; Feed::Status m_feedStatus; }; diff --git a/src/librssguard/network-web/networkfactory.h b/src/librssguard/network-web/networkfactory.h index d0f5bdbca..f1178f53d 100644 --- a/src/librssguard/network-web/networkfactory.h +++ b/src/librssguard/network-web/networkfactory.h @@ -96,6 +96,7 @@ class RSSGUARD_DLLSPEC NetworkFactory { Http2Status http2_status = Http2Status::DontSet); }; +Q_DECLARE_METATYPE(NetworkResult) Q_DECLARE_METATYPE(NetworkFactory::NetworkAuthentication) #endif // NETWORKFACTORY_H