work on throttling and 429s

src/librssguard-standard/src/definitions.h

@@ -8,10 +8,6 @@
 
 #define ADVANCED_FEED_ADD_DIALOG_CODE 64
 
-#define HTTP_CODE_NOT_MODIFIED      304
-#define HTTP_CODE_TOO_MANY_REQUESTS 429
-#define HTTP_CODE_UNAVAILABLE       503
-
 #define RSS_REGEX_MATCHER      "<link[^>]+type=\"application\\/(?:rss\\+xml)\"[^>]*>"
 #define RSS_HREF_REGEX_MATCHER "href=\"([^\"]+)\""
 

src/librssguard-standard/src/standardserviceroot.cpp

@@ -195,10 +195,6 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const {
   return ServiceRoot::additionalFlags() | Qt::ItemFlag::ItemIsDragEnabled | Qt::ItemFlag::ItemIsDropEnabled;
 }
 
-void StandardServiceRoot::clearFeedOverload(StandardFeed* feed) {
-  m_overloadedHosts.remove(QUrl(feed->source()).host());
-}
-
 QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
                                                       const QHash<ServiceRoot::BagOfMessages, QStringList>&
                                                         stated_messages,
@@ -207,13 +203,6 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
   Q_UNUSED(tagged_messages)
 
   StandardFeed* f = static_cast<StandardFeed*>(feed);
-
-  if (checkIfFeedOverloaded(f)) {
-    qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(f->source())
-               << "was signalled temporarily being down. Returning no articles for now.";
-    return {};
-  }
-
   QByteArray feed_contents;
   QString formatted_feed_contents;
   int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
@@ -244,27 +233,12 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
                                                                   f->http2Status());
 
     if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) {
-      if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS ||
-          network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) {
-
-        QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after")));
-
-        m_overloadedHosts.insert(QUrl(f->source()).host(), safe_dt);
-
-        qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source())
-                   << "indicates that there is too many requests right now on the same host.";
-        return {};
-      }
-      else {
       qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError)
                  << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
       throw FeedFetchException(Feed::Status::NetworkError,
                                NetworkFactory::networkErrorText(network_result.m_networkError));
     }
-    }
     else {
-      clearFeedOverload(f);
-
       f->setLastEtag(network_result.m_headers.value(QSL("etag")));
 
       if (network_result.m_httpCode == HTTP_CODE_NOT_MODIFIED && feed_contents.trimmed().isEmpty()) {
@@ -592,19 +566,6 @@ void StandardServiceRoot::exportFeeds() {
   form.data()->exec();
 }
 
-bool StandardServiceRoot::checkIfFeedOverloaded(StandardFeed* feed) const {
-  if (feed->sourceType() == StandardFeed::SourceType::Url ||
-      feed->sourceType() == StandardFeed::SourceType::EmbeddedBrowser) {
-    QString hostname = QUrl(feed->source()).host();
-    QDateTime retry_after = m_overloadedHosts.value(hostname);
-
-    return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc();
-  }
-  else {
-    return false;
-  }
-}
-
 QList<QAction*> StandardServiceRoot::serviceMenu() {
   if (m_serviceMenu.isEmpty()) {
     ServiceRoot::serviceMenu();

src/librssguard-standard/src/standardserviceroot.h

@@ -54,9 +54,6 @@ class StandardServiceRoot : public ServiceRoot {
     void exportFeeds();
 
   private:
-    void clearFeedOverload(StandardFeed* feed);
-    bool checkIfFeedOverloaded(StandardFeed* feed) const;
-
     // Takes structure residing under given root item and adds feeds/categories from
     // it to active structure.
     // NOTE: This is used for import/export of the model.
@@ -64,8 +61,6 @@ class StandardServiceRoot : public ServiceRoot {
 
     QPointer<StandardFeed> m_feedForMetadata = {};
     QList<QAction*> m_feedContextMenu = {};
-
-    QHash<QString, QDateTime> m_overloadedHosts;
 };
 
 #endif // STANDARDSERVICEROOT_H

src/librssguard/core/feeddownloader.cpp

@@ -170,6 +170,17 @@ void FeedDownloader::updateFeeds(const QList<Feed*>& feeds) {
   }
 }
 
+void FeedDownloader::clearFeedOverload(Feed* feed) {
+  m_overloadedHosts.remove(QUrl(feed->source()).host());
+}
+
+bool FeedDownloader::checkIfFeedOverloaded(Feed* feed) const {
+  QString hostname = QUrl(feed->source()).host();
+  QDateTime retry_after = m_overloadedHosts.value(hostname);
+
+  return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc();
+}
+
 FeedUpdateResult FeedDownloader::updateThreadedFeed(const FeedUpdateRequest& fd) {
   if (m_erroredAccounts.contains(fd.account)) {
     // This feed is errored because its account errored when preparing feed update.
@@ -214,11 +225,24 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
                                    Feed* feed,
                                    const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
                                    const QHash<QString, QStringList>& tagged_messages) {
-  feed->setStatus(Feed::Status::Fetching);
-
   const bool update_feed_list =
     qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateFeedListDuringFetching)).toBool();
 
+  if (checkIfFeedOverloaded(feed)) {
+    qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(feed->source())
+               << "was signalled temporarily being down. Returning no articles for now.";
+
+    feed->setStatus(Feed::Status::NetworkError, tr("feed is in network cooldown mode"));
+
+    if (update_feed_list) {
+      acc->itemChanged({feed});
+    }
+
+    return;
+  }
+
+  feed->setStatus(Feed::Status::Fetching);
+
   if (update_feed_list) {
     acc->itemChanged({feed});
   }
@@ -241,6 +265,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
              << QUOTE_W_SPACE_COMMA(feed->customId()) << "operation took" << NONQUOTE_W_SPACE(tmr.nsecsElapsed() / 1000)
              << "microseconds.";
 
+    clearFeedOverload(feed);
+
     bool fix_future_datetimes =
       qApp->settings()->value(GROUP(Messages), SETTING(Messages::FixupFutureArticleDateTimes)).toBool();
 
@@ -420,6 +446,19 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
                 << "message:" << QUOTE_W_SPACE_DOT(feed_ex.message());
 
     feed->setStatus(feed_ex.feedStatus(), feed_ex.message());
+
+    if (feed_ex.feedStatus() == Feed::Status::NetworkError && !feed_ex.data().isNull()) {
+      NetworkResult network_result = feed_ex.data().value<NetworkResult>();
+
+      if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS ||
+          network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) {
+        QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after")));
+        m_overloadedHosts.insert(QUrl(feed->source()).host(), safe_dt);
+
+        qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source())
+                   << "indicates that there is too many requests right now on the same host.";
+      }
+    }
   }
   catch (const ApplicationException& app_ex) {
     qCriticalNN << LOGSEC_NETWORK << "Unknown error when fetching feed:"

src/librssguard/core/feeddownloader.h

@@ -63,6 +63,9 @@ class FeedDownloader : public QObject {
     void updateProgress(const Feed* feed, int current, int total);
 
   private:
+    void clearFeedOverload(Feed* feed);
+    bool checkIfFeedOverloaded(Feed* feed) const;
+
     void skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex);
     void updateOneFeed(ServiceRoot* acc,
                        Feed* feed,
@@ -82,6 +85,7 @@ class FeedDownloader : public QObject {
     QList<FeedUpdateRequest> m_feeds = {};
     QFutureWatcher<FeedUpdateResult> m_watcherLookup;
     FeedDownloadResults m_results;
+    QHash<QString, QDateTime> m_overloadedHosts;
 };
 
 #endif // FEEDDOWNLOADER_H

src/librssguard/definitions/definitions.h

@@ -154,6 +154,10 @@
 
 #define CLI_THREADS "threads"
 
+#define HTTP_CODE_NOT_MODIFIED      304
+#define HTTP_CODE_TOO_MANY_REQUESTS 429
+#define HTTP_CODE_UNAVAILABLE       503
+
 #define HTTP_HEADERS_ACCEPT         "Accept"
 #define HTTP_HEADERS_CONTENT_TYPE   "Content-Type"
 #define HTTP_HEADERS_CONTENT_LENGTH "Content-Length"

src/librssguard/exceptions/feedfetchexception.cpp

@@ -2,9 +2,13 @@
 
 #include "exceptions/feedfetchexception.h"
 
-FeedFetchException::FeedFetchException(Feed::Status feed_status, const QString& message)
+FeedFetchException::FeedFetchException(Feed::Status feed_status, const QString& message, const QVariant& data)
-  : ApplicationException(message), m_feedStatus(feed_status) {}
+  : ApplicationException(message), m_data(data), m_feedStatus(feed_status) {}
 
 Feed::Status FeedFetchException::feedStatus() const {
   return m_feedStatus;
 }
+
+QVariant FeedFetchException::data() const {
+  return m_data;
+}

src/librssguard/exceptions/feedfetchexception.h

@@ -8,11 +8,16 @@
 
 class RSSGUARD_DLLSPEC FeedFetchException : public ApplicationException {
   public:
-    explicit FeedFetchException(Feed::Status feed_status, const QString& message = {});
+    // "data" parameter should contain this, depending on "feed_status":
+    //   - Feed::Status::NetworkError -> NetworkResult instance
+    //   - other feed status values -> arbitrary data
+    explicit FeedFetchException(Feed::Status feed_status, const QString& message = {}, const QVariant& data = {});
 
     Feed::Status feedStatus() const;
+    QVariant data() const;
 
   private:
+    QVariant m_data;
     Feed::Status m_feedStatus;
 };
 

src/librssguard/network-web/networkfactory.h

@@ -96,6 +96,7 @@ class RSSGUARD_DLLSPEC NetworkFactory {
                                                  Http2Status http2_status = Http2Status::DontSet);
 };
 
+Q_DECLARE_METATYPE(NetworkResult)
 Q_DECLARE_METATYPE(NetworkFactory::NetworkAuthentication)
 
 #endif // NETWORKFACTORY_H