initial untested implementation for #1322

src/librssguard-standard/src/definitions.h

@@ -8,6 +8,10 @@
 
 #define ADVANCED_FEED_ADD_DIALOG_CODE 64
 
+#define HTTP_CODE_NOT_MODIFIED      304
+#define HTTP_CODE_TOO_MANY_REQUESTS 429
+#define HTTP_CODE_UNAVAILABLE       503
+
 #define RSS_REGEX_MATCHER      "<link[^>]+type=\"application\\/(?:rss\\+xml)\"[^>]*>"
 #define RSS_HREF_REGEX_MATCHER "href=\"([^\"]+)\""
 

src/librssguard-standard/src/standardserviceroot.cpp

@@ -195,6 +195,10 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const {
   return ServiceRoot::additionalFlags() | Qt::ItemFlag::ItemIsDragEnabled | Qt::ItemFlag::ItemIsDropEnabled;
 }
 
+void StandardServiceRoot::clearFeedOverload(StandardFeed* feed) {
+  m_overloadedHosts.remove(QUrl(feed->source()).host());
+}
+
 QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
                                                       const QHash<ServiceRoot::BagOfMessages, QStringList>&
                                                         stated_messages,
@@ -203,6 +207,13 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
   Q_UNUSED(tagged_messages)
 
   StandardFeed* f = static_cast<StandardFeed*>(feed);
+
+  if (checkIfFeedOverloaded(f)) {
+    qWarningNN << LOGSEC_CORE << "Feed with source" << QUOTE_W_SPACE(f->source())
+               << "was signalled temporarily being down. Returning no articles for now.";
+    return {};
+  }
+
   QByteArray feed_contents;
   QString formatted_feed_contents;
   int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
@@ -233,12 +244,27 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
                                                                   f->http2Status());
 
     if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) {
-      qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError)
+      if (network_result.m_httpCode == HTTP_CODE_TOO_MANY_REQUESTS ||
-                 << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
+          network_result.m_httpCode == HTTP_CODE_UNAVAILABLE) {
-      throw FeedFetchException(Feed::Status::NetworkError,
+
-                               NetworkFactory::networkErrorText(network_result.m_networkError));
+        QDateTime safe_dt = NetworkFactory::extractRetryAfter(network_result.m_headers.value(QSL("retry-after")));
+
+        m_overloadedHosts.insert(QUrl(f->source()).host(), safe_dt);
+
+        qWarningNN << LOGSEC_CORE << "Feed" << QUOTE_W_SPACE_DOT(feed->source())
+                   << "indicates that there is too many requests right now on the same host.";
+        return {};
+      }
+      else {
+        qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError)
+                   << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
+        throw FeedFetchException(Feed::Status::NetworkError,
+                                 NetworkFactory::networkErrorText(network_result.m_networkError));
+      }
     }
     else {
+      clearFeedOverload(f);
+
       f->setLastEtag(network_result.m_headers.value(QSL("etag")));
 
       if (network_result.m_httpCode == HTTP_CODE_NOT_MODIFIED && feed_contents.trimmed().isEmpty()) {
@@ -566,6 +592,19 @@ void StandardServiceRoot::exportFeeds() {
   form.data()->exec();
 }
 
+bool StandardServiceRoot::checkIfFeedOverloaded(StandardFeed* feed) const {
+  if (feed->sourceType() == StandardFeed::SourceType::Url ||
+      feed->sourceType() == StandardFeed::SourceType::EmbeddedBrowser) {
+    QString hostname = QUrl(feed->source()).host();
+    QDateTime retry_after = m_overloadedHosts.value(hostname);
+
+    return retry_after.isValid() && retry_after > QDateTime::currentDateTimeUtc();
+  }
+  else {
+    return false;
+  }
+}
+
 QList<QAction*> StandardServiceRoot::serviceMenu() {
   if (m_serviceMenu.isEmpty()) {
     ServiceRoot::serviceMenu();

src/librssguard-standard/src/standardserviceroot.h

@@ -54,6 +54,9 @@ class StandardServiceRoot : public ServiceRoot {
     void exportFeeds();
 
   private:
+    void clearFeedOverload(StandardFeed* feed);
+    bool checkIfFeedOverloaded(StandardFeed* feed) const;
+
     // Takes structure residing under given root item and adds feeds/categories from
     // it to active structure.
     // NOTE: This is used for import/export of the model.
@@ -61,6 +64,8 @@ class StandardServiceRoot : public ServiceRoot {
 
     QPointer<StandardFeed> m_feedForMetadata = {};
     QList<QAction*> m_feedContextMenu = {};
+
+    QHash<QString, QDateTime> m_overloadedHosts;
 };
 
 #endif // STANDARDSERVICEROOT_H

src/librssguard/definitions/definitions.h

@@ -154,8 +154,6 @@
 
 #define CLI_THREADS "threads"
 
-#define HTTP_CODE_NOT_MODIFIED 304
-
 #define HTTP_HEADERS_ACCEPT         "Accept"
 #define HTTP_HEADERS_CONTENT_TYPE   "Content-Type"
 #define HTTP_HEADERS_CONTENT_LENGTH "Content-Length"

src/librssguard/network-web/networkfactory.cpp

@@ -13,6 +13,10 @@
 #include <QTextDocument>
 #include <QTimer>
 
+QDateTime NetworkFactory::extractRetryAfter(const QString& retry_after_value) {
+  return {};
+}
+
 QStringList NetworkFactory::extractFeedLinksFromHtmlPage(const QUrl& url, const QString& html) {
   QStringList feeds;
   QRegularExpression rx(QSL(FEED_REGEX_MATCHER), QRegularExpression::PatternOption::CaseInsensitiveOption);

src/librssguard/network-web/networkfactory.h

@@ -48,6 +48,8 @@ class RSSGUARD_DLLSPEC NetworkFactory {
       Disabled = 2
     };
 
+    static QDateTime extractRetryAfter(const QString& retry_after_value);
+
     static QStringList extractFeedLinksFromHtmlPage(const QUrl& url, const QString& html);
     static QPair<QByteArray, QByteArray> generateBasicAuthHeader(NetworkAuthentication protection,
                                                                  const QString& username,