From fc147a44ed7fe48573b352576e4cf6076605ae9e Mon Sep 17 00:00:00 2001
From: Martin Rotter <rotter.martinos@gmail.com>
Date: Thu, 13 Feb 2025 12:49:17 +0100
Subject: [PATCH] fix #1154

---
 .../src/gui/formeditstandardaccount.cpp       | 14 ++--
 .../src/gui/formstandardfeeddetails.cpp       |  2 +
 .../src/gui/formstandardfeeddetails.h         |  3 +-
 .../src/gui/standardaccountdetails.cpp        | 20 ++++++
 .../src/gui/standardaccountdetails.h          |  1 +
 .../src/gui/standardaccountdetails.ui         | 43 +++++++++++-
 .../src/gui/standardfeeddetails.cpp           |  4 ++
 .../src/gui/standardfeeddetails.h             |  2 +
 src/librssguard-standard/src/standardfeed.cpp |  7 ++
 src/librssguard-standard/src/standardfeed.h   |  1 +
 .../src/standardfeedsimportexportmodel.cpp    |  1 +
 .../src/standardserviceroot.cpp               | 65 ++++++++++++++++++-
 .../src/standardserviceroot.h                 | 16 +++++
 13 files changed, 171 insertions(+), 8 deletions(-)

diff --git a/src/librssguard-standard/src/gui/formeditstandardaccount.cpp b/src/librssguard-standard/src/gui/formeditstandardaccount.cpp
index a53fb1652..2a49f4af8 100644
--- a/src/librssguard-standard/src/gui/formeditstandardaccount.cpp
+++ b/src/librssguard-standard/src/gui/formeditstandardaccount.cpp
@@ -19,21 +19,27 @@ FormEditStandardAccount::~FormEditStandardAccount() = default;
 void FormEditStandardAccount::loadAccountData() {
   FormAccountDetails::loadAccountData();
 
+  StandardServiceRoot* acc = account<StandardServiceRoot>();
+
   if (m_creatingNew) {
     m_standardDetails->m_ui.m_txtTitle->setText(StandardServiceRoot::defaultTitle());
   }
   else {
-    m_standardDetails->m_ui.m_txtTitle->setText(m_account->title());
+    m_standardDetails->m_ui.m_txtTitle->setText(acc->title());
   }
 
-  m_standardDetails->m_ui.m_btnIcon->setIcon(m_account->fullIcon());
+  m_standardDetails->m_ui.m_btnIcon->setIcon(acc->fullIcon());
+  m_standardDetails->m_ui.m_spinFeedSpacing->setValue(acc->spacingSameHostsRequests());
 }
 
 void FormEditStandardAccount::apply() {
   FormAccountDetails::apply();
 
-  m_account->setIcon(m_standardDetails->m_ui.m_btnIcon->icon());
-  m_account->setTitle(m_standardDetails->m_ui.m_txtTitle->text());
+  StandardServiceRoot* acc = account<StandardServiceRoot>();
+
+  acc->setIcon(m_standardDetails->m_ui.m_btnIcon->icon());
+  acc->setTitle(m_standardDetails->m_ui.m_txtTitle->text());
+  acc->setSpacingSameHostsRequests(m_standardDetails->m_ui.m_spinFeedSpacing->value());
 
   m_account->saveAccountDataToDatabase();
   m_account->itemChanged({m_account});
diff --git a/src/librssguard-standard/src/gui/formstandardfeeddetails.cpp b/src/librssguard-standard/src/gui/formstandardfeeddetails.cpp
index ae1d37167..33cd78199 100644
--- a/src/librssguard-standard/src/gui/formstandardfeeddetails.cpp
+++ b/src/librssguard-standard/src/gui/formstandardfeeddetails.cpp
@@ -52,6 +52,7 @@ void FormStandardFeedDetails::guessFeed() {
   m_standardFeedDetails->guessFeed(m_standardFeedDetails->sourceType(),
                                    m_standardFeedDetails->m_ui.m_txtSource->textEdit()->toPlainText(),
                                    m_standardFeedDetails->m_ui.m_txtPostProcessScript->textEdit()->toPlainText(),
+                                   qobject_cast<StandardServiceRoot*>(m_serviceRoot),
                                    m_authDetails->authenticationType(),
                                    m_authDetails->username(),
                                    m_authDetails->password(),
@@ -63,6 +64,7 @@ void FormStandardFeedDetails::guessIconOnly() {
   m_standardFeedDetails->guessIconOnly(m_standardFeedDetails->sourceType(),
                                        m_standardFeedDetails->m_ui.m_txtSource->textEdit()->toPlainText(),
                                        m_standardFeedDetails->m_ui.m_txtPostProcessScript->textEdit()->toPlainText(),
+                                       qobject_cast<StandardServiceRoot*>(m_serviceRoot),
                                        m_authDetails->authenticationType(),
                                        m_authDetails->username(),
                                        m_authDetails->password(),
diff --git a/src/librssguard-standard/src/gui/formstandardfeeddetails.h b/src/librssguard-standard/src/gui/formstandardfeeddetails.h
index 7fd6cca72..908fd0250 100644
--- a/src/librssguard-standard/src/gui/formstandardfeeddetails.h
+++ b/src/librssguard-standard/src/gui/formstandardfeeddetails.h
@@ -3,12 +3,13 @@
 #ifndef FORMSSFEEDDETAILS_H
 #define FORMSSFEEDDETAILS_H
 
+#include "src/standardserviceroot.h"
+
 #include <librssguard/services/abstract/gui/formfeeddetails.h>
 
 class StandardFeedDetails;
 class StandardFeedExpDetails;
 class HttpHeadersDetails;
-class StandardServiceRoot;
 class AuthenticationDetails;
 class StandardFeed;
 
diff --git a/src/librssguard-standard/src/gui/standardaccountdetails.cpp b/src/librssguard-standard/src/gui/standardaccountdetails.cpp
index 953b5fe85..3478ce1bf 100644
--- a/src/librssguard-standard/src/gui/standardaccountdetails.cpp
+++ b/src/librssguard-standard/src/gui/standardaccountdetails.cpp
@@ -26,6 +26,17 @@ StandardAccountDetails::StandardAccountDetails(QWidget* parent) : QWidget(parent
   icon_menu->addAction(action_default_icon);
 
   m_ui.m_btnIcon->setMenu(icon_menu);
+
+  m_ui.m_helpFeedSpacing
+    ->setHelpText(tr("When you fetch many feeds from same website/host, then %1 could be (likely "
+                     "temporarily) banned for making too many network requests at once.\n\n"
+                     "If that is the case, then you need to set some time gaps when fetching those feeds.")
+                    .arg(QSL(APP_NAME)),
+                  false);
+
+  connect(m_ui.m_spinFeedSpacing, &QSpinBox::valueChanged, this, &StandardAccountDetails::onFeedSpacingChanged);
+
+  onFeedSpacingChanged(m_ui.m_spinFeedSpacing->value());
 }
 
 void StandardAccountDetails::onLoadIconFromFile() {
@@ -63,3 +74,12 @@ void StandardAccountDetails::onLoadIconFromFile() {
 void StandardAccountDetails::onUseDefaultIcon() {
   m_ui.m_btnIcon->setIcon(StandardServiceEntryPoint().icon());
 }
+
+void StandardAccountDetails::onFeedSpacingChanged(int spacing) {
+  if (spacing <= 0) {
+    m_ui.m_spinFeedSpacing->setSuffix(tr(" = no spacing"));
+  }
+  else {
+    m_ui.m_spinFeedSpacing->setSuffix(tr(" seconds", nullptr, spacing));
+  }
+}
diff --git a/src/librssguard-standard/src/gui/standardaccountdetails.h b/src/librssguard-standard/src/gui/standardaccountdetails.h
index bab0f9503..1f9d303c9 100644
--- a/src/librssguard-standard/src/gui/standardaccountdetails.h
+++ b/src/librssguard-standard/src/gui/standardaccountdetails.h
@@ -22,6 +22,7 @@ class StandardAccountDetails : public QWidget {
   private slots:
     void onLoadIconFromFile();
     void onUseDefaultIcon();
+    void onFeedSpacingChanged(int spacing);
 
   private:
     Ui::StandardAccountDetails m_ui;
diff --git a/src/librssguard-standard/src/gui/standardaccountdetails.ui b/src/librssguard-standard/src/gui/standardaccountdetails.ui
index 5141517b5..0dd973df4 100644
--- a/src/librssguard-standard/src/gui/standardaccountdetails.ui
+++ b/src/librssguard-standard/src/gui/standardaccountdetails.ui
@@ -19,6 +19,9 @@
      <property name="text">
       <string>Title</string>
      </property>
+     <property name="buddy">
+      <cstring>m_txtTitle</cstring>
+     </property>
     </widget>
    </item>
    <item row="0" column="1">
@@ -29,6 +32,9 @@
      <property name="text">
       <string>Icon</string>
      </property>
+     <property name="buddy">
+      <cstring>m_btnIcon</cstring>
+     </property>
     </widget>
    </item>
    <item row="1" column="1">
@@ -43,12 +49,47 @@
       <string>Select icon for your account.</string>
      </property>
      <property name="popupMode">
-      <enum>QToolButton::InstantPopup</enum>
+      <enum>QToolButton::ToolButtonPopupMode::InstantPopup</enum>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <widget class="QLabel" name="label_3">
+     <property name="text">
+      <string>Feed fetch spacing</string>
+     </property>
+     <property name="buddy">
+      <cstring>m_spinFeedSpacing</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QSpinBox" name="m_spinFeedSpacing">
+     <property name="maximum">
+      <number>120</number>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="0" colspan="2">
+    <widget class="HelpSpoiler" name="m_helpFeedSpacing" native="true">
+     <property name="minimumSize">
+      <size>
+       <width>50</width>
+       <height>0</height>
+      </size>
      </property>
     </widget>
    </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>HelpSpoiler</class>
+   <extends>QWidget</extends>
+   <header>helpspoiler.h</header>
+   <container>1</container>
+  </customwidget>
+ </customwidgets>
  <resources/>
  <connections/>
 </ui>
diff --git a/src/librssguard-standard/src/gui/standardfeeddetails.cpp b/src/librssguard-standard/src/gui/standardfeeddetails.cpp
index 072fb2fa4..14ba1035d 100644
--- a/src/librssguard-standard/src/gui/standardfeeddetails.cpp
+++ b/src/librssguard-standard/src/gui/standardfeeddetails.cpp
@@ -140,6 +140,7 @@ StandardFeedDetails::StandardFeedDetails(QWidget* parent) : QWidget(parent) {
 void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
                                         const QString& source,
                                         const QString& post_process_script,
+                                        StandardServiceRoot* account,
                                         NetworkFactory::NetworkAuthentication protection,
                                         const QString& username,
                                         const QString& password,
@@ -149,6 +150,7 @@ void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
     auto metadata = StandardFeed::guessFeed(source_type,
                                             source,
                                             post_process_script,
+                                            account,
                                             protection,
                                             true,
                                             username,
@@ -185,6 +187,7 @@ void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
 void StandardFeedDetails::guessFeed(StandardFeed::SourceType source_type,
                                     const QString& source,
                                     const QString& post_process_script,
+                                    StandardServiceRoot* account,
                                     NetworkFactory::NetworkAuthentication protection,
                                     const QString& username,
                                     const QString& password,
@@ -194,6 +197,7 @@ void StandardFeedDetails::guessFeed(StandardFeed::SourceType source_type,
     auto metadata = StandardFeed::guessFeed(source_type,
                                             source,
                                             post_process_script,
+                                            account,
                                             protection,
                                             true,
                                             username,
diff --git a/src/librssguard-standard/src/gui/standardfeeddetails.h b/src/librssguard-standard/src/gui/standardfeeddetails.h
index 82bd5107b..ad89166dd 100644
--- a/src/librssguard-standard/src/gui/standardfeeddetails.h
+++ b/src/librssguard-standard/src/gui/standardfeeddetails.h
@@ -27,6 +27,7 @@ class StandardFeedDetails : public QWidget {
     void guessIconOnly(StandardFeed::SourceType source_type,
                        const QString& source,
                        const QString& post_process_script,
+                       StandardServiceRoot* account,
                        NetworkFactory::NetworkAuthentication protection,
                        const QString& username,
                        const QString& password,
@@ -36,6 +37,7 @@ class StandardFeedDetails : public QWidget {
     void guessFeed(StandardFeed::SourceType source_type,
                    const QString& source,
                    const QString& post_process_script,
+                   StandardServiceRoot* account,
                    NetworkFactory::NetworkAuthentication protection,
                    const QString& username,
                    const QString& password,
diff --git a/src/librssguard-standard/src/standardfeed.cpp b/src/librssguard-standard/src/standardfeed.cpp
index 33125f18a..23edcf4bc 100644
--- a/src/librssguard-standard/src/standardfeed.cpp
+++ b/src/librssguard-standard/src/standardfeed.cpp
@@ -245,6 +245,7 @@ void StandardFeed::fetchMetadataForItself() {
     auto metadata = guessFeed(sourceType(),
                               source(),
                               postProcessScript(),
+                              serviceRoot(),
                               protection(),
                               true,
                               username(),
@@ -298,6 +299,7 @@ void StandardFeed::setSourceType(SourceType source_type) {
 QPair<StandardFeed*, NetworkResult> StandardFeed::guessFeed(StandardFeed::SourceType source_type,
                                                             const QString& source,
                                                             const QString& post_process_script,
+                                                            StandardServiceRoot* account,
                                                             NetworkFactory::NetworkFactory::NetworkAuthentication
                                                               protection,
                                                             bool fetch_icons,
@@ -310,6 +312,9 @@ QPair<StandardFeed*, NetworkResult> StandardFeed::guessFeed(StandardFeed::Source
   NetworkResult network_result;
 
   if (source_type == StandardFeed::SourceType::Url) {
+    QString host = QUrl(source).host();
+    account->spaceHost(host, source);
+
     QList<QPair<QByteArray, QByteArray>> headers = http_headers;
     headers << NetworkFactory::generateBasicAuthHeader(protection, username, password);
 
@@ -324,6 +329,8 @@ QPair<StandardFeed*, NetworkResult> StandardFeed::guessFeed(StandardFeed::Source
                                                              {},
                                                              custom_proxy);
 
+    // account->resetHostSpacing(host);
+
     if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) {
       throw NetworkException(network_result.m_networkError);
     }
diff --git a/src/librssguard-standard/src/standardfeed.h b/src/librssguard-standard/src/standardfeed.h
index 9a9f5275b..5e92616e3 100644
--- a/src/librssguard-standard/src/standardfeed.h
+++ b/src/librssguard-standard/src/standardfeed.h
@@ -85,6 +85,7 @@ class StandardFeed : public Feed {
     static QPair<StandardFeed*, NetworkResult> guessFeed(SourceType source_type,
                                                          const QString& url,
                                                          const QString& post_process_script,
+                                                         StandardServiceRoot* account,
                                                          NetworkFactory::NetworkAuthentication protection,
                                                          bool fetch_icons = true,
                                                          const QString& username = {},
diff --git a/src/librssguard-standard/src/standardfeedsimportexportmodel.cpp b/src/librssguard-standard/src/standardfeedsimportexportmodel.cpp
index a530fd616..b6adcd104 100644
--- a/src/librssguard-standard/src/standardfeedsimportexportmodel.cpp
+++ b/src/librssguard-standard/src/standardfeedsimportexportmodel.cpp
@@ -199,6 +199,7 @@ bool FeedsImportExportModel::produceFeed(const FeedLookup& feed_lookup) {
         auto new_feed_data = StandardFeed::guessFeed(source_type,
                                                      feed_lookup.url,
                                                      pp_script,
+                                                     m_account,
                                                      NetworkFactory::NetworkAuthentication::NoAuthentication,
                                                      !feed_lookup.do_not_fetch_icons,
                                                      {},
diff --git a/src/librssguard-standard/src/standardserviceroot.cpp b/src/librssguard-standard/src/standardserviceroot.cpp
index 55a54cbb5..2ce512c1f 100644
--- a/src/librssguard-standard/src/standardserviceroot.cpp
+++ b/src/librssguard-standard/src/standardserviceroot.cpp
@@ -45,7 +45,7 @@
 #include <QStack>
 #include <QTextCodec>
 
-StandardServiceRoot::StandardServiceRoot(RootItem* parent) : ServiceRoot(parent) {
+StandardServiceRoot::StandardServiceRoot(RootItem* parent) : ServiceRoot(parent), m_spacingSameHostsRequests(0) {
   setIcon(StandardServiceEntryPoint().icon());
   setDescription(tr("This is the obligatory service account for standard RSS/RDF/ATOM feeds."));
 }
@@ -195,6 +195,49 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const {
   return ServiceRoot::additionalFlags() | Qt::ItemFlag::ItemIsDragEnabled | Qt::ItemFlag::ItemIsDropEnabled;
 }
 
+void StandardServiceRoot::spaceHost(const QString& host, const QString& url) {
+  if (m_spacingSameHostsRequests <= 0 || host.simplified().isEmpty()) {
+    // Spacing not enabled or host information unavailable.
+    return;
+  }
+
+  m_spacingMutex.lock();
+
+  QDateTime host_last_fetched = m_spacingHosts.value(host);
+  QDateTime now = QDateTime::currentDateTimeUtc();
+  int secs_to_wait = 0;
+
+  if (host_last_fetched.isValid()) {
+    // No last fetch time saved yet.
+    QDateTime last = host_last_fetched.addSecs(m_spacingSameHostsRequests);
+
+    if (last < now) {
+      // This host was last fetched sometimes in the past and not within the critical spacing window.
+      // We can therefore fetch now.
+      secs_to_wait = 0;
+    }
+    else {
+      secs_to_wait = now.secsTo(last);
+    }
+  }
+
+  resetHostSpacing(host, now.addSecs(secs_to_wait));
+
+  m_spacingMutex.unlock();
+
+  if (secs_to_wait > 0) {
+    qDebugNN << LOGSEC_CORE << "Freezing feed with URL" << QUOTE_W_SPACE(url) << "for" << NONQUOTE_W_SPACE(secs_to_wait)
+             << "seconds, because its host was used for fetching another feed during the spacing period.";
+    QThread::sleep(ulong(secs_to_wait));
+    qDebugNN << LOGSEC_CORE << "Freezing feed with URL" << QUOTE_W_SPACE(url) << "is done.";
+  }
+}
+
+void StandardServiceRoot::resetHostSpacing(const QString& host, const QDateTime& next_dt) {
+  m_spacingHosts.insert(host, next_dt);
+  qDebugNN << LOGSEC_CORE << "Setting spacing for" << QUOTE_W_SPACE(host) << "to" << QUOTE_W_SPACE_DOT(next_dt);
+}
+
 QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
                                                       const QHash<ServiceRoot::BagOfMessages, QStringList>&
                                                         stated_messages,
@@ -203,11 +246,14 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
   Q_UNUSED(tagged_messages)
 
   StandardFeed* f = static_cast<StandardFeed*>(feed);
+  QString host = QUrl(f->source()).host();
   QByteArray feed_contents;
   QString formatted_feed_contents;
   int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
 
   if (f->sourceType() == StandardFeed::SourceType::Url) {
+    spaceHost(host, f->source());
+
     qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";
 
     QList<QPair<QByteArray, QByteArray>> headers = StandardFeed::httpHeadersToList(f->httpHeaders());
@@ -232,11 +278,15 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
                                                                   networkProxy(),
                                                                   f->http2Status());
 
+    // Update last datetime this host was used.
+    // resetHostSpacing(host);
+
     if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) {
       qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError)
                  << "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
       throw FeedFetchException(Feed::Status::NetworkError,
-                               NetworkFactory::networkErrorText(network_result.m_networkError));
+                               NetworkFactory::networkErrorText(network_result.m_networkError),
+                               QVariant::fromValue(network_result));
     }
     else {
       f->setLastEtag(network_result.m_headers.value(QSL("etag")));
@@ -404,6 +454,7 @@ QVariantHash StandardServiceRoot::customDatabaseData() const {
 
   data[QSL("title")] = title();
   data[QSL("icon")] = IconFactory::toByteArray(icon());
+  data[QSL("requests_spacing")] = spacingSameHostsRequests();
 
   return data;
 }
@@ -418,6 +469,8 @@ void StandardServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   if (!icon_data.isEmpty()) {
     setIcon(IconFactory::fromByteArray(icon_data));
   }
+
+  setSpacingSameHostsRequests(data.value(QSL("requests_spacing")).toInt());
 }
 
 QString StandardServiceRoot::defaultTitle() {
@@ -530,6 +583,14 @@ bool StandardServiceRoot::mergeImportExportModel(FeedsImportExportModel* model,
   return !some_feed_category_error;
 }
 
+int StandardServiceRoot::spacingSameHostsRequests() const {
+  return m_spacingSameHostsRequests;
+}
+
+void StandardServiceRoot::setSpacingSameHostsRequests(int spacing) {
+  m_spacingSameHostsRequests = spacing;
+}
+
 void StandardServiceRoot::addNewCategory(RootItem* selected_item) {
   if (!qApp->feedUpdateLock()->tryLock()) {
     // Lock was not obtained because
diff --git a/src/librssguard-standard/src/standardserviceroot.h b/src/librssguard-standard/src/standardserviceroot.h
index 5133d4e0c..393dbabf5 100644
--- a/src/librssguard-standard/src/standardserviceroot.h
+++ b/src/librssguard-standard/src/standardserviceroot.h
@@ -8,6 +8,7 @@
 #include <librssguard/services/abstract/serviceroot.h>
 
 #include <QCoreApplication>
+#include <QMutex>
 #include <QPair>
 
 class StandardCategory;
@@ -43,6 +44,17 @@ class StandardServiceRoot : public ServiceRoot {
     QList<QAction*> serviceMenu();
     QList<QAction*> getContextMenuForFeed(StandardFeed* feed);
 
+    void spaceHost(const QString& host, const QString& url);
+    void resetHostSpacing(const QString& host, const QDateTime &next_dt);
+
+    // If set to number > 0, then requests to fetch feeds
+    // will be spaced by the given number (in seconds).
+    // This is used to avoid too many concurrent network
+    // requests from to the same server and getting HTTP/429
+    // errors or other DDoS-attack-related bans.
+    int spacingSameHostsRequests() const;
+    void setSpacingSameHostsRequests(int spacing);
+
     static QString defaultTitle();
 
   public slots:
@@ -61,6 +73,10 @@ class StandardServiceRoot : public ServiceRoot {
 
     QPointer<StandardFeed> m_feedForMetadata = {};
     QList<QAction*> m_feedContextMenu = {};
+
+    int m_spacingSameHostsRequests;
+    QHash<QString, QDateTime> m_spacingHosts;
+    QMutex m_spacingMutex;
 };
 
 #endif // STANDARDSERVICEROOT_H