pogmommy/rssguard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1154

Browse source
This commit is contained in:
Martin Rotter 2025-02-13 12:49:17 +01:00
parent 25c53260cf
commit fc147a44ed
13 changed files with 171 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/librssguard-standard/src/gui/formeditstandardaccount.cpp
View file

@ -19,21 +19,27 @@ FormEditStandardAccount::~FormEditStandardAccount() = default;
void FormEditStandardAccount::loadAccountData() {
FormAccountDetails::loadAccountData();
StandardServiceRoot* acc = account<StandardServiceRoot>();
if (m_creatingNew) {
m_standardDetails->m_ui.m_txtTitle->setText(StandardServiceRoot::defaultTitle());
}
else {
m_standardDetails->m_ui.m_txtTitle->setText(m_account->title());
m_standardDetails->m_ui.m_txtTitle->setText(acc->title());
}
m_standardDetails->m_ui.m_btnIcon->setIcon(m_account->fullIcon());
m_standardDetails->m_ui.m_btnIcon->setIcon(acc->fullIcon());
m_standardDetails->m_ui.m_spinFeedSpacing->setValue(acc->spacingSameHostsRequests());
}
void FormEditStandardAccount::apply() {
FormAccountDetails::apply();
m_account->setIcon(m_standardDetails->m_ui.m_btnIcon->icon());
m_account->setTitle(m_standardDetails->m_ui.m_txtTitle->text());
StandardServiceRoot* acc = account<StandardServiceRoot>();
acc->setIcon(m_standardDetails->m_ui.m_btnIcon->icon());
acc->setTitle(m_standardDetails->m_ui.m_txtTitle->text());
acc->setSpacingSameHostsRequests(m_standardDetails->m_ui.m_spinFeedSpacing->value());
m_account->saveAccountDataToDatabase();
m_account->itemChanged({m_account});

2
src/librssguard-standard/src/gui/formstandardfeeddetails.cpp
View file

@ -52,6 +52,7 @@ void FormStandardFeedDetails::guessFeed() {
m_standardFeedDetails->guessFeed(m_standardFeedDetails->sourceType(),
m_standardFeedDetails->m_ui.m_txtSource->textEdit()->toPlainText(),
m_standardFeedDetails->m_ui.m_txtPostProcessScript->textEdit()->toPlainText(),
qobject_cast<StandardServiceRoot*>(m_serviceRoot),
m_authDetails->authenticationType(),
m_authDetails->username(),
m_authDetails->password(),
@ -63,6 +64,7 @@ void FormStandardFeedDetails::guessIconOnly() {
m_standardFeedDetails->guessIconOnly(m_standardFeedDetails->sourceType(),
m_standardFeedDetails->m_ui.m_txtSource->textEdit()->toPlainText(),
m_standardFeedDetails->m_ui.m_txtPostProcessScript->textEdit()->toPlainText(),
qobject_cast<StandardServiceRoot*>(m_serviceRoot),
m_authDetails->authenticationType(),
m_authDetails->username(),
m_authDetails->password(),

3
src/librssguard-standard/src/gui/formstandardfeeddetails.h
View file

@ -3,12 +3,13 @@
#ifndef FORMSSFEEDDETAILS_H
#define FORMSSFEEDDETAILS_H
#include "src/standardserviceroot.h"
#include <librssguard/services/abstract/gui/formfeeddetails.h>
class StandardFeedDetails;
class StandardFeedExpDetails;
class HttpHeadersDetails;
class StandardServiceRoot;
class AuthenticationDetails;
class StandardFeed;

20
src/librssguard-standard/src/gui/standardaccountdetails.cpp
View file

@ -26,6 +26,17 @@ StandardAccountDetails::StandardAccountDetails(QWidget* parent) : QWidget(parent
icon_menu->addAction(action_default_icon);
m_ui.m_btnIcon->setMenu(icon_menu);
m_ui.m_helpFeedSpacing
->setHelpText(tr("When you fetch many feeds from same website/host, then %1 could be (likely "
"temporarily) banned for making too many network requests at once.\n\n"
"If that is the case, then you need to set some time gaps when fetching those feeds.")
.arg(QSL(APP_NAME)),
false);
connect(m_ui.m_spinFeedSpacing, &QSpinBox::valueChanged, this, &StandardAccountDetails::onFeedSpacingChanged);
onFeedSpacingChanged(m_ui.m_spinFeedSpacing->value());
}
void StandardAccountDetails::onLoadIconFromFile() {
@ -63,3 +74,12 @@ void StandardAccountDetails::onLoadIconFromFile() {
void StandardAccountDetails::onUseDefaultIcon() {
m_ui.m_btnIcon->setIcon(StandardServiceEntryPoint().icon());
}
void StandardAccountDetails::onFeedSpacingChanged(int spacing) {
if (spacing <= 0) {
m_ui.m_spinFeedSpacing->setSuffix(tr(" = no spacing"));
}
else {
m_ui.m_spinFeedSpacing->setSuffix(tr(" seconds", nullptr, spacing));
}
}

1
src/librssguard-standard/src/gui/standardaccountdetails.h
View file

@ -22,6 +22,7 @@ class StandardAccountDetails : public QWidget {
private slots:
void onLoadIconFromFile();
void onUseDefaultIcon();
void onFeedSpacingChanged(int spacing);
private:
Ui::StandardAccountDetails m_ui;

43
src/librssguard-standard/src/gui/standardaccountdetails.ui
View file

@ -19,6 +19,9 @@
<property name="text">
<string>Title</string>
</property>
<property name="buddy">
<cstring>m_txtTitle</cstring>
</property>
</widget>
</item>
<item row="0" column="1">
@ -29,6 +32,9 @@
<property name="text">
<string>Icon</string>
</property>
<property name="buddy">
<cstring>m_btnIcon</cstring>
</property>
</widget>
</item>
<item row="1" column="1">
@ -43,12 +49,47 @@
<string>Select icon for your account.</string>
</property>
<property name="popupMode">
<enum>QToolButton::InstantPopup</enum>
<enum>QToolButton::ToolButtonPopupMode::InstantPopup</enum>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_3">
<property name="text">
<string>Feed fetch spacing</string>
</property>
<property name="buddy">
<cstring>m_spinFeedSpacing</cstring>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QSpinBox" name="m_spinFeedSpacing">
<property name="maximum">
<number>120</number>
</property>
</widget>
</item>
<item row="3" column="0" colspan="2">
<widget class="HelpSpoiler" name="m_helpFeedSpacing" native="true">
<property name="minimumSize">
<size>
<width>50</width>
<height>0</height>
</size>
</property>
</widget>
</item>
</layout>
</widget>
<customwidgets>
<customwidget>
<class>HelpSpoiler</class>
<extends>QWidget</extends>
<header>helpspoiler.h</header>
<container>1</container>
</customwidget>
</customwidgets>
<resources/>
<connections/>
</ui>

4
src/librssguard-standard/src/gui/standardfeeddetails.cpp
View file

@ -140,6 +140,7 @@ StandardFeedDetails::StandardFeedDetails(QWidget* parent) : QWidget(parent) {
void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
const QString& source,
const QString& post_process_script,
StandardServiceRoot* account,
NetworkFactory::NetworkAuthentication protection,
const QString& username,
const QString& password,
@ -149,6 +150,7 @@ void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
auto metadata = StandardFeed::guessFeed(source_type,
source,
post_process_script,
account,
protection,
true,
username,
@ -185,6 +187,7 @@ void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
void StandardFeedDetails::guessFeed(StandardFeed::SourceType source_type,
const QString& source,
const QString& post_process_script,
StandardServiceRoot* account,
NetworkFactory::NetworkAuthentication protection,
const QString& username,
const QString& password,
@ -194,6 +197,7 @@ void StandardFeedDetails::guessFeed(StandardFeed::SourceType source_type,
auto metadata = StandardFeed::guessFeed(source_type,
source,
post_process_script,
account,
protection,
true,
username,

2
src/librssguard-standard/src/gui/standardfeeddetails.h
View file

@ -27,6 +27,7 @@ class StandardFeedDetails : public QWidget {
void guessIconOnly(StandardFeed::SourceType source_type,
const QString& source,
const QString& post_process_script,
StandardServiceRoot* account,
NetworkFactory::NetworkAuthentication protection,
const QString& username,
const QString& password,
@ -36,6 +37,7 @@ class StandardFeedDetails : public QWidget {
void guessFeed(StandardFeed::SourceType source_type,
const QString& source,
const QString& post_process_script,
StandardServiceRoot* account,
NetworkFactory::NetworkAuthentication protection,
const QString& username,
const QString& password,

7
src/librssguard-standard/src/standardfeed.cpp
View file

@ -245,6 +245,7 @@ void StandardFeed::fetchMetadataForItself() {
auto metadata = guessFeed(sourceType(),
source(),
postProcessScript(),
serviceRoot(),
protection(),
true,
username(),
@ -298,6 +299,7 @@ void StandardFeed::setSourceType(SourceType source_type) {
QPair<StandardFeed*, NetworkResult> StandardFeed::guessFeed(StandardFeed::SourceType source_type,
const QString& source,
const QString& post_process_script,
StandardServiceRoot* account,
NetworkFactory::NetworkFactory::NetworkAuthentication
protection,
bool fetch_icons,
@ -310,6 +312,9 @@ QPair<StandardFeed*, NetworkResult> StandardFeed::guessFeed(StandardFeed::Source
NetworkResult network_result;
if (source_type == StandardFeed::SourceType::Url) {
QString host = QUrl(source).host();
account->spaceHost(host, source);
QList<QPair<QByteArray, QByteArray>> headers = http_headers;
headers << NetworkFactory::generateBasicAuthHeader(protection, username, password);
@ -324,6 +329,8 @@ QPair<StandardFeed*, NetworkResult> StandardFeed::guessFeed(StandardFeed::Source
{},
custom_proxy);
// account->resetHostSpacing(host);
if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) {
throw NetworkException(network_result.m_networkError);
}

1
src/librssguard-standard/src/standardfeed.h
View file

@ -85,6 +85,7 @@ class StandardFeed : public Feed {
static QPair<StandardFeed*, NetworkResult> guessFeed(SourceType source_type,
const QString& url,
const QString& post_process_script,
StandardServiceRoot* account,
NetworkFactory::NetworkAuthentication protection,
bool fetch_icons = true,
const QString& username = {},

1
src/librssguard-standard/src/standardfeedsimportexportmodel.cpp
View file

@ -199,6 +199,7 @@ bool FeedsImportExportModel::produceFeed(const FeedLookup& feed_lookup) {
auto new_feed_data = StandardFeed::guessFeed(source_type,
feed_lookup.url,
pp_script,
m_account,
NetworkFactory::NetworkAuthentication::NoAuthentication,
!feed_lookup.do_not_fetch_icons,
{},

65
src/librssguard-standard/src/standardserviceroot.cpp
View file

@ -45,7 +45,7 @@
#include <QStack>
#include <QTextCodec>
StandardServiceRoot::StandardServiceRoot(RootItem* parent) : ServiceRoot(parent) {
StandardServiceRoot::StandardServiceRoot(RootItem* parent) : ServiceRoot(parent), m_spacingSameHostsRequests(0) {
setIcon(StandardServiceEntryPoint().icon());
setDescription(tr("This is the obligatory service account for standard RSS/RDF/ATOM feeds."));
}
@ -195,6 +195,49 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const {
return ServiceRoot::additionalFlags() | Qt::ItemFlag::ItemIsDragEnabled | Qt::ItemFlag::ItemIsDropEnabled;
}
void StandardServiceRoot::spaceHost(const QString& host, const QString& url) {
if (m_spacingSameHostsRequests <= 0 || host.simplified().isEmpty()) {
// Spacing not enabled or host information unavailable.
return;
}
m_spacingMutex.lock();
QDateTime host_last_fetched = m_spacingHosts.value(host);
QDateTime now = QDateTime::currentDateTimeUtc();
int secs_to_wait = 0;
if (host_last_fetched.isValid()) {
// No last fetch time saved yet.
QDateTime last = host_last_fetched.addSecs(m_spacingSameHostsRequests);
if (last < now) {
// This host was last fetched sometimes in the past and not within the critical spacing window.
// We can therefore fetch now.
secs_to_wait = 0;
}
else {
secs_to_wait = now.secsTo(last);
}
}
resetHostSpacing(host, now.addSecs(secs_to_wait));
m_spacingMutex.unlock();
if (secs_to_wait > 0) {
qDebugNN << LOGSEC_CORE << "Freezing feed with URL" << QUOTE_W_SPACE(url) << "for" << NONQUOTE_W_SPACE(secs_to_wait)
<< "seconds, because its host was used for fetching another feed during the spacing period.";
QThread::sleep(ulong(secs_to_wait));
qDebugNN << LOGSEC_CORE << "Freezing feed with URL" << QUOTE_W_SPACE(url) << "is done.";
}
}
void StandardServiceRoot::resetHostSpacing(const QString& host, const QDateTime& next_dt) {
m_spacingHosts.insert(host, next_dt);
qDebugNN << LOGSEC_CORE << "Setting spacing for" << QUOTE_W_SPACE(host) << "to" << QUOTE_W_SPACE_DOT(next_dt);
}
QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>&
stated_messages,
@ -203,11 +246,14 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
Q_UNUSED(tagged_messages)
StandardFeed* f = static_cast<StandardFeed*>(feed);
QString host = QUrl(f->source()).host();
QByteArray feed_contents;
QString formatted_feed_contents;
int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
if (f->sourceType() == StandardFeed::SourceType::Url) {
spaceHost(host, f->source());
qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";
QList<QPair<QByteArray, QByteArray>> headers = StandardFeed::httpHeadersToList(f->httpHeaders());
@ -232,11 +278,15 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
networkProxy(),
f->http2Status());
// Update last datetime this host was used.
// resetHostSpacing(host);
if (network_result.m_networkError != QNetworkReply::NetworkError::NoError) {
qWarningNN << LOGSEC_CORE << "Error" << QUOTE_W_SPACE(network_result.m_networkError)
<< "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
throw FeedFetchException(Feed::Status::NetworkError,
NetworkFactory::networkErrorText(network_result.m_networkError));
NetworkFactory::networkErrorText(network_result.m_networkError),
QVariant::fromValue(network_result));
}
else {
f->setLastEtag(network_result.m_headers.value(QSL("etag")));
@ -404,6 +454,7 @@ QVariantHash StandardServiceRoot::customDatabaseData() const {
data[QSL("title")] = title();
data[QSL("icon")] = IconFactory::toByteArray(icon());
data[QSL("requests_spacing")] = spacingSameHostsRequests();
return data;
}
@ -418,6 +469,8 @@ void StandardServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
if (!icon_data.isEmpty()) {
setIcon(IconFactory::fromByteArray(icon_data));
}
setSpacingSameHostsRequests(data.value(QSL("requests_spacing")).toInt());
}
QString StandardServiceRoot::defaultTitle() {
@ -530,6 +583,14 @@ bool StandardServiceRoot::mergeImportExportModel(FeedsImportExportModel* model,
return !some_feed_category_error;
}
int StandardServiceRoot::spacingSameHostsRequests() const {
return m_spacingSameHostsRequests;
}
void StandardServiceRoot::setSpacingSameHostsRequests(int spacing) {
m_spacingSameHostsRequests = spacing;
}
void StandardServiceRoot::addNewCategory(RootItem* selected_item) {
if (!qApp->feedUpdateLock()->tryLock()) {
// Lock was not obtained because

16
src/librssguard-standard/src/standardserviceroot.h
View file

@ -8,6 +8,7 @@
#include <librssguard/services/abstract/serviceroot.h>
#include <QCoreApplication>
#include <QMutex>
#include <QPair>
class StandardCategory;
@ -43,6 +44,17 @@ class StandardServiceRoot : public ServiceRoot {
QList<QAction*> serviceMenu();
QList<QAction*> getContextMenuForFeed(StandardFeed* feed);
void spaceHost(const QString& host, const QString& url);
void resetHostSpacing(const QString& host, const QDateTime &next_dt);
// If set to number > 0, then requests to fetch feeds
// will be spaced by the given number (in seconds).
// This is used to avoid too many concurrent network
// requests from to the same server and getting HTTP/429
// errors or other DDoS-attack-related bans.
int spacingSameHostsRequests() const;
void setSpacingSameHostsRequests(int spacing);
static QString defaultTitle();
public slots:
@ -61,6 +73,10 @@ class StandardServiceRoot : public ServiceRoot {
QPointer<StandardFeed> m_feedForMetadata = {};
QList<QAction*> m_feedContextMenu = {};
int m_spacingSameHostsRequests;
QHash<QString, QDateTime> m_spacingHosts;
QMutex m_spacingMutex;
};
#endif // STANDARDSERVICEROOT_H