pogmommy/rssguard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feedly now supports ISA - intelligent sync. algorithm

Browse source
This commit is contained in:
Martin Rotter 2022-03-23 12:58:36 +01:00
parent a639f06891
commit ad2ec0eb1b
9 changed files with 275 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
resources/docs/Documentation.md
View file

@ -60,7 +60,7 @@ I organized the supported web-based feed readers into an elegant table:
| Service | Two-way Synchronization | [Intelligent Synchronization Algorithm](#intel) (ISA) <sup>1</sup> | Synchronized Labels <sup>2</sup> <a id="sfrl"></a> | OAuth <sup>4</sup> |
| :--- | :---: | :---: | :---: | :---:
| Feedly | ✅ | ❌ | ✅ | ✅ (only for official binaries)
| Feedly | ✅ | ✅ | ✅ | ✅ (only for official binaries)
| Gmail | ✅ | ✅ | ❌ | ✅
| Google Reader API <sup>3</sup> | ✅ | ✅ | ✅ | ✅ (only for Inoreader)
| Nextcloud News | ✅ | ❌ | ❌ | ❌

2
src/librssguard/services/feedly/definitions.h
View file

@ -28,6 +28,8 @@
#define FEEDLY_API_URL_COLLETIONS "collections"
#define FEEDLY_API_URL_TAGS "tags"
#define FEEDLY_API_URL_STREAM_CONTENTS "streams/contents?streamId=%1"
#define FEEDLY_API_URL_STREAM_IDS "streams/%1/ids"
#define FEEDLY_API_URL_MARKERS "markers"
#define FEEDLY_API_URL_ENTRIES "entries/.mget"
#endif // FEEDLY_DEFINITIONS_H

195
src/librssguard/services/feedly/feedlynetwork.cpp
View file

@ -33,7 +33,8 @@ FeedlyNetwork::FeedlyNetwork(QObject* parent)
QSL(FEEDLY_API_SCOPE), this)),
#endif
m_username(QString()),
m_developerAccessToken(QString()), m_batchSize(FEEDLY_DEFAULT_BATCH_SIZE), m_downloadOnlyUnreadMessages(false) {
m_developerAccessToken(QString()), m_batchSize(FEEDLY_DEFAULT_BATCH_SIZE), m_downloadOnlyUnreadMessages(false),
m_intelligentSynchronization(true) {
#if defined(FEEDLY_OFFICIAL_SUPPORT)
m_oauth->setRedirectUrl(QSL(OAUTH_REDIRECT_URI) + QL1C(':') + QString::number(FEEDLY_API_REDIRECT_URI_PORT),
@ -45,6 +46,66 @@ FeedlyNetwork::FeedlyNetwork(QObject* parent)
#endif
}
QList<Message> FeedlyNetwork::messages(const QString& stream_id,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages) {
if (!m_intelligentSynchronization) {
return streamContents(stream_id);
}
// 1. Get unread IDs for a feed.
// 2. Get read IDs for a feed.
// 3. Download messages/contents for missing or changed IDs.
QStringList remote_all_ids_list, remote_unread_ids_list;
remote_unread_ids_list = streamIds(stream_id, true, batchSize());
if (!downloadOnlyUnreadMessages()) {
remote_all_ids_list = streamIds(stream_id, false, batchSize());
}
// 1.
auto local_unread_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Unread);
QSet<QString> local_unread_ids = FROM_LIST_TO_SET(QSet<QString>, local_unread_ids_list);
QSet<QString> remote_unread_ids = FROM_LIST_TO_SET(QSet<QString>, remote_unread_ids_list);
// 2.
auto local_read_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Read);
QSet<QString> local_read_ids = FROM_LIST_TO_SET(QSet<QString>, local_read_ids_list);
QSet<QString> remote_read_ids = FROM_LIST_TO_SET(QSet<QString>, remote_all_ids_list) - remote_unread_ids;
// 3.
QSet<QString> to_download;
// Undownloaded unread articles.
to_download += remote_unread_ids - local_unread_ids;
// Undownloaded read articles.
if (!m_downloadOnlyUnreadMessages) {
to_download += remote_read_ids - local_read_ids;
}
// Read articles newly marked as unread in service.
auto moved_read = local_read_ids.intersect(remote_unread_ids);
to_download += moved_read;
// Unread articles newly marked as read in service.
if (!m_downloadOnlyUnreadMessages) {
auto moved_unread = local_unread_ids.intersect(remote_read_ids);
to_download += moved_unread;
}
qDebugNN << LOGSEC_FEEDLY << "Will download" << QUOTE_W_SPACE(to_download.size()) << "articles.";
if (to_download.isEmpty()) {
return {};
}
else {
return entries(QStringList(to_download.values()));
}
}
void FeedlyNetwork::untagEntries(const QString& tag_id, const QStringList& msg_custom_ids) {
if (msg_custom_ids.isEmpty()) {
return;
@ -167,6 +228,49 @@ void FeedlyNetwork::markers(const QString& action, const QStringList& msg_custom
}
}
QList<Message> FeedlyNetwork::entries(const QStringList& ids) {
const QString bear = bearer();
if (bear.isEmpty()) {
qCriticalNN << LOGSEC_FEEDLY << "Cannot obtain personal collections, because bearer is empty.";
throw NetworkException(QNetworkReply::NetworkError::AuthenticationRequiredError);
}
QList<Message> msgs;
int next_message = 0;
const QString target_url = fullUrl(Service::Entries);
const int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
do {
QJsonArray json;
for (int window = next_message + 1000; next_message < window && next_message < ids.size(); next_message++ ) {
json.append(QJsonValue(ids.at(next_message)));
}
QByteArray output;
auto result = NetworkFactory::performNetworkOperation(target_url,
timeout,
QJsonDocument(json).toJson(QJsonDocument::JsonFormat::Compact),
output,
QNetworkAccessManager::Operation::PostOperation,
{ bearerHeader(bear) },
false,
{},
{},
m_service->networkProxy());
if (result.m_networkError != QNetworkReply::NetworkError::NoError) {
throw NetworkException(result.m_networkError, output);
}
msgs += decodeStreamContents(output, false, QString());
}
while (next_message < ids.size());
return msgs;
}
QList<Message> FeedlyNetwork::streamContents(const QString& stream_id) {
QString bear = bearer();
@ -216,7 +320,7 @@ QList<Message> FeedlyNetwork::streamContents(const QString& stream_id) {
throw NetworkException(result.m_networkError, output);
}
messages += decodeStreamContents(output, continuation);
messages += decodeStreamContents(output, true, continuation);
}
while (!continuation.isEmpty() &&
(m_batchSize <= 0 || messages.size() < m_batchSize) &&
@ -225,14 +329,83 @@ QList<Message> FeedlyNetwork::streamContents(const QString& stream_id) {
return messages;
}
QList<Message> FeedlyNetwork::decodeStreamContents(const QByteArray& stream_contents, QString& continuation) const {
QStringList FeedlyNetwork::streamIds(const QString& stream_id, bool unread_only, int batch_size) {
QString bear = bearer();
if (bear.isEmpty()) {
qCriticalNN << LOGSEC_FEEDLY << "Cannot obtain stream IDs, because bearer is empty.";
throw NetworkException(QNetworkReply::NetworkError::AuthenticationRequiredError);
}
int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
QByteArray output;
QString continuation;
QStringList messages;
// We download in batches.
do {
QString target_url = fullUrl(Service::StreamIds).arg(QString(QUrl::toPercentEncoding(stream_id)));
if (batch_size > 0) {
target_url += QSL("?count=%1").arg(QString::number(batch_size));
}
else {
// User wants to download all messages. Make sure we use large batches
// to limit network requests.
target_url += QSL("?count=%1").arg(QString::number(10000));
}
if (unread_only) {
target_url += QSL("&unreadOnly=true");
}
if (!continuation.isEmpty()) {
target_url += QSL("&continuation=%1").arg(continuation);
}
auto result = NetworkFactory::performNetworkOperation(target_url,
timeout,
{},
output,
QNetworkAccessManager::Operation::GetOperation,
{ bearerHeader(bear) },
false,
{},
{},
m_service->networkProxy());
if (result.m_networkError != QNetworkReply::NetworkError::NoError) {
throw NetworkException(result.m_networkError, output);
}
messages += decodeStreamIds(output, continuation);
}
while (!continuation.isEmpty() && (batch_size <= 0 || messages.size() < batch_size));
return messages;
}
QStringList FeedlyNetwork::decodeStreamIds(const QByteArray& stream_ids, QString& continuation) const {
QStringList messages;
QJsonDocument json = QJsonDocument::fromJson(stream_ids);
continuation = json.object()[QSL("continuation")].toString();
for (const QJsonValue& id_val : json.object()[QSL("ids")].toArray()) {
messages << id_val.toString();
}
return messages;
}
QList<Message> FeedlyNetwork::decodeStreamContents(const QByteArray& stream_contents, bool nested_items, QString& continuation) const {
QList<Message> messages;
QJsonDocument json = QJsonDocument::fromJson(stream_contents);
auto active_labels = m_service->labelsNode() != nullptr ? m_service->labelsNode()->labels() : QList<Label*>();
continuation = json.object()[QSL("continuation")].toString();
auto items = json.object()[QSL("items")].toArray();
auto items = nested_items ? json.object()[QSL("items")].toArray() : json.array();
for (const QJsonValue& entry : qAsConst(items)) {
const QJsonObject& entry_obj = entry.toObject();
@ -573,6 +746,12 @@ QString FeedlyNetwork::fullUrl(FeedlyNetwork::Service service) const {
case Service::StreamContents:
return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_STREAM_CONTENTS);
case Service::StreamIds:
return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_STREAM_IDS);
case Service::Entries:
return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_ENTRIES);
case Service::Markers:
return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_MARKERS);
@ -595,6 +774,14 @@ QPair<QByteArray, QByteArray> FeedlyNetwork::bearerHeader(const QString& bearer)
return { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(), bearer.toLocal8Bit() };
}
void FeedlyNetwork::setIntelligentSynchronization(bool intelligent_sync) {
m_intelligentSynchronization = intelligent_sync;
}
bool FeedlyNetwork::intelligentSynchronization() const {
return m_intelligentSynchronization;
}
bool FeedlyNetwork::downloadOnlyUnreadMessages() const {
return m_downloadOnlyUnreadMessages;
}

19
src/librssguard/services/feedly/feedlynetwork.h
View file

@ -7,6 +7,7 @@
#include "network-web/networkfactory.h"
#include "services/abstract/feed.h"
#include "services/abstract/serviceroot.h"
#if defined(FEEDLY_OFFICIAL_SUPPORT)
class OAuth2Service;
@ -20,11 +21,16 @@ class FeedlyNetwork : public QObject {
public:
explicit FeedlyNetwork(QObject* parent = nullptr);
QList<Message> messages(const QString& stream_id,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages);
// API operations.
void untagEntries(const QString& tag_id, const QStringList& msg_custom_ids);
void tagEntries(const QString& tag_id, const QStringList& msg_custom_ids);
void markers(const QString& action, const QStringList& msg_custom_ids);
QList<Message> entries(const QStringList& ids);
QList<Message> streamContents(const QString& stream_id);
QStringList streamIds(const QString& stream_id, bool unread_only, int batch_size);
QVariantHash profile(const QNetworkProxy& network_proxy);
QList<RootItem*> tags();
RootItem* collections(bool obtain_icons);
@ -39,6 +45,9 @@ class FeedlyNetwork : public QObject {
bool downloadOnlyUnreadMessages() const;
void setDownloadOnlyUnreadMessages(bool download_only_unread_messages);
bool intelligentSynchronization() const;
void setIntelligentSynchronization(bool intelligent_sync);
int batchSize() const;
void setBatchSize(int batch_size);
@ -61,12 +70,15 @@ class FeedlyNetwork : public QObject {
Tags,
StreamContents,
Markers,
TagEntries
TagEntries,
StreamIds,
Entries
};
QString fullUrl(Service service) const;
QString bearer() const;
QList<Message> decodeStreamContents(const QByteArray& stream_contents, QString& continuation) const;
QStringList decodeStreamIds(const QByteArray& stream_ids, QString& continuation) const;
QList<Message> decodeStreamContents(const QByteArray& stream_contents, bool nested_items, QString& continuation) const;
RootItem* decodeCollections(const QByteArray& json, bool obtain_icons, const QNetworkProxy& proxy, int timeout = 0) const;
QPair<QByteArray, QByteArray> bearerHeader(const QString& bearer) const;
@ -85,6 +97,9 @@ class FeedlyNetwork : public QObject {
// Only download unread messages.
bool m_downloadOnlyUnreadMessages;
// Better synchronization algorithm.
bool m_intelligentSynchronization;
};
#endif // FEEDLYNETWORK_H

9
src/librssguard/services/feedly/feedlyserviceroot.cpp
View file

@ -56,6 +56,7 @@ QVariantHash FeedlyServiceRoot::customDatabaseData() const {
data[QSL("batch_size")] = m_network->batchSize();
data[QSL("download_only_unread")] = m_network->downloadOnlyUnreadMessages();
data[QSL("intelligent_synchronization")] = m_network->intelligentSynchronization();
return data;
}
@ -70,16 +71,16 @@ void FeedlyServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
m_network->setBatchSize(data[QSL("batch_size")].toInt());
m_network->setDownloadOnlyUnreadMessages(data[QSL("download_only_unread")].toBool());
m_network->setIntelligentSynchronization(data[QSL("intelligent_synchronization")].toBool());
}
QList<Message> FeedlyServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
try {
return m_network->streamContents(feed->customId());
return m_network->messages(feed->customId(), stated_messages);
}
catch (const ApplicationException& ex) {
throw FeedFetchException(Feed::Status::NetworkError, ex.message());
@ -258,3 +259,7 @@ RootItem* FeedlyServiceRoot::obtainNewTreeForSyncIn() const {
return nullptr;
}
}
bool FeedlyServiceRoot::wantsBaggedIdsOfExistingMessages() const {
return m_network->intelligentSynchronization();
}

1
src/librssguard/services/feedly/feedlyserviceroot.h
View file

@ -23,6 +23,7 @@ class FeedlyServiceRoot : public ServiceRoot, public CacheForServiceRoot {
virtual LabelOperation supportedLabelOperations() const;
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual bool wantsBaggedIdsOfExistingMessages() const;
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);

10
src/librssguard/services/feedly/gui/feedlyaccountdetails.cpp
View file

@ -48,6 +48,13 @@ FeedlyAccountDetails::FeedlyAccountDetails(QWidget* parent) : QWidget(parent), m
"end up with thousands of articles which you will never read anyway."),
true);
m_ui.m_lblNewAlgorithm->setHelpText(tr("If you select intelligent synchronization, then only not-yet-fetched "
"or updated articles are downloaded. Network usage is greatly reduced and "
"overall synchronization speed is greatly improved, but "
"first feed fetching could be slow anyway if your feed contains "
"huge number of articles."),
false);
connect(m_ui.m_btnGetToken, &QPushButton::clicked, this, &FeedlyAccountDetails::getDeveloperAccessToken);
connect(m_ui.m_txtUsername->lineEdit(), &BaseLineEdit::textChanged, this, &FeedlyAccountDetails::onUsernameChanged);
connect(m_ui.m_txtDeveloperAccessToken->lineEdit(), &BaseLineEdit::textChanged,
@ -56,7 +63,8 @@ FeedlyAccountDetails::FeedlyAccountDetails(QWidget* parent) : QWidget(parent), m
setTabOrder(m_ui.m_txtUsername->lineEdit(), m_ui.m_btnGetToken);
setTabOrder(m_ui.m_btnGetToken, m_ui.m_txtDeveloperAccessToken->lineEdit());
setTabOrder(m_ui.m_txtDeveloperAccessToken->lineEdit(), m_ui.m_checkDownloadOnlyUnreadMessages);
setTabOrder(m_ui.m_checkDownloadOnlyUnreadMessages, m_ui.m_spinLimitMessages);
setTabOrder(m_ui.m_checkDownloadOnlyUnreadMessages, m_ui.m_cbNewAlgorithm);
setTabOrder(m_ui.m_cbNewAlgorithm, m_ui.m_spinLimitMessages);
setTabOrder(m_ui.m_spinLimitMessages, m_ui.m_btnTestSetup);
onDeveloperAccessTokenChanged();

71
src/librssguard/services/feedly/gui/feedlyaccountdetails.ui
View file

@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>421</width>
<height>235</height>
<height>321</height>
</rect>
</property>
<layout class="QFormLayout" name="formLayout">
@ -45,7 +45,24 @@
<item row="2" column="0" colspan="2">
<widget class="HelpSpoiler" name="m_lblInfo" native="true"/>
</item>
<item row="3" column="0" colspan="2">
<widget class="QCheckBox" name="m_checkDownloadOnlyUnreadMessages">
<property name="text">
<string>Download unread articles only</string>
</property>
</widget>
</item>
<item row="4" column="0" colspan="2">
<widget class="QCheckBox" name="m_cbNewAlgorithm">
<property name="text">
<string>Intelligent synchronization algorithm</string>
</property>
</widget>
</item>
<item row="5" column="0" colspan="2">
<widget class="HelpSpoiler" name="m_lblNewAlgorithm" native="true"/>
</item>
<item row="6" column="0" colspan="2">
<layout class="QFormLayout" name="formLayout_3">
<item row="0" column="0">
<widget class="QLabel" name="label">
@ -70,19 +87,9 @@
</layout>
</item>
<item row="7" column="0" colspan="2">
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>400</width>
<height>86</height>
</size>
</property>
</spacer>
<widget class="HelpSpoiler" name="m_lblLimitMessagesInfo" native="true"/>
</item>
<item row="6" column="0" colspan="2">
<item row="8" column="0">
<layout class="QFormLayout" name="formLayout_2">
<item row="0" column="0">
<widget class="QPushButton" name="m_btnTestSetup">
@ -100,31 +107,34 @@
</item>
</layout>
</item>
<item row="3" column="0" colspan="2">
<widget class="QCheckBox" name="m_checkDownloadOnlyUnreadMessages">
<property name="text">
<string>Download unread articles only</string>
<item row="9" column="0" colspan="2">
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
</widget>
</item>
<item row="5" column="0" colspan="2">
<widget class="HelpSpoiler" name="m_lblLimitMessagesInfo" native="true"/>
<property name="sizeHint" stdset="0">
<size>
<width>400</width>
<height>86</height>
</size>
</property>
</spacer>
</item>
</layout>
</widget>
<customwidgets>
<customwidget>
<class>LineEditWithStatus</class>
<extends>QWidget</extends>
<header>lineeditwithstatus.h</header>
<container>1</container>
</customwidget>
<customwidget>
<class>LabelWithStatus</class>
<extends>QWidget</extends>
<header>labelwithstatus.h</header>
<container>1</container>
</customwidget>
<customwidget>
<class>LineEditWithStatus</class>
<extends>QWidget</extends>
<header>lineeditwithstatus.h</header>
<container>1</container>
</customwidget>
<customwidget>
<class>MessageCountSpinBox</class>
<extends>QSpinBox</extends>
@ -137,6 +147,13 @@
<container>1</container>
</customwidget>
</customwidgets>
<tabstops>
<tabstop>m_btnGetToken</tabstop>
<tabstop>m_checkDownloadOnlyUnreadMessages</tabstop>
<tabstop>m_cbNewAlgorithm</tabstop>
<tabstop>m_spinLimitMessages</tabstop>
<tabstop>m_btnTestSetup</tabstop>
</tabstops>
<resources/>
<connections/>
</ui>

4
src/librssguard/services/feedly/gui/formeditfeedlyaccount.cpp
View file

@ -31,12 +31,13 @@ void FormEditFeedlyAccount::apply() {
#endif
bool using_another_acc =
m_details->m_ui.m_txtUsername->lineEdit()->text() !=account<FeedlyServiceRoot>()->network()->username();
m_details->m_ui.m_txtUsername->lineEdit()->text() != account<FeedlyServiceRoot>()->network()->username();
account<FeedlyServiceRoot>()->network()->setUsername(m_details->m_ui.m_txtUsername->lineEdit()->text());
account<FeedlyServiceRoot>()->network()->setDownloadOnlyUnreadMessages(m_details->m_ui.m_checkDownloadOnlyUnreadMessages->isChecked());
account<FeedlyServiceRoot>()->network()->setBatchSize(m_details->m_ui.m_spinLimitMessages->value());
account<FeedlyServiceRoot>()->network()->setDeveloperAccessToken(m_details->m_ui.m_txtDeveloperAccessToken->lineEdit()->text());
account<FeedlyServiceRoot>()->network()->setIntelligentSynchronization(m_details->m_ui.m_cbNewAlgorithm->isChecked());
account<FeedlyServiceRoot>()->saveAccountDataToDatabase();
accept();
@ -62,6 +63,7 @@ void FormEditFeedlyAccount::loadAccountData() {
m_details->m_ui.m_txtDeveloperAccessToken->lineEdit()->setText(account<FeedlyServiceRoot>()->network()->developerAccessToken());
m_details->m_ui.m_checkDownloadOnlyUnreadMessages->setChecked(account<FeedlyServiceRoot>()->network()->downloadOnlyUnreadMessages());
m_details->m_ui.m_spinLimitMessages->setValue(account<FeedlyServiceRoot>()->network()->batchSize());
m_details->m_ui.m_cbNewAlgorithm->setChecked(account<FeedlyServiceRoot>()->network()->intelligentSynchronization());
}
void FormEditFeedlyAccount::performTest() {