diff --git a/resources/docs/Documentation.md b/resources/docs/Documentation.md index 4f96cbe14..f87c4af35 100644 --- a/resources/docs/Documentation.md +++ b/resources/docs/Documentation.md @@ -60,7 +60,7 @@ I organized the supported web-based feed readers into an elegant table: | Service | Two-way Synchronization | [Intelligent Synchronization Algorithm](#intel) (ISA) 1 | Synchronized Labels 2 | OAuth 4 | | :--- | :---: | :---: | :---: | :---: -| Feedly | ✅ | ❌ | ✅ | ✅ (only for official binaries) +| Feedly | ✅ | ✅ | ✅ | ✅ (only for official binaries) | Gmail | ✅ | ✅ | ❌ | ✅ | Google Reader API 3 | ✅ | ✅ | ✅ | ✅ (only for Inoreader) | Nextcloud News | ✅ | ❌ | ❌ | ❌ diff --git a/src/librssguard/services/feedly/definitions.h b/src/librssguard/services/feedly/definitions.h index b0abfe989..4025451fd 100644 --- a/src/librssguard/services/feedly/definitions.h +++ b/src/librssguard/services/feedly/definitions.h @@ -28,6 +28,8 @@ #define FEEDLY_API_URL_COLLETIONS "collections" #define FEEDLY_API_URL_TAGS "tags" #define FEEDLY_API_URL_STREAM_CONTENTS "streams/contents?streamId=%1" +#define FEEDLY_API_URL_STREAM_IDS "streams/%1/ids" #define FEEDLY_API_URL_MARKERS "markers" +#define FEEDLY_API_URL_ENTRIES "entries/.mget" #endif // FEEDLY_DEFINITIONS_H diff --git a/src/librssguard/services/feedly/feedlynetwork.cpp b/src/librssguard/services/feedly/feedlynetwork.cpp index eda935410..f008395c7 100644 --- a/src/librssguard/services/feedly/feedlynetwork.cpp +++ b/src/librssguard/services/feedly/feedlynetwork.cpp @@ -33,7 +33,8 @@ FeedlyNetwork::FeedlyNetwork(QObject* parent) QSL(FEEDLY_API_SCOPE), this)), #endif m_username(QString()), - m_developerAccessToken(QString()), m_batchSize(FEEDLY_DEFAULT_BATCH_SIZE), m_downloadOnlyUnreadMessages(false) { + m_developerAccessToken(QString()), m_batchSize(FEEDLY_DEFAULT_BATCH_SIZE), m_downloadOnlyUnreadMessages(false), + m_intelligentSynchronization(true) { #if defined(FEEDLY_OFFICIAL_SUPPORT) m_oauth->setRedirectUrl(QSL(OAUTH_REDIRECT_URI) + QL1C(':') + QString::number(FEEDLY_API_REDIRECT_URI_PORT), @@ -45,6 +46,66 @@ FeedlyNetwork::FeedlyNetwork(QObject* parent) #endif } +QList FeedlyNetwork::messages(const QString& stream_id, + const QHash& stated_messages) { + if (!m_intelligentSynchronization) { + return streamContents(stream_id); + } + + // 1. Get unread IDs for a feed. + // 2. Get read IDs for a feed. + // 3. Download messages/contents for missing or changed IDs. + QStringList remote_all_ids_list, remote_unread_ids_list; + + remote_unread_ids_list = streamIds(stream_id, true, batchSize()); + + if (!downloadOnlyUnreadMessages()) { + remote_all_ids_list = streamIds(stream_id, false, batchSize()); + } + + // 1. + auto local_unread_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Unread); + QSet local_unread_ids = FROM_LIST_TO_SET(QSet, local_unread_ids_list); + QSet remote_unread_ids = FROM_LIST_TO_SET(QSet, remote_unread_ids_list); + + // 2. + auto local_read_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Read); + QSet local_read_ids = FROM_LIST_TO_SET(QSet, local_read_ids_list); + QSet remote_read_ids = FROM_LIST_TO_SET(QSet, remote_all_ids_list) - remote_unread_ids; + + // 3. + QSet to_download; + + // Undownloaded unread articles. + to_download += remote_unread_ids - local_unread_ids; + + // Undownloaded read articles. + if (!m_downloadOnlyUnreadMessages) { + to_download += remote_read_ids - local_read_ids; + } + + // Read articles newly marked as unread in service. + auto moved_read = local_read_ids.intersect(remote_unread_ids); + + to_download += moved_read; + + // Unread articles newly marked as read in service. + if (!m_downloadOnlyUnreadMessages) { + auto moved_unread = local_unread_ids.intersect(remote_read_ids); + + to_download += moved_unread; + } + + qDebugNN << LOGSEC_FEEDLY << "Will download" << QUOTE_W_SPACE(to_download.size()) << "articles."; + + if (to_download.isEmpty()) { + return {}; + } + else { + return entries(QStringList(to_download.values())); + } +} + void FeedlyNetwork::untagEntries(const QString& tag_id, const QStringList& msg_custom_ids) { if (msg_custom_ids.isEmpty()) { return; @@ -167,6 +228,49 @@ void FeedlyNetwork::markers(const QString& action, const QStringList& msg_custom } } +QList FeedlyNetwork::entries(const QStringList& ids) { + const QString bear = bearer(); + + if (bear.isEmpty()) { + qCriticalNN << LOGSEC_FEEDLY << "Cannot obtain personal collections, because bearer is empty."; + throw NetworkException(QNetworkReply::NetworkError::AuthenticationRequiredError); + } + + QList msgs; + int next_message = 0; + const QString target_url = fullUrl(Service::Entries); + const int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); + + do { + QJsonArray json; + + for (int window = next_message + 1000; next_message < window && next_message < ids.size(); next_message++ ) { + json.append(QJsonValue(ids.at(next_message))); + } + + QByteArray output; + auto result = NetworkFactory::performNetworkOperation(target_url, + timeout, + QJsonDocument(json).toJson(QJsonDocument::JsonFormat::Compact), + output, + QNetworkAccessManager::Operation::PostOperation, + { bearerHeader(bear) }, + false, + {}, + {}, + m_service->networkProxy()); + + if (result.m_networkError != QNetworkReply::NetworkError::NoError) { + throw NetworkException(result.m_networkError, output); + } + + msgs += decodeStreamContents(output, false, QString()); + } + while (next_message < ids.size()); + + return msgs; +} + QList FeedlyNetwork::streamContents(const QString& stream_id) { QString bear = bearer(); @@ -216,7 +320,7 @@ QList FeedlyNetwork::streamContents(const QString& stream_id) { throw NetworkException(result.m_networkError, output); } - messages += decodeStreamContents(output, continuation); + messages += decodeStreamContents(output, true, continuation); } while (!continuation.isEmpty() && (m_batchSize <= 0 || messages.size() < m_batchSize) && @@ -225,14 +329,83 @@ QList FeedlyNetwork::streamContents(const QString& stream_id) { return messages; } -QList FeedlyNetwork::decodeStreamContents(const QByteArray& stream_contents, QString& continuation) const { +QStringList FeedlyNetwork::streamIds(const QString& stream_id, bool unread_only, int batch_size) { + QString bear = bearer(); + + if (bear.isEmpty()) { + qCriticalNN << LOGSEC_FEEDLY << "Cannot obtain stream IDs, because bearer is empty."; + throw NetworkException(QNetworkReply::NetworkError::AuthenticationRequiredError); + } + + int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); + QByteArray output; + QString continuation; + QStringList messages; + + // We download in batches. + do { + QString target_url = fullUrl(Service::StreamIds).arg(QString(QUrl::toPercentEncoding(stream_id))); + + if (batch_size > 0) { + target_url += QSL("?count=%1").arg(QString::number(batch_size)); + } + else { + // User wants to download all messages. Make sure we use large batches + // to limit network requests. + target_url += QSL("?count=%1").arg(QString::number(10000)); + } + + if (unread_only) { + target_url += QSL("&unreadOnly=true"); + } + + if (!continuation.isEmpty()) { + target_url += QSL("&continuation=%1").arg(continuation); + } + + auto result = NetworkFactory::performNetworkOperation(target_url, + timeout, + {}, + output, + QNetworkAccessManager::Operation::GetOperation, + { bearerHeader(bear) }, + false, + {}, + {}, + m_service->networkProxy()); + + if (result.m_networkError != QNetworkReply::NetworkError::NoError) { + throw NetworkException(result.m_networkError, output); + } + + messages += decodeStreamIds(output, continuation); + } + while (!continuation.isEmpty() && (batch_size <= 0 || messages.size() < batch_size)); + + return messages; +} + +QStringList FeedlyNetwork::decodeStreamIds(const QByteArray& stream_ids, QString& continuation) const { + QStringList messages; + QJsonDocument json = QJsonDocument::fromJson(stream_ids); + + continuation = json.object()[QSL("continuation")].toString(); + + for (const QJsonValue& id_val : json.object()[QSL("ids")].toArray()) { + messages << id_val.toString(); + } + + return messages; +} + +QList FeedlyNetwork::decodeStreamContents(const QByteArray& stream_contents, bool nested_items, QString& continuation) const { QList messages; QJsonDocument json = QJsonDocument::fromJson(stream_contents); auto active_labels = m_service->labelsNode() != nullptr ? m_service->labelsNode()->labels() : QList(); continuation = json.object()[QSL("continuation")].toString(); - auto items = json.object()[QSL("items")].toArray(); + auto items = nested_items ? json.object()[QSL("items")].toArray() : json.array(); for (const QJsonValue& entry : qAsConst(items)) { const QJsonObject& entry_obj = entry.toObject(); @@ -573,6 +746,12 @@ QString FeedlyNetwork::fullUrl(FeedlyNetwork::Service service) const { case Service::StreamContents: return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_STREAM_CONTENTS); + case Service::StreamIds: + return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_STREAM_IDS); + + case Service::Entries: + return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_ENTRIES); + case Service::Markers: return QSL(FEEDLY_API_URL_BASE) + QSL(FEEDLY_API_URL_MARKERS); @@ -595,6 +774,14 @@ QPair FeedlyNetwork::bearerHeader(const QString& bearer) return { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(), bearer.toLocal8Bit() }; } +void FeedlyNetwork::setIntelligentSynchronization(bool intelligent_sync) { + m_intelligentSynchronization = intelligent_sync; +} + +bool FeedlyNetwork::intelligentSynchronization() const { + return m_intelligentSynchronization; +} + bool FeedlyNetwork::downloadOnlyUnreadMessages() const { return m_downloadOnlyUnreadMessages; } diff --git a/src/librssguard/services/feedly/feedlynetwork.h b/src/librssguard/services/feedly/feedlynetwork.h index c18102f05..4e6503bcf 100644 --- a/src/librssguard/services/feedly/feedlynetwork.h +++ b/src/librssguard/services/feedly/feedlynetwork.h @@ -7,6 +7,7 @@ #include "network-web/networkfactory.h" #include "services/abstract/feed.h" +#include "services/abstract/serviceroot.h" #if defined(FEEDLY_OFFICIAL_SUPPORT) class OAuth2Service; @@ -20,11 +21,16 @@ class FeedlyNetwork : public QObject { public: explicit FeedlyNetwork(QObject* parent = nullptr); + QList messages(const QString& stream_id, + const QHash& stated_messages); + // API operations. void untagEntries(const QString& tag_id, const QStringList& msg_custom_ids); void tagEntries(const QString& tag_id, const QStringList& msg_custom_ids); void markers(const QString& action, const QStringList& msg_custom_ids); + QList entries(const QStringList& ids); QList streamContents(const QString& stream_id); + QStringList streamIds(const QString& stream_id, bool unread_only, int batch_size); QVariantHash profile(const QNetworkProxy& network_proxy); QList tags(); RootItem* collections(bool obtain_icons); @@ -39,6 +45,9 @@ class FeedlyNetwork : public QObject { bool downloadOnlyUnreadMessages() const; void setDownloadOnlyUnreadMessages(bool download_only_unread_messages); + bool intelligentSynchronization() const; + void setIntelligentSynchronization(bool intelligent_sync); + int batchSize() const; void setBatchSize(int batch_size); @@ -61,12 +70,15 @@ class FeedlyNetwork : public QObject { Tags, StreamContents, Markers, - TagEntries + TagEntries, + StreamIds, + Entries }; QString fullUrl(Service service) const; QString bearer() const; - QList decodeStreamContents(const QByteArray& stream_contents, QString& continuation) const; + QStringList decodeStreamIds(const QByteArray& stream_ids, QString& continuation) const; + QList decodeStreamContents(const QByteArray& stream_contents, bool nested_items, QString& continuation) const; RootItem* decodeCollections(const QByteArray& json, bool obtain_icons, const QNetworkProxy& proxy, int timeout = 0) const; QPair bearerHeader(const QString& bearer) const; @@ -85,6 +97,9 @@ class FeedlyNetwork : public QObject { // Only download unread messages. bool m_downloadOnlyUnreadMessages; + + // Better synchronization algorithm. + bool m_intelligentSynchronization; }; #endif // FEEDLYNETWORK_H diff --git a/src/librssguard/services/feedly/feedlyserviceroot.cpp b/src/librssguard/services/feedly/feedlyserviceroot.cpp index e6923e380..1922e2316 100644 --- a/src/librssguard/services/feedly/feedlyserviceroot.cpp +++ b/src/librssguard/services/feedly/feedlyserviceroot.cpp @@ -56,6 +56,7 @@ QVariantHash FeedlyServiceRoot::customDatabaseData() const { data[QSL("batch_size")] = m_network->batchSize(); data[QSL("download_only_unread")] = m_network->downloadOnlyUnreadMessages(); + data[QSL("intelligent_synchronization")] = m_network->intelligentSynchronization(); return data; } @@ -70,16 +71,16 @@ void FeedlyServiceRoot::setCustomDatabaseData(const QVariantHash& data) { m_network->setBatchSize(data[QSL("batch_size")].toInt()); m_network->setDownloadOnlyUnreadMessages(data[QSL("download_only_unread")].toBool()); + m_network->setIntelligentSynchronization(data[QSL("intelligent_synchronization")].toBool()); } QList FeedlyServiceRoot::obtainNewMessages(Feed* feed, const QHash& stated_messages, const QHash& tagged_messages) { - Q_UNUSED(stated_messages) Q_UNUSED(tagged_messages) try { - return m_network->streamContents(feed->customId()); + return m_network->messages(feed->customId(), stated_messages); } catch (const ApplicationException& ex) { throw FeedFetchException(Feed::Status::NetworkError, ex.message()); @@ -258,3 +259,7 @@ RootItem* FeedlyServiceRoot::obtainNewTreeForSyncIn() const { return nullptr; } } + +bool FeedlyServiceRoot::wantsBaggedIdsOfExistingMessages() const { + return m_network->intelligentSynchronization(); +} diff --git a/src/librssguard/services/feedly/feedlyserviceroot.h b/src/librssguard/services/feedly/feedlyserviceroot.h index dc92146ef..0f432a032 100644 --- a/src/librssguard/services/feedly/feedlyserviceroot.h +++ b/src/librssguard/services/feedly/feedlyserviceroot.h @@ -23,6 +23,7 @@ class FeedlyServiceRoot : public ServiceRoot, public CacheForServiceRoot { virtual LabelOperation supportedLabelOperations() const; virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); + virtual bool wantsBaggedIdsOfExistingMessages() const; virtual QList obtainNewMessages(Feed* feed, const QHash& stated_messages, const QHash& tagged_messages); diff --git a/src/librssguard/services/feedly/gui/feedlyaccountdetails.cpp b/src/librssguard/services/feedly/gui/feedlyaccountdetails.cpp index a9ae65b15..b1b191a02 100644 --- a/src/librssguard/services/feedly/gui/feedlyaccountdetails.cpp +++ b/src/librssguard/services/feedly/gui/feedlyaccountdetails.cpp @@ -48,6 +48,13 @@ FeedlyAccountDetails::FeedlyAccountDetails(QWidget* parent) : QWidget(parent), m "end up with thousands of articles which you will never read anyway."), true); + m_ui.m_lblNewAlgorithm->setHelpText(tr("If you select intelligent synchronization, then only not-yet-fetched " + "or updated articles are downloaded. Network usage is greatly reduced and " + "overall synchronization speed is greatly improved, but " + "first feed fetching could be slow anyway if your feed contains " + "huge number of articles."), + false); + connect(m_ui.m_btnGetToken, &QPushButton::clicked, this, &FeedlyAccountDetails::getDeveloperAccessToken); connect(m_ui.m_txtUsername->lineEdit(), &BaseLineEdit::textChanged, this, &FeedlyAccountDetails::onUsernameChanged); connect(m_ui.m_txtDeveloperAccessToken->lineEdit(), &BaseLineEdit::textChanged, @@ -56,7 +63,8 @@ FeedlyAccountDetails::FeedlyAccountDetails(QWidget* parent) : QWidget(parent), m setTabOrder(m_ui.m_txtUsername->lineEdit(), m_ui.m_btnGetToken); setTabOrder(m_ui.m_btnGetToken, m_ui.m_txtDeveloperAccessToken->lineEdit()); setTabOrder(m_ui.m_txtDeveloperAccessToken->lineEdit(), m_ui.m_checkDownloadOnlyUnreadMessages); - setTabOrder(m_ui.m_checkDownloadOnlyUnreadMessages, m_ui.m_spinLimitMessages); + setTabOrder(m_ui.m_checkDownloadOnlyUnreadMessages, m_ui.m_cbNewAlgorithm); + setTabOrder(m_ui.m_cbNewAlgorithm, m_ui.m_spinLimitMessages); setTabOrder(m_ui.m_spinLimitMessages, m_ui.m_btnTestSetup); onDeveloperAccessTokenChanged(); diff --git a/src/librssguard/services/feedly/gui/feedlyaccountdetails.ui b/src/librssguard/services/feedly/gui/feedlyaccountdetails.ui index 75ef6746f..dca300fa8 100644 --- a/src/librssguard/services/feedly/gui/feedlyaccountdetails.ui +++ b/src/librssguard/services/feedly/gui/feedlyaccountdetails.ui @@ -7,7 +7,7 @@ 0 0 421 - 235 + 321 @@ -45,7 +45,24 @@ + + + + Download unread articles only + + + + + + Intelligent synchronization algorithm + + + + + + + @@ -70,19 +87,9 @@ - - - Qt::Vertical - - - - 400 - 86 - - - + - + @@ -100,31 +107,34 @@ - - - - Download unread articles only + + + + Qt::Vertical - - - - + + + 400 + 86 + + + - - LineEditWithStatus - QWidget -
lineeditwithstatus.h
- 1 -
LabelWithStatus QWidget
labelwithstatus.h
1
+ + LineEditWithStatus + QWidget +
lineeditwithstatus.h
+ 1 +
MessageCountSpinBox QSpinBox @@ -137,6 +147,13 @@ 1
+ + m_btnGetToken + m_checkDownloadOnlyUnreadMessages + m_cbNewAlgorithm + m_spinLimitMessages + m_btnTestSetup + diff --git a/src/librssguard/services/feedly/gui/formeditfeedlyaccount.cpp b/src/librssguard/services/feedly/gui/formeditfeedlyaccount.cpp index ba82e3e60..61474740d 100644 --- a/src/librssguard/services/feedly/gui/formeditfeedlyaccount.cpp +++ b/src/librssguard/services/feedly/gui/formeditfeedlyaccount.cpp @@ -31,12 +31,13 @@ void FormEditFeedlyAccount::apply() { #endif bool using_another_acc = - m_details->m_ui.m_txtUsername->lineEdit()->text() !=account()->network()->username(); + m_details->m_ui.m_txtUsername->lineEdit()->text() != account()->network()->username(); account()->network()->setUsername(m_details->m_ui.m_txtUsername->lineEdit()->text()); account()->network()->setDownloadOnlyUnreadMessages(m_details->m_ui.m_checkDownloadOnlyUnreadMessages->isChecked()); account()->network()->setBatchSize(m_details->m_ui.m_spinLimitMessages->value()); account()->network()->setDeveloperAccessToken(m_details->m_ui.m_txtDeveloperAccessToken->lineEdit()->text()); + account()->network()->setIntelligentSynchronization(m_details->m_ui.m_cbNewAlgorithm->isChecked()); account()->saveAccountDataToDatabase(); accept(); @@ -62,6 +63,7 @@ void FormEditFeedlyAccount::loadAccountData() { m_details->m_ui.m_txtDeveloperAccessToken->lineEdit()->setText(account()->network()->developerAccessToken()); m_details->m_ui.m_checkDownloadOnlyUnreadMessages->setChecked(account()->network()->downloadOnlyUnreadMessages()); m_details->m_ui.m_spinLimitMessages->setValue(account()->network()->batchSize()); + m_details->m_ui.m_cbNewAlgorithm->setChecked(account()->network()->intelligentSynchronization()); } void FormEditFeedlyAccount::performTest() {