From 8cc268ca9a0d9a44bce0b935fd5f0ca26184872b Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Fri, 29 Jan 2021 11:11:35 +0100 Subject: [PATCH] Working msg download - fresh rss. --- src/librssguard/core/messagesmodel.cpp | 10 +- .../services/greader/definitions.h | 9 +- .../services/greader/greaderfeed.cpp | 5 +- .../services/greader/greadernetwork.cpp | 112 ++++++++++++++++-- .../services/greader/greadernetwork.h | 6 +- 5 files changed, 123 insertions(+), 19 deletions(-) diff --git a/src/librssguard/core/messagesmodel.cpp b/src/librssguard/core/messagesmodel.cpp index 8c2697a95..b6c711740 100644 --- a/src/librssguard/core/messagesmodel.cpp +++ b/src/librssguard/core/messagesmodel.cpp @@ -242,7 +242,7 @@ QVariant MessagesModel::data(const QModelIndex& idx, int role) const { // This message is not in cache, return real data from live query. switch (role) { // Human readable data for viewing. - case Qt::DisplayRole: { + case Qt::ItemDataRole::DisplayRole: { int index_column = idx.column(); if (index_column == MSG_DB_DCREATED_INDEX) { @@ -276,10 +276,10 @@ QVariant MessagesModel::data(const QModelIndex& idx, int role) const { } } - case Qt::EditRole: + case Qt::ItemDataRole::EditRole: return m_cache->containsData(idx.row()) ? m_cache->data(idx) : QSqlQueryModel::data(idx, role); - case Qt::FontRole: { + case Qt::ItemDataRole::FontRole: { QModelIndex idx_read = index(idx.row(), MSG_DB_READ_INDEX); QVariant data_read = data(idx_read, Qt::EditRole); const bool is_bin = qobject_cast(loadedItem()) != nullptr; @@ -306,7 +306,7 @@ QVariant MessagesModel::data(const QModelIndex& idx, int role) const { } } - case Qt::ForegroundRole: + case Qt::ItemDataRole::ForegroundRole: switch (m_messageHighlighter) { case MessageHighlighter::HighlightImportant: { QModelIndex idx_important = index(idx.row(), MSG_DB_IMPORTANT_INDEX); @@ -327,7 +327,7 @@ QVariant MessagesModel::data(const QModelIndex& idx, int role) const { return QVariant(); } - case Qt::DecorationRole: { + case Qt::ItemDataRole::DecorationRole: { const int index_column = idx.column(); if (index_column == MSG_DB_READ_INDEX) { diff --git a/src/librssguard/services/greader/definitions.h b/src/librssguard/services/greader/definitions.h index 4bef79418..b95ba5349 100755 --- a/src/librssguard/services/greader/definitions.h +++ b/src/librssguard/services/greader/definitions.h @@ -3,8 +3,10 @@ #define GREADER_UNLIMITED_BATCH_SIZE -1 -// FreshRSS. -#define FRESHRSS_BASE_URL_PATH "api/greader.php/" +// States. +#define GREADER_API_STATE_READING_LIST "state/com.google/reading-list" +#define GREADER_API_STATE_READ "state/com.google/read" +#define GREADER_API_STATE_IMPORTANT "state/com.google/starred" // API. #define GREADER_API_CLIENT_LOGIN "accounts/ClientLogin?Email=%1&Passwd=%2" @@ -12,4 +14,7 @@ #define GREADER_API_SUBSCRIPTION_LIST "reader/api/0/subscription/list?output=json" #define GREADER_API_STREAM_CONTENTS "reader/api/0/stream/contents/%1?output=json&n=%2" +// FreshRSS. +#define FRESHRSS_BASE_URL_PATH "api/greader.php/" + #endif // GREADER_DEFINITIONS_H diff --git a/src/librssguard/services/greader/greaderfeed.cpp b/src/librssguard/services/greader/greaderfeed.cpp index f8770d3d8..dba728534 100755 --- a/src/librssguard/services/greader/greaderfeed.cpp +++ b/src/librssguard/services/greader/greaderfeed.cpp @@ -17,7 +17,10 @@ GreaderServiceRoot* GreaderFeed::serviceRoot() const { QList GreaderFeed::obtainNewMessages(bool* error_during_obtaining) { Feed::Status error = Feed::Status::Normal; - QList messages = serviceRoot()->network()->messages(getParentServiceRoot(), customId(), error); + QList messages = serviceRoot()->network()->streamContents(getParentServiceRoot(), + customId(), + error, + getParentServiceRoot()->networkProxy()); setStatus(error); diff --git a/src/librssguard/services/greader/greadernetwork.cpp b/src/librssguard/services/greader/greadernetwork.cpp index d297313ff..a2bd38540 100755 --- a/src/librssguard/services/greader/greadernetwork.cpp +++ b/src/librssguard/services/greader/greadernetwork.cpp @@ -2,8 +2,10 @@ #include "services/greader/greadernetwork.h" +#include "3rd-party/boolinq/boolinq.h" #include "miscellaneous/application.h" #include "network-web/networkfactory.h" +#include "network-web/webfactory.h" #include "services/abstract/category.h" #include "services/abstract/label.h" #include "services/abstract/labelsnode.h" @@ -20,19 +22,22 @@ GreaderNetwork::GreaderNetwork(QObject* parent) clearCredentials(); } -QList GreaderNetwork::streamContents(ServiceRoot* root, const QString& stream_id, Feed::Status& error) { - QString full_url = generateFullUrl(Operations::StreamContents).arg(stream_id, batchSize()); +QList GreaderNetwork::streamContents(ServiceRoot* root, const QString& stream_id, + Feed::Status& error, const QNetworkProxy& proxy) { + QString full_url = generateFullUrl(Operations::StreamContents).arg(stream_id, + QString::number(batchSize())); auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); if (!ensureLogin(proxy)) { - return nullptr; + error = Feed::Status::AuthError; + return {}; } - QByteArray output_labels; - auto result_labels = NetworkFactory::performNetworkOperation(full_url, + QByteArray output_stream; + auto result_stream = NetworkFactory::performNetworkOperation(full_url, timeout, {}, - output_labels, + output_stream, QNetworkAccessManager::Operation::GetOperation, { authHeader() }, false, @@ -40,7 +45,19 @@ QList GreaderNetwork::streamContents(ServiceRoot* root, const QString& {}, proxy); - return {}; + if (result_stream.first != QNetworkReply::NetworkError::NoError) { + qCriticalNN << LOGSEC_GREADER + << "Cannot download messages for " + << QUOTE_NO_SPACE(stream_id) + << ", network error:" + << QUOTE_W_SPACE_DOT(result_stream.first); + error = Feed::Status::NetworkError; + return {}; + } + else { + error = Feed::Status::Normal; + return decodeStreamContents(root, output_stream, stream_id); + } } RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNetworkProxy& proxy) { @@ -84,12 +101,12 @@ RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNe return nullptr; } - auto root = decodeFeedCategoriesData(output_labels, output_feeds, obtain_icons); + auto root = decodeTagsSubscriptions(output_labels, output_feeds, obtain_icons); return root; } -RootItem* GreaderNetwork::decodeFeedCategoriesData(const QString& categories, const QString& feeds, bool obtain_icons) { +RootItem* GreaderNetwork::decodeTagsSubscriptions(const QString& categories, const QString& feeds, bool obtain_icons) { auto* parent = new RootItem(); auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); QJsonArray json = QJsonDocument::fromJson(categories.toUtf8()).object()["tags"].toArray(); @@ -305,6 +322,80 @@ bool GreaderNetwork::ensureLogin(const QNetworkProxy& proxy) { return true; } +QList GreaderNetwork::decodeStreamContents(ServiceRoot* root, + const QString& stream_json_data, + const QString& stream_id) { + QList messages; + QJsonArray json = QJsonDocument::fromJson(stream_json_data.toUtf8()).object()["items"].toArray(); + auto active_labels = root->labelsNode() != nullptr ? root->labelsNode()->labels() : QList(); + + messages.reserve(json.count()); + + for (const QJsonValue& obj : json) { + auto message_obj = obj.toObject(); + Message message; + + message.m_title = qApp->web()->unescapeHtml(message_obj["title"].toString()); + message.m_author = qApp->web()->unescapeHtml(message_obj["author"].toString()); + message.m_created = QDateTime::fromSecsSinceEpoch(message_obj["published"].toInt(), Qt::UTC); + message.m_createdFromFeed = true; + message.m_customId = message_obj["id"].toString(); + + auto alternates = message_obj["alternate"].toArray(); + auto enclosures = message_obj["enclosure"].toArray(); + auto categories = message_obj["categories"].toArray(); + + for (const QJsonValue& alt : alternates) { + auto alt_obj = alt.toObject(); + QString mime = alt_obj["type"].toString(); + QString href = alt_obj["href"].toString(); + + if (mime.isEmpty() || mime == QL1S("text/html")) { + message.m_url = href; + } + else { + message.m_enclosures.append(Enclosure(href, mime)); + } + } + + for (const QJsonValue& enc : enclosures) { + auto enc_obj = enc.toObject(); + QString mime = enc_obj["type"].toString(); + QString href = enc_obj["href"].toString(); + + message.m_enclosures.append(Enclosure(href, mime)); + } + + for (const QJsonValue& cat : categories) { + QString category = cat.toString(); + + if (category.contains(GREADER_API_STATE_READ)) { + message.m_isRead = !category.contains(GREADER_API_STATE_READING_LIST); + } + else if (category.contains(GREADER_API_STATE_IMPORTANT)) { + message.m_isImportant = category.contains(GREADER_API_STATE_IMPORTANT); + } + else if (category.contains(QSL("label"))) { + Label* label = boolinq::from(active_labels.begin(), active_labels.end()).firstOrDefault([category](Label* lbl) { + return lbl->customId() == category; + }); + + if (label != nullptr) { + // We found live Label object for our assigned label. + message.m_assignedLabels.append(label); + } + } + } + + message.m_contents = message_obj["summary"].toObject()["content"].toString(); + message.m_feedId = stream_id; + + messages.append(message); + } + + return messages; +} + int GreaderNetwork::batchSize() const { return m_batchSize; } @@ -346,5 +437,8 @@ QString GreaderNetwork::generateFullUrl(GreaderNetwork::Operations operation) co case Operations::SubscriptionList: return sanitizedBaseUrl() + GREADER_API_SUBSCRIPTION_LIST; + + case Operations::StreamContents: + return sanitizedBaseUrl() + GREADER_API_STREAM_CONTENTS; } } diff --git a/src/librssguard/services/greader/greadernetwork.h b/src/librssguard/services/greader/greadernetwork.h index dbbce27cc..76b3a16ad 100755 --- a/src/librssguard/services/greader/greadernetwork.h +++ b/src/librssguard/services/greader/greadernetwork.h @@ -23,7 +23,8 @@ class GreaderNetwork : public QObject { explicit GreaderNetwork(QObject* parent = nullptr); // Stream contents for a feed/label/etc. - QList streamContents(ServiceRoot* root, const QString& stream_id, Feed::Status& error); + QList streamContents(ServiceRoot* root, const QString& stream_id, + Feed::Status& error, const QNetworkProxy& proxy); // Downloads and structures full tree for sync-in. RootItem* categoriesFeedsLabelsTree(bool obtain_icons, const QNetworkProxy& proxy); @@ -57,7 +58,8 @@ class GreaderNetwork : public QObject { // Make sure we are logged in and if we are not, return error. bool ensureLogin(const QNetworkProxy& proxy); - RootItem* decodeFeedCategoriesData(const QString& categories, const QString& feeds, bool obtain_icons); + QList decodeStreamContents(ServiceRoot* root, const QString& stream_json_data, const QString& stream_id); + RootItem* decodeTagsSubscriptions(const QString& categories, const QString& feeds, bool obtain_icons); QString sanitizedBaseUrl() const; QString generateFullUrl(Operations operation) const;