diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index a972710df..7dc4ed606 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -21,7 +21,8 @@ #include FeedDownloader::FeedDownloader() - : QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false), m_mutex(new QMutex()), m_feedsUpdated(0), m_feedsOriginalCount(0) { + : QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false), m_mutex(new QMutex()), + m_feedsUpdated(0), m_feedsOriginalCount(0) { qRegisterMetaType("FeedDownloadResults"); } @@ -40,8 +41,8 @@ void FeedDownloader::synchronizeAccountCaches(const QList& m_isCacheSynchronizationRunning = true; for (CacheForServiceRoot* cache : caches) { - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "Synchronizing cache back to server on thread" << QUOTE_W_SPACE_DOT(QThread::currentThreadId()); + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Synchronizing cache back to server on thread" + << QUOTE_W_SPACE_DOT(QThread::currentThreadId()); cache->saveAllCachedData(false); if (m_stopCacheSynchronization) { @@ -72,9 +73,8 @@ void FeedDownloader::updateFeeds(const QList& feeds) { qDebugNN << LOGSEC_FEEDDOWNLOADER << "No feeds to update in worker thread, aborting update."; } else { - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "Starting feed updates from worker in thread: '" - << QThread::currentThreadId() << "'."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Starting feed updates from worker in thread: '" << QThread::currentThreadId() + << "'."; // Job starts now. emit updateStarted(); @@ -102,11 +102,11 @@ void FeedDownloader::updateFeeds(const QList& feeds) { synchronizeAccountCaches(caches.values(), false); + QHash errored_roots; auto roots = feeds_per_root.uniqueKeys(); bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd")); for (auto* rt : roots) { // Obtain lists of local IDs. @@ -124,41 +124,60 @@ void FeedDownloader::updateFeeds(const QList& feeds) { QHash per_feed_states; per_feed_states.insert(ServiceRoot::BagOfMessages::Read, - DatabaseQueries::bagOfMessages(database, - ServiceRoot::BagOfMessages::Read, - fd)); + DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Read, fd)); per_feed_states.insert(ServiceRoot::BagOfMessages::Unread, - DatabaseQueries::bagOfMessages(database, - ServiceRoot::BagOfMessages::Unread, - fd)); + DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Unread, fd)); per_feed_states.insert(ServiceRoot::BagOfMessages::Starred, - DatabaseQueries::bagOfMessages(database, - ServiceRoot::BagOfMessages::Starred, - fd)); + DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Starred, fd)); per_acc_states.insert(fd->customId(), per_feed_states); } stated_messages.insert(rt, per_acc_states); } - rt->aboutToBeginFeedFetching(feeds_per_root.values(rt), - stated_messages.value(rt), - tagged_messages.value(rt)); + try { + rt->aboutToBeginFeedFetching(feeds_per_root.values(rt), stated_messages.value(rt), tagged_messages.value(rt)); + } + catch (const ApplicationException& ex) { + // Common error showed, all feeds from the root are errored now! + errored_roots.insert(rt, ex); + } } while (!m_feeds.isEmpty()) { auto n_f = m_feeds.takeFirst(); + auto n_r = n_f->getParentServiceRoot(); - updateOneFeed(n_f->getParentServiceRoot(), - n_f, - stated_messages.value(n_f->getParentServiceRoot()).value(n_f->customId()), - tagged_messages.value(n_f->getParentServiceRoot())); + if (errored_roots.contains(n_r)) { + // This feed is errored because its account errored when preparing feed update. + ApplicationException root_ex = errored_roots.value(n_r); + + skipFeedUpdateWithError(n_r, n_f, root_ex); + } + else { + updateOneFeed(n_r, n_f, stated_messages.value(n_r).value(n_f->customId()), tagged_messages.value(n_r)); + } } } finalizeUpdate(); } +void FeedDownloader::skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex) { + const FeedFetchException* fetch_ex = dynamic_cast(&ex); + + if (fetch_ex != nullptr) { + feed->setStatus(fetch_ex->feedStatus(), fetch_ex->message()); + } + else { + feed->setStatus(Feed::Status::OtherError, ex.message()); + } + + acc->itemChanged({feed}); + + emit updateProgress(feed, ++m_feedsUpdated, m_feedsOriginalCount); +} + void FeedDownloader::stopRunningUpdate() { m_stopCacheSynchronization = true; m_feeds.clear(); @@ -169,29 +188,25 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, Feed* feed, const QHash& stated_messages, const QHash& tagged_messages) { - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "Downloading new messages for feed ID '" - << feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '" - << QThread::currentThreadId() << "'."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloading new messages for feed ID '" << feed->customId() << "' URL: '" + << feed->source() << "' title: '" << feed->title() << "' in thread: '" << QThread::currentThreadId() << "'."; int acc_id = feed->getParentServiceRoot()->accountId(); - QElapsedTimer tmr; tmr.start(); + QElapsedTimer tmr; + tmr.start(); try { bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); - QList msgs = feed->getParentServiceRoot()->obtainNewMessages(feed, - stated_messages, - tagged_messages); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd")); + QList msgs = feed->getParentServiceRoot()->obtainNewMessages(feed, stated_messages, tagged_messages); - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '" - << feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '" + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '" << feed->customId() + << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '" << QThread::currentThreadId() << "'. Operation took " << tmr.nsecsElapsed() / 1000 << " microseconds."; - bool fix_future_datetimes = qApp->settings()->value(GROUP(Messages), - SETTING(Messages::FixupFutureArticleDateTimes)).toBool(); + bool fix_future_datetimes = + qApp->settings()->value(GROUP(Messages), SETTING(Messages::FixupFutureArticleDateTimes)).toBool(); // Now, sanitize messages (tweak encoding etc.). for (auto& msg : msgs) { @@ -214,7 +229,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, MessageFilter::initializeFilteringEngine(filter_engine, &msg_obj); - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Setting up JS evaluation took " << tmr.nsecsElapsed() / 1000 << " microseconds."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Setting up JS evaluation took " << tmr.nsecsElapsed() / 1000 + << " microseconds."; QList read_msgs, important_msgs; @@ -247,8 +263,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, try { MessageObject::FilteringAction decision = msg_filter->filterMessage(&filter_engine); - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "Running filter script, it took " << tmr.nsecsElapsed() / 1000 << " microseconds."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Running filter script, it took " << tmr.nsecsElapsed() / 1000 + << " microseconds."; switch (decision) { case MessageObject::FilteringAction::Accept: @@ -265,8 +281,7 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, } catch (const FilteringException& ex) { qCriticalNN << LOGSEC_FEEDDOWNLOADER - << "Error when evaluating filtering JS function: " - << QUOTE_W_SPACE_DOT(ex.message()) + << "Error when evaluating filtering JS function: " << QUOTE_W_SPACE_DOT(ex.message()) << " Accepting message."; continue; } @@ -277,13 +292,15 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, } if (!msg_backup.m_isRead && msg_orig->m_isRead) { - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId << "' was marked as read by message scripts."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId + << "' was marked as read by message scripts."; read_msgs << *msg_orig; } if (!msg_backup.m_isImportant && msg_orig->m_isImportant) { - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId << "' was marked as important by message scripts."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId + << "' was marked as important by message scripts."; important_msgs << *msg_orig; } @@ -294,10 +311,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, // Label is not there anymore, it was deassigned. lbl->deassignFromMessage(*msg_orig); - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "It was detected that label" << QUOTE_W_SPACE(lbl->customId()) - << "was DEASSIGNED from message" << QUOTE_W_SPACE(msg_orig->m_customId) - << "by message filter(s)."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "It was detected that label" << QUOTE_W_SPACE(lbl->customId()) + << "was DEASSIGNED from message" << QUOTE_W_SPACE(msg_orig->m_customId) << "by message filter(s)."; } } @@ -307,10 +322,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, // was newly assigned. lbl->assignToMessage(*msg_orig); - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "It was detected that label" << QUOTE_W_SPACE(lbl->customId()) - << "was ASSIGNED to message" << QUOTE_W_SPACE(msg_orig->m_customId) - << "by message filter(s)."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "It was detected that label" << QUOTE_W_SPACE(lbl->customId()) + << "was ASSIGNED to message" << QUOTE_W_SPACE(msg_orig->m_customId) << "by message filter(s)."; } } @@ -322,8 +335,7 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, if (!read_msgs.isEmpty()) { // Now we push new read states to the service. if (feed->getParentServiceRoot()->onBeforeSetMessagesRead(feed, read_msgs, RootItem::ReadStatus::Read)) { - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "Notified services about messages marked as read by message filters."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Notified services about messages marked as read by message filters."; } else { qCriticalNN << LOGSEC_FEEDDOWNLOADER @@ -333,9 +345,11 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, if (!important_msgs.isEmpty()) { // Now we push new read states to the service. - auto list = boolinq::from(important_msgs).select([](const Message& msg) { - return ImportanceChange(msg, RootItem::Importance::Important); - }).toStdList(); + auto list = boolinq::from(important_msgs) + .select([](const Message& msg) { + return ImportanceChange(msg, RootItem::Importance::Important); + }) + .toStdList(); QList chngs = FROM_STD_LIST(QList, list); if (feed->getParentServiceRoot()->onBeforeSwitchMessageImportance(feed, chngs)) { @@ -352,56 +366,47 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, removeDuplicateMessages(msgs); // Now make sure, that messages are actually stored to SQL in a locked state. - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Saving messages of feed ID '" - << feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '" - << QThread::currentThreadId() << "'."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Saving messages of feed ID '" << feed->customId() << "' URL: '" + << feed->source() << "' title: '" << feed->title() << "' in thread: '" << QThread::currentThreadId() + << "'."; tmr.restart(); auto updated_messages = acc->updateMessages(msgs, feed, false); - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "Updating messages in DB took " << tmr.nsecsElapsed() / 1000 << " microseconds."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Updating messages in DB took " << tmr.nsecsElapsed() / 1000 + << " microseconds."; if (feed->status() != Feed::Status::NewMessages) { - feed->setStatus(updated_messages.first > 0 || updated_messages.second > 0 - ? Feed::Status::NewMessages - : Feed::Status::Normal); + feed->setStatus(updated_messages.first > 0 || updated_messages.second > 0 ? Feed::Status::NewMessages + : Feed::Status::Normal); } - qDebugNN << LOGSEC_FEEDDOWNLOADER - << updated_messages << " messages for feed " - << feed->customId() << " stored in DB."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << updated_messages << " messages for feed " << feed->customId() + << " stored in DB."; if (updated_messages.first > 0) { - m_results.appendUpdatedFeed({ feed->title(), updated_messages.first }); + m_results.appendUpdatedFeed({feed->title(), updated_messages.first}); } } catch (const FeedFetchException& feed_ex) { - qCriticalNN << LOGSEC_NETWORK - << "Error when fetching feed:" - << QUOTE_W_SPACE(feed_ex.feedStatus()) - << "message:" - << QUOTE_W_SPACE_DOT(feed_ex.message()); + qCriticalNN << LOGSEC_NETWORK << "Error when fetching feed:" << QUOTE_W_SPACE(feed_ex.feedStatus()) + << "message:" << QUOTE_W_SPACE_DOT(feed_ex.message()); feed->setStatus(feed_ex.feedStatus(), feed_ex.message()); } catch (const ApplicationException& app_ex) { - qCriticalNN << LOGSEC_NETWORK - << "Unknown error when fetching feed:" - << "message:" - << QUOTE_W_SPACE_DOT(app_ex.message()); + qCriticalNN << LOGSEC_NETWORK << "Unknown error when fetching feed:" + << "message:" << QUOTE_W_SPACE_DOT(app_ex.message()); feed->setStatus(Feed::Status::OtherError, app_ex.message()); } - feed->getParentServiceRoot()->itemChanged({ feed }); + feed->getParentServiceRoot()->itemChanged({feed}); m_feedsUpdated++; - qDebugNN << LOGSEC_FEEDDOWNLOADER - << "Made progress in feed updates, total feeds count " - << m_feedsUpdated << "/" << m_feedsOriginalCount << " (id of feed is " - << feed->id() << ")."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Made progress in feed updates, total feeds count " << m_feedsUpdated << "/" + << m_feedsOriginalCount << " (id of feed is " << feed->id() << ")."; emit updateProgress(feed, m_feedsUpdated, m_feedsOriginalCount); } @@ -429,22 +434,22 @@ void FeedDownloader::removeDuplicateMessages(QList& messages) { if (message.m_id > 0) { is_duplicate = [](const Message& a, const Message& b) { - return a.m_id == b.m_id; - }; + return a.m_id == b.m_id; + }; } else if (message.m_customId.isEmpty()) { is_duplicate = [](const Message& a, const Message& b) { - return std::tie(a.m_title, a.m_url, a.m_author) == std::tie(b.m_title, b.m_url, b.m_author); - }; + return std::tie(a.m_title, a.m_url, a.m_author) == std::tie(b.m_title, b.m_url, b.m_author); + }; } else { is_duplicate = [](const Message& a, const Message& b) { - return a.m_customId == b.m_customId; - }; + return a.m_customId == b.m_customId; + }; } auto next_idx = idx + 1; // Index of next message to check after removing all duplicates. - auto last_idx = idx; // Index of the last kept duplicate. + auto last_idx = idx; // Index of the last kept duplicate. idx = next_idx; @@ -506,9 +511,11 @@ void FeedDownloadResults::appendUpdatedFeed(const QPair& feed) { } void FeedDownloadResults::sort() { - std::sort(m_updatedFeeds.begin(), m_updatedFeeds.end(), [](const QPair& lhs, const QPair& rhs) { - return lhs.second > rhs.second; - }); + std::sort(m_updatedFeeds.begin(), + m_updatedFeeds.end(), + [](const QPair& lhs, const QPair& rhs) { + return lhs.second > rhs.second; + }); } void FeedDownloadResults::clear() { diff --git a/src/librssguard/core/feeddownloader.h b/src/librssguard/core/feeddownloader.h index d35556cbf..4b959fbb8 100644 --- a/src/librssguard/core/feeddownloader.h +++ b/src/librssguard/core/feeddownloader.h @@ -8,6 +8,7 @@ #include #include "core/message.h" +#include "exceptions/applicationexception.h" #include "services/abstract/cacheforserviceroot.h" #include "services/abstract/feed.h" @@ -25,7 +26,6 @@ class FeedDownloadResults { void clear(); private: - // QString represents title if the feed, int represents count of newly downloaded messages. QList> m_updatedFeeds; }; @@ -33,7 +33,7 @@ class FeedDownloadResults { // This class offers means to "update" feeds and "special" categories. // NOTE: This class is used within separate thread. class FeedDownloader : public QObject { - Q_OBJECT + Q_OBJECT public: explicit FeedDownloader(); @@ -54,6 +54,7 @@ class FeedDownloader : public QObject { void updateProgress(const Feed* feed, int current, int total); private: + void skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex); void updateOneFeed(ServiceRoot* acc, Feed* feed, const QHash& stated_messages, diff --git a/src/librssguard/exceptions/applicationexception.cpp b/src/librssguard/exceptions/applicationexception.cpp index 6a51f08a0..581a47ed3 100644 --- a/src/librssguard/exceptions/applicationexception.cpp +++ b/src/librssguard/exceptions/applicationexception.cpp @@ -4,6 +4,8 @@ ApplicationException::ApplicationException(QString message) : m_message(std::move(message)) {} +ApplicationException::~ApplicationException() {} + QString ApplicationException::message() const { return m_message; } diff --git a/src/librssguard/exceptions/applicationexception.h b/src/librssguard/exceptions/applicationexception.h index b6d098ee1..ed73df6aa 100644 --- a/src/librssguard/exceptions/applicationexception.h +++ b/src/librssguard/exceptions/applicationexception.h @@ -8,6 +8,7 @@ class ApplicationException { public: explicit ApplicationException(QString message = {}); + virtual ~ApplicationException(); QString message() const; diff --git a/src/librssguard/network-web/downloader.cpp b/src/librssguard/network-web/downloader.cpp index 0aa5a310d..211e75cef 100644 --- a/src/librssguard/network-web/downloader.cpp +++ b/src/librssguard/network-web/downloader.cpp @@ -18,7 +18,7 @@ Downloader::Downloader(QObject* parent) : QObject(parent), m_activeReply(nullptr), m_downloadManager(new SilentNetworkAccessManager(this)), m_timer(new QTimer(this)), m_inputData(QByteArray()), m_inputMultipartData(nullptr), m_targetProtected(false), m_targetUsername(QString()), m_targetPassword(QString()), m_lastOutputData({}), - m_lastOutputError(QNetworkReply::NetworkError::NoError), m_lastHttpStatusCode(0) { + m_lastOutputError(QNetworkReply::NetworkError::NoError), m_lastHttpStatusCode(0), m_lastHeaders({}) { m_timer->setInterval(DOWNLOAD_TIMEOUT); m_timer->setSingleShot(true); @@ -215,6 +215,7 @@ void Downloader::finished() { m_lastContentType = reply->header(QNetworkRequest::KnownHeaders::ContentTypeHeader); m_lastOutputError = reply->error(); m_lastHttpStatusCode = reply->attribute(QNetworkRequest::Attribute::HttpStatusCodeAttribute).toInt(); + m_lastHeaders = reply->rawHeaderPairs(); // original_url = m_activeReply->property("original_url").toUrl(); @@ -336,6 +337,10 @@ void Downloader::runGetRequest(const QNetworkRequest& request) { connect(m_activeReply, &QNetworkReply::finished, this, &Downloader::finished); } +QList Downloader::lastHeaders() const { + return m_lastHeaders; +} + int Downloader::lastHttpStatusCode() const { return m_lastHttpStatusCode; } diff --git a/src/librssguard/network-web/downloader.h b/src/librssguard/network-web/downloader.h index b48a60315..f2d3c4398 100644 --- a/src/librssguard/network-web/downloader.h +++ b/src/librssguard/network-web/downloader.h @@ -30,6 +30,7 @@ class Downloader : public QObject { QVariant lastContentType() const; QList lastCookies() const; int lastHttpStatusCode() const; + QList lastHeaders() const; void setProxy(const QNetworkProxy& proxy); @@ -115,6 +116,7 @@ class Downloader : public QObject { int m_lastHttpStatusCode; QVariant m_lastContentType; QList m_lastCookies; + QList m_lastHeaders; }; #endif // DOWNLOADER_H diff --git a/src/librssguard/network-web/networkfactory.cpp b/src/librssguard/network-web/networkfactory.cpp index 445584b17..139e4a554 100644 --- a/src/librssguard/network-web/networkfactory.cpp +++ b/src/librssguard/network-web/networkfactory.cpp @@ -271,6 +271,7 @@ NetworkResult NetworkFactory::performNetworkOperation(const QString& url, result.m_contentType = downloader.lastContentType().toString(); result.m_cookies = downloader.lastCookies(); result.m_httpCode = downloader.lastHttpStatusCode(); + result.m_headers = downloader.lastHeaders(); return result; } @@ -311,12 +312,14 @@ NetworkResult NetworkFactory::performNetworkOperation(const QString& url, result.m_contentType = downloader.lastContentType().toString(); result.m_cookies = downloader.lastCookies(); result.m_httpCode = downloader.lastHttpStatusCode(); + result.m_headers = downloader.lastHeaders(); return result; } NetworkResult::NetworkResult() - : m_networkError(QNetworkReply::NetworkError::NoError), m_httpCode(0), m_contentType(QString()), m_cookies({}) {} + : m_networkError(QNetworkReply::NetworkError::NoError), m_httpCode(0), m_contentType(QString()), m_cookies({}), + m_headers({}) {} NetworkResult::NetworkResult(QNetworkReply::NetworkError err, int http_code, diff --git a/src/librssguard/network-web/networkfactory.h b/src/librssguard/network-web/networkfactory.h index 0723669e9..09421067d 100644 --- a/src/librssguard/network-web/networkfactory.h +++ b/src/librssguard/network-web/networkfactory.h @@ -18,6 +18,7 @@ struct NetworkResult { int m_httpCode; QString m_contentType; QList m_cookies; + QList m_headers; explicit NetworkResult(); explicit NetworkResult(QNetworkReply::NetworkError err, diff --git a/src/librssguard/services/greader/greadernetwork.cpp b/src/librssguard/services/greader/greadernetwork.cpp index 70fde5a65..f918b2c41 100644 --- a/src/librssguard/services/greader/greadernetwork.cpp +++ b/src/librssguard/services/greader/greadernetwork.cpp @@ -23,9 +23,8 @@ GreaderNetwork::GreaderNetwork(QObject* parent) : QObject(parent), m_root(nullptr), m_service(GreaderServiceRoot::Service::FreshRss), m_username(QString()), m_password(QString()), m_baseUrl(QString()), m_batchSize(GREADER_DEFAULT_BATCH_SIZE), - m_downloadOnlyUnreadMessages(false), m_prefetchedMessages({}), m_prefetchedStatus(Feed::Status::Normal), - m_performGlobalFetching(false), m_intelligentSynchronization(true), - m_newerThanFilter(QDate::currentDate().addYears(-1)), + m_downloadOnlyUnreadMessages(false), m_prefetchedMessages({}), m_performGlobalFetching(false), + m_intelligentSynchronization(true), m_newerThanFilter(QDate::currentDate().addYears(-1)), m_oauth(new OAuth2Service(QSL(INO_OAUTH_AUTH_URL), QSL(INO_OAUTH_TOKEN_URL), {}, {}, QSL(INO_OAUTH_SCOPE), this)) { initializeOauth(); clearCredentials(); @@ -125,17 +124,12 @@ QVariantHash GreaderNetwork::userInfo(const QNetworkProxy& proxy) { proxy); if (res.m_networkError != QNetworkReply::NetworkError::NoError) { - throw NetworkException(res.m_networkError); + throw NetworkException(res.m_networkError, output); } return QJsonDocument::fromJson(output).object().toVariantHash(); } -void GreaderNetwork::clearPrefetchedMessages() { - m_prefetchedMessages.clear(); - m_prefetchedStatus = Feed::Status::Normal; -} - void GreaderNetwork::prepareFeedFetching(GreaderServiceRoot* root, const QList& feeds, const QHash>& @@ -145,114 +139,95 @@ void GreaderNetwork::prepareFeedFetching(GreaderServiceRoot* root, Q_UNUSED(tagged_messages) m_prefetchedMessages.clear(); - m_prefetchedStatus = Feed::Status::Normal; - try { - double perc_of_fetching = (feeds.size() * 1.0) / root->getSubTreeFeeds().size(); + double perc_of_fetching = (feeds.size() * 1.0) / root->getSubTreeFeeds().size(); - m_performGlobalFetching = perc_of_fetching > GREADER_GLOBAL_UPDATE_THRES; + m_performGlobalFetching = perc_of_fetching > GREADER_GLOBAL_UPDATE_THRES; - qDebugNN << LOGSEC_GREADER << "Percentage of feeds for fetching:" << QUOTE_W_SPACE_DOT(perc_of_fetching * 100.0); + qDebugNN << LOGSEC_GREADER << "Percentage of feeds for fetching:" << QUOTE_W_SPACE_DOT(perc_of_fetching * 100.0); - auto remote_starred_ids_list = itemIds(QSL(GREADER_API_FULL_STATE_IMPORTANT), false, proxy, -1, m_newerThanFilter); + auto remote_starred_ids_list = itemIds(QSL(GREADER_API_FULL_STATE_IMPORTANT), false, proxy, -1, m_newerThanFilter); - for (int i = 0; i < remote_starred_ids_list.size(); i++) { - remote_starred_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_starred_ids_list.at(i))); + for (int i = 0; i < remote_starred_ids_list.size(); i++) { + remote_starred_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_starred_ids_list.at(i))); + } + + QSet remote_starred_ids = FROM_LIST_TO_SET(QSet, remote_starred_ids_list); + QSet local_starred_ids; + QList> all_states = stated_messages.values(); + + for (auto& lst : all_states) { + auto s = lst.value(ServiceRoot::BagOfMessages::Starred); + + local_starred_ids.unite(FROM_LIST_TO_SET(QSet, s)); + } + + auto starred_to_download((remote_starred_ids - local_starred_ids).unite(local_starred_ids - remote_starred_ids)); + auto to_download = starred_to_download; + + if (m_performGlobalFetching) { + qWarningNN << LOGSEC_GREADER << "Performing global contents fetching."; + + QStringList remote_all_ids_list = + m_downloadOnlyUnreadMessages + ? QStringList() + : itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), false, proxy, -1, m_newerThanFilter); + QStringList remote_unread_ids_list = + itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), true, proxy, -1, m_newerThanFilter); + + for (int i = 0; i < remote_all_ids_list.size(); i++) { + remote_all_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_all_ids_list.at(i))); } - QSet remote_starred_ids = FROM_LIST_TO_SET(QSet, remote_starred_ids_list); - QSet local_starred_ids; - QList> all_states = stated_messages.values(); + for (int i = 0; i < remote_unread_ids_list.size(); i++) { + remote_unread_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_unread_ids_list.at(i))); + } + + QSet remote_all_ids = FROM_LIST_TO_SET(QSet, remote_all_ids_list); + QSet remote_unread_ids = FROM_LIST_TO_SET(QSet, remote_unread_ids_list); + QSet remote_read_ids = remote_all_ids - remote_unread_ids; + QSet local_unread_ids; + QSet local_read_ids; for (auto& lst : all_states) { - auto s = lst.value(ServiceRoot::BagOfMessages::Starred); + auto u = lst.value(ServiceRoot::BagOfMessages::Unread); + auto r = lst.value(ServiceRoot::BagOfMessages::Read); - local_starred_ids.unite(FROM_LIST_TO_SET(QSet, s)); + local_unread_ids.unite(FROM_LIST_TO_SET(QSet, u)); + local_read_ids.unite(FROM_LIST_TO_SET(QSet, r)); } - auto starred_to_download((remote_starred_ids - local_starred_ids).unite(local_starred_ids - remote_starred_ids)); - auto to_download = starred_to_download; - - if (m_performGlobalFetching) { - qWarningNN << LOGSEC_GREADER << "Performing global contents fetching."; - - QStringList remote_all_ids_list = - m_downloadOnlyUnreadMessages - ? QStringList() - : itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), false, proxy, -1, m_newerThanFilter); - QStringList remote_unread_ids_list = - itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), true, proxy, -1, m_newerThanFilter); - - for (int i = 0; i < remote_all_ids_list.size(); i++) { - remote_all_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_all_ids_list.at(i))); - } - - for (int i = 0; i < remote_unread_ids_list.size(); i++) { - remote_unread_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_unread_ids_list.at(i))); - } - - QSet remote_all_ids = FROM_LIST_TO_SET(QSet, remote_all_ids_list); - QSet remote_unread_ids = FROM_LIST_TO_SET(QSet, remote_unread_ids_list); - QSet remote_read_ids = remote_all_ids - remote_unread_ids; - QSet local_unread_ids; - QSet local_read_ids; - - for (auto& lst : all_states) { - auto u = lst.value(ServiceRoot::BagOfMessages::Unread); - auto r = lst.value(ServiceRoot::BagOfMessages::Read); - - local_unread_ids.unite(FROM_LIST_TO_SET(QSet, u)); - local_read_ids.unite(FROM_LIST_TO_SET(QSet, r)); - } - - if (!m_downloadOnlyUnreadMessages) { - to_download += remote_all_ids - local_read_ids - local_unread_ids; - } - else { - to_download += remote_unread_ids - local_read_ids - local_unread_ids; - } - - auto moved_read = local_read_ids.intersect(remote_unread_ids); - - to_download += moved_read; - - if (!m_downloadOnlyUnreadMessages) { - auto moved_unread = local_unread_ids.intersect(remote_read_ids); - - to_download += moved_unread; - } + if (!m_downloadOnlyUnreadMessages) { + to_download += remote_all_ids - local_read_ids - local_unread_ids; } else { - qWarningNN << LOGSEC_GREADER << "Performing feed-based contents fetching."; + to_download += remote_unread_ids - local_read_ids - local_unread_ids; } - Feed::Status error; - QList to_download_list(to_download.values()); + auto moved_read = local_read_ids.intersect(remote_unread_ids); - if (!to_download_list.isEmpty()) { - if (m_service == GreaderServiceRoot::Service::Reedah) { - for (int i = 0; i < to_download_list.size(); i++) { - to_download_list.replace(i, convertLongStreamIdToShortStreamId(to_download_list.at(i))); - } + to_download += moved_read; + + if (!m_downloadOnlyUnreadMessages) { + auto moved_unread = local_unread_ids.intersect(remote_read_ids); + + to_download += moved_unread; + } + } + else { + qWarningNN << LOGSEC_GREADER << "Performing feed-based contents fetching."; + } + + QList to_download_list(to_download.values()); + + if (!to_download_list.isEmpty()) { + if (m_service == GreaderServiceRoot::Service::Reedah) { + for (int i = 0; i < to_download_list.size(); i++) { + to_download_list.replace(i, convertLongStreamIdToShortStreamId(to_download_list.at(i))); } - - m_prefetchedMessages = itemContents(root, to_download_list, error, proxy); } - } - catch (const FeedFetchException& fex) { - m_prefetchedStatus = fex.feedStatus(); - qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for common stream:" << QUOTE_W_SPACE_DOT(fex.message()); - } - catch (const NetworkException& nex) { - m_prefetchedStatus = Feed::Status::NetworkError; - - qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for common stream:" << QUOTE_W_SPACE_DOT(nex.message()); - } - catch (const ApplicationException& aex) { - m_prefetchedStatus = Feed::Status::OtherError; - - qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for common stream:" << QUOTE_W_SPACE_DOT(aex.message()); + m_prefetchedMessages = itemContents(root, to_download_list, proxy); } } @@ -261,50 +236,19 @@ QList GreaderNetwork::getMessagesIntelligently(ServiceRoot* root, const QHash& stated_messages, const QHash& tagged_messages, - Feed::Status& error, const QNetworkProxy& proxy) { Q_UNUSED(tagged_messages) QList msgs; - if (m_prefetchedStatus != Feed::Status::Normal) { - error = m_prefetchedStatus; - return msgs; - } - if (!m_performGlobalFetching) { // 1. Get unread IDs for a feed. // 2. Get read IDs for a feed. // 3. Download messages/contents for missing or changed IDs. // 4. Add prefetched starred msgs. - QStringList remote_all_ids_list, remote_unread_ids_list; - - try { - remote_all_ids_list = - m_downloadOnlyUnreadMessages ? QStringList() : itemIds(stream_id, false, proxy, -1, m_newerThanFilter); - remote_unread_ids_list = itemIds(stream_id, true, proxy, -1, m_newerThanFilter); - } - catch (const FeedFetchException& fex) { - error = fex.feedStatus(); - - qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for specific stream:" << QUOTE_W_SPACE_DOT(fex.message()); - - return msgs; - } - catch (const NetworkException& nex) { - error = Feed::Status::NetworkError; - - qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for specific stream:" << QUOTE_W_SPACE_DOT(nex.message()); - - return msgs; - } - catch (const ApplicationException& aex) { - error = Feed::Status::OtherError; - - qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for specific stream:" << QUOTE_W_SPACE_DOT(aex.message()); - - return msgs; - } + QStringList remote_all_ids_list = + m_downloadOnlyUnreadMessages ? QStringList() : itemIds(stream_id, false, proxy, -1, m_newerThanFilter); + QStringList remote_unread_ids_list = itemIds(stream_id, true, proxy, -1, m_newerThanFilter); // Convert item IDs to long form. for (int i = 0; i < remote_all_ids_list.size(); i++) { @@ -356,7 +300,7 @@ QList GreaderNetwork::getMessagesIntelligently(ServiceRoot* root, } } - msgs = itemContents(root, to_download_list, error, proxy); + msgs = itemContents(root, to_download_list, proxy); } } @@ -443,7 +387,7 @@ QStringList GreaderNetwork::itemIds(const QString& stream_id, if (result_stream.m_networkError != QNetworkReply::NetworkError::NoError) { qCriticalNN << LOGSEC_GREADER << "Cannot download item IDs for " << QUOTE_NO_SPACE(stream_id) << ", network error:" << QUOTE_W_SPACE_DOT(result_stream.m_networkError); - throw NetworkException(result_stream.m_networkError); + throw NetworkException(result_stream.m_networkError, output_stream); } else { ids.append(decodeItemIds(output_stream, continuation)); @@ -456,13 +400,11 @@ QStringList GreaderNetwork::itemIds(const QString& stream_id, QList GreaderNetwork::itemContents(ServiceRoot* root, const QList& stream_ids, - Feed::Status& error, const QNetworkProxy& proxy) { QString continuation; if (!ensureLogin(proxy)) { - error = Feed::Status::AuthError; - return {}; + throw FeedFetchException(Feed::Status::AuthError, tr("login failed")); } QList msgs; @@ -518,8 +460,7 @@ QList GreaderNetwork::itemContents(ServiceRoot* root, if (result_stream.m_networkError != QNetworkReply::NetworkError::NoError) { qCriticalNN << LOGSEC_GREADER << "Cannot download messages for " << batch_ids << ", network error:" << QUOTE_W_SPACE_DOT(result_stream.m_networkError); - error = Feed::Status::NetworkError; - return {}; + throw NetworkException(result_stream.m_networkError, output_stream); } else { msgs.append(decodeStreamContents(root, output_stream, QString(), continuation)); @@ -528,19 +469,14 @@ QList GreaderNetwork::itemContents(ServiceRoot* root, while (!continuation.isEmpty()); } - error = Feed::Status::Normal; return msgs; } -QList GreaderNetwork::streamContents(ServiceRoot* root, - const QString& stream_id, - Feed::Status& error, - const QNetworkProxy& proxy) { +QList GreaderNetwork::streamContents(ServiceRoot* root, const QString& stream_id, const QNetworkProxy& proxy) { QString continuation; if (!ensureLogin(proxy)) { - error = Feed::Status::AuthError; - return {}; + throw FeedFetchException(Feed::Status::AuthError, tr("login failed")); } QList msgs; @@ -589,8 +525,7 @@ QList GreaderNetwork::streamContents(ServiceRoot* root, if (result_stream.m_networkError != QNetworkReply::NetworkError::NoError) { qCriticalNN << LOGSEC_GREADER << "Cannot download messages for " << QUOTE_NO_SPACE(stream_id) << ", network error:" << QUOTE_W_SPACE_DOT(result_stream.m_networkError); - error = Feed::Status::NetworkError; - return {}; + throw NetworkException(result_stream.m_networkError, output_stream); } else { msgs.append(decodeStreamContents(root, output_stream, stream_id, continuation)); @@ -598,7 +533,6 @@ QList GreaderNetwork::streamContents(ServiceRoot* root, } while (!continuation.isEmpty() && msgs.size() < target_msgs_size); - error = Feed::Status::Normal; return msgs; } @@ -608,7 +542,7 @@ RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNe if (!ensureLogin(proxy)) { qCriticalNN << LOGSEC_GREADER << "Cannot get feed tree, not logged-in."; - throw ApplicationException(tr("you are not logged-in, maybe wrong credentials")); + throw ApplicationException(tr("login failed")); } QByteArray output_labels; @@ -627,8 +561,7 @@ RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNe qCriticalNN << LOGSEC_GREADER << "Cannot get labels tree, network error:" << QUOTE_W_SPACE_DOT(result_labels.m_networkError); - throw NetworkException(result_labels.m_networkError, - tr("cannot get list of labels, HTTP code '%1'").arg(result_labels.m_httpCode)); + throw NetworkException(result_labels.m_networkError, output_labels); } full_url = generateFullUrl(Operations::SubscriptionList); @@ -648,8 +581,7 @@ RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNe qCriticalNN << LOGSEC_GREADER << "Cannot get feed tree, network error:" << QUOTE_W_SPACE_DOT(result_feeds.m_networkError); - throw NetworkException(result_labels.m_networkError, - tr("cannot get list of feeds, HTTP code '%1'").arg(result_feeds.m_httpCode)); + throw NetworkException(result_labels.m_networkError, output_feeds); } return decodeTagsSubscriptions(output_labels, output_feeds, obtain_icons, proxy); @@ -1097,6 +1029,10 @@ void GreaderNetwork::clearCredentials() { m_authAuth = m_authSid = m_authToken = QString(); } +void GreaderNetwork::clearPrefetchedMessages() { + m_prefetchedMessages.clear(); +} + QString GreaderNetwork::sanitizedBaseUrl() const { QString base_url = m_service == GreaderServiceRoot::Service::Inoreader ? QSL(GREADER_URL_INOREADER) : m_baseUrl; diff --git a/src/librssguard/services/greader/greadernetwork.h b/src/librssguard/services/greader/greadernetwork.h index 5d3d8ac4c..ec13b3a28 100644 --- a/src/librssguard/services/greader/greadernetwork.h +++ b/src/librssguard/services/greader/greadernetwork.h @@ -47,7 +47,6 @@ class GreaderNetwork : public QObject { const QString& stream_id, const QHash& stated_messages, const QHash& tagged_messages, - Feed::Status& error, const QNetworkProxy& proxy); RootItem* categoriesFeedsLabelsTree(bool obtain_icons, const QNetworkProxy& proxy); @@ -95,12 +94,8 @@ class GreaderNetwork : public QObject { QDate newer_than = {}); QList itemContents(ServiceRoot* root, const QList& stream_ids, - Feed::Status& error, const QNetworkProxy& proxy); - QList streamContents(ServiceRoot* root, - const QString& stream_id, - Feed::Status& error, - const QNetworkProxy& proxy); + QList streamContents(ServiceRoot* root, const QString& stream_id, const QNetworkProxy& proxy); QNetworkReply::NetworkError clientLogin(const QNetworkProxy& proxy); QDate newerThanFilter() const; @@ -148,7 +143,6 @@ class GreaderNetwork : public QObject { QString m_authAuth; QString m_authToken; QList m_prefetchedMessages; - Feed::Status m_prefetchedStatus; bool m_performGlobalFetching; bool m_intelligentSynchronization; QDate m_newerThanFilter; diff --git a/src/librssguard/services/greader/greaderserviceroot.cpp b/src/librssguard/services/greader/greaderserviceroot.cpp index f16a17539..4d64ee1eb 100644 --- a/src/librssguard/services/greader/greaderserviceroot.cpp +++ b/src/librssguard/services/greader/greaderserviceroot.cpp @@ -130,27 +130,17 @@ QList GreaderServiceRoot::obtainNewMessages(Feed* feed, const QHash& stated_messages, const QHash& tagged_messages) { - Feed::Status error = Feed::Status::Normal; QList msgs; if (m_network->intelligentSynchronization()) { - msgs = m_network->getMessagesIntelligently(this, - feed->customId(), - stated_messages, - tagged_messages, - error, - networkProxy()); + msgs = + m_network->getMessagesIntelligently(this, feed->customId(), stated_messages, tagged_messages, networkProxy()); } else { - msgs = m_network->streamContents(this, feed->customId(), error, networkProxy()); + msgs = m_network->streamContents(this, feed->customId(), networkProxy()); } - if (error != Feed::Status::NewMessages && error != Feed::Status::Normal) { - throw FeedFetchException(error); - } - else { - return msgs; - } + return msgs; } bool GreaderServiceRoot::wantsBaggedIdsOfExistingMessages() const {