From 1cdb975fcd9b885672bc8ae7422222355982ea11 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 4 Jan 2023 06:57:14 +0100 Subject: [PATCH 01/15] save --- resources/desktop/rssguard.metainfo.xml.in | 2 +- .../services/standard/standardfeedsimportexportmodel.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/desktop/rssguard.metainfo.xml.in b/resources/desktop/rssguard.metainfo.xml.in index d135fa012..8f3f27b2c 100644 --- a/resources/desktop/rssguard.metainfo.xml.in +++ b/resources/desktop/rssguard.metainfo.xml.in @@ -60,7 +60,7 @@ - + @APP_LOW_NAME@ diff --git a/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp b/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp index ca0b682e2..32f8af2d2 100644 --- a/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp +++ b/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp @@ -31,7 +31,7 @@ FeedsImportExportModel::FeedsImportExportModel(QObject* parent) int number_error = boolinq::from(res).count(false); emit layoutChanged(); - emit parsingFinished(number_error, m_lookup.size() - number_error); + emit parsingFinished(number_error, res.size() - number_error); // Done, remove lookups. m_lookup.clear(); @@ -184,7 +184,7 @@ bool FeedsImportExportModel::produceFeed(const FeedLookup& feed_lookup) { new_feed->setPostProcessScript(feed_lookup.post_process_script); } else { - new_feed = new StandardFeed(feed_lookup.parent); + new_feed = new StandardFeed(); if (feed_lookup.opml_element.isNull()) { new_feed->setSource(feed_lookup.url); From f60c6a2bc25ccd162e47900c168bf3607fda5d92 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 4 Jan 2023 10:27:01 +0100 Subject: [PATCH 02/15] initial PoC work - 160 feeds in 1 minute --- src/librssguard/core/feeddownloader.cpp | 99 +++++++++++-------- src/librssguard/core/feeddownloader.h | 14 ++- src/librssguard/services/abstract/feed.cpp | 4 +- .../services/abstract/importantnode.cpp | 6 +- src/librssguard/services/abstract/label.cpp | 28 +++--- .../services/abstract/recyclebin.cpp | 21 ++-- .../services/abstract/serviceroot.cpp | 3 +- .../services/abstract/unreadnode.cpp | 6 +- 8 files changed, 103 insertions(+), 78 deletions(-) diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 0e1913fa4..26502d5c8 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -16,20 +16,17 @@ #include #include -#include #include #include +#include FeedDownloader::FeedDownloader() - : QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false), m_mutex(new QMutex()), - m_feedsUpdated(0), m_feedsOriginalCount(0) { + : QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false), m_feedsUpdated(0), + m_feedsOriginalCount(0) { qRegisterMetaType("FeedDownloadResults"); } FeedDownloader::~FeedDownloader() { - m_mutex->tryLock(); - m_mutex->unlock(); - delete m_mutex; qDebugNN << LOGSEC_FEEDDOWNLOADER << "Destroying FeedDownloader instance."; } @@ -62,14 +59,12 @@ void FeedDownloader::synchronizeAccountCaches(const QList& } void FeedDownloader::updateFeeds(const QList& feeds) { - QMutexLocker locker(m_mutex); - + m_erroredAccounts.clear(); m_results.clear(); - m_feeds = feeds; - m_feedsOriginalCount = m_feeds.size(); - m_feedsUpdated = 0; + m_feeds.clear(); - const QDateTime update_time = QDateTime::currentDateTimeUtc(); + m_feedsOriginalCount = feeds.size(); + m_feedsUpdated = 0; if (feeds.isEmpty()) { qDebugNN << LOGSEC_FEEDDOWNLOADER << "No feeds to update in worker thread, aborting update."; @@ -104,24 +99,25 @@ void FeedDownloader::updateFeeds(const QList& feeds) { synchronizeAccountCaches(caches.values(), false); - QHash errored_roots; auto roots = feeds_per_root.uniqueKeys(); bool is_main_thread = QThread::currentThread() == qApp->thread(); QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) : qApp->database()->driver()->connection(QSL("feed_upd")); for (auto* rt : roots) { + auto fds = feeds_per_root.values(rt); + // Obtain lists of local IDs. if (rt->wantsBaggedIdsOfExistingMessages()) { - // Tagged messages for the account. - tagged_messages.insert(rt, DatabaseQueries::bagsOfMessages(database, rt->labelsNode()->labels())); + // Tags per account. + auto per_acc_tags = DatabaseQueries::bagsOfMessages(database, rt->labelsNode()->labels()); + + tagged_messages.insert(rt, per_acc_tags); QHash> per_acc_states; // This account has activated intelligent downloading of messages. // Prepare bags. - auto fds = feeds_per_root.values(rt); - for (Feed* fd : fds) { QHash per_feed_states; @@ -131,42 +127,65 @@ void FeedDownloader::updateFeeds(const QList& feeds) { DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Unread, fd)); per_feed_states.insert(ServiceRoot::BagOfMessages::Starred, DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Starred, fd)); + per_acc_states.insert(fd->customId(), per_feed_states); + + FeedUpdate fu; + + fu.account = rt; + fu.feed = fd; + fu.stated_messages = per_feed_states; + fu.tagged_messages = per_acc_tags; + + m_feeds.append(fu); } stated_messages.insert(rt, per_acc_states); } + else { + for (Feed* fd : fds) { + FeedUpdate fu; + + fu.account = rt; + fu.feed = fd; + + m_feeds.append(fu); + } + } try { - rt->aboutToBeginFeedFetching(feeds_per_root.values(rt), stated_messages.value(rt), tagged_messages.value(rt)); + rt->aboutToBeginFeedFetching(fds, stated_messages.value(rt), tagged_messages.value(rt)); } catch (const ApplicationException& ex) { // Common error showed, all feeds from the root are errored now! - errored_roots.insert(rt, ex); + m_erroredAccounts.insert(rt, ex); } } - while (!m_feeds.isEmpty()) { - auto n_f = m_feeds.takeFirst(); - auto n_r = n_f->getParentServiceRoot(); + std::function func = [=](const FeedUpdate& fd) -> void { + updateThreadedFeed(fd); + }; - if (errored_roots.contains(n_r)) { - // This feed is errored because its account errored when preparing feed update. - ApplicationException root_ex = errored_roots.value(n_r); - - skipFeedUpdateWithError(n_r, n_f, root_ex); - } - else { - updateOneFeed(n_r, n_f, stated_messages.value(n_r).value(n_f->customId()), tagged_messages.value(n_r)); - } - - n_f->setLastUpdated(QDateTime::currentDateTimeUtc()); - } + QtConcurrent::blockingMap(m_feeds, func); } finalizeUpdate(); } +void FeedDownloader::updateThreadedFeed(const FeedUpdate& fd) { + if (m_erroredAccounts.contains(fd.account)) { + // This feed is errored because its account errored when preparing feed update. + ApplicationException root_ex = m_erroredAccounts.value(fd.account); + + skipFeedUpdateWithError(fd.account, fd.feed, root_ex); + } + else { + updateOneFeed(fd.account, fd.feed, fd.stated_messages, fd.tagged_messages); + } + + fd.feed->setLastUpdated(QDateTime::currentDateTimeUtc()); +} + void FeedDownloader::skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex) { const FeedFetchException* fetch_ex = dynamic_cast(&ex); @@ -178,8 +197,6 @@ void FeedDownloader::skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const } acc->itemChanged({feed}); - - emit updateProgress(feed, ++m_feedsUpdated, m_feedsOriginalCount); } void FeedDownloader::stopRunningUpdate() { @@ -192,17 +209,19 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, Feed* feed, const QHash& stated_messages, const QHash& tagged_messages) { - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloading new messages for feed ID '" << feed->customId() << "' URL: '" - << feed->source() << "' title: '" << feed->title() << "' in thread: '" << QThread::currentThreadId() << "'."; + qlonglong thread_id = qlonglong(QThread::currentThreadId()); - int acc_id = feed->getParentServiceRoot()->accountId(); + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloading new messages for feed ID '" << feed->customId() << "' URL: '" + << feed->source() << "' title: '" << feed->title() << "' in thread: '" << thread_id << "'."; + + int acc_id = acc->accountId(); QElapsedTimer tmr; tmr.start(); try { bool is_main_thread = QThread::currentThread() == qApp->thread(); QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd")); + : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); QList msgs = feed->getParentServiceRoot()->obtainNewMessages(feed, stated_messages, tagged_messages); qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '" << feed->customId() diff --git a/src/librssguard/core/feeddownloader.h b/src/librssguard/core/feeddownloader.h index 600e8291e..3b2aeca40 100644 --- a/src/librssguard/core/feeddownloader.h +++ b/src/librssguard/core/feeddownloader.h @@ -13,7 +13,6 @@ #include "services/abstract/feed.h" class MessageFilter; -class QMutex; // Represents results of batch feed updates. class FeedDownloadResults { @@ -30,6 +29,13 @@ class FeedDownloadResults { QList> m_updatedFeeds; }; +struct FeedUpdate { + Feed* feed = nullptr; + ServiceRoot* account = nullptr; + QHash stated_messages; + QHash tagged_messages; +}; + // This class offers means to "update" feeds and "special" categories. // NOTE: This class is used within separate thread. class FeedDownloader : public QObject { @@ -62,11 +68,13 @@ class FeedDownloader : public QObject { void finalizeUpdate(); void removeDuplicateMessages(QList& messages); + void updateThreadedFeed(const FeedUpdate& fd); + private: bool m_isCacheSynchronizationRunning; bool m_stopCacheSynchronization; - QList m_feeds = {}; - QMutex* m_mutex; + QHash m_erroredAccounts; + QList m_feeds = {}; FeedDownloadResults m_results; int m_feedsUpdated; int m_feedsOriginalCount; diff --git a/src/librssguard/services/abstract/feed.cpp b/src/librssguard/services/abstract/feed.cpp index 882d74df7..6550baf53 100644 --- a/src/librssguard/services/abstract/feed.cpp +++ b/src/librssguard/services/abstract/feed.cpp @@ -197,8 +197,10 @@ void Feed::appendMessageFilter(MessageFilter* filter) { void Feed::updateCounts(bool including_total_count) { bool is_main_thread = QThread::currentThread() == qApp->thread(); + qlonglong thread_id = qlonglong(QThread::currentThreadId()); QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd")); + : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); + int account_id = getParentServiceRoot()->accountId(); if (including_total_count) { diff --git a/src/librssguard/services/abstract/importantnode.cpp b/src/librssguard/services/abstract/importantnode.cpp index bddb5d3a3..684b1a683 100644 --- a/src/librssguard/services/abstract/importantnode.cpp +++ b/src/librssguard/services/abstract/importantnode.cpp @@ -26,9 +26,9 @@ QList ImportantNode::undeletedMessages() const { void ImportantNode::updateCounts(bool including_total_count) { bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); + qlonglong thread_id = qlonglong(QThread::currentThreadId()); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); int account_id = getParentServiceRoot()->accountId(); if (including_total_count) { diff --git a/src/librssguard/services/abstract/label.cpp b/src/librssguard/services/abstract/label.cpp index 38d249409..b292873c3 100644 --- a/src/librssguard/services/abstract/label.cpp +++ b/src/librssguard/services/abstract/label.cpp @@ -2,10 +2,10 @@ #include "services/abstract/label.h" -#include "gui/dialogs/formaddeditlabel.h" -#include "miscellaneous/application.h" #include "database/databasefactory.h" #include "database/databasequeries.h" +#include "gui/dialogs/formaddeditlabel.h" +#include "miscellaneous/application.h" #include "services/abstract/cacheforserviceroot.h" #include "services/abstract/labelsnode.h" #include "services/abstract/serviceroot.h" @@ -77,9 +77,9 @@ bool Label::deleteViaGui() { void Label::updateCounts(bool including_total_count) { bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); + qlonglong thread_id = qlonglong(QThread::currentThreadId()); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); int account_id = getParentServiceRoot()->accountId(); if (including_total_count) { @@ -111,27 +111,25 @@ QIcon Label::generateIcon(const QColor& color) { void Label::assignToMessage(const Message& msg) { bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd")); - if (getParentServiceRoot()->onBeforeLabelMessageAssignmentChanged({ this }, { msg }, true)) { + if (getParentServiceRoot()->onBeforeLabelMessageAssignmentChanged({this}, {msg}, true)) { DatabaseQueries::assignLabelToMessage(database, this, msg); - getParentServiceRoot()->onAfterLabelMessageAssignmentChanged({ this }, { msg }, true); + getParentServiceRoot()->onAfterLabelMessageAssignmentChanged({this}, {msg}, true); } } void Label::deassignFromMessage(const Message& msg) { bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd")); - if (getParentServiceRoot()->onBeforeLabelMessageAssignmentChanged({ this }, { msg }, false)) { + if (getParentServiceRoot()->onBeforeLabelMessageAssignmentChanged({this}, {msg}, false)) { DatabaseQueries::deassignLabelFromMessage(database, this, msg); - getParentServiceRoot()->onAfterLabelMessageAssignmentChanged({ this }, { msg }, false); + getParentServiceRoot()->onAfterLabelMessageAssignmentChanged({this}, {msg}, false); } } diff --git a/src/librssguard/services/abstract/recyclebin.cpp b/src/librssguard/services/abstract/recyclebin.cpp index 90703f708..e54e839dc 100644 --- a/src/librssguard/services/abstract/recyclebin.cpp +++ b/src/librssguard/services/abstract/recyclebin.cpp @@ -11,8 +11,7 @@ #include -RecycleBin::RecycleBin(RootItem* parent_item) : RootItem(parent_item), m_totalCount(0), - m_unreadCount(0) { +RecycleBin::RecycleBin(RootItem* parent_item) : RootItem(parent_item), m_totalCount(0), m_unreadCount(0) { setKind(RootItem::Kind::Bin); setId(ID_RECYCLE_BIN); setIcon(qApp->icons()->fromTheme(QSL("user-trash"))); @@ -34,9 +33,9 @@ int RecycleBin::countOfAllMessages() const { void RecycleBin::updateCounts(bool update_total_count) { bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); + qlonglong thread_id = qlonglong(QThread::currentThreadId()); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); m_unreadCount = DatabaseQueries::getMessageCountsForBin(database, getParentServiceRoot()->accountId(), false); @@ -47,12 +46,9 @@ void RecycleBin::updateCounts(bool update_total_count) { QList RecycleBin::contextMenuFeedsList() { if (m_contextMenu.isEmpty()) { - QAction* restore_action = new QAction(qApp->icons()->fromTheme(QSL("view-refresh")), - tr("Restore recycle bin"), - this); - QAction* empty_action = new QAction(qApp->icons()->fromTheme(QSL("edit-clear")), - tr("Empty recycle bin"), - this); + QAction* restore_action = + new QAction(qApp->icons()->fromTheme(QSL("view-refresh")), tr("Restore recycle bin"), this); + QAction* empty_action = new QAction(qApp->icons()->fromTheme(QSL("edit-clear")), tr("Empty recycle bin"), this); connect(restore_action, &QAction::triggered, this, &RecycleBin::restore); connect(empty_action, &QAction::triggered, this, &RecycleBin::empty); @@ -99,7 +95,8 @@ bool RecycleBin::cleanMessages(bool clear_only_read) { updateCounts(true); parent_root->itemChanged(QList() << this); parent_root->requestReloadMessageList(true); - return true;; + return true; + ; } else { return false; diff --git a/src/librssguard/services/abstract/serviceroot.cpp b/src/librssguard/services/abstract/serviceroot.cpp index e9ab1d0bf..b6e468345 100644 --- a/src/librssguard/services/abstract/serviceroot.cpp +++ b/src/librssguard/services/abstract/serviceroot.cpp @@ -959,8 +959,9 @@ QPair ServiceRoot::updateMessages(QList& messages, Feed* feed qDebugNN << LOGSEC_CORE << "Updating messages in DB. Main thread:" << QUOTE_W_SPACE_DOT(is_main_thread); bool ok = false; + qlonglong thread_id = qlonglong(QThread::currentThreadId()); QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd")); + : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); updated_messages = DatabaseQueries::updateMessages(database, messages, feed, force_update, &ok); diff --git a/src/librssguard/services/abstract/unreadnode.cpp b/src/librssguard/services/abstract/unreadnode.cpp index 92ec3616f..f143ff34c 100644 --- a/src/librssguard/services/abstract/unreadnode.cpp +++ b/src/librssguard/services/abstract/unreadnode.cpp @@ -26,9 +26,9 @@ void UnreadNode::updateCounts(bool including_total_count) { Q_UNUSED(including_total_count) bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); + qlonglong thread_id = qlonglong(QThread::currentThreadId()); + QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) + : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); int account_id = getParentServiceRoot()->accountId(); m_totalCount = m_unreadCount = DatabaseQueries::getUnreadMessageCounts(database, account_id); From 10e84709e1933f0c3b2d8a8de470b94f6b08efb6 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 4 Jan 2023 11:13:24 +0100 Subject: [PATCH 03/15] poc enhancement 160 feeds in 10 seconds --- src/librssguard/core/feeddownloader.cpp | 4 ++-- src/librssguard/miscellaneous/application.cpp | 6 ++++++ src/librssguard/services/abstract/serviceroot.cpp | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 26502d5c8..77be2695a 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -426,13 +426,13 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, feed->setStatus(Feed::Status::OtherError, app_ex.message()); } - feed->getParentServiceRoot()->itemChanged({feed}); + // feed->getParentServiceRoot()->itemChanged({feed}); m_feedsUpdated++; qDebugNN << LOGSEC_FEEDDOWNLOADER << "Made progress in feed updates, total feeds count " << m_feedsUpdated << "/" << m_feedsOriginalCount << " (id of feed is " << feed->id() << ")."; - emit updateProgress(feed, m_feedsUpdated, m_feedsOriginalCount); + // emit updateProgress(feed, m_feedsUpdated, m_feedsOriginalCount); } void FeedDownloader::finalizeUpdate() { diff --git a/src/librssguard/miscellaneous/application.cpp b/src/librssguard/miscellaneous/application.cpp index 951165d17..968850454 100644 --- a/src/librssguard/miscellaneous/application.cpp +++ b/src/librssguard/miscellaneous/application.cpp @@ -215,6 +215,12 @@ Application::Application(const QString& id, int& argc, char** argv, const QStrin QTimer::singleShot(1000, system(), &SystemFactory::checkForUpdatesOnStartup); + auto ideal_th_count = QThread::idealThreadCount(); + + if (ideal_th_count > 1) { + QThreadPool::globalInstance()->setMaxThreadCount(2 * ideal_th_count); + } + qDebugNN << LOGSEC_CORE << "OpenSSL version:" << QUOTE_W_SPACE_DOT(QSslSocket::sslLibraryVersionString()); qDebugNN << LOGSEC_CORE << "OpenSSL supported:" << QUOTE_W_SPACE_DOT(QSslSocket::supportsSsl()); qDebugNN << LOGSEC_CORE << "Global thread pool has" diff --git a/src/librssguard/services/abstract/serviceroot.cpp b/src/librssguard/services/abstract/serviceroot.cpp index b6e468345..b59b64952 100644 --- a/src/librssguard/services/abstract/serviceroot.cpp +++ b/src/librssguard/services/abstract/serviceroot.cpp @@ -992,7 +992,7 @@ QPair ServiceRoot::updateMessages(QList& messages, Feed* feed // Some messages were really added to DB, reload feed in model. items_to_update.append(feed); - getParentServiceRoot()->itemChanged(items_to_update); + // getParentServiceRoot()->itemChanged(items_to_update); return updated_messages; } From e87ecd91f286886a1506e6d1a5a639015382022f Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 4 Jan 2023 14:11:23 +0100 Subject: [PATCH 04/15] reasonably working --- src/librssguard/core/feeddownloader.cpp | 83 +++++++++++-------- src/librssguard/core/feeddownloader.h | 14 ++-- src/librssguard/database/databasedriver.cpp | 11 +++ src/librssguard/database/databasedriver.h | 26 +++--- src/librssguard/miscellaneous/application.cpp | 2 +- src/librssguard/miscellaneous/feedreader.cpp | 7 +- src/librssguard/miscellaneous/feedreader.h | 1 + 7 files changed, 85 insertions(+), 59 deletions(-) diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 77be2695a..ec512e8f0 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -21,9 +21,24 @@ #include FeedDownloader::FeedDownloader() - : QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false), m_feedsUpdated(0), - m_feedsOriginalCount(0) { + : QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false) { qRegisterMetaType("FeedDownloadResults"); + + connect(&m_watcherLookup, &QFutureWatcher::resultReadyAt, this, [=](int idx) { + FeedUpdateResult res = m_watcherLookup.resultAt(idx); + + emit updateProgress(res.feed, m_watcherLookup.progressValue(), m_watcherLookup.progressMaximum()); + }); + + /* +connect(&m_watcherLookup, &QFutureWatcher::progressValueChanged, this, [=](int prog) { +// +}); +*/ + + connect(&m_watcherLookup, &QFutureWatcher::finished, this, [=]() { + finalizeUpdate(); + }); } FeedDownloader::~FeedDownloader() { @@ -63,9 +78,6 @@ void FeedDownloader::updateFeeds(const QList& feeds) { m_results.clear(); m_feeds.clear(); - m_feedsOriginalCount = feeds.size(); - m_feedsUpdated = 0; - if (feeds.isEmpty()) { qDebugNN << LOGSEC_FEEDDOWNLOADER << "No feeds to update in worker thread, aborting update."; } @@ -130,7 +142,7 @@ void FeedDownloader::updateFeeds(const QList& feeds) { per_acc_states.insert(fd->customId(), per_feed_states); - FeedUpdate fu; + FeedUpdateRequest fu; fu.account = rt; fu.feed = fd; @@ -144,7 +156,7 @@ void FeedDownloader::updateFeeds(const QList& feeds) { } else { for (Feed* fd : fds) { - FeedUpdate fu; + FeedUpdateRequest fu; fu.account = rt; fu.feed = fd; @@ -162,17 +174,16 @@ void FeedDownloader::updateFeeds(const QList& feeds) { } } - std::function func = [=](const FeedUpdate& fd) -> void { - updateThreadedFeed(fd); + std::function func = + [=](const FeedUpdateRequest& fd) -> FeedUpdateResult { + return updateThreadedFeed(fd); }; - QtConcurrent::blockingMap(m_feeds, func); + m_watcherLookup.setFuture(QtConcurrent::mapped(m_feeds, func)); } - - finalizeUpdate(); } -void FeedDownloader::updateThreadedFeed(const FeedUpdate& fd) { +FeedUpdateResult FeedDownloader::updateThreadedFeed(const FeedUpdateRequest& fd) { if (m_erroredAccounts.contains(fd.account)) { // This feed is errored because its account errored when preparing feed update. ApplicationException root_ex = m_erroredAccounts.value(fd.account); @@ -184,6 +195,12 @@ void FeedDownloader::updateThreadedFeed(const FeedUpdate& fd) { } fd.feed->setLastUpdated(QDateTime::currentDateTimeUtc()); + + FeedUpdateResult res; + + res.feed = fd.feed; + + return res; } void FeedDownloader::skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex) { @@ -195,14 +212,15 @@ void FeedDownloader::skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const else { feed->setStatus(Feed::Status::OtherError, ex.message()); } - - acc->itemChanged({feed}); } void FeedDownloader::stopRunningUpdate() { m_stopCacheSynchronization = true; + + m_watcherLookup.cancel(); + m_watcherLookup.waitForFinished(); + m_feeds.clear(); - m_feedsOriginalCount = m_feedsUpdated = 0; } void FeedDownloader::updateOneFeed(ServiceRoot* acc, @@ -211,8 +229,9 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, const QHash& tagged_messages) { qlonglong thread_id = qlonglong(QThread::currentThreadId()); - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloading new messages for feed ID '" << feed->customId() << "' URL: '" - << feed->source() << "' title: '" << feed->title() << "' in thread: '" << thread_id << "'."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloading new messages for feed ID" << QUOTE_W_SPACE(feed->customId()) + << "URL:" << QUOTE_W_SPACE(feed->source()) << "title:" << QUOTE_W_SPACE(feed->title()) << "in thread " + << QUOTE_W_SPACE_DOT(thread_id); int acc_id = acc->accountId(); QElapsedTimer tmr; @@ -224,9 +243,9 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); QList msgs = feed->getParentServiceRoot()->obtainNewMessages(feed, stated_messages, tagged_messages); - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '" << feed->customId() - << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '" - << QThread::currentThreadId() << "'. Operation took " << tmr.nsecsElapsed() / 1000 << " microseconds."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded" << NONQUOTE_W_SPACE(msgs.size()) << "messages for feed ID" + << QUOTE_W_SPACE_COMMA(feed->customId()) << "operation took" << NONQUOTE_W_SPACE(tmr.nsecsElapsed() / 1000) + << "microseconds."; bool fix_future_datetimes = qApp->settings()->value(GROUP(Messages), SETTING(Messages::FixupFutureArticleDateTimes)).toBool(); @@ -390,16 +409,11 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, removeDuplicateMessages(msgs); - // Now make sure, that messages are actually stored to SQL in a locked state. - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Saving messages of feed ID '" << feed->customId() << "' URL: '" - << feed->source() << "' title: '" << feed->title() << "' in thread: '" << QThread::currentThreadId() - << "'."; - tmr.restart(); auto updated_messages = acc->updateMessages(msgs, feed, false); - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Updating messages in DB took " << tmr.nsecsElapsed() / 1000 - << " microseconds."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Updating messages in DB took" << NONQUOTE_W_SPACE(tmr.nsecsElapsed() / 1000) + << "microseconds."; if (feed->status() != Feed::Status::NewMessages) { feed->setStatus(updated_messages.first > 0 || updated_messages.second > 0 ? Feed::Status::NewMessages @@ -426,17 +440,14 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, feed->setStatus(Feed::Status::OtherError, app_ex.message()); } - // feed->getParentServiceRoot()->itemChanged({feed}); - - m_feedsUpdated++; - - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Made progress in feed updates, total feeds count " << m_feedsUpdated << "/" - << m_feedsOriginalCount << " (id of feed is " << feed->id() << ")."; - // emit updateProgress(feed, m_feedsUpdated, m_feedsOriginalCount); + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Made progress in feed updates, total feeds count " + << m_watcherLookup.progressValue() + 1 << "/" << m_feeds.size() << " (id of feed is " << feed->id() << ")."; } void FeedDownloader::finalizeUpdate() { - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Finished feed updates in thread: '" << QThread::currentThreadId() << "'."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Finished feed updates in thread" + << QUOTE_W_SPACE_DOT(QThread::currentThreadId()); + m_results.sort(); // Update of feeds has finished. diff --git a/src/librssguard/core/feeddownloader.h b/src/librssguard/core/feeddownloader.h index 3b2aeca40..ea9b7ae6d 100644 --- a/src/librssguard/core/feeddownloader.h +++ b/src/librssguard/core/feeddownloader.h @@ -5,6 +5,7 @@ #include +#include #include #include "core/message.h" @@ -29,13 +30,17 @@ class FeedDownloadResults { QList> m_updatedFeeds; }; -struct FeedUpdate { +struct FeedUpdateRequest { Feed* feed = nullptr; ServiceRoot* account = nullptr; QHash stated_messages; QHash tagged_messages; }; +struct FeedUpdateResult { + Feed* feed = nullptr; +}; + // This class offers means to "update" feeds and "special" categories. // NOTE: This class is used within separate thread. class FeedDownloader : public QObject { @@ -68,16 +73,15 @@ class FeedDownloader : public QObject { void finalizeUpdate(); void removeDuplicateMessages(QList& messages); - void updateThreadedFeed(const FeedUpdate& fd); + FeedUpdateResult updateThreadedFeed(const FeedUpdateRequest& fd); private: bool m_isCacheSynchronizationRunning; bool m_stopCacheSynchronization; QHash m_erroredAccounts; - QList m_feeds = {}; + QList m_feeds = {}; + QFutureWatcher m_watcherLookup; FeedDownloadResults m_results; - int m_feedsUpdated; - int m_feedsOriginalCount; }; #endif // FEEDDOWNLOADER_H diff --git a/src/librssguard/database/databasedriver.cpp b/src/librssguard/database/databasedriver.cpp index 7c893e22f..6003efabf 100644 --- a/src/librssguard/database/databasedriver.cpp +++ b/src/librssguard/database/databasedriver.cpp @@ -10,9 +10,20 @@ #include #include #include +#include DatabaseDriver::DatabaseDriver(QObject* parent) : QObject(parent) {} +QSqlDatabase DatabaseDriver::threadSafeConnection(const QString& connection_name, DesiredStorageType desired_type) { + qlonglong thread_id = qlonglong(QThread::currentThreadId()); + bool is_main_thread = QThread::currentThread() == qApp->thread(); + + QSqlDatabase database = + connection(is_main_thread ? connection_name : QSL("db_connection_%1").arg(thread_id), desired_type); + + return database; +} + void DatabaseDriver::updateDatabaseSchema(QSqlQuery& query, int source_db_schema_version, const QString& database_name) { diff --git a/src/librssguard/database/databasedriver.h b/src/librssguard/database/databasedriver.h index 257acbbd8..2c6ad8ba1 100644 --- a/src/librssguard/database/databasedriver.h +++ b/src/librssguard/database/databasedriver.h @@ -9,25 +9,21 @@ #include class DatabaseDriver : public QObject { - Q_OBJECT + Q_OBJECT public: - // Describes available types of database backend. - enum class DriverType { - SQLite, - MySQL - }; + enum class DriverType { SQLite, MySQL }; // Describes what type of database user wants. - enum class DesiredStorageType { - StrictlyFileBased, - StrictlyInMemory, - FromSettings - }; + enum class DesiredStorageType { StrictlyFileBased, StrictlyInMemory, FromSettings }; explicit DatabaseDriver(QObject* parent = nullptr); + QSqlDatabase threadSafeConnection(const QString& connection_name, + DatabaseDriver::DesiredStorageType desired_type = + DatabaseDriver::DesiredStorageType::FromSettings); + // API. virtual QString location() const = 0; virtual QString humanDriverType() const = 0; @@ -43,19 +39,17 @@ class DatabaseDriver : public QObject { virtual bool finishRestoration() = 0; virtual qint64 databaseDataSize() = 0; virtual QSqlDatabase connection(const QString& connection_name, - DatabaseDriver::DesiredStorageType desired_type = DatabaseDriver::DesiredStorageType::FromSettings) = 0; + DatabaseDriver::DesiredStorageType desired_type = + DatabaseDriver::DesiredStorageType::FromSettings) = 0; protected: - void updateDatabaseSchema(QSqlQuery& query, - int source_db_schema_version, - const QString& database_name = {}); + void updateDatabaseSchema(QSqlQuery& query, int source_db_schema_version, const QString& database_name = {}); void setSchemaVersion(QSqlQuery& query, int new_schema_version, bool empty_table); QStringList prepareScript(const QString& base_sql_folder, const QString& sql_file, const QString& database_name = {}); - }; #endif // DATABASEDRIVER_H diff --git a/src/librssguard/miscellaneous/application.cpp b/src/librssguard/miscellaneous/application.cpp index 968850454..a7c0015e0 100644 --- a/src/librssguard/miscellaneous/application.cpp +++ b/src/librssguard/miscellaneous/application.cpp @@ -218,7 +218,7 @@ Application::Application(const QString& id, int& argc, char** argv, const QStrin auto ideal_th_count = QThread::idealThreadCount(); if (ideal_th_count > 1) { - QThreadPool::globalInstance()->setMaxThreadCount(2 * ideal_th_count); + QThreadPool::globalInstance()->setMaxThreadCount((std::min)(128, 2 * ideal_th_count)); } qDebugNN << LOGSEC_CORE << "OpenSSL version:" << QUOTE_W_SPACE_DOT(QSslSocket::sslLibraryVersionString()); diff --git a/src/librssguard/miscellaneous/feedreader.cpp b/src/librssguard/miscellaneous/feedreader.cpp index 0e7727ab4..3be4bbd56 100644 --- a/src/librssguard/miscellaneous/feedreader.cpp +++ b/src/librssguard/miscellaneous/feedreader.cpp @@ -122,7 +122,7 @@ void FeedReader::initializeFeedDownloader() { connect(m_feedDownloaderThread, &QThread::finished, m_feedDownloaderThread, &QThread::deleteLater); connect(m_feedDownloaderThread, &QThread::finished, m_feedDownloader, &FeedDownloader::deleteLater); - connect(m_feedDownloader, &FeedDownloader::updateFinished, this, &FeedReader::feedUpdatesFinished); + connect(m_feedDownloader, &FeedDownloader::updateFinished, this, &FeedReader::onFeedUpdatesFinished); connect(m_feedDownloader, &FeedDownloader::updateProgress, this, &FeedReader::feedUpdatesProgress); connect(m_feedDownloader, &FeedDownloader::updateStarted, this, &FeedReader::feedUpdatesStarted); connect(m_feedDownloader, &FeedDownloader::updateFinished, qApp->feedUpdateLock(), &Mutex::unlock); @@ -348,6 +348,11 @@ void FeedReader::executeNextAutoUpdate() { } } +void FeedReader::onFeedUpdatesFinished(FeedDownloadResults updated_feeds) { + m_feedsModel->reloadWholeLayout(); + emit feedUpdatesFinished(updated_feeds); +} + QList FeedReader::messageFilters() const { return m_messageFilters; } diff --git a/src/librssguard/miscellaneous/feedreader.h b/src/librssguard/miscellaneous/feedreader.h index be32e0303..62c055429 100644 --- a/src/librssguard/miscellaneous/feedreader.h +++ b/src/librssguard/miscellaneous/feedreader.h @@ -72,6 +72,7 @@ class RSSGUARD_DLLSPEC FeedReader : public QObject { private slots: void executeNextAutoUpdate(); + void onFeedUpdatesFinished(FeedDownloadResults updated_feeds); signals: void feedUpdatesStarted(); From 454cca671135e0e99287ff16b01d32736a513387 Mon Sep 17 00:00:00 2001 From: martinrotter Date: Wed, 4 Jan 2023 14:17:38 +0100 Subject: [PATCH 05/15] Add workaround for a macOS-only Qt crash (#866) QTBUG: https://bugreports.qt.io/browse/QTBUG-102107 Co-authored-by: guihkx <626206+guihkx@users.noreply.github.com> --- src/librssguard/gui/dialogs/formmain.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/librssguard/gui/dialogs/formmain.cpp b/src/librssguard/gui/dialogs/formmain.cpp index c0c0f669d..5c3b42c98 100644 --- a/src/librssguard/gui/dialogs/formmain.cpp +++ b/src/librssguard/gui/dialogs/formmain.cpp @@ -267,6 +267,12 @@ void FormMain::prepareMenus() { if (QSysInfo::currentCpuArchitecture().contains(QSL("arm"), Qt::CaseSensitivity::CaseInsensitive)) { qWarningNN << LOGSEC_GUI << "Disabling native menu bar."; m_ui->m_menuBar->setNativeMenuBar(false); +#if defined(Q_OS_MACOS) + // This works around a macOS-only Qt crash. + // QTBUG: https://bugreports.qt.io/browse/QTBUG-102107 + // TODO: Remove this workaround once the upstream bug gets addressed. + m_ui->m_menuBar->setCornerWidget(nullptr); +#endif } } From 7e14390e2c3c743fa7b53b2d7edd1d01884c38c3 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Thu, 5 Jan 2023 09:44:46 +0100 Subject: [PATCH 06/15] make cookie jar thread safe, make qsettings thread safe --- resources/desktop/rssguard.metainfo.xml.in | 2 +- src/librssguard/core/feeddownloader.cpp | 29 ++++++---------------- src/librssguard/miscellaneous/settings.cpp | 3 ++- src/librssguard/miscellaneous/settings.h | 9 ++++++- src/librssguard/network-web/cookiejar.cpp | 11 ++++++++ src/librssguard/network-web/cookiejar.h | 4 +++ 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/resources/desktop/rssguard.metainfo.xml.in b/resources/desktop/rssguard.metainfo.xml.in index 8f3f27b2c..5da01b359 100644 --- a/resources/desktop/rssguard.metainfo.xml.in +++ b/resources/desktop/rssguard.metainfo.xml.in @@ -60,7 +60,7 @@ - + @APP_LOW_NAME@ diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index ec512e8f0..77cc6c82c 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -82,23 +82,14 @@ void FeedDownloader::updateFeeds(const QList& feeds) { qDebugNN << LOGSEC_FEEDDOWNLOADER << "No feeds to update in worker thread, aborting update."; } else { - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Starting feed updates from worker in thread: '" << QThread::currentThreadId() - << "'."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Starting feed updates from worker in thread" + << QUOTE_W_SPACE_DOT(QThread::currentThreadId()); // Job starts now. emit updateStarted(); QSet caches; QMultiHash feeds_per_root; - // 1. key - account. - // 2. key - feed custom ID. - // 3. key - msg state. - QHash>> stated_messages; - - // 1. key - account. - // 2. key - label custom ID. - QHash> tagged_messages; - for (auto* fd : feeds) { CacheForServiceRoot* fd_cache = fd->getParentServiceRoot()->toCache(); @@ -112,21 +103,17 @@ void FeedDownloader::updateFeeds(const QList& feeds) { synchronizeAccountCaches(caches.values(), false); auto roots = feeds_per_root.uniqueKeys(); - bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd")); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); for (auto* rt : roots) { auto fds = feeds_per_root.values(rt); + QHash per_acc_tags; + QHash> per_acc_states; // Obtain lists of local IDs. if (rt->wantsBaggedIdsOfExistingMessages()) { // Tags per account. - auto per_acc_tags = DatabaseQueries::bagsOfMessages(database, rt->labelsNode()->labels()); - - tagged_messages.insert(rt, per_acc_tags); - - QHash> per_acc_states; + per_acc_tags = DatabaseQueries::bagsOfMessages(database, rt->labelsNode()->labels()); // This account has activated intelligent downloading of messages. // Prepare bags. @@ -151,8 +138,6 @@ void FeedDownloader::updateFeeds(const QList& feeds) { m_feeds.append(fu); } - - stated_messages.insert(rt, per_acc_states); } else { for (Feed* fd : fds) { @@ -166,7 +151,7 @@ void FeedDownloader::updateFeeds(const QList& feeds) { } try { - rt->aboutToBeginFeedFetching(fds, stated_messages.value(rt), tagged_messages.value(rt)); + rt->aboutToBeginFeedFetching(fds, per_acc_states, per_acc_tags); } catch (const ApplicationException& ex) { // Common error showed, all feeds from the root are errored now! diff --git a/src/librssguard/miscellaneous/settings.cpp b/src/librssguard/miscellaneous/settings.cpp index 9edb1e1e8..ea135a409 100644 --- a/src/librssguard/miscellaneous/settings.cpp +++ b/src/librssguard/miscellaneous/settings.cpp @@ -438,7 +438,8 @@ DVALUE(QStringList) Browser::ExternalToolsDef = QStringList(); DKEY CategoriesExpandStates::ID = "categories_expand_states"; Settings::Settings(const QString& file_name, Format format, SettingsProperties::SettingsType type, QObject* parent) - : QSettings(file_name, format, parent), m_initializationStatus(type) { + : QSettings(file_name, format, parent), m_lock(QReadWriteLock(QReadWriteLock::RecursionMode::Recursive)), + m_initializationStatus(type) { Messages::PreviewerFontStandardDef = QFont(QApplication::font().family(), 12).toString(); } diff --git a/src/librssguard/miscellaneous/settings.h b/src/librssguard/miscellaneous/settings.h index cfbd56591..f8d25f301 100644 --- a/src/librssguard/miscellaneous/settings.h +++ b/src/librssguard/miscellaneous/settings.h @@ -14,7 +14,9 @@ #include #include #include +#include #include +#include #define KEY extern const QString #define DKEY const QString @@ -518,12 +520,13 @@ class Settings : public QSettings { static SettingsProperties determineProperties(); private: - // Constructor. explicit Settings(const QString& file_name, Format format, SettingsProperties::SettingsType type, QObject* parent = nullptr); + private: + mutable QReadWriteLock m_lock; SettingsProperties::SettingsType m_initializationStatus; }; @@ -545,10 +548,12 @@ inline QVariant Settings::value(const QString& section, const QString& key, cons } inline void Settings::setValue(const QString& section, const QString& key, const QVariant& value) { + QWriteLocker lck(&m_lock); QSettings::setValue(QString(QSL("%1/%2")).arg(section, key), value); } inline void Settings::setValue(const QString& key, const QVariant& value) { + QWriteLocker lck(&m_lock); QSettings::setValue(key, value); } @@ -557,6 +562,8 @@ inline bool Settings::contains(const QString& section, const QString& key) const } inline void Settings::remove(const QString& section, const QString& key) { + QWriteLocker lck(&m_lock); + if (key.isEmpty()) { beginGroup(section); QSettings::remove({}); diff --git a/src/librssguard/network-web/cookiejar.cpp b/src/librssguard/network-web/cookiejar.cpp index b819f7bae..fd0414b25 100644 --- a/src/librssguard/network-web/cookiejar.cpp +++ b/src/librssguard/network-web/cookiejar.cpp @@ -120,10 +120,12 @@ void CookieJar::saveCookies() { } QList CookieJar::cookiesForUrl(const QUrl& url) const { + QReadLocker l(&m_lock); return QNetworkCookieJar::cookiesForUrl(url); } bool CookieJar::setCookiesFromUrl(const QList& cookie_list, const QUrl& url) { + QWriteLocker l(&m_lock); return QNetworkCookieJar::setCookiesFromUrl(cookie_list, url); } @@ -188,11 +190,13 @@ bool CookieJar::insertCookie(const QNetworkCookie& cookie) { return {}; } else { + QWriteLocker l(&m_lock); return insertCookieInternal(cookie, false, true); } } bool CookieJar::deleteCookie(const QNetworkCookie& cookie) { + QWriteLocker l(&m_lock); return deleteCookieInternal(cookie, false); } @@ -206,5 +210,12 @@ void CookieJar::updateSettings() { } bool CookieJar::updateCookie(const QNetworkCookie& cookie) { + QWriteLocker l(&m_lock); return updateCookieInternal(cookie, false); } + +/* +bool CookieJar::validateCookie(const QNetworkCookie &cookie, const QUrl &url) const { + return QNetworkCookieJar::validateCookie(cookie, url); +} +*/ diff --git a/src/librssguard/network-web/cookiejar.h b/src/librssguard/network-web/cookiejar.h index 59f97e4ad..5409685f5 100644 --- a/src/librssguard/network-web/cookiejar.h +++ b/src/librssguard/network-web/cookiejar.h @@ -5,6 +5,8 @@ #include +#include + #if defined(USE_WEBENGINE) class QWebEngineCookieStore; #endif @@ -18,6 +20,7 @@ class CookieJar : public QNetworkCookieJar { virtual bool insertCookie(const QNetworkCookie& cookie); virtual bool updateCookie(const QNetworkCookie& cookie); virtual bool deleteCookie(const QNetworkCookie& cookie); + // virtual bool validateCookie(const QNetworkCookie& cookie, const QUrl& url) const; void updateSettings(); @@ -37,6 +40,7 @@ class CookieJar : public QNetworkCookieJar { QWebEngineCookieStore* m_webEngineCookies; #endif + mutable QReadWriteLock m_lock{QReadWriteLock::RecursionMode::Recursive}; bool m_ignoreAllCookies; }; From 0571bec5120725d7f090739ca10ea6252d94cad6 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Thu, 5 Jan 2023 13:56:04 +0100 Subject: [PATCH 07/15] bump version, fix end of feed fetching --- CMakeLists.txt | 2 +- src/librssguard/core/feeddownloader.cpp | 1 + src/librssguard/definitions/definitions.h | 2 +- src/librssguard/gui/dialogs/formmain.cpp | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 824071a34..02fc06768 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,7 +65,7 @@ set(APP_AUTHOR "Martin Rotter") set(APP_COPYRIGHT "\\251 2011-${YEAR} ${APP_AUTHOR}") set(APP_REVERSE_NAME "io.github.martinrotter.rssguard") set(APP_DONATE_URL "https://github.com/sponsors/martinrotter") -set(APP_VERSION "4.2.7") +set(APP_VERSION "4.3.0") set(APP_URL "https://github.com/martinrotter/rssguard") set(APP_URL_DOCUMENTATION "https://github.com/martinrotter/rssguard/blob/${APP_VERSION}/resources/docs/Documentation.md") diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 77cc6c82c..02b683622 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -434,6 +434,7 @@ void FeedDownloader::finalizeUpdate() { << QUOTE_W_SPACE_DOT(QThread::currentThreadId()); m_results.sort(); + m_feeds.clear(); // Update of feeds has finished. // NOTE: This means that now "update lock" can be unlocked diff --git a/src/librssguard/definitions/definitions.h b/src/librssguard/definitions/definitions.h index dec825cfd..59660aedf 100644 --- a/src/librssguard/definitions/definitions.h +++ b/src/librssguard/definitions/definitions.h @@ -317,7 +317,7 @@ #define OS_ID "OpenBSD" #elif defined(Q_OS_OS2) #define OS_ID "OS2" -#elif defined(Q_OS_OSX) +#elif defined(Q_OS_MACOS) #define OS_ID "macOS" #elif defined(Q_OS_WIN) #define OS_ID "Windows" diff --git a/src/librssguard/gui/dialogs/formmain.cpp b/src/librssguard/gui/dialogs/formmain.cpp index 5c3b42c98..d563af6a0 100644 --- a/src/librssguard/gui/dialogs/formmain.cpp +++ b/src/librssguard/gui/dialogs/formmain.cpp @@ -267,6 +267,7 @@ void FormMain::prepareMenus() { if (QSysInfo::currentCpuArchitecture().contains(QSL("arm"), Qt::CaseSensitivity::CaseInsensitive)) { qWarningNN << LOGSEC_GUI << "Disabling native menu bar."; m_ui->m_menuBar->setNativeMenuBar(false); + #if defined(Q_OS_MACOS) // This works around a macOS-only Qt crash. // QTBUG: https://bugreports.qt.io/browse/QTBUG-102107 From 15e4f8e31f46cb8b827d0bd75129f6c2c346216e Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Thu, 5 Jan 2023 15:02:57 +0100 Subject: [PATCH 08/15] Save --- src/librssguard/network-web/cookiejar.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/librssguard/network-web/cookiejar.cpp b/src/librssguard/network-web/cookiejar.cpp index fd0414b25..1285df533 100644 --- a/src/librssguard/network-web/cookiejar.cpp +++ b/src/librssguard/network-web/cookiejar.cpp @@ -112,7 +112,6 @@ void CookieJar::saveCookies() { if (cookie.isSessionCookie()) { continue; } - sett->setPassword(GROUP(Cookies), QSL("%1-%2").arg(QString::number(i++), QString::fromUtf8(cookie.name())), cookie.toRawForm(QNetworkCookie::RawForm::Full)); From b0cc456087201c5178d4e5b5717be2dc73f3ee41 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Fri, 6 Jan 2023 13:40:17 +0100 Subject: [PATCH 09/15] usa thread safe db where makes sense --- resources/desktop/rssguard.metainfo.xml.in | 2 +- src/librssguard/core/feeddownloader.cpp | 4 +--- src/librssguard/services/abstract/feed.cpp | 6 +----- src/librssguard/services/abstract/importantnode.cpp | 5 +---- src/librssguard/services/abstract/label.cpp | 13 +++---------- src/librssguard/services/abstract/recyclebin.cpp | 5 +---- src/librssguard/services/abstract/serviceroot.cpp | 10 +++------- src/librssguard/services/abstract/unreadnode.cpp | 5 +---- 8 files changed, 12 insertions(+), 38 deletions(-) diff --git a/resources/desktop/rssguard.metainfo.xml.in b/resources/desktop/rssguard.metainfo.xml.in index 5da01b359..9f4b74f4e 100644 --- a/resources/desktop/rssguard.metainfo.xml.in +++ b/resources/desktop/rssguard.metainfo.xml.in @@ -60,7 +60,7 @@ - + @APP_LOW_NAME@ diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 02b683622..a78cfb3b9 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -223,9 +223,7 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, tmr.start(); try { - bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); QList msgs = feed->getParentServiceRoot()->obtainNewMessages(feed, stated_messages, tagged_messages); qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded" << NONQUOTE_W_SPACE(msgs.size()) << "messages for feed ID" diff --git a/src/librssguard/services/abstract/feed.cpp b/src/librssguard/services/abstract/feed.cpp index 6550baf53..2ce2240b2 100644 --- a/src/librssguard/services/abstract/feed.cpp +++ b/src/librssguard/services/abstract/feed.cpp @@ -196,11 +196,7 @@ void Feed::appendMessageFilter(MessageFilter* filter) { } void Feed::updateCounts(bool including_total_count) { - bool is_main_thread = QThread::currentThread() == qApp->thread(); - qlonglong thread_id = qlonglong(QThread::currentThreadId()); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); - + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); int account_id = getParentServiceRoot()->accountId(); if (including_total_count) { diff --git a/src/librssguard/services/abstract/importantnode.cpp b/src/librssguard/services/abstract/importantnode.cpp index 684b1a683..9526f96d4 100644 --- a/src/librssguard/services/abstract/importantnode.cpp +++ b/src/librssguard/services/abstract/importantnode.cpp @@ -25,10 +25,7 @@ QList ImportantNode::undeletedMessages() const { } void ImportantNode::updateCounts(bool including_total_count) { - bool is_main_thread = QThread::currentThread() == qApp->thread(); - qlonglong thread_id = qlonglong(QThread::currentThreadId()); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); int account_id = getParentServiceRoot()->accountId(); if (including_total_count) { diff --git a/src/librssguard/services/abstract/label.cpp b/src/librssguard/services/abstract/label.cpp index b292873c3..9fb7e1c83 100644 --- a/src/librssguard/services/abstract/label.cpp +++ b/src/librssguard/services/abstract/label.cpp @@ -76,10 +76,7 @@ bool Label::deleteViaGui() { } void Label::updateCounts(bool including_total_count) { - bool is_main_thread = QThread::currentThread() == qApp->thread(); - qlonglong thread_id = qlonglong(QThread::currentThreadId()); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); int account_id = getParentServiceRoot()->accountId(); if (including_total_count) { @@ -110,9 +107,7 @@ QIcon Label::generateIcon(const QColor& color) { } void Label::assignToMessage(const Message& msg) { - bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd")); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); if (getParentServiceRoot()->onBeforeLabelMessageAssignmentChanged({this}, {msg}, true)) { DatabaseQueries::assignLabelToMessage(database, this, msg); @@ -122,9 +117,7 @@ void Label::assignToMessage(const Message& msg) { } void Label::deassignFromMessage(const Message& msg) { - bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd")); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); if (getParentServiceRoot()->onBeforeLabelMessageAssignmentChanged({this}, {msg}, false)) { DatabaseQueries::deassignLabelFromMessage(database, this, msg); diff --git a/src/librssguard/services/abstract/recyclebin.cpp b/src/librssguard/services/abstract/recyclebin.cpp index e54e839dc..d80c4ce1a 100644 --- a/src/librssguard/services/abstract/recyclebin.cpp +++ b/src/librssguard/services/abstract/recyclebin.cpp @@ -32,10 +32,7 @@ int RecycleBin::countOfAllMessages() const { } void RecycleBin::updateCounts(bool update_total_count) { - bool is_main_thread = QThread::currentThread() == qApp->thread(); - qlonglong thread_id = qlonglong(QThread::currentThreadId()); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); m_unreadCount = DatabaseQueries::getMessageCountsForBin(database, getParentServiceRoot()->accountId(), false); diff --git a/src/librssguard/services/abstract/serviceroot.cpp b/src/librssguard/services/abstract/serviceroot.cpp index b59b64952..c5e6d541e 100644 --- a/src/librssguard/services/abstract/serviceroot.cpp +++ b/src/librssguard/services/abstract/serviceroot.cpp @@ -954,14 +954,10 @@ QPair ServiceRoot::updateMessages(QList& messages, Feed* feed } QList items_to_update; - bool is_main_thread = QThread::currentThread() == qApp->thread(); - - qDebugNN << LOGSEC_CORE << "Updating messages in DB. Main thread:" << QUOTE_W_SPACE_DOT(is_main_thread); - bool ok = false; - qlonglong thread_id = qlonglong(QThread::currentThreadId()); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); + + qDebugNN << LOGSEC_CORE << "Updating messages in DB."; updated_messages = DatabaseQueries::updateMessages(database, messages, feed, force_update, &ok); diff --git a/src/librssguard/services/abstract/unreadnode.cpp b/src/librssguard/services/abstract/unreadnode.cpp index f143ff34c..d71be05f5 100644 --- a/src/librssguard/services/abstract/unreadnode.cpp +++ b/src/librssguard/services/abstract/unreadnode.cpp @@ -25,10 +25,7 @@ QList UnreadNode::undeletedMessages() const { void UnreadNode::updateCounts(bool including_total_count) { Q_UNUSED(including_total_count) - bool is_main_thread = QThread::currentThread() == qApp->thread(); - qlonglong thread_id = qlonglong(QThread::currentThreadId()); - QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className()) - : qApp->database()->driver()->connection(QSL("feed_upd_%1").arg(thread_id)); + QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); int account_id = getParentServiceRoot()->accountId(); m_totalCount = m_unreadCount = DatabaseQueries::getUnreadMessageCounts(database, account_id); From ffc9c7490034b6ef97bb6b836e91a98cc5cb17eb Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Mon, 9 Jan 2023 14:44:41 +0100 Subject: [PATCH 10/15] some cleanups, remove DB transactins support as it is now useless --- resources/desktop/rssguard.metainfo.xml.in | 2 +- src/librssguard/database/databasecleaner.cpp | 2 +- src/librssguard/database/databasequeries.cpp | 24 +--- .../gui/settings/settingsdatabase.cpp | 112 ++++++++++-------- .../gui/settings/settingsdatabase.ui | 26 +--- src/librssguard/miscellaneous/settings.cpp | 3 - src/librssguard/miscellaneous/settings.h | 3 - src/librssguard/services/abstract/feed.cpp | 2 - .../services/abstract/importantnode.cpp | 2 - src/librssguard/services/abstract/label.cpp | 1 - .../services/abstract/recyclebin.cpp | 2 - .../services/abstract/serviceroot.cpp | 11 +- .../services/abstract/unreadnode.cpp | 2 - .../services/gmail/gmailnetworkfactory.cpp | 1 - .../services/reddit/redditnetworkfactory.cpp | 1 - 15 files changed, 73 insertions(+), 121 deletions(-) diff --git a/resources/desktop/rssguard.metainfo.xml.in b/resources/desktop/rssguard.metainfo.xml.in index 9f4b74f4e..f1e7fbf94 100644 --- a/resources/desktop/rssguard.metainfo.xml.in +++ b/resources/desktop/rssguard.metainfo.xml.in @@ -60,7 +60,7 @@ - + @APP_LOW_NAME@ diff --git a/src/librssguard/database/databasecleaner.cpp b/src/librssguard/database/databasecleaner.cpp index 6265a6d25..db32291ae 100644 --- a/src/librssguard/database/databasecleaner.cpp +++ b/src/librssguard/database/databasecleaner.cpp @@ -11,7 +11,7 @@ DatabaseCleaner::DatabaseCleaner(QObject* parent) : QObject(parent) {} void DatabaseCleaner::purgeDatabaseData(CleanerOrders which_data) { - qDebugNN << LOGSEC_DB << "Performing database cleanup in thread: '" << QThread::currentThreadId() << "'."; + qDebugNN << LOGSEC_DB << "Performing database cleanup in thread:" << QUOTE_W_SPACE_DOT(QThread::currentThreadId()); // Inform everyone about the start of the process. emit purgeStarted(); diff --git a/src/librssguard/database/databasequeries.cpp b/src/librssguard/database/databasequeries.cpp index 5de60aa49..17a1422bf 100644 --- a/src/librssguard/database/databasequeries.cpp +++ b/src/librssguard/database/databasequeries.cpp @@ -1086,7 +1086,6 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, return {0, 0}; } - bool use_transactions = qApp->settings()->value(GROUP(Database), SETTING(Database::UseTransactions)).toBool(); QPair updated_messages = {0, 0}; int account_id = feed->getParentServiceRoot()->accountId(); auto feed_custom_id = feed->customId(); @@ -1098,7 +1097,6 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, QSqlQuery query_select_with_id(db); QSqlQuery query_update(db); QSqlQuery query_insert(db); - QSqlQuery query_begin_transaction(db); // Here we have query which will check for existence of the "same" message in given feed. // The two message are the "same" if: @@ -1148,12 +1146,6 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, "contents = :contents, enclosures = :enclosures, feed = :feed " "WHERE id = :id;")); - if (use_transactions && !db.transaction()) { - qCriticalNN << LOGSEC_DB << "Transaction start for message downloader failed:" - << QUOTE_W_SPACE_DOT(query_begin_transaction.lastError().text()); - return updated_messages; - } - QVector msgs_to_insert; for (Message& message : messages) { @@ -1473,20 +1465,8 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, qWarningNN << LOGSEC_DB << "Failed to set custom ID for all messages:" << QUOTE_W_SPACE_DOT(db.lastError().text()); } - if (use_transactions && !db.commit()) { - qCriticalNN << LOGSEC_DB - << "Transaction commit for message downloader failed:" << QUOTE_W_SPACE_DOT(db.lastError().text()); - db.rollback(); - - if (ok != nullptr) { - *ok = false; - updated_messages = {0, 0}; - } - } - else { - if (ok != nullptr) { - *ok = true; - } + if (ok != nullptr) { + *ok = true; } return updated_messages; diff --git a/src/librssguard/gui/settings/settingsdatabase.cpp b/src/librssguard/gui/settings/settingsdatabase.cpp index 1b2cacc92..6eb310251 100644 --- a/src/librssguard/gui/settings/settingsdatabase.cpp +++ b/src/librssguard/gui/settings/settingsdatabase.cpp @@ -12,54 +12,70 @@ SettingsDatabase::SettingsDatabase(Settings* settings, QWidget* parent) : SettingsPanel(settings, parent), m_ui(new Ui::SettingsDatabase) { m_ui->setupUi(this); - m_ui->m_lblDataStorageWarning->setHelpText(tr("Note that switching to another data storage type will " - "NOT copy existing your data from currently active data " - "storage to newly selected one."), - true); - m_ui->m_lblMysqlInfo->setHelpText(tr("Note that speed of used MySQL server and latency of used connection " "medium HEAVILY influences the final performance of this application. " "Using slow database connections leads to bad performance when browsing " "feeds or messages."), false); - m_ui->m_lblSqliteInMemoryWarnings->setHelpText(tr("Usage of in-memory working database has several advantages " - "and pitfalls. Make sure that you are familiar with these " - "before you turn this feature on.\n" - "\n" - "Advantages:\n" - " • higher speed for feed/message manipulations " - "(especially with thousands of messages displayed),\n" - " • whole database stored in RAM, thus your hard drive can " - "rest more.\n" - "\n" - "Disadvantages:\n" - " • if application crashes, your changes from last session are lost,\n" - " • application startup and shutdown can take little longer " - "(max. 2 seconds).\n" - "\n" - "Authors of this application are NOT responsible for lost data."), - true); + m_ui->m_lblSqliteInMemoryWarnings + ->setHelpText(tr("Usage of in-memory working database has several advantages " + "and pitfalls. Make sure that you are familiar with these " + "before you turn this feature on.\n" + "\n" + "Advantages:\n" + " • higher speed for feed/message manipulations " + "(especially with thousands of messages displayed),\n" + " • whole database stored in RAM, thus your hard drive can " + "rest more.\n" + "\n" + "Disadvantages:\n" + " • if application crashes, your changes from last session are lost,\n" + " • application startup and shutdown can take little longer " + "(max. 2 seconds).\n" + "\n" + "Authors of this application are NOT responsible for lost data."), + true); m_ui->m_txtMysqlPassword->lineEdit()->setPasswordMode(true); - connect(m_ui->m_cmbDatabaseDriver, static_cast(&QComboBox::currentIndexChanged), this, + connect(m_ui->m_cmbDatabaseDriver, + static_cast(&QComboBox::currentIndexChanged), + this, &SettingsDatabase::dirtifySettings); connect(m_ui->m_checkSqliteUseInMemoryDatabase, &QCheckBox::toggled, this, &SettingsDatabase::dirtifySettings); connect(m_ui->m_txtMysqlDatabase->lineEdit(), &QLineEdit::textChanged, this, &SettingsDatabase::dirtifySettings); connect(m_ui->m_txtMysqlHostname->lineEdit(), &QLineEdit::textChanged, this, &SettingsDatabase::dirtifySettings); connect(m_ui->m_txtMysqlPassword->lineEdit(), &QLineEdit::textChanged, this, &SettingsDatabase::dirtifySettings); - connect(m_ui->m_checkUseTransactions, &QCheckBox::toggled, this, &SettingsDatabase::dirtifySettings); connect(m_ui->m_txtMysqlUsername->lineEdit(), &QLineEdit::textChanged, this, &SettingsDatabase::dirtifySettings); - connect(m_ui->m_spinMysqlPort, static_cast(&QSpinBox::valueChanged), this, &SettingsDatabase::dirtifySettings); - connect(m_ui->m_cmbDatabaseDriver, static_cast(&QComboBox::currentIndexChanged), this, + connect(m_ui->m_spinMysqlPort, + static_cast(&QSpinBox::valueChanged), + this, + &SettingsDatabase::dirtifySettings); + connect(m_ui->m_cmbDatabaseDriver, + static_cast(&QComboBox::currentIndexChanged), + this, &SettingsDatabase::selectSqlBackend); - connect(m_ui->m_txtMysqlUsername->lineEdit(), &BaseLineEdit::textChanged, this, &SettingsDatabase::onMysqlUsernameChanged); - connect(m_ui->m_txtMysqlHostname->lineEdit(), &BaseLineEdit::textChanged, this, &SettingsDatabase::onMysqlHostnameChanged); - connect(m_ui->m_txtMysqlPassword->lineEdit(), &BaseLineEdit::textChanged, this, &SettingsDatabase::onMysqlPasswordChanged); - connect(m_ui->m_txtMysqlDatabase->lineEdit(), &BaseLineEdit::textChanged, this, &SettingsDatabase::onMysqlDatabaseChanged); + connect(m_ui->m_txtMysqlUsername->lineEdit(), + &BaseLineEdit::textChanged, + this, + &SettingsDatabase::onMysqlUsernameChanged); + connect(m_ui->m_txtMysqlHostname->lineEdit(), + &BaseLineEdit::textChanged, + this, + &SettingsDatabase::onMysqlHostnameChanged); + connect(m_ui->m_txtMysqlPassword->lineEdit(), + &BaseLineEdit::textChanged, + this, + &SettingsDatabase::onMysqlPasswordChanged); + connect(m_ui->m_txtMysqlDatabase->lineEdit(), + &BaseLineEdit::textChanged, + this, + &SettingsDatabase::onMysqlDatabaseChanged); connect(m_ui->m_btnMysqlTestSetup, &QPushButton::clicked, this, &SettingsDatabase::mysqlTestConnection); - connect(m_ui->m_cmbDatabaseDriver, static_cast(&QComboBox::currentIndexChanged), this, + connect(m_ui->m_cmbDatabaseDriver, + static_cast(&QComboBox::currentIndexChanged), + this, &SettingsDatabase::requireRestart); connect(m_ui->m_checkSqliteUseInMemoryDatabase, &QCheckBox::toggled, this, &SettingsDatabase::requireRestart); connect(m_ui->m_spinMysqlPort, &QSpinBox::editingFinished, this, &SettingsDatabase::requireRestart); @@ -139,17 +155,14 @@ void SettingsDatabase::selectSqlBackend(int index) { m_ui->m_stackedDatabaseDriver->setCurrentIndex(1); } else { - qWarningNN << LOGSEC_GUI - << "GUI for given database driver '" - << selected_db_driver - << "' is not available."; + qWarningNN << LOGSEC_GUI << "GUI for given database driver '" << selected_db_driver << "' is not available."; } } void SettingsDatabase::loadSettings() { onBeginLoadSettings(); - m_ui->m_checkUseTransactions->setChecked(qApp->settings()->value(GROUP(Database), SETTING(Database::UseTransactions)).toBool()); - m_ui->m_lblMysqlTestResult->setStatus(WidgetWithStatus::StatusType::Information, tr("No connection test triggered so far."), + m_ui->m_lblMysqlTestResult->setStatus(WidgetWithStatus::StatusType::Information, + tr("No connection test triggered so far."), tr("You did not executed any connection test yet.")); // Load SQLite. @@ -158,7 +171,8 @@ void SettingsDatabase::loadSettings() { m_ui->m_cmbDatabaseDriver->addItem(lite_driver->humanDriverType(), lite_driver->qtDriverCode()); // Load in-memory database status. - m_ui->m_checkSqliteUseInMemoryDatabase->setChecked(settings()->value(GROUP(Database), SETTING(Database::UseInMemory)).toBool()); + m_ui->m_checkSqliteUseInMemoryDatabase + ->setChecked(settings()->value(GROUP(Database), SETTING(Database::UseInMemory)).toBool()); auto* mysq_driver = qApp->database()->driverForType(DatabaseDriver::DriverType::MySQL); @@ -176,16 +190,19 @@ void SettingsDatabase::loadSettings() { m_ui->m_txtMysqlUsername->lineEdit()->setPlaceholderText(tr("Username to login with")); m_ui->m_txtMysqlPassword->lineEdit()->setPlaceholderText(tr("Password for your username")); m_ui->m_txtMysqlDatabase->lineEdit()->setPlaceholderText(tr("Working database which you have full access to.")); - m_ui->m_txtMysqlHostname->lineEdit()->setText(settings()->value(GROUP(Database), SETTING(Database::MySQLHostname)).toString()); - m_ui->m_txtMysqlUsername->lineEdit()->setText(settings()->value(GROUP(Database), SETTING(Database::MySQLUsername)).toString()); - m_ui->m_txtMysqlPassword->lineEdit()->setText(settings()->password(GROUP(Database), - SETTING(Database::MySQLPassword)).toString()); - m_ui->m_txtMysqlDatabase->lineEdit()->setText(settings()->value(GROUP(Database), SETTING(Database::MySQLDatabase)).toString()); + m_ui->m_txtMysqlHostname->lineEdit() + ->setText(settings()->value(GROUP(Database), SETTING(Database::MySQLHostname)).toString()); + m_ui->m_txtMysqlUsername->lineEdit() + ->setText(settings()->value(GROUP(Database), SETTING(Database::MySQLUsername)).toString()); + m_ui->m_txtMysqlPassword->lineEdit() + ->setText(settings()->password(GROUP(Database), SETTING(Database::MySQLPassword)).toString()); + m_ui->m_txtMysqlDatabase->lineEdit() + ->setText(settings()->value(GROUP(Database), SETTING(Database::MySQLDatabase)).toString()); m_ui->m_spinMysqlPort->setValue(settings()->value(GROUP(Database), SETTING(Database::MySQLPort)).toInt()); } - int index_current_backend = m_ui->m_cmbDatabaseDriver->findData(settings()->value(GROUP(Database), - SETTING(Database::ActiveDriver)).toString()); + int index_current_backend = + m_ui->m_cmbDatabaseDriver->findData(settings()->value(GROUP(Database), SETTING(Database::ActiveDriver)).toString()); if (index_current_backend >= 0) { m_ui->m_cmbDatabaseDriver->setCurrentIndex(index_current_backend); @@ -201,11 +218,10 @@ void SettingsDatabase::saveSettings() { const bool original_inmemory = settings()->value(GROUP(Database), SETTING(Database::UseInMemory)).toBool(); const bool new_inmemory = m_ui->m_checkSqliteUseInMemoryDatabase->isChecked(); - qApp->settings()->setValue(GROUP(Database), Database::UseTransactions, m_ui->m_checkUseTransactions->isChecked()); - // Save data storage settings. QString original_db_driver = settings()->value(GROUP(Database), SETTING(Database::ActiveDriver)).toString(); - QString selected_db_driver = m_ui->m_cmbDatabaseDriver->itemData(m_ui->m_cmbDatabaseDriver->currentIndex()).toString(); + QString selected_db_driver = + m_ui->m_cmbDatabaseDriver->itemData(m_ui->m_cmbDatabaseDriver->currentIndex()).toString(); // Save SQLite. settings()->setValue(GROUP(Database), Database::UseInMemory, new_inmemory); diff --git a/src/librssguard/gui/settings/settingsdatabase.ui b/src/librssguard/gui/settings/settingsdatabase.ui index 88bb21cad..f63f5e3a5 100644 --- a/src/librssguard/gui/settings/settingsdatabase.ui +++ b/src/librssguard/gui/settings/settingsdatabase.ui @@ -11,20 +11,7 @@ - - - - Note that turning this option ON will make saving of new messages FASTER, but it might rarely cause some issues with messages saving. - - - Use DB transactions when storing downloaded messages - - - - - - - + Database driver @@ -34,10 +21,10 @@ - + - + 0 @@ -204,7 +191,7 @@ - + Qt::Vertical @@ -218,11 +205,6 @@ - m_lblDatabaseDriver - m_cmbDatabaseDriver - m_stackedDatabaseDriver - m_checkUseTransactions - m_lblDataStorageWarning diff --git a/src/librssguard/miscellaneous/settings.cpp b/src/librssguard/miscellaneous/settings.cpp index ea135a409..68431ac98 100644 --- a/src/librssguard/miscellaneous/settings.cpp +++ b/src/librssguard/miscellaneous/settings.cpp @@ -374,9 +374,6 @@ DVALUE(int) Proxy::PortDef = 80; // Database. DKEY Database::ID = "database"; -DKEY Database::UseTransactions = "use_transactions"; -DVALUE(bool) Database::UseTransactionsDef = false; - DKEY Database::UseInMemory = "use_in_memory_db"; DVALUE(bool) Database::UseInMemoryDef = false; diff --git a/src/librssguard/miscellaneous/settings.h b/src/librssguard/miscellaneous/settings.h index f8d25f301..beacc018d 100644 --- a/src/librssguard/miscellaneous/settings.h +++ b/src/librssguard/miscellaneous/settings.h @@ -401,9 +401,6 @@ namespace Proxy { // Database. namespace Database { KEY ID; - KEY UseTransactions; - - VALUE(bool) UseTransactionsDef; KEY UseInMemory; diff --git a/src/librssguard/services/abstract/feed.cpp b/src/librssguard/services/abstract/feed.cpp index 2ce2240b2..56b062cc2 100644 --- a/src/librssguard/services/abstract/feed.cpp +++ b/src/librssguard/services/abstract/feed.cpp @@ -17,8 +17,6 @@ #include "services/abstract/serviceroot.h" #include "services/abstract/unreadnode.h" -#include - Feed::Feed(RootItem* parent) : RootItem(parent), m_source(QString()), m_status(Status::Normal), m_statusString(QString()), m_autoUpdateType(AutoUpdateType::DefaultAutoUpdate), m_autoUpdateInterval(DEFAULT_AUTO_UPDATE_INTERVAL), diff --git a/src/librssguard/services/abstract/importantnode.cpp b/src/librssguard/services/abstract/importantnode.cpp index 9526f96d4..0c1a3d078 100644 --- a/src/librssguard/services/abstract/importantnode.cpp +++ b/src/librssguard/services/abstract/importantnode.cpp @@ -8,8 +8,6 @@ #include "services/abstract/cacheforserviceroot.h" #include "services/abstract/serviceroot.h" -#include - ImportantNode::ImportantNode(RootItem* parent_item) : RootItem(parent_item) { setKind(RootItem::Kind::Important); setId(ID_IMPORTANT); diff --git a/src/librssguard/services/abstract/label.cpp b/src/librssguard/services/abstract/label.cpp index 9fb7e1c83..cd83d52f1 100644 --- a/src/librssguard/services/abstract/label.cpp +++ b/src/librssguard/services/abstract/label.cpp @@ -12,7 +12,6 @@ #include #include -#include Label::Label(const QString& name, const QColor& color, RootItem* parent_item) : Label(parent_item) { setColor(color); diff --git a/src/librssguard/services/abstract/recyclebin.cpp b/src/librssguard/services/abstract/recyclebin.cpp index d80c4ce1a..fdb822c41 100644 --- a/src/librssguard/services/abstract/recyclebin.cpp +++ b/src/librssguard/services/abstract/recyclebin.cpp @@ -9,8 +9,6 @@ #include "services/abstract/cacheforserviceroot.h" #include "services/abstract/serviceroot.h" -#include - RecycleBin::RecycleBin(RootItem* parent_item) : RootItem(parent_item), m_totalCount(0), m_unreadCount(0) { setKind(RootItem::Kind::Bin); setId(ID_RECYCLE_BIN); diff --git a/src/librssguard/services/abstract/serviceroot.cpp b/src/librssguard/services/abstract/serviceroot.cpp index c5e6d541e..57bcf3f85 100644 --- a/src/librssguard/services/abstract/serviceroot.cpp +++ b/src/librssguard/services/abstract/serviceroot.cpp @@ -19,8 +19,6 @@ #include "services/abstract/recyclebin.h" #include "services/abstract/unreadnode.h" -#include - ServiceRoot::ServiceRoot(RootItem* parent) : RootItem(parent), m_recycleBin(new RecycleBin(this)), m_importantNode(new ImportantNode(this)), m_labelsNode(new LabelsNode(this)), m_unreadNode(new UnreadNode(this)), m_accountId(NO_PARENT_CATEGORY), @@ -953,7 +951,6 @@ QPair ServiceRoot::updateMessages(QList& messages, Feed* feed return updated_messages; } - QList items_to_update; bool ok = false; QSqlDatabase database = qApp->database()->driver()->threadSafeConnection(metaObject()->className()); @@ -967,28 +964,22 @@ QPair ServiceRoot::updateMessages(QList& messages, Feed* feed if (recycleBin() != nullptr) { recycleBin()->updateCounts(true); - items_to_update.append(recycleBin()); } if (importantNode() != nullptr) { importantNode()->updateCounts(true); - items_to_update.append(importantNode()); } if (unreadNode() != nullptr) { unreadNode()->updateCounts(true); - items_to_update.append(unreadNode()); } if (labelsNode() != nullptr) { labelsNode()->updateCounts(true); - items_to_update.append(labelsNode()); } } - // Some messages were really added to DB, reload feed in model. - items_to_update.append(feed); - // getParentServiceRoot()->itemChanged(items_to_update); + // NOTE: Do not update model items here. We update only once when all feeds are fetched. return updated_messages; } diff --git a/src/librssguard/services/abstract/unreadnode.cpp b/src/librssguard/services/abstract/unreadnode.cpp index d71be05f5..55d55785e 100644 --- a/src/librssguard/services/abstract/unreadnode.cpp +++ b/src/librssguard/services/abstract/unreadnode.cpp @@ -6,8 +6,6 @@ #include "miscellaneous/application.h" #include "miscellaneous/iconfactory.h" -#include - UnreadNode::UnreadNode(RootItem* parent_item) : RootItem(parent_item) { setKind(RootItem::Kind::Unread); setId(ID_UNREAD); diff --git a/src/librssguard/services/gmail/gmailnetworkfactory.cpp b/src/librssguard/services/gmail/gmailnetworkfactory.cpp index cc0229760..6e1a1ce61 100644 --- a/src/librssguard/services/gmail/gmailnetworkfactory.cpp +++ b/src/librssguard/services/gmail/gmailnetworkfactory.cpp @@ -23,7 +23,6 @@ #include #include #include -#include #include GmailNetworkFactory::GmailNetworkFactory(QObject* parent) diff --git a/src/librssguard/services/reddit/redditnetworkfactory.cpp b/src/librssguard/services/reddit/redditnetworkfactory.cpp index 1b3712e4d..4e37b4040 100644 --- a/src/librssguard/services/reddit/redditnetworkfactory.cpp +++ b/src/librssguard/services/reddit/redditnetworkfactory.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include RedditNetworkFactory::RedditNetworkFactory(QObject* parent) From 6d4ebac4da6511019e10de4888d2f17ac34d1058 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Tue, 10 Jan 2023 09:07:59 +0100 Subject: [PATCH 11/15] use qt 6.4.2 --- resources/desktop/rssguard.metainfo.xml.in | 2 +- resources/scripts/github-actions/build-linux-mac.sh | 2 +- resources/scripts/github-actions/build-windows.ps1 | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/resources/desktop/rssguard.metainfo.xml.in b/resources/desktop/rssguard.metainfo.xml.in index f1e7fbf94..ee4b6c709 100644 --- a/resources/desktop/rssguard.metainfo.xml.in +++ b/resources/desktop/rssguard.metainfo.xml.in @@ -60,7 +60,7 @@ - + @APP_LOW_NAME@ diff --git a/resources/scripts/github-actions/build-linux-mac.sh b/resources/scripts/github-actions/build-linux-mac.sh index 341d44613..97e9dd7eb 100755 --- a/resources/scripts/github-actions/build-linux-mac.sh +++ b/resources/scripts/github-actions/build-linux-mac.sh @@ -51,7 +51,7 @@ else USE_QT6="ON" QTPATH="$(pwd)/Qt" - QTVERSION="6.4.1" + QTVERSION="6.4.2" QTBIN="$QTPATH/$QTVERSION/$QTOS/bin" pip3 install aqtinstall diff --git a/resources/scripts/github-actions/build-windows.ps1 b/resources/scripts/github-actions/build-windows.ps1 index f202c5b8b..2c2a79446 100755 --- a/resources/scripts/github-actions/build-windows.ps1 +++ b/resources/scripts/github-actions/build-windows.ps1 @@ -20,12 +20,11 @@ $AllProtocols = [System.Net.SecurityProtocolType]'Tls11,Tls12' $ProgressPreference = 'SilentlyContinue' # Get and prepare needed dependencies. - if ($use_qt5 -eq "ON") { $qt_version = "5.15.2" } else { - $qt_version = "6.3.2" + $qt_version = "6.4.2" } $maria_version = "10.6.11" From 714d1998bac037672cb59346d1724209c795d9ca Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Tue, 10 Jan 2023 10:23:43 +0100 Subject: [PATCH 12/15] enhance metainfo XML file cmake integration, remove now redundant git hook --- CMakeLists.txt | 1 + resources/desktop/rssguard.metainfo.xml.in | 4 ++-- resources/scripts/hooks/pre-commit | 17 ----------------- 3 files changed, 3 insertions(+), 19 deletions(-) delete mode 100755 resources/scripts/hooks/pre-commit diff --git a/CMakeLists.txt b/CMakeLists.txt index 02fc06768..30c75a2a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,7 @@ cmake_minimum_required(VERSION 3.9.0) # Global variables describing the project. string(TIMESTAMP YEAR "%Y") +string(TIMESTAMP DATE "%Y-%m-%d") set(APP_NAME "RSS Guard") set(APP_EMAIL "rotter.martinos@gmail.com") diff --git a/resources/desktop/rssguard.metainfo.xml.in b/resources/desktop/rssguard.metainfo.xml.in index ee4b6c709..c4a115754 100644 --- a/resources/desktop/rssguard.metainfo.xml.in +++ b/resources/desktop/rssguard.metainfo.xml.in @@ -1,5 +1,5 @@ - + @APP_REVERSE_NAME@ CC0-1.0 @@ -60,7 +60,7 @@ - + @APP_LOW_NAME@ diff --git a/resources/scripts/hooks/pre-commit b/resources/scripts/hooks/pre-commit deleted file mode 100755 index 255964435..000000000 --- a/resources/scripts/hooks/pre-commit +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -changelog_file="resources/text/CHANGELOG" -datestring="$(date +%F)" -versionstring="$(head -n 1 "$changelog_file")" - -for appdata_file in resources/desktop/*.metainfo.xml.in; do - appdata_file_n="${appdata_file}.n" - - # Set version and date. - cat "$appdata_file" | sed -e "s@release version\=\"[A-Za-z0-9.]*\"@release version\=\"$versionstring\"@g" | sed -e "s@ date\=\"[0-9\-]*\"@ date\=\"$datestring\"@g" > "$appdata_file_n" - mv "$appdata_file_n" "$appdata_file" - - git add "$appdata_file" -done - -exit 0 \ No newline at end of file From 1f988b8a608f978717219f1448b4a873f6492713 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 11 Jan 2023 12:31:07 +0100 Subject: [PATCH 13/15] fix some DB race conditions, fix some bulk insert SQL formatting bugs along the way --- src/librssguard/core/feeddownloader.cpp | 14 +++++--- src/librssguard/core/feeddownloader.h | 1 + src/librssguard/database/databasequeries.cpp | 32 ++++++++++--------- src/librssguard/database/databasequeries.h | 1 + .../gui/dialogs/formmessagefiltersmanager.cpp | 2 +- .../services/abstract/serviceroot.cpp | 6 ++-- .../services/abstract/serviceroot.h | 3 +- .../standard/gui/formstandardimportexport.cpp | 3 +- 8 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index a78cfb3b9..e35bc52c0 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -317,15 +317,15 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, } if (!msg_original.m_isRead && msg_tweaked_by_filter->m_isRead) { - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_original.m_customId - << "' was marked as read by message scripts."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID:" << QUOTE_W_SPACE(msg_original.m_customId) + << "was marked as read by message scripts."; read_msgs << *msg_tweaked_by_filter; } if (!msg_original.m_isImportant && msg_tweaked_by_filter->m_isImportant) { - qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_original.m_customId - << "' was marked as important by message scripts."; + qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID:" << QUOTE_W_SPACE(msg_original.m_customId) + << "was marked as important by message scripts."; important_msgs << *msg_tweaked_by_filter; } @@ -333,6 +333,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, // Process changed labels. for (Label* lbl : qAsConst(msg_original.m_assignedLabels)) { if (!msg_tweaked_by_filter->m_assignedLabels.contains(lbl)) { + QMutexLocker lck(&m_mutexDb); + // Label is not there anymore, it was deassigned. lbl->deassignFromMessage(*msg_tweaked_by_filter); @@ -344,6 +346,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, for (Label* lbl : qAsConst(msg_tweaked_by_filter->m_assignedLabels)) { if (!msg_original.m_assignedLabels.contains(lbl)) { + QMutexLocker lck(&m_mutexDb); + // Label is in new message, but is not in old message, it // was newly assigned. lbl->assignToMessage(*msg_tweaked_by_filter); @@ -393,7 +397,7 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc, removeDuplicateMessages(msgs); tmr.restart(); - auto updated_messages = acc->updateMessages(msgs, feed, false); + auto updated_messages = acc->updateMessages(msgs, feed, false, &m_mutexDb); qDebugNN << LOGSEC_FEEDDOWNLOADER << "Updating messages in DB took" << NONQUOTE_W_SPACE(tmr.nsecsElapsed() / 1000) << "microseconds."; diff --git a/src/librssguard/core/feeddownloader.h b/src/librssguard/core/feeddownloader.h index ea9b7ae6d..09e29fd5e 100644 --- a/src/librssguard/core/feeddownloader.h +++ b/src/librssguard/core/feeddownloader.h @@ -78,6 +78,7 @@ class FeedDownloader : public QObject { private: bool m_isCacheSynchronizationRunning; bool m_stopCacheSynchronization; + QMutex m_mutexDb; QHash m_erroredAccounts; QList m_feeds = {}; QFutureWatcher m_watcherLookup; diff --git a/src/librssguard/database/databasequeries.cpp b/src/librssguard/database/databasequeries.cpp index 17a1422bf..b91ccdef7 100644 --- a/src/librssguard/database/databasequeries.cpp +++ b/src/librssguard/database/databasequeries.cpp @@ -1080,6 +1080,7 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, QList& messages, Feed* feed, bool force_update, + QMutex* db_mutex, bool* ok) { if (messages.isEmpty()) { *ok = true; @@ -1096,7 +1097,6 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, QSqlQuery query_select_with_custom_id_for_feed(db); QSqlQuery query_select_with_id(db); QSqlQuery query_update(db); - QSqlQuery query_insert(db); // Here we have query which will check for existence of the "same" message in given feed. // The two message are the "same" if: @@ -1130,14 +1130,6 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, .prepare(QSL("SELECT date_created, is_read, is_important, contents, feed, title, author FROM Messages " "WHERE id = :id AND account_id = :account_id;")); - // Used to insert new messages. - query_insert.setForwardOnly(true); - query_insert.prepare(QSL("INSERT INTO Messages " - "(feed, title, is_read, is_important, is_deleted, url, author, score, date_created, " - "contents, enclosures, custom_id, custom_hash, account_id) " - "VALUES (:feed, :title, :is_read, :is_important, :is_deleted, :url, :author, :score, " - ":date_created, :contents, :enclosures, :custom_id, :custom_hash, :account_id);")); - // Used to update existing messages. query_update.setForwardOnly(true); query_update.prepare(QSL("UPDATE Messages " @@ -1321,6 +1313,8 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, (!ignore_contents_changes && message.m_contents != contents_existing_message); if (cond_1 || cond_2 || cond_3 || force_update) { + QMutexLocker lck(db_mutex); + // Message exists and is changed, update it. query_update.bindValue(QSL(":title"), unnulifyString(message.m_title)); query_update.bindValue(QSL(":is_read"), int(message.m_isRead)); @@ -1349,8 +1343,8 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, updated_messages.second++; } else if (query_update.lastError().isValid()) { - qWarningNN << LOGSEC_DB - << "Failed to update message in DB:" << QUOTE_W_SPACE_DOT(query_update.lastError().text()); + qCriticalNN << LOGSEC_DB + << "Failed to update message in DB:" << QUOTE_W_SPACE_DOT(query_update.lastError().text()); } query_update.finish(); @@ -1407,7 +1401,7 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, .replace(QSL(":date_created"), QString::number(msg->m_created.toMSecsSinceEpoch())) .replace(QSL(":contents"), DatabaseFactory::escapeQuery(unnulifyString(msg->m_contents))) .replace(QSL(":enclosures"), Enclosures::encodeEnclosuresToString(msg->m_enclosures)) - .replace(QSL(":custom_id"), unnulifyString(msg->m_customId)) + .replace(QSL(":custom_id"), DatabaseFactory::escapeQuery(unnulifyString(msg->m_customId))) .replace(QSL(":custom_hash"), unnulifyString(msg->m_customHash)) .replace(QSL(":score"), QString::number(msg->m_score)) .replace(QSL(":account_id"), QString::number(account_id))); @@ -1415,6 +1409,9 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, if (!vals.isEmpty()) { QString final_bulk = bulk_insert.arg(vals.join(QSL(", "))); + + QMutexLocker lck(db_mutex); + auto bulk_query = db.exec(final_bulk); auto bulk_error = bulk_query.lastError(); @@ -1422,6 +1419,8 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, QString txt = bulk_error.text() + bulk_error.databaseText() + bulk_error.driverText(); qCriticalNN << LOGSEC_DB << "Failed bulk insert of articles:" << QUOTE_W_SPACE_DOT(txt); + + IOFactory::writeFile("aaa.sql", final_bulk.toUtf8()); } else { // OK, we bulk-inserted many messages but the thing is that they do not @@ -1446,23 +1445,26 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, for (Message& message : messages) { if (!message.m_assignedLabels.isEmpty()) { if (!message.m_customId.isEmpty() || message.m_id > 0) { + QMutexLocker lck(db_mutex); setLabelsForMessage(db, message.m_assignedLabels, message); } else { - qWarningNN << LOGSEC_DB << "Cannot set labels for message" << QUOTE_W_SPACE(message.m_title) - << "because we don't have ID or custom ID."; + qCriticalNN << LOGSEC_DB << "Cannot set labels for message" << QUOTE_W_SPACE(message.m_title) + << "because we don't have ID or custom ID."; } } } // Now, fixup custom IDS for messages which initially did not have them, // just to keep the data consistent. + QMutexLocker lck(db_mutex); + if (db.exec("UPDATE Messages " "SET custom_id = id " "WHERE custom_id IS NULL OR custom_id = '';") .lastError() .isValid()) { - qWarningNN << LOGSEC_DB << "Failed to set custom ID for all messages:" << QUOTE_W_SPACE_DOT(db.lastError().text()); + qCriticalNN << LOGSEC_DB << "Failed to set custom ID for all messages:" << QUOTE_W_SPACE_DOT(db.lastError().text()); } if (ok != nullptr) { diff --git a/src/librssguard/database/databasequeries.h b/src/librssguard/database/databasequeries.h index 37913768d..28d7d4197 100644 --- a/src/librssguard/database/databasequeries.h +++ b/src/librssguard/database/databasequeries.h @@ -140,6 +140,7 @@ class DatabaseQueries { QList& messages, Feed* feed, bool force_update, + QMutex* db_mutex, bool* ok = nullptr); static bool deleteAccount(const QSqlDatabase& db, ServiceRoot* account); static bool deleteAccountData(const QSqlDatabase& db, diff --git a/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp b/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp index 14f34a198..2f2ab3ddb 100644 --- a/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp +++ b/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp @@ -448,7 +448,7 @@ void FormMessageFiltersManager::processCheckedFeeds() { } // Update messages in DB and reload selection. - it->getParentServiceRoot()->updateMessages(msgs, it->toFeed(), true); + it->getParentServiceRoot()->updateMessages(msgs, it->toFeed(), true, nullptr); displayMessagesOfFeed(); } } diff --git a/src/librssguard/services/abstract/serviceroot.cpp b/src/librssguard/services/abstract/serviceroot.cpp index 57bcf3f85..14929d1bd 100644 --- a/src/librssguard/services/abstract/serviceroot.cpp +++ b/src/librssguard/services/abstract/serviceroot.cpp @@ -943,7 +943,7 @@ ServiceRoot::LabelOperation operator&(ServiceRoot::LabelOperation lhs, ServiceRo return static_cast(static_cast(lhs) & static_cast(rhs)); } -QPair ServiceRoot::updateMessages(QList& messages, Feed* feed, bool force_update) { +QPair ServiceRoot::updateMessages(QList& messages, Feed* feed, bool force_update, QMutex* db_mutex) { QPair updated_messages = {0, 0}; if (messages.isEmpty()) { @@ -956,9 +956,11 @@ QPair ServiceRoot::updateMessages(QList& messages, Feed* feed qDebugNN << LOGSEC_CORE << "Updating messages in DB."; - updated_messages = DatabaseQueries::updateMessages(database, messages, feed, force_update, &ok); + updated_messages = DatabaseQueries::updateMessages(database, messages, feed, force_update, db_mutex, &ok); if (updated_messages.first > 0 || updated_messages.second > 0) { + QMutexLocker lck(db_mutex); + // Something was added or updated in the DB, update numbers. feed->updateCounts(true); diff --git a/src/librssguard/services/abstract/serviceroot.h b/src/librssguard/services/abstract/serviceroot.h index 5ff7da93f..6845b606e 100644 --- a/src/librssguard/services/abstract/serviceroot.h +++ b/src/librssguard/services/abstract/serviceroot.h @@ -14,6 +14,7 @@ #include class QAction; +class QMutex; class FeedsModel; class RecycleBin; class ImportantNode; @@ -197,7 +198,7 @@ class ServiceRoot : public RootItem { void completelyRemoveAllData(); // Returns counts of updated messages . - QPair updateMessages(QList& messages, Feed* feed, bool force_update); + QPair updateMessages(QList& messages, Feed* feed, bool force_update, QMutex* db_mutex); QIcon feedIconForMessage(const QString& feed_custom_id) const; diff --git a/src/librssguard/services/standard/gui/formstandardimportexport.cpp b/src/librssguard/services/standard/gui/formstandardimportexport.cpp index cbc2018c4..4140afab4 100644 --- a/src/librssguard/services/standard/gui/formstandardimportexport.cpp +++ b/src/librssguard/services/standard/gui/formstandardimportexport.cpp @@ -248,7 +248,8 @@ void FormStandardImportExport::parseImportFile(const QString& file_name, bool fe QFile input_file(file_name); QByteArray input_data; - if (input_file.open(QIODevice::Text | QIODevice::Unbuffered | QIODevice::ReadOnly)) { + if (input_file.open(QIODevice::OpenModeFlag::Text | QIODevice::OpenModeFlag::Unbuffered | + QIODevice::OpenModeFlag::ReadOnly)) { input_data = input_file.readAll(); input_file.close(); } From ebfd27c7426444ac00b2788f6d4644a08078b350 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 11 Jan 2023 13:43:51 +0100 Subject: [PATCH 14/15] update unread counts after batched feed fetch --- src/librssguard/database/databasequeries.cpp | 2 -- src/librssguard/miscellaneous/feedreader.cpp | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/librssguard/database/databasequeries.cpp b/src/librssguard/database/databasequeries.cpp index b91ccdef7..239608cac 100644 --- a/src/librssguard/database/databasequeries.cpp +++ b/src/librssguard/database/databasequeries.cpp @@ -1419,8 +1419,6 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, QString txt = bulk_error.text() + bulk_error.databaseText() + bulk_error.driverText(); qCriticalNN << LOGSEC_DB << "Failed bulk insert of articles:" << QUOTE_W_SPACE_DOT(txt); - - IOFactory::writeFile("aaa.sql", final_bulk.toUtf8()); } else { // OK, we bulk-inserted many messages but the thing is that they do not diff --git a/src/librssguard/miscellaneous/feedreader.cpp b/src/librssguard/miscellaneous/feedreader.cpp index 3be4bbd56..1b63ed045 100644 --- a/src/librssguard/miscellaneous/feedreader.cpp +++ b/src/librssguard/miscellaneous/feedreader.cpp @@ -350,6 +350,8 @@ void FeedReader::executeNextAutoUpdate() { void FeedReader::onFeedUpdatesFinished(FeedDownloadResults updated_feeds) { m_feedsModel->reloadWholeLayout(); + m_feedsModel->notifyWithCounts(); + emit feedUpdatesFinished(updated_feeds); } From ff3d1e5e8e5eb428d8f4ff973abb9cd4c9201c3b Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 11 Jan 2023 14:47:51 +0100 Subject: [PATCH 15/15] bit more robust db sync --- src/librssguard/database/databasequeries.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/librssguard/database/databasequeries.cpp b/src/librssguard/database/databasequeries.cpp index 239608cac..727348604 100644 --- a/src/librssguard/database/databasequeries.cpp +++ b/src/librssguard/database/databasequeries.cpp @@ -1150,6 +1150,8 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, QString title_existing_message; QString author_existing_message; + QMutexLocker lck(db_mutex); + if (message.m_id > 0) { // We recognize directly existing message. // NOTE: Particularly for manual message filter execution. @@ -1313,8 +1315,6 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, (!ignore_contents_changes && message.m_contents != contents_existing_message); if (cond_1 || cond_2 || cond_3 || force_update) { - QMutexLocker lck(db_mutex); - // Message exists and is changed, update it. query_update.bindValue(QSL(":title"), unnulifyString(message.m_title)); query_update.bindValue(QSL(":is_read"), int(message.m_isRead));