Concurrent import (#864)

* save

* save

* save

CMakeLists.txt

@@ -128,6 +128,7 @@ set(QT_COMPONENTS
   Sql
   Widgets
   Xml
+  Concurrent
 )
 
 if(NOT OS2)

resources/desktop/rssguard.metainfo.xml.in

@@ -60,7 +60,7 @@
   <content_rating type="oars-1.0" />
   <content_rating type="oars-1.1" />
   <releases>
-    <release version="4.2.7" date="2022-12-21" />
+    <release version="4.2.7" date="2023-01-03" />
   </releases>
   <provides>
     <binary>@APP_LOW_NAME@</binary>

src/librssguard/CMakeLists.txt

@@ -625,6 +625,7 @@ target_link_libraries(rssguard PUBLIC
   Qt${QT_VERSION_MAJOR}::Sql
   Qt${QT_VERSION_MAJOR}::Widgets
   Qt${QT_VERSION_MAJOR}::Xml
+  Qt${QT_VERSION_MAJOR}::Concurrent
 )
 
 if(QT_VERSION_MAJOR EQUAL 6)

src/librssguard/miscellaneous/application.cpp

@@ -216,6 +216,8 @@ Application::Application(const QString& id, int& argc, char** argv, const QStrin
 
   qDebugNN << LOGSEC_CORE << "OpenSSL version:" << QUOTE_W_SPACE_DOT(QSslSocket::sslLibraryVersionString());
   qDebugNN << LOGSEC_CORE << "OpenSSL supported:" << QUOTE_W_SPACE_DOT(QSslSocket::supportsSsl());
+  qDebugNN << LOGSEC_CORE << "Global thread pool has"
+           << NONQUOTE_W_SPACE(QThreadPool::globalInstance()->maxThreadCount()) << "threads.";
 }
 
 Application::~Application() {

src/librssguard/network-web/basenetworkaccessmanager.cpp

@@ -17,7 +17,6 @@ BaseNetworkAccessManager::BaseNetworkAccessManager(QObject* parent)
 }
 
 void BaseNetworkAccessManager::loadSettings() {
-  QNetworkProxy new_proxy;
   const QNetworkProxy::ProxyType selected_proxy_type =
     static_cast<QNetworkProxy::ProxyType>(qApp->settings()->value(GROUP(Proxy), SETTING(Proxy::Type)).toInt());
 
@@ -55,10 +54,10 @@ QNetworkReply* BaseNetworkAccessManager::createRequest(QNetworkAccessManager::Op
 
 #if defined(Q_OS_WIN)
   new_request.setAttribute(QNetworkRequest::Attribute::HttpPipeliningAllowedAttribute, true);
+#endif
 
 #if QT_VERSION >= 0x050F00 // Qt >= 5.15.0
   new_request.setAttribute(QNetworkRequest::Attribute::Http2AllowedAttribute, m_enableHttp2);
-#endif
 #endif
 
   new_request.setRawHeader(HTTP_HEADERS_COOKIE, QSL("JSESSIONID= ").toLocal8Bit());

src/librssguard/services/standard/gui/formstandardimportexport.cpp

@@ -125,15 +125,19 @@ void FormStandardImportExport::onParsingStarted() {
   m_ui->m_buttonBox->button(QDialogButtonBox::StandardButton::Ok)->setEnabled(false);
 }
 
-void FormStandardImportExport::onParsingFinished(int count_failed, int count_succeeded, bool parsing_error) {
+void FormStandardImportExport::onParsingFinished(int count_failed, int count_succeeded) {
-  Q_UNUSED(count_failed)
-  Q_UNUSED(count_succeeded)
-
   m_ui->m_progressBar->setVisible(false);
   m_ui->m_progressBar->setValue(0);
   m_model->checkAllItems();
 
-  if (!parsing_error) {
+  if (count_failed > 0 && count_succeeded == 0) {
+    m_ui->m_groupFeeds->setEnabled(false);
+    m_ui->m_groupFetchMetadata->setEnabled(false);
+    m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Error,
+                                 tr("Some feeds were not loaded properly or import file is corrupted."),
+                                 tr("Some feeds were not loaded properly or import file is corrupted."));
+  }
+  else {
     m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Ok, tr("Feeds were loaded."), tr("Feeds were loaded."));
     m_ui->m_groupFeeds->setEnabled(true);
     m_ui->m_groupFetchMetadata->setEnabled(true);
@@ -141,13 +145,6 @@ void FormStandardImportExport::onParsingFinished(int count_failed, int count_suc
     m_ui->m_treeFeeds->setModel(m_model);
     m_ui->m_treeFeeds->expandAll();
   }
-  else {
-    m_ui->m_groupFeeds->setEnabled(false);
-    m_ui->m_groupFetchMetadata->setEnabled(false);
-    m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Error,
-                                 tr("Error, file is not well-formed. Select another file."),
-                                 tr("Error occurred. File is not well-formed. Select another file."));
-  }
 
   m_ui->m_buttonBox->button(QDialogButtonBox::StandardButton::Ok)->setEnabled(true);
 }

src/librssguard/services/standard/gui/formstandardimportexport.h

@@ -31,7 +31,7 @@ class FormStandardImportExport : public QDialog {
     void selectFile();
 
     void onParsingStarted();
-    void onParsingFinished(int count_failed, int count_succeeded, bool parsing_error);
+    void onParsingFinished(int count_failed, int count_succeeded);
     void onParsingProgress(int completed, int total);
 
     void onPostProcessScriptChanged(const QString& new_pp);

src/librssguard/services/standard/standardfeedsimportexportmodel.cpp

@@ -2,6 +2,7 @@
 
 #include "services/standard/standardfeedsimportexportmodel.h"
 
+#include "3rd-party/boolinq/boolinq.h"
 #include "definitions/definitions.h"
 #include "exceptions/applicationexception.h"
 #include "miscellaneous/application.h"
@@ -16,11 +17,35 @@
 #include <QDomElement>
 #include <QLocale>
 #include <QStack>
+#include <QtConcurrent/QtConcurrentMap>
 
 FeedsImportExportModel::FeedsImportExportModel(QObject* parent)
-  : AccountCheckSortedModel(parent), m_mode(Mode::Import) {}
+  : AccountCheckSortedModel(parent), m_mode(Mode::Import) {
+
+  connect(&m_watcherLookup, &QFutureWatcher<bool>::progressValueChanged, this, [=](int prog) {
+    emit parsingProgress(prog, m_lookup.size());
+  });
+
+  connect(&m_watcherLookup, &QFutureWatcher<bool>::finished, this, [=]() {
+    auto res = m_watcherLookup.future().results();
+    int number_error = boolinq::from(res).count([](bool rs) {
+      return !rs;
+    });
+
+    emit layoutChanged();
+    emit parsingFinished(number_error, m_lookup.size() - number_error);
+
+    // Done, remove lookups.
+    m_lookup.clear();
+  });
+}
 
 FeedsImportExportModel::~FeedsImportExportModel() {
+  if (m_watcherLookup.isRunning()) {
+    m_watcherLookup.cancel();
+    m_watcherLookup.waitForFinished();
+  }
+
   if (sourceModel() != nullptr && sourceModel()->rootItem() != nullptr && m_mode == Mode::Import) {
     // Delete all model items, but only if we are in import mode. Export mode shares
     // root item with main feed model, thus cannot be deleted from memory now.
@@ -145,27 +170,107 @@ bool FeedsImportExportModel::exportToOMPL20(QByteArray& result, bool export_icon
   return true;
 }
 
+bool FeedsImportExportModel::produceFeed(const FeedLookup& feed_lookup) {
+  StandardFeed* new_feed = nullptr;
+
+  try {
+    if (feed_lookup.fetch_metadata_online) {
+      new_feed = StandardFeed::guessFeed(StandardFeed::SourceType::Url,
+                                         feed_lookup.url,
+                                         feed_lookup.post_process_script,
+                                         {},
+                                         {},
+                                         feed_lookup.custom_proxy);
+
+      new_feed->setSource(feed_lookup.url);
+      new_feed->setPostProcessScript(feed_lookup.post_process_script);
+    }
+    else {
+      new_feed = new StandardFeed(feed_lookup.parent);
+
+      if (feed_lookup.opml_element.isNull()) {
+        new_feed->setSource(feed_lookup.url);
+        new_feed->setTitle(feed_lookup.url);
+        new_feed->setIcon(qApp->icons()->fromTheme(QSL("application-rss+xml")));
+        new_feed->setEncoding(QSL(DEFAULT_FEED_ENCODING));
+        new_feed->setPostProcessScript(feed_lookup.post_process_script);
+      }
+      else {
+        QString feed_title = feed_lookup.opml_element.attribute(QSL("text"));
+        QString feed_encoding = feed_lookup.opml_element.attribute(QSL("encoding"), QSL(DEFAULT_FEED_ENCODING));
+        QString feed_type = feed_lookup.opml_element.attribute(QSL("version"), QSL(DEFAULT_FEED_TYPE)).toUpper();
+        QString feed_description = feed_lookup.opml_element.attribute(QSL("description"));
+        QIcon feed_icon =
+          qApp->icons()->fromByteArray(feed_lookup.opml_element.attribute(QSL("rssguard:icon")).toLocal8Bit());
+        StandardFeed::SourceType source_type =
+          StandardFeed::SourceType(feed_lookup.opml_element.attribute(QSL("rssguard:xmlUrlType")).toInt());
+        QString post_process = feed_lookup.opml_element.attribute(QSL("rssguard:postProcess"));
+
+        new_feed->setTitle(feed_title);
+        new_feed->setDescription(feed_description);
+        new_feed->setEncoding(feed_encoding);
+        new_feed->setSource(feed_lookup.url);
+        new_feed->setSourceType(source_type);
+        new_feed->setPostProcessScript(feed_lookup.post_process_script.isEmpty() ? post_process
+                                                                                 : feed_lookup.post_process_script);
+
+        if (!feed_icon.isNull()) {
+          new_feed->setIcon(feed_icon);
+        }
+
+        if (feed_type == QL1S("RSS1")) {
+          new_feed->setType(StandardFeed::Type::Rdf);
+        }
+        else if (feed_type == QL1S("JSON")) {
+          new_feed->setType(StandardFeed::Type::Json);
+        }
+        else if (feed_type == QL1S("ATOM")) {
+          new_feed->setType(StandardFeed::Type::Atom10);
+        }
+        else {
+          new_feed->setType(StandardFeed::Type::Rss2X);
+        }
+      }
+    }
+
+    QMutexLocker mtx(&m_mtxLookup);
+    feed_lookup.parent->appendChild(new_feed);
+
+    return true;
+  }
+  catch (const ApplicationException& ex) {
+    qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(feed_lookup.url)
+                << "with error:" << QUOTE_W_SPACE_DOT(ex.message());
+
+    if (new_feed != nullptr) {
+      new_feed->deleteLater();
+    }
+
+    return false;
+  }
+}
+
 void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
                                             bool fetch_metadata_online,
                                             const QString& post_process_script) {
   emit parsingStarted();
   emit layoutAboutToBeChanged();
-
   setRootItem(nullptr);
   emit layoutChanged();
+
   QDomDocument opml_document;
 
   if (!opml_document.setContent(data)) {
-    emit parsingFinished(0, 0, true);
+    emit parsingFinished(0, 0);
   }
 
   if (opml_document.documentElement().isNull() || opml_document.documentElement().tagName() != QSL("opml") ||
       opml_document.documentElement().elementsByTagName(QSL("body")).size() != 1) {
     // This really is not an OPML file.
-    emit parsingFinished(0, 0, true);
+    emit parsingFinished(0, 0);
   }
 
-  int completed = 0, total = 0, succeded = 0, failed = 0;
+  int completed = 0, total = 0;
   auto* root_item = new StandardServiceRoot();
   QStack<RootItem*> model_items;
   QNetworkProxy custom_proxy;
@@ -180,6 +285,8 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
   elements_to_process.push(opml_document.documentElement().elementsByTagName(QSL("body")).at(0).toElement());
   total = opml_document.elementsByTagName(QSL("outline")).size();
 
+  QList<FeedLookup> lookup;
+
   while (!elements_to_process.isEmpty()) {
     RootItem* active_model_item = model_items.pop();
     QDomElement active_element = elements_to_process.pop();
@@ -197,76 +304,18 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
           // This is FEED.
           // Add feed and end this iteration.
           QString feed_url = child_element.attribute(QSL("xmlUrl"));
-          bool add_offline_anyway = true;
 
           if (!feed_url.isEmpty()) {
-            try {
+            FeedLookup f;
-              if (fetch_metadata_online) {
-                StandardFeed* guessed = StandardFeed::guessFeed(StandardFeed::SourceType::Url,
-                                                                feed_url,
-                                                                post_process_script,
-                                                                {},
-                                                                {},
-                                                                custom_proxy);
 
-                guessed->setSource(feed_url);
+            f.custom_proxy = custom_proxy;
-                guessed->setPostProcessScript(post_process_script);
+            f.fetch_metadata_online = fetch_metadata_online;
+            f.opml_element = child_element;
+            f.parent = active_model_item;
+            f.post_process_script = post_process_script;
+            f.url = feed_url;
 
-                active_model_item->appendChild(guessed);
+            lookup.append(f);
-                succeded++;
-                add_offline_anyway = false;
-              }
-            }
-            catch (const ApplicationException& ex) {
-              qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(feed_url)
-                          << "with error:" << QUOTE_W_SPACE_DOT(ex.message());
-            }
-
-            if (add_offline_anyway) {
-              QString feed_title = child_element.attribute(QSL("text"));
-              QString feed_encoding = child_element.attribute(QSL("encoding"), QSL(DEFAULT_FEED_ENCODING));
-              QString feed_type = child_element.attribute(QSL("version"), QSL(DEFAULT_FEED_TYPE)).toUpper();
-              QString feed_description = child_element.attribute(QSL("description"));
-              QIcon feed_icon =
-                qApp->icons()->fromByteArray(child_element.attribute(QSL("rssguard:icon")).toLocal8Bit());
-              StandardFeed::SourceType source_type =
-                StandardFeed::SourceType(child_element.attribute(QSL("rssguard:xmlUrlType")).toInt());
-              QString post_process = child_element.attribute(QSL("rssguard:postProcess"));
-              auto* new_feed = new StandardFeed(active_model_item);
-
-              new_feed->setTitle(feed_title);
-              new_feed->setDescription(feed_description);
-              new_feed->setEncoding(feed_encoding);
-              new_feed->setSource(feed_url);
-              new_feed->setSourceType(source_type);
-              new_feed->setPostProcessScript(post_process);
-
-              if (!feed_icon.isNull()) {
-                new_feed->setIcon(feed_icon);
-              }
-
-              if (feed_type == QL1S("RSS1")) {
-                new_feed->setType(StandardFeed::Type::Rdf);
-              }
-              else if (feed_type == QL1S("JSON")) {
-                new_feed->setType(StandardFeed::Type::Json);
-              }
-              else if (feed_type == QL1S("ATOM")) {
-                new_feed->setType(StandardFeed::Type::Atom10);
-              }
-              else {
-                new_feed->setType(StandardFeed::Type::Rss2X);
-              }
-
-              active_model_item->appendChild(new_feed);
-
-              if (fetch_metadata_online) {
-                failed++;
-              }
-              else {
-                succeded++;
-              }
-            }
           }
         }
         else {
@@ -313,8 +362,18 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
 
   setRootItem(root_item);
 
-  emit layoutChanged();
+  m_lookup.clear();
-  emit parsingFinished(failed, succeded, false);
+  m_lookup.append(lookup);
+
+  QFuture<bool> fut = QtConcurrent::mapped(m_lookup, [=](const FeedLookup& lookup) {
+    return produceFeed(lookup);
+  });
+
+  m_watcherLookup.setFuture(fut);
+
+  if (!fetch_metadata_online) {
+    m_watcherLookup.waitForFinished();
+  }
 }
 
 bool FeedsImportExportModel::exportToTxtURLPerLine(QByteArray& result) {
@@ -335,7 +394,7 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data,
   setRootItem(nullptr);
   emit layoutChanged();
 
-  int completed = 0, succeded = 0, failed = 0;
+  int completed = 0;
   auto* root_item = new StandardServiceRoot();
   QNetworkProxy custom_proxy;
 
@@ -344,51 +403,22 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data,
   }
 
   QList<QByteArray> urls = data.split('\n');
+  QList<FeedLookup> lookup;
 
   for (const QByteArray& url : urls) {
     if (!url.isEmpty()) {
-      bool add_offline_anyway = true;
+      FeedLookup f;
 
-      try {
+      f.custom_proxy = custom_proxy;
-        if (fetch_metadata_online) {
+      f.fetch_metadata_online = fetch_metadata_online;
-          StandardFeed* guessed =
+      f.parent = root_item;
-            StandardFeed::guessFeed(StandardFeed::SourceType::Url, url, post_process_script, {}, {}, custom_proxy);
+      f.post_process_script = post_process_script;
+      f.url = url;
 
-          guessed->setSource(url);
+      lookup.append(f);
-          guessed->setPostProcessScript(post_process_script);
-
-          root_item->appendChild(guessed);
-          succeded++;
-          add_offline_anyway = false;
-        }
-      }
-      catch (const ApplicationException& ex) {
-        qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(url)
-                    << "with error:" << QUOTE_W_SPACE_DOT(ex.message());
-      }
-
-      if (add_offline_anyway) {
-        auto* feed = new StandardFeed();
-
-        feed->setSource(url);
-        feed->setTitle(url);
-        feed->setIcon(qApp->icons()->fromTheme(QSL("application-rss+xml")));
-        feed->setEncoding(QSL(DEFAULT_FEED_ENCODING));
-        root_item->appendChild(feed);
-
-        if (fetch_metadata_online) {
-          failed++;
-        }
-        else {
-          succeded++;
-        }
-      }
-
-      qApp->processEvents();
     }
     else {
       qWarningNN << LOGSEC_CORE << "Detected empty URL when parsing input TXT [one URL per line] data.";
-      failed++;
     }
 
     emit parsingProgress(++completed, urls.size());
@@ -398,8 +428,19 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data,
   emit layoutAboutToBeChanged();
 
   setRootItem(root_item);
-  emit layoutChanged();
+
-  emit parsingFinished(failed, succeded, false);
+  m_lookup.clear();
+  m_lookup.append(lookup);
+
+  QFuture<bool> fut = QtConcurrent::mapped(m_lookup, [=](const FeedLookup& lookup) {
+    return produceFeed(lookup);
+  });
+
+  m_watcherLookup.setFuture(fut);
+
+  if (!fetch_metadata_online) {
+    m_watcherLookup.waitForFinished();
+  }
 }
 
 FeedsImportExportModel::Mode FeedsImportExportModel::mode() const {

src/librssguard/services/standard/standardfeedsimportexportmodel.h

@@ -5,6 +5,21 @@
 
 #include "services/abstract/accountcheckmodel.h"
 
+#include <QDomElement>
+#include <QFutureWatcher>
+#include <QNetworkProxy>
+
+class StandardFeed;
+
+struct FeedLookup {
+    RootItem* parent;
+    QDomElement opml_element;
+    QString url;
+    bool fetch_metadata_online;
+    QNetworkProxy custom_proxy;
+    QString post_process_script;
+};
+
 class FeedsImportExportModel : public AccountCheckSortedModel {
     Q_OBJECT
 
@@ -32,9 +47,15 @@ class FeedsImportExportModel : public AccountCheckSortedModel {
   signals:
     void parsingStarted();
     void parsingProgress(int completed, int total);
-    void parsingFinished(int count_failed, int count_succeeded, bool parsing_error);
+    void parsingFinished(int count_failed, int count_succeeded);
 
   private:
+    bool produceFeed(const FeedLookup& feed_lookup);
+
+  private:
+    QMutex m_mtxLookup;
+    QList<FeedLookup> m_lookup;
+    QFutureWatcher<bool> m_watcherLookup;
     Mode m_mode;
 };