add support for fetching URL content via embedded browser

src/librssguard/network-web/webengine/webenginepage.cpp

@@ -30,6 +30,24 @@ WebEngineViewer* WebEnginePage::view() const {
 #endif
 }
 
+QString WebEnginePage::pageHtml(const QString& url) {
+  QEventLoop loop;
+  QString html;
+
+  connect(this, &WebEnginePage::loadFinished, &loop, &QEventLoop::quit);
+
+  load(url);
+  loop.exec();
+
+  toHtml([&](const QString& htm) {
+    html = htm;
+    loop.exit();
+  });
+  loop.exec();
+
+  return html;
+}
+
 void WebEnginePage::hideUnwantedElements() {
   if (!qApp->web()->adBlock()->isEnabled()) {
     return;

src/librssguard/network-web/webengine/webenginepage.h

@@ -8,20 +8,25 @@
 class WebEngineViewer;
 
 class WebEnginePage : public QWebEnginePage {
-  Q_OBJECT
+    Q_OBJECT
 
   public:
     explicit WebEnginePage(QObject* parent = nullptr);
 
     WebEngineViewer* view() const;
 
+  public slots:
+    QString pageHtml(const QString& url);
+
   private slots:
     void hideUnwantedElements();
 
   protected:
     virtual bool acceptNavigationRequest(const QUrl& url, NavigationType type, bool is_main_frame);
-    virtual void javaScriptConsoleMessage(JavaScriptConsoleMessageLevel level, const QString& message,
+    virtual void javaScriptConsoleMessage(JavaScriptConsoleMessageLevel level,
-                                          int line_number, const QString& source_id);
+                                          const QString& message,
+                                          int line_number,
+                                          const QString& source_id);
 };
 
 #endif // WEBENGINEPAGE_H

src/librssguard/services/standard/gui/standardfeeddetails.cpp

@@ -41,6 +41,8 @@ StandardFeedDetails::StandardFeedDetails(QWidget* parent) : QWidget(parent) {
                                 QVariant::fromValue(StandardFeed::SourceType::Script));
   m_ui.m_cmbSourceType->addItem(StandardFeed::sourceTypeToString(StandardFeed::SourceType::LocalFile),
                                 QVariant::fromValue(StandardFeed::SourceType::LocalFile));
+  m_ui.m_cmbSourceType->addItem(StandardFeed::sourceTypeToString(StandardFeed::SourceType::EmbeddedBrowser),
+                                QVariant::fromValue(StandardFeed::SourceType::EmbeddedBrowser));
 
   // Add standard feed types.
   m_ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Atom10),
@@ -251,6 +253,7 @@ void StandardFeedDetails::onDescriptionChanged(const QString& new_description) {
 
 void StandardFeedDetails::onUrlChanged(const QString& new_url) {
   switch (sourceType()) {
+    case StandardFeed::SourceType::EmbeddedBrowser:
     case StandardFeed::SourceType::Url: {
       if (QUrl(new_url).isValid()) {
         m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Ok, tr("The URL is ok."));

src/librssguard/services/standard/standardfeed.cpp

@@ -13,6 +13,10 @@
 #include "services/standard/gui/formstandardfeeddetails.h"
 #include "services/standard/standardserviceroot.h"
 
+#if defined(NO_LITE)
+#include "network-web/webengine/webenginepage.h"
+#endif
+
 #include "services/standard/parsers/atomparser.h"
 #include "services/standard/parsers/jsonparser.h"
 #include "services/standard/parsers/rdfparser.h"
@@ -199,6 +203,9 @@ QString StandardFeed::sourceTypeToString(StandardFeed::SourceType type) {
     case StandardFeed::SourceType::LocalFile:
       return tr("Local file");
 
+    case StandardFeed::SourceType::EmbeddedBrowser:
+      return tr("Built-in web browser");
+
     default:
       return tr("Unknown");
   }
@@ -287,6 +294,15 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
       throw NetworkException(network_result.m_networkError);
     }
   }
+  else if (source_type == StandardFeed::SourceType::EmbeddedBrowser) {
+#if defined(NO_LITE)
+    WebEnginePage page;
+
+    feed_contents = page.pageHtml(source).toUtf8();
+#else
+    throw ApplicationException(tr("this source type cannot be used on 'lite' %1 build").arg(QSL(APP_NAME)));
+#endif
+  }
   else if (source_type == StandardFeed::SourceType::LocalFile) {
     feed_contents = IOFactory::readFile(source);
   }

src/librssguard/services/standard/standardfeed.h

@@ -27,7 +27,8 @@ class StandardFeed : public Feed {
     enum class SourceType {
       Url = 0,
       Script = 1,
-      LocalFile = 2
+      LocalFile = 2,
+      EmbeddedBrowser = 3
     };
 
     enum class Type {

src/librssguard/services/standard/standardserviceroot.cpp

@@ -13,6 +13,7 @@
 #include "miscellaneous/mutex.h"
 #include "miscellaneous/settings.h"
 #include "network-web/networkfactory.h"
+
 #include "services/abstract/gui/formcategorydetails.h"
 #include "services/standard/definitions.h"
 #include "services/standard/gui/formdiscoverfeeds.h"
@@ -29,6 +30,10 @@
 #include "services/standard/standardfeedsimportexportmodel.h"
 #include "services/standard/standardserviceentrypoint.h"
 
+#if defined(NO_LITE)
+#include "network-web/webengine/webenginepage.h"
+#endif
+
 #if defined(ENABLE_COMPRESSED_SITEMAP)
 #include "3rd-party/qcompressor/qcompressor.h"
 #endif
@@ -234,6 +239,26 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
       f->setLastEtag(network_result.m_headers.value(QSL("etag")));
     }
   }
+  else if (f->sourceType() == StandardFeed::SourceType::EmbeddedBrowser) {
+#if defined(NO_LITE)
+    WebEnginePage* page = new WebEnginePage();
+
+    page->moveToThread(qApp->thread());
+
+    QString html;
+    QMetaObject::invokeMethod(page,
+                              "pageHtml",
+                              Qt::ConnectionType::BlockingQueuedConnection,
+                              Q_RETURN_ARG(QString, html),
+                              Q_ARG(QString, f->source()));
+
+    feed_contents = html.toUtf8();
+
+    page->deleteLater();
+#else
+    throw ApplicationException(tr("this source type cannot be used on 'lite' %1 build").arg(QSL(APP_NAME)));
+#endif
+  }
   else if (f->sourceType() == StandardFeed::SourceType::LocalFile) {
     feed_contents = IOFactory::readFile(feed->source());
   }