starting to work on fetching full article content

resources/rssguard.qrc

@@ -26,6 +26,7 @@
 
     <file>scripts/adblock/adblock-server.js</file>
     <file>scripts/readability/readabilize-article.js</file>
+    <file>scripts/article-extractor/extract-article.mjs</file>
 
     <file>scripts/filters/blacklist.js</file>
     <file>scripts/filters/whitelist.js</file>

resources/scripts/article-extractor/extract-article.mjs

@@ -0,0 +1,11 @@
+import { extract } from '@extractus/article-extractor'
+
+const input = process.argv[2];
+
+try {
+  const article = await extract(input);
+  console.log(JSON.stringify(article));
+}
+catch (err) {
+  console.error(err);
+}

src/librssguard/CMakeLists.txt

@@ -253,6 +253,8 @@ set(SOURCES
   network-web/adblock/adblockrequestinfo.h
   network-web/apiserver.cpp
   network-web/apiserver.h
+  network-web/articleparse.cpp
+  network-web/articleparse.h
   network-web/basenetworkaccessmanager.cpp
   network-web/basenetworkaccessmanager.h
   network-web/cookiejar.cpp

src/librssguard/gui/webbrowser.cpp

@@ -12,10 +12,12 @@
 #include "miscellaneous/application.h"
 #include "miscellaneous/iconfactory.h"
 #include "miscellaneous/settings.h"
+#include "network-web/articleparse.h"
 #include "network-web/networkfactory.h"
 #include "network-web/readability.h"
 #include "network-web/webfactory.h"
 
+#include <QJsonObject>
 #include <QKeyEvent>
 #include <QProgressBar>
 #include <QScrollBar>
@@ -37,6 +39,9 @@ WebBrowser::WebBrowser(WebViewer* viewer, QWidget* parent)
 #endif
     m_actionReadabilePage(new QAction(qApp->icons()->fromTheme(QSL("text-html")),
                                       tr("View website in reader mode"),
+                                      this)),
+    m_actionGetFullArticle(new QAction(qApp->icons()->fromTheme(QSL("download"), QSL("browser-download")),
+                                       tr("Load full source article"),
                                        this)) {
   if (m_webView == nullptr) {
     m_webView = qApp->createWebView();
@@ -87,6 +92,7 @@ void WebBrowser::createConnections() {
 
   connect(m_actionOpenInSystemBrowser, &QAction::triggered, this, &WebBrowser::openCurrentSiteInSystemBrowser);
   connect(m_actionReadabilePage, &QAction::triggered, this, &WebBrowser::readabilePage);
+  connect(m_actionGetFullArticle, &QAction::triggered, this, &WebBrowser::getFullArticle);
 
 #if defined(ENABLE_MEDIAPLAYER)
   connect(m_actionPlayPageInMediaPlayer, &QAction::triggered, this, &WebBrowser::playCurrentSiteInMediaPlayer);
@@ -99,6 +105,9 @@ void WebBrowser::createConnections() {
 
   connect(qApp->web()->readability(), &Readability::htmlReadabled, this, &WebBrowser::setReadabledHtml);
   connect(qApp->web()->readability(), &Readability::errorOnHtmlReadabiliting, this, &WebBrowser::readabilityFailed);
+
+  connect(qApp->web()->articleParse(), &ArticleParse::articleParsed, this, &WebBrowser::setFullArticleHtml);
+  connect(qApp->web()->articleParse(), &ArticleParse::errorOnArticlePArsing, this, &WebBrowser::fullArticleFailed);
 }
 
 void WebBrowser::updateUrl(const QUrl& url) {
@@ -182,6 +191,11 @@ void WebBrowser::readabilePage() {
   qApp->web()->readability()->makeHtmlReadable(this, m_webView->html(), m_webView->url().toString());
 }
 
+void WebBrowser::getFullArticle() {
+  m_actionGetFullArticle->setEnabled(false);
+  qApp->web()->articleParse()->parseArticle(this, m_webView->url().toString());
+}
+
 bool WebBrowser::eventFilter(QObject* watched, QEvent* event) {
   Q_UNUSED(watched)
 
@@ -292,6 +306,21 @@ void WebBrowser::readabilityFailed(QObject* sndr, const QString& error) {
   }
 }
 
+void WebBrowser::setFullArticleHtml(QObject* sndr, const QString& json_answer) {
+  if (sndr == this && !json_answer.isEmpty()) {
+    QJsonDocument json_doc = QJsonDocument::fromJson(json_answer.toUtf8());
+    QString better_html = json_doc["content"].toString();
+
+    m_webView->setReadabledHtml(better_html, m_webView->url());
+  }
+}
+
+void WebBrowser::fullArticleFailed(QObject* sndr, const QString& error) {
+  if (sndr == this && !error.isEmpty()) {
+    m_webView->setReadabledHtml(error, m_webView->url());
+  }
+}
+
 void WebBrowser::initializeLayout() {
   m_toolBar->setFloatable(false);
   m_toolBar->setMovable(false);
@@ -324,9 +353,11 @@ void WebBrowser::initializeLayout() {
 
   m_actionOpenInSystemBrowser->setEnabled(false);
   m_actionReadabilePage->setEnabled(false);
+  m_actionGetFullArticle->setEnabled(false);
 
   // Add needed actions into toolbar.
   m_toolBar->addAction(m_actionOpenInSystemBrowser);
+  m_toolBar->addAction(m_actionGetFullArticle);
   m_toolBar->addAction(m_actionReadabilePage);
 
 #if defined(ENABLE_MEDIAPLAYER)
@@ -358,6 +389,7 @@ void WebBrowser::onLoadingStarted() {
   m_loadingProgress->show();
   m_actionOpenInSystemBrowser->setEnabled(false);
   m_actionReadabilePage->setEnabled(false);
+  m_actionGetFullArticle->setEnabled(false);
 
 #if defined(ENABLE_MEDIAPLAYER)
   m_actionPlayPageInMediaPlayer->setEnabled(false);
@@ -375,6 +407,7 @@ void WebBrowser::onLoadingFinished(bool success) {
 
     if (url.isValid() && !url.host().isEmpty()) {
       m_actionOpenInSystemBrowser->setEnabled(true);
+      m_actionGetFullArticle->setEnabled(true);
       m_actionReadabilePage->setEnabled(true);
 
 #if defined(ENABLE_MEDIAPLAYER)
@@ -384,6 +417,7 @@ void WebBrowser::onLoadingFinished(bool success) {
     else {
       m_actionOpenInSystemBrowser->setEnabled(false);
       m_actionReadabilePage->setEnabled(false);
+      m_actionGetFullArticle->setEnabled(false);
 
 #if defined(ENABLE_MEDIAPLAYER)
       m_actionPlayPageInMediaPlayer->setEnabled(false);

src/librssguard/gui/webbrowser.h

@@ -74,9 +74,14 @@ class RSSGUARD_DLLSPEC WebBrowser : public TabContent {
     void newWindowRequested(WebViewer* viewer);
 
     void readabilePage();
+    void getFullArticle();
+
     void setReadabledHtml(QObject* sndr, const QString& better_html);
     void readabilityFailed(QObject* sndr, const QString& error);
 
+    void setFullArticleHtml(QObject* sndr, const QString& json_answer);
+    void fullArticleFailed(QObject* sndr, const QString& error);
+
   signals:
     void windowCloseRequested();
     void iconChanged(int index, const QIcon& icon);
@@ -106,6 +111,7 @@ class RSSGUARD_DLLSPEC WebBrowser : public TabContent {
 #endif
 
     QAction* m_actionReadabilePage;
+    QAction* m_actionGetFullArticle;
 
     QList<Message> m_messages;
     QPointer<RootItem> m_root;

src/librssguard/network-web/articleparse.cpp

@@ -0,0 +1,140 @@
+// For license of this file, see <project-root-folder>/LICENSE.md.
+
+#include "network-web/articleparse.h"
+
+#include "3rd-party/boolinq/boolinq.h"
+#include "exceptions/applicationexception.h"
+#include "miscellaneous/application.h"
+
+#include <QDir>
+
+#define EXTRACTOR_PACKAGE "@extractus/article-extractor"
+#define EXTRACTOR_VERSION "8.0.7"
+
+ArticleParse::ArticleParse(QObject* parent) : QObject{parent}, m_modulesInstalling(false), m_modulesInstalled(false) {
+  connect(qApp->nodejs(), &NodeJs::packageInstalledUpdated, this, &ArticleParse::onPackageReady);
+  connect(qApp->nodejs(), &NodeJs::packageError, this, &ArticleParse::onPackageError);
+}
+
+void ArticleParse::onPackageReady(const QList<NodeJs::PackageMetadata>& pkgs, bool already_up_to_date) {
+  Q_UNUSED(already_up_to_date)
+
+  bool concerns_extractor = boolinq::from(pkgs).any([](const NodeJs::PackageMetadata& pkg) {
+    return pkg.m_name == QSL(EXTRACTOR_PACKAGE);
+  });
+
+  if (!concerns_extractor) {
+    return;
+  }
+
+  m_modulesInstalled = true;
+  m_modulesInstalling = false;
+
+  qApp->showGuiMessage(Notification::Event::NodePackageUpdated,
+                       {tr("Packages for article-extractor are installed"),
+                        tr("Press the button once more!"),
+                        QSystemTrayIcon::MessageIcon::Information},
+                       {true, true, false});
+
+  // Emit this just to allow readability again for user.
+  emit articleParsed(nullptr, tr("Packages for article-extractor are installed. You can now use this feature!"));
+}
+
+void ArticleParse::onPackageError(const QList<NodeJs::PackageMetadata>& pkgs, const QString& error) {
+  bool concerns_extractor = boolinq::from(pkgs).any([](const NodeJs::PackageMetadata& pkg) {
+    return pkg.m_name == QSL(EXTRACTOR_PACKAGE);
+  });
+
+  if (!concerns_extractor) {
+    return;
+  }
+
+  m_modulesInstalled = m_modulesInstalling = false;
+
+  qApp->showGuiMessage(Notification::Event::NodePackageUpdated,
+                       {tr("Packages for article-extractor are NOT installed"),
+                        tr("There is error: %1").arg(error),
+                        QSystemTrayIcon::MessageIcon::Critical},
+                       {true, true, false});
+
+  // Emit this just to allow readability again for user.
+  emit articleParsed(nullptr, tr("Packages for article-extractor are NOT installed. There is error: %1").arg(error));
+}
+
+void ArticleParse::parseArticle(QObject* sndr, const QString& url) {
+  if (!m_modulesInstalled) {
+    // NOTE: Here we use MJS file directly placed in its NODE package folder
+    // because NODE_PATH is not supported for MJS files.
+    m_scriptFilename = qApp->nodejs()->processedPackageFolder() + QDir::separator() + QSL("extract-article.mjs");
+
+    if (!IOFactory::copyFile(QSL(":/scripts/article-extractor/extract-article.mjs"), m_scriptFilename)) {
+      qCriticalNN << LOGSEC_ADBLOCK << "Failed to copy article-extractor script to TEMP.";
+    }
+
+    try {
+      NodeJs::PackageStatus st_extractor =
+        qApp->nodejs()->packageStatus({QSL(EXTRACTOR_PACKAGE), QSL(EXTRACTOR_VERSION)});
+
+      if (st_extractor != NodeJs::PackageStatus::UpToDate) {
+        if (!m_modulesInstalling) {
+          // We make sure to update modules.
+          m_modulesInstalling = true;
+
+          qApp->showGuiMessage(Notification::Event::NodePackageUpdated,
+                               {tr("Node.js libraries not installed"),
+                                tr("%1 will now install some needed libraries, this will take only a few seconds. "
+                                   "You will be notified when installation is complete.")
+                                  .arg(QSL(APP_NAME)),
+                                QSystemTrayIcon::MessageIcon::Warning},
+                               {true, true, false});
+          qApp->nodejs()->installUpdatePackages({{QSL(EXTRACTOR_PACKAGE), QSL(EXTRACTOR_VERSION)}});
+        }
+
+        return;
+      }
+      else {
+        m_modulesInstalled = true;
+      }
+    }
+    catch (const ApplicationException& ex) {
+      qApp->showGuiMessage(Notification::Event::NodePackageUpdated,
+                           {tr("Node.js libraries not installed"),
+                            tr("Node.js is not configured properly. Go to \"Settings\" -> \"Node.js\" and check "
+                               "if your Node.js is properly configured."),
+                            QSystemTrayIcon::MessageIcon::Critical},
+                           {true, true, false});
+
+      qCriticalNN << LOGSEC_CORE << "Failed to check for Node.js package status:" << QUOTE_W_SPACE_DOT(ex.message());
+
+      // Emit this just to allow readability again for user.
+      emit articleParsed(sndr,
+                         tr("Node.js is not configured properly. Go to \"Settings\" -> \"Node.js\" and check "
+                            "if your Node.js is properly configured."));
+    }
+  }
+
+  QProcess* proc = new QProcess(this);
+
+  connect(proc,
+          QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
+          this,
+          [=](int exit_code, QProcess::ExitStatus exit_status) {
+            onParsingFinished(sndr, exit_code, exit_status);
+          });
+
+  qApp->nodejs()->runScript(proc, m_scriptFilename, {url});
+}
+
+void ArticleParse::onParsingFinished(QObject* sndr, int exit_code, QProcess::ExitStatus exit_status) {
+  QProcess* proc = qobject_cast<QProcess*>(sender());
+
+  if (exit_status == QProcess::ExitStatus::NormalExit && exit_code == EXIT_SUCCESS) {
+    emit articleParsed(sndr, QString::fromUtf8(proc->readAllStandardOutput()));
+  }
+  else {
+    QString err = QString::fromUtf8(proc->readAllStandardError());
+    emit errorOnArticlePArsing(sndr, err);
+  }
+
+  proc->deleteLater();
+}

src/librssguard/network-web/articleparse.h

@@ -0,0 +1,34 @@
+// For license of this file, see <project-root-folder>/LICENSE.md.
+
+#ifndef ARTICLEPARSE_H
+#define ARTICLEPARSE_H
+
+#include "miscellaneous/nodejs.h"
+
+#include <QObject>
+#include <QProcess>
+
+class ArticleParse : public QObject {
+    Q_OBJECT
+
+  public:
+    explicit ArticleParse(QObject* parent = nullptr);
+
+    void parseArticle(QObject* sndr, const QString& url);
+
+  private slots:
+    void onParsingFinished(QObject* sndr, int exit_code, QProcess::ExitStatus exit_status);
+    void onPackageReady(const QList<NodeJs::PackageMetadata>& pkgs, bool already_up_to_date);
+    void onPackageError(const QList<NodeJs::PackageMetadata>& pkgs, const QString& error);
+
+  signals:
+    void articleParsed(QObject* sndr, const QString& better_html);
+    void errorOnArticlePArsing(QObject* sndr, const QString& error);
+
+  private:
+    bool m_modulesInstalling;
+    bool m_modulesInstalled;
+    QString m_scriptFilename;
+};
+
+#endif // ARTICLEPARSE_H

src/librssguard/network-web/webfactory.cpp

@@ -8,6 +8,7 @@
 #include "miscellaneous/settings.h"
 #include "network-web/adblock/adblockmanager.h"
 #include "network-web/apiserver.h"
+#include "network-web/articleparse.h"
 #include "network-web/cookiejar.h"
 #include "network-web/readability.h"
 
@@ -54,6 +55,7 @@ WebFactory::WebFactory(QObject* parent) : QObject(parent), m_apiServer(nullptr),
 
   m_cookieJar = new CookieJar(this);
   m_readability = new Readability(this);
+  m_articleParse = new ArticleParse(this);
 
 #if defined(NO_LITE)
 #if QT_VERSION >= 0x050D00 // Qt >= 5.13.0
@@ -568,6 +570,10 @@ Readability* WebFactory::readability() const {
   return m_readability;
 }
 
+ArticleParse* WebFactory::articleParse() const {
+  return m_articleParse;
+}
+
 void WebFactory::startApiServer() {
   m_apiServer = new ApiServer(this);
   m_apiServer->setListenAddressPort(QSL("http://localhost:54123"), true);

src/librssguard/network-web/webfactory.h

@@ -20,6 +20,7 @@ class AdBlockManager;
 class CookieJar;
 class ApiServer;
 class Readability;
+class ArticleParse;
 
 class RSSGUARD_DLLSPEC WebFactory : public QObject {
     Q_OBJECT
@@ -50,6 +51,7 @@ class RSSGUARD_DLLSPEC WebFactory : public QObject {
 
     CookieJar* cookieJar() const;
     Readability* readability() const;
+    ArticleParse* articleParse() const;
 
     void startApiServer();
     void stopApiServer();
@@ -95,6 +97,7 @@ class RSSGUARD_DLLSPEC WebFactory : public QObject {
     ApiServer* m_apiServer;
     CookieJar* m_cookieJar;
     Readability* m_readability;
+    ArticleParse* m_articleParse;
     QString m_customUserAgent;
 };