Working on #265.

resources/docs/Feed-formats.md

@@ -4,6 +4,7 @@ RSS Guard is a modular application which supports plugins. It offers well-mainta
 * [Tiny Tiny RSS](https://tt-rss.org) plugin: Adds ability to synchronize messages with TT-RSS instances, either self-hosted or via 3rd-party external service.
 * [Inoreader](https://www.inoreader.com) plugin: Adds ability to synchronize messages with Inoreader. All you need to do is create free account on their website and start rocking.
 * [Nextcloud News](https://apps.nextcloud.com/apps/news) plugin: Nextcloud News is a Nextcloud app which adds feed reader abilities into your Nextcloud instances.
+* Google Reader API plugin: This plugin was added in RSS Guard 3.9.0 and offers two-way synchronization with services which implement Google Reader API. At this point, plugin was tested and works with Bazqux, The Old Reader and FreshRSS.
 * [Gmail](https://www.google.com/gmail) plugin: Yes, you are reading it right. RSS Guard can be used as very lightweight and simple e-mail client. This plugins uses [Gmail API](https://developers.google.com/gmail/api) and offers even e-mail sending.
 
 All plugins share almost all core RSS Guard's features, including labels, recycle bins, podcasts fetching or newspaper view. They are implemented in a very transparent way, making it easy to maintain them or add new ones.
@@ -27,4 +28,33 @@ OPML files can be exported/imported in simple dialog.
 <img src="images/im-ex-feeds.png" width="80%">
 <img src="images/im-ex-feeds-dialog.png" width="50%">
 
-You just select output file (in case of OPML export), check desired feeds and hit `Export to file`.
+You just select output file (in case of OPML export), check desired feeds and hit `Export to file`.
+
+### Websites scraping and other related advanced features
+RSS Guard 3.9.0+ offers extra advanced features which were inspired by [Liferea](https://lzone.de/liferea/).
+
+**Only proceed if you consider yourself as power user and you know you are doing!**
+
+You can select source type of each feed. If you select `URL`, then RSS Guard simply downloads feed file from given location.
+
+However, if you choose `Script` option, then you cannot provide URL of your feed and you rely on custom script to obtain your script and provide its contents to **standard output**. Resulting data written to standard output **MUST** be valid feed file, for example RSS or ATOM XML file.
+
+<img src="images/scrape-source-type.png" width="50%">
+
+Any errors in your script must be written to **error output**.
+
+Note that you must provide full execution line to your custom script, including interpreter binary path and name. Some examples of valid execution lines are:
+
+| Command | Explanation |
+|---------|-------------|
+| `bash -c "curl 'https://github.com/martinrotter.atom'"` | Downloads ATOM feed file with Bash and Curl. |
+| `Powershell "Invoke-WebRequest 'https://github.com/martinrotter.atom' | Select-Object -ExpandProperty Content"` | Downloads ATOM feed file with Powershell. |
+| `php tweeper.php https://twitter.com/NSACareers` | Downloads RSS feed file with [Tweeper](https://git.ao2.it/tweeper.git/). Tweeper is utility which is able to produce RSS feed from Twitter. |
+
+<img src="images/scrape-source.png" width="50%">
+
+Note that the above examples are cross-platform and you can use the exact same command on Windows, Linux or Mac OS X, if your operating system is properly configured.
+
+RSS Guard offers placeholder `%data%` which is automatically replaced with full path to RSS Guard's [user data folder](Documentation.md#portable-user-data). You can, therefore, use something like this as source script line: `bash %data%/scripts/download-feed.sh`.
+
+Also, working directory of process executing the script is set to RSS Guard's user data folder.

src/librssguard/definitions/definitions.h

@@ -39,6 +39,7 @@
 #define MSG_FILTERING_HELP                    "https://github.com/martinrotter/rssguard/blob/master/resources/docs/Message-filters.md#message-filtering"
 #define DEFAULT_FEED_TYPE                     "RSS"
 #define URL_REGEXP "^(http|https|feed|ftp):\\/\\/[\\w\\-_]+(\\.[\\w\\-_]+)+([\\w\\-\\.,@?^=%&:/~\\+#]*[\\w\\-\\@?^=%&/~\\+#])?$"
+#define SCRIPT_SOURCE_TYPE_REGEXP             "^.+#.*$"
 #define TEXT_TITLE_LIMIT                      30
 #define RESELECT_MESSAGE_THRESSHOLD           500
 #define ICON_SIZE_SETTINGS                    16

src/librssguard/miscellaneous/databasequeries.cpp

@@ -2089,15 +2089,16 @@ int DatabaseQueries::addStandardFeed(const QSqlDatabase& db, int parent_id, int
                                      const QString& description, const QDateTime& creation_date, const QIcon& icon,
                                      const QString& encoding, const QString& url, bool is_protected,
                                      const QString& username, const QString& password,
-                                     Feed::AutoUpdateType auto_update_type,
+                                     Feed::AutoUpdateType auto_update_type, int auto_update_interval,
-                                     int auto_update_interval, StandardFeed::Type feed_format, bool* ok) {
+                                     StandardFeed::SourceType source_type, const QString& post_process_script,
+                                     StandardFeed::Type feed_format, bool* ok) {
   QSqlQuery q(db);
 
   qDebug() << "Adding feed with title '" << title.toUtf8() << "' to DB.";
   q.setForwardOnly(true);
   q.prepare("INSERT INTO Feeds "
-            "(title, description, date_created, icon, category, encoding, url, protected, username, password, update_type, update_interval, type, account_id) "
+            "(title, description, date_created, icon, category, encoding, url, source_type, post_process, protected, username, password, update_type, update_interval, type, account_id) "
-            "VALUES (:title, :description, :date_created, :icon, :category, :encoding, :url, :protected, :username, :password, :update_type, :update_interval, :type, :account_id);");
+            "VALUES (:title, :description, :date_created, :icon, :category, :encoding, :url, :source_type, :post_process, :protected, :username, :password, :update_type, :update_interval, :type, :account_id);");
   q.bindValue(QSL(":title"), title.toUtf8());
   q.bindValue(QSL(":description"), description.toUtf8());
   q.bindValue(QSL(":date_created"), creation_date.toMSecsSinceEpoch());
@@ -2105,6 +2106,8 @@ int DatabaseQueries::addStandardFeed(const QSqlDatabase& db, int parent_id, int
   q.bindValue(QSL(":category"), parent_id);
   q.bindValue(QSL(":encoding"), encoding);
   q.bindValue(QSL(":url"), url);
+  q.bindValue(QSL(":source_type"), int(source_type));
+  q.bindValue(QSL(":post_process"), post_process_script);
   q.bindValue(QSL(":protected"), is_protected ? 1 : 0);
   q.bindValue(QSL(":username"), username);
   q.bindValue(QSL(":account_id"), account_id);
@@ -2153,12 +2156,13 @@ bool DatabaseQueries::editStandardFeed(const QSqlDatabase& db, int parent_id, in
                                        const QString& encoding, const QString& url, bool is_protected,
                                        const QString& username, const QString& password,
                                        Feed::AutoUpdateType auto_update_type,
-                                       int auto_update_interval, StandardFeed::Type feed_format) {
+                                       int auto_update_interval, StandardFeed::SourceType source_type,
+                                       const QString& post_process_script, StandardFeed::Type feed_format) {
   QSqlQuery q(db);
 
   q.setForwardOnly(true);
   q.prepare("UPDATE Feeds "
-            "SET title = :title, description = :description, icon = :icon, category = :category, encoding = :encoding, url = :url, protected = :protected, username = :username, password = :password, update_type = :update_type, update_interval = :update_interval, type = :type "
+            "SET title = :title, description = :description, icon = :icon, category = :category, encoding = :encoding, url = :url, source_type = :source_type, post_process = :post_process, protected = :protected, username = :username, password = :password, update_type = :update_type, update_interval = :update_interval, type = :type "
             "WHERE id = :id;");
   q.bindValue(QSL(":title"), title);
   q.bindValue(QSL(":description"), description);
@@ -2166,6 +2170,8 @@ bool DatabaseQueries::editStandardFeed(const QSqlDatabase& db, int parent_id, in
   q.bindValue(QSL(":category"), parent_id);
   q.bindValue(QSL(":encoding"), encoding);
   q.bindValue(QSL(":url"), url);
+  q.bindValue(QSL(":source_type"), int(source_type));
+  q.bindValue(QSL(":post_process"), post_process_script);
   q.bindValue(QSL(":protected"), is_protected ? 1 : 0);
   q.bindValue(QSL(":username"), username);
 

src/librssguard/miscellaneous/databasequeries.h

@@ -136,13 +136,15 @@ class DatabaseQueries {
                                const QString& description, const QDateTime& creation_date, const QIcon& icon,
                                const QString& encoding, const QString& url, bool is_protected,
                                const QString& username, const QString& password,
-                               Feed::AutoUpdateType auto_update_type,
+                               Feed::AutoUpdateType auto_update_type, int auto_update_interval,
-                               int auto_update_interval, StandardFeed::Type feed_format, bool* ok = nullptr);
+                               StandardFeed::SourceType source_type, const QString& post_process_script,
+                               StandardFeed::Type feed_format, bool* ok = nullptr);
     static bool editStandardFeed(const QSqlDatabase& db, int parent_id, int feed_id, const QString& title,
                                  const QString& description, const QIcon& icon,
                                  const QString& encoding, const QString& url, bool is_protected,
                                  const QString& username, const QString& password, Feed::AutoUpdateType auto_update_type,
-                                 int auto_update_interval, StandardFeed::Type feed_format);
+                                 int auto_update_interval, StandardFeed::SourceType source_type,
+                                 const QString& post_process_script, StandardFeed::Type feed_format);
     static QList<ServiceRoot*> getStandardAccounts(const QSqlDatabase& db, bool* ok = nullptr);
 
     template<typename T>

src/librssguard/services/abstract/gui/formaccountdetails.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>400</width>
+    <width>500</width>
-    <height>300</height>
+    <height>450</height>
    </rect>
   </property>
   <layout class="QFormLayout" name="formLayout_2">

src/librssguard/services/abstract/gui/formfeeddetails.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>471</width>
+    <width>500</width>
-    <height>352</height>
+    <height>450</height>
    </rect>
   </property>
   <property name="windowTitle">

src/librssguard/services/standard/gui/formstandardfeeddetails.cpp

@@ -22,7 +22,7 @@ FormStandardFeedDetails::FormStandardFeedDetails(ServiceRoot* service_root, QWid
   insertCustomTab(m_authDetails, tr("Network"), 2);
   activateTab(0);
 
-  connect(m_standardFeedDetails->ui.m_btnFetchMetadata, &QPushButton::clicked, this, &FormStandardFeedDetails::guessFeed);
+  connect(m_standardFeedDetails->m_ui.m_btnFetchMetadata, &QPushButton::clicked, this, &FormStandardFeedDetails::guessFeed);
   connect(m_standardFeedDetails->m_actionFetchIcon, &QAction::triggered, this, &FormStandardFeedDetails::guessIconOnly);
 }
 
@@ -47,34 +47,36 @@ int FormStandardFeedDetails::addEditFeed(StandardFeed* input_feed, RootItem* par
 }
 
 void FormStandardFeedDetails::guessFeed() {
-  m_standardFeedDetails->guessFeed(m_standardFeedDetails->ui.m_txtUrl->lineEdit()->text(),
+  m_standardFeedDetails->guessFeed(m_standardFeedDetails->m_ui.m_txtSource->lineEdit()->text(),
                                    m_authDetails->m_txtUsername->lineEdit()->text(),
                                    m_authDetails->m_txtPassword->lineEdit()->text());
 }
 
 void FormStandardFeedDetails::guessIconOnly() {
-  m_standardFeedDetails->guessIconOnly(m_standardFeedDetails->ui.m_txtUrl->lineEdit()->text(),
+  m_standardFeedDetails->guessIconOnly(m_standardFeedDetails->m_ui.m_txtSource->lineEdit()->text(),
                                        m_authDetails->m_txtUsername->lineEdit()->text(),
                                        m_authDetails->m_txtPassword->lineEdit()->text());
 }
 
 void FormStandardFeedDetails::apply() {
   RootItem* parent =
-    static_cast<RootItem*>(m_standardFeedDetails->ui.m_cmbParentCategory->itemData(
+    static_cast<RootItem*>(m_standardFeedDetails->m_ui.m_cmbParentCategory->itemData(
-                             m_standardFeedDetails->ui.m_cmbParentCategory->currentIndex()).value<void*>());
+                             m_standardFeedDetails->m_ui.m_cmbParentCategory->currentIndex()).value<void*>());
 
   StandardFeed::Type type =
-    static_cast<StandardFeed::Type>(m_standardFeedDetails->ui.m_cmbType->itemData(m_standardFeedDetails->ui.m_cmbType->currentIndex()).value<int>());
+    static_cast<StandardFeed::Type>(m_standardFeedDetails->m_ui.m_cmbType->itemData(m_standardFeedDetails->m_ui.m_cmbType->currentIndex()).value<int>());
   auto* new_feed = new StandardFeed();
 
   // Setup data for new_feed.
-  new_feed->setTitle(m_standardFeedDetails->ui.m_txtTitle->lineEdit()->text());
+  new_feed->setTitle(m_standardFeedDetails->m_ui.m_txtTitle->lineEdit()->text());
   new_feed->setCreationDate(QDateTime::currentDateTime());
-  new_feed->setDescription(m_standardFeedDetails->ui.m_txtDescription->lineEdit()->text());
+  new_feed->setDescription(m_standardFeedDetails->m_ui.m_txtDescription->lineEdit()->text());
-  new_feed->setIcon(m_standardFeedDetails->ui.m_btnIcon->icon());
+  new_feed->setIcon(m_standardFeedDetails->m_ui.m_btnIcon->icon());
-  new_feed->setEncoding(m_standardFeedDetails->ui.m_cmbEncoding->currentText());
+  new_feed->setEncoding(m_standardFeedDetails->m_ui.m_cmbEncoding->currentText());
   new_feed->setType(type);
-  new_feed->setUrl(m_standardFeedDetails->ui.m_txtUrl->lineEdit()->text());
+  new_feed->setSourceType(m_standardFeedDetails->sourceType());
+  new_feed->setPostProcessScript(m_standardFeedDetails->m_ui.m_txtPostProcessScript->lineEdit()->text());
+  new_feed->setUrl(m_standardFeedDetails->m_ui.m_txtSource->lineEdit()->text());
   new_feed->setPasswordProtected(m_authDetails->m_gbAuthentication->isChecked());
   new_feed->setUsername(m_authDetails->m_txtUsername->lineEdit()->text());
   new_feed->setPassword(m_authDetails->m_txtPassword->lineEdit()->text());

src/librssguard/services/standard/gui/standardfeeddetails.cpp

@@ -2,33 +2,44 @@
 
 #include "services/standard/gui/standardfeeddetails.h"
 
+#include "gui/guiutilities.h"
 #include "miscellaneous/iconfactory.h"
 #include "network-web/networkfactory.h"
 #include "services/abstract/category.h"
-#include "services/standard/standardfeed.h"
 
 #include <QClipboard>
 #include <QFileDialog>
 #include <QMenu>
 #include <QMimeData>
+#include <QtGlobal>
 #include <QTextCodec>
 
 StandardFeedDetails::StandardFeedDetails(QWidget* parent) : QWidget(parent) {
-  ui.setupUi(this);
+  m_ui.setupUi(this);
 
-  ui.m_txtTitle->lineEdit()->setPlaceholderText(tr("Feed title"));
+  m_ui.m_txtTitle->lineEdit()->setPlaceholderText(tr("Feed title"));
-  ui.m_txtTitle->lineEdit()->setToolTip(tr("Set title for your feed."));
+  m_ui.m_txtTitle->lineEdit()->setToolTip(tr("Set title for your feed."));
-  ui.m_txtDescription->lineEdit()->setPlaceholderText(tr("Feed description"));
+  m_ui.m_txtDescription->lineEdit()->setPlaceholderText(tr("Feed description"));
-  ui.m_txtDescription->lineEdit()->setToolTip(tr("Set description for your feed."));
+  m_ui.m_txtDescription->lineEdit()->setToolTip(tr("Set description for your feed."));
-  ui.m_txtUrl->lineEdit()->setPlaceholderText(tr("Full feed url including scheme"));
+  m_ui.m_txtSource->lineEdit()->setPlaceholderText(tr("Full feed source identifier"));
-  ui.m_txtUrl->lineEdit()->setToolTip(tr("Set url for your feed."));
+  m_ui.m_txtSource->lineEdit()->setToolTip(tr("Full feed source identifier which can be URL."));
+  m_ui.m_txtPostProcessScript->lineEdit()->setPlaceholderText(tr("Full command to execute"));
+  m_ui.m_txtPostProcessScript->lineEdit()->setToolTip(tr("You can enter full command including interpreter here."));
+
+  // Add source types.
+  m_ui.m_cmbSourceType->addItem(StandardFeed::sourceTypeToString(StandardFeed::SourceType::Url),
+                                QVariant::fromValue(StandardFeed::SourceType::Url));
+  m_ui.m_cmbSourceType->addItem(StandardFeed::sourceTypeToString(StandardFeed::SourceType::Script),
+                                QVariant::fromValue(StandardFeed::SourceType::Script));
+  m_ui.m_txtPostProcessScript->setStatus(WidgetWithStatus::StatusType::Ok,
+                                         tr("Here you can enter script executaion line, including interpreter."));
 
   // Add standard feed types.
-  ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Atom10), QVariant::fromValue(int(StandardFeed::Type::Atom10)));
+  m_ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Atom10), QVariant::fromValue(int(StandardFeed::Type::Atom10)));
-  ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rdf), QVariant::fromValue(int(StandardFeed::Type::Rdf)));
+  m_ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rdf), QVariant::fromValue(int(StandardFeed::Type::Rdf)));
-  ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss0X), QVariant::fromValue(int(StandardFeed::Type::Rss0X)));
+  m_ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss0X), QVariant::fromValue(int(StandardFeed::Type::Rss0X)));
-  ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss2X), QVariant::fromValue(int(StandardFeed::Type::Rss2X)));
+  m_ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss2X), QVariant::fromValue(int(StandardFeed::Type::Rss2X)));
-  ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Json), QVariant::fromValue(int(StandardFeed::Type::Json)));
+  m_ui.m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Json), QVariant::fromValue(int(StandardFeed::Type::Json)));
 
   // Load available encodings.
   const QList<QByteArray> encodings = QTextCodec::availableCodecs();
@@ -43,7 +54,7 @@ StandardFeedDetails::StandardFeedDetails(QWidget* parent) : QWidget(parent) {
     return lhs.toLower() < rhs.toLower();
   });
 
-  ui.m_cmbEncoding->addItems(encoded_encodings);
+  m_ui.m_cmbEncoding->addItems(encoded_encodings);
 
   // Setup menu & actions for icon selection.
   m_iconMenu = new QMenu(tr("Icon selection"), this);
@@ -59,20 +70,36 @@ StandardFeedDetails::StandardFeedDetails(QWidget* parent) : QWidget(parent) {
   m_iconMenu->addAction(m_actionFetchIcon);
   m_iconMenu->addAction(m_actionLoadIconFromFile);
   m_iconMenu->addAction(m_actionUseDefaultIcon);
-  ui.m_btnIcon->setMenu(m_iconMenu);
+  m_ui.m_btnIcon->setMenu(m_iconMenu);
-  ui.m_txtUrl->lineEdit()->setFocus(Qt::TabFocusReason);
+  m_ui.m_txtSource->lineEdit()->setFocus(Qt::TabFocusReason);
 
   // Set feed metadata fetch label.
-  ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Information,
+  m_ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Information,
-                                   tr("No metadata fetched so far."),
+                                     tr("No metadata fetched so far."),
-                                   tr("No metadata fetched so far."));
+                                     tr("No metadata fetched so far."));
 
-  connect(ui.m_txtTitle->lineEdit(), &BaseLineEdit::textChanged, this, &StandardFeedDetails::onTitleChanged);
+  connect(m_ui.m_txtTitle->lineEdit(), &BaseLineEdit::textChanged, this, &StandardFeedDetails::onTitleChanged);
-  connect(ui.m_txtDescription->lineEdit(), &BaseLineEdit::textChanged, this, &StandardFeedDetails::onDescriptionChanged);
+  connect(m_ui.m_txtDescription->lineEdit(), &BaseLineEdit::textChanged, this, &StandardFeedDetails::onDescriptionChanged);
-  connect(ui.m_txtUrl->lineEdit(), &BaseLineEdit::textChanged, this, &StandardFeedDetails::onUrlChanged);
+  connect(m_ui.m_cmbSourceType, QOverload<int>::of(&QComboBox::currentIndexChanged),
+          this, [this]() {
+    onUrlChanged(m_ui.m_txtSource->lineEdit()->text());
+  });
+  connect(m_ui.m_txtSource->lineEdit(), &BaseLineEdit::textChanged, this, &StandardFeedDetails::onUrlChanged);
   connect(m_actionLoadIconFromFile, &QAction::triggered, this, &StandardFeedDetails::onLoadIconFromFile);
   connect(m_actionUseDefaultIcon, &QAction::triggered, this, &StandardFeedDetails::onUseDefaultIcon);
 
+  setTabOrder(m_ui.m_cmbParentCategory, m_ui.m_cmbType);
+  setTabOrder(m_ui.m_cmbType, m_ui.m_cmbEncoding);
+  setTabOrder(m_ui.m_cmbEncoding, m_ui.m_txtTitle->lineEdit());
+  setTabOrder(m_ui.m_txtTitle->lineEdit(), m_ui.m_txtDescription->lineEdit());
+  setTabOrder(m_ui.m_txtDescription->lineEdit(), m_ui.m_cmbSourceType);
+  setTabOrder(m_ui.m_cmbSourceType, m_ui.m_txtSource->lineEdit());
+  setTabOrder(m_ui.m_txtSource->lineEdit(), m_ui.m_txtPostProcessScript->lineEdit());
+  setTabOrder(m_ui.m_txtPostProcessScript->lineEdit(), m_ui.m_btnFetchMetadata);
+  setTabOrder(m_ui.m_btnFetchMetadata, m_ui.m_btnIcon);
+
+  GuiUtilities::setLabelAsNotice(*m_ui.m_lblScriptInfo, false);
+
   onTitleChanged(QString());
   onDescriptionChanged(QString());
   onUrlChanged(QString());
@@ -87,17 +114,17 @@ void StandardFeedDetails::guessIconOnly(const QString& url, const QString& usern
 
   if (result.first != nullptr) {
     // Icon or whole feed was guessed.
-    ui.m_btnIcon->setIcon(result.first->icon());
+    m_ui.m_btnIcon->setIcon(result.first->icon());
 
     if (result.second == QNetworkReply::NoError) {
-      ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Ok,
+      m_ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Ok,
-                                       tr("Icon fetched successfully."),
+                                         tr("Icon fetched successfully."),
-                                       tr("Icon metadata fetched."));
+                                         tr("Icon metadata fetched."));
     }
     else {
-      ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Warning,
+      m_ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Warning,
-                                       tr("Result: %1.").arg(NetworkFactory::networkErrorText(result.second)),
+                                         tr("Result: %1.").arg(NetworkFactory::networkErrorText(result.second)),
-                                       tr("Icon metadata not fetched."));
+                                         tr("Icon metadata not fetched."));
     }
 
     // Remove temporary feed object.
@@ -105,9 +132,9 @@ void StandardFeedDetails::guessIconOnly(const QString& url, const QString& usern
   }
   else {
     // No feed guessed, even no icon available.
-    ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Error,
+    m_ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Error,
-                                     tr("Error: %1.").arg(NetworkFactory::networkErrorText(result.second)),
+                                       tr("Error: %1.").arg(NetworkFactory::networkErrorText(result.second)),
-                                     tr("No icon fetched."));
+                                       tr("No icon fetched."));
   }
 }
 
@@ -120,28 +147,28 @@ void StandardFeedDetails::guessFeed(const QString& url, const QString& username,
 
   if (result.first != nullptr) {
     // Icon or whole feed was guessed.
-    ui.m_btnIcon->setIcon(result.first->icon());
+    m_ui.m_btnIcon->setIcon(result.first->icon());
-    ui.m_txtTitle->lineEdit()->setText(result.first->title());
+    m_ui.m_txtTitle->lineEdit()->setText(result.first->title());
-    ui.m_txtDescription->lineEdit()->setText(result.first->description());
+    m_ui.m_txtDescription->lineEdit()->setText(result.first->description());
-    ui.m_cmbType->setCurrentIndex(ui.m_cmbType->findData(QVariant::fromValue((int) result.first->type())));
+    m_ui.m_cmbType->setCurrentIndex(m_ui.m_cmbType->findData(QVariant::fromValue((int) result.first->type())));
-    int encoding_index = ui.m_cmbEncoding->findText(result.first->encoding(), Qt::MatchFixedString);
+    int encoding_index = m_ui.m_cmbEncoding->findText(result.first->encoding(), Qt::MatchFixedString);
 
     if (encoding_index >= 0) {
-      ui.m_cmbEncoding->setCurrentIndex(encoding_index);
+      m_ui.m_cmbEncoding->setCurrentIndex(encoding_index);
     }
     else {
-      ui.m_cmbEncoding->setCurrentIndex(ui.m_cmbEncoding->findText(DEFAULT_FEED_ENCODING, Qt::MatchFixedString));
+      m_ui.m_cmbEncoding->setCurrentIndex(m_ui.m_cmbEncoding->findText(DEFAULT_FEED_ENCODING, Qt::MatchFixedString));
     }
 
     if (result.second == QNetworkReply::NoError) {
-      ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Ok,
+      m_ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Ok,
-                                       tr("All metadata fetched successfully."),
+                                         tr("All metadata fetched successfully."),
-                                       tr("Feed and icon metadata fetched."));
+                                         tr("Feed and icon metadata fetched."));
     }
     else {
-      ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Warning,
+      m_ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Warning,
-                                       tr("Result: %1.").arg(NetworkFactory::networkErrorText(result.second)),
+                                         tr("Result: %1.").arg(NetworkFactory::networkErrorText(result.second)),
-                                       tr("Feed or icon metadata not fetched."));
+                                         tr("Feed or icon metadata not fetched."));
     }
 
     // Remove temporary feed object.
@@ -149,43 +176,58 @@ void StandardFeedDetails::guessFeed(const QString& url, const QString& username,
   }
   else {
     // No feed guessed, even no icon available.
-    ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Error,
+    m_ui.m_lblFetchMetadata->setStatus(WidgetWithStatus::StatusType::Error,
-                                     tr("Error: %1.").arg(NetworkFactory::networkErrorText(result.second)),
+                                       tr("Error: %1.").arg(NetworkFactory::networkErrorText(result.second)),
-                                     tr("No metadata fetched."));
+                                       tr("No metadata fetched."));
   }
 }
 
 void StandardFeedDetails::onTitleChanged(const QString& new_title) {
   if (new_title.simplified().size() >= MIN_CATEGORY_NAME_LENGTH) {
-    ui.m_txtTitle->setStatus(LineEditWithStatus::StatusType::Ok, tr("Feed name is ok."));
+    m_ui.m_txtTitle->setStatus(LineEditWithStatus::StatusType::Ok, tr("Feed name is ok."));
   }
   else {
-    ui.m_txtTitle->setStatus(LineEditWithStatus::StatusType::Error, tr("Feed name is too short."));
+    m_ui.m_txtTitle->setStatus(LineEditWithStatus::StatusType::Error, tr("Feed name is too short."));
   }
 }
 
 void StandardFeedDetails::onDescriptionChanged(const QString& new_description) {
   if (new_description.simplified().isEmpty()) {
-    ui.m_txtDescription->setStatus(LineEditWithStatus::StatusType::Warning, tr("Description is empty."));
+    m_ui.m_txtDescription->setStatus(LineEditWithStatus::StatusType::Warning, tr("Description is empty."));
   }
   else {
-    ui.m_txtDescription->setStatus(LineEditWithStatus::StatusType::Ok, tr("The description is ok."));
+    m_ui.m_txtDescription->setStatus(LineEditWithStatus::StatusType::Ok, tr("The description is ok."));
   }
 }
 
 void StandardFeedDetails::onUrlChanged(const QString& new_url) {
-  if (QRegularExpression(URL_REGEXP).match(new_url).hasMatch()) {
+  if (sourceType() == StandardFeed::SourceType::Url) {
-    // New url is well-formed.
+    if (QRegularExpression(URL_REGEXP).match(new_url).hasMatch()) {
-    ui.m_txtUrl->setStatus(LineEditWithStatus::StatusType::Ok, tr("The URL is ok."));
+      m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Ok, tr("The URL is ok."));
+    }
+    else if (!new_url.simplified().isEmpty()) {
+      m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Warning,
+                                  tr("The URL does not meet standard pattern. "
+                                     "Does your URL start with \"http://\" or \"https://\" prefix."));
+    }
+    else {
+      m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Error, tr("The URL is empty."));
+    }
   }
-  else if (!new_url.simplified().isEmpty()) {
+  else if (sourceType() == StandardFeed::SourceType::Script) {
-    // New url is not well-formed but is not empty on the other hand.
+    if (QRegularExpression(SCRIPT_SOURCE_TYPE_REGEXP).match(new_url).hasMatch()) {
-    ui.m_txtUrl->setStatus(LineEditWithStatus::StatusType::Warning,
+      m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Ok, tr("The source is ok."));
-                           tr(R"(The URL does not meet standard pattern. Does your URL start with "http://" or "https://" prefix.)"));
+    }
+    else if (!new_url.simplified().isEmpty()) {
+      m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Warning,
+                                  tr("The source needs to include \"#\" separator."));
+    }
+    else {
+      m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Error, tr("The source is empty."));
+    }
   }
   else {
-    // New url is empty.
+    m_ui.m_txtSource->setStatus(LineEditWithStatus::StatusType::Ok, tr("The source is ok."));
-    ui.m_txtUrl->setStatus(LineEditWithStatus::StatusType::Error, tr("The URL is empty."));
   }
 }
 
@@ -206,63 +248,69 @@ void StandardFeedDetails::onLoadIconFromFile() {
   dialog.setLabelText(QFileDialog::DialogLabel::FileType, tr("Icon type:"));
 
   if (dialog.exec() == QDialog::DialogCode::Accepted) {
-    ui.m_btnIcon->setIcon(QIcon(dialog.selectedFiles().value(0)));
+    m_ui.m_btnIcon->setIcon(QIcon(dialog.selectedFiles().value(0)));
   }
 }
 
 void StandardFeedDetails::onUseDefaultIcon() {
-  ui.m_btnIcon->setIcon(QIcon());
+  m_ui.m_btnIcon->setIcon(QIcon());
+}
+
+StandardFeed::SourceType StandardFeedDetails::sourceType() const {
+  return m_ui.m_cmbSourceType->currentData().value<StandardFeed::SourceType>();
 }
 
 void StandardFeedDetails::prepareForNewFeed(RootItem* parent_to_select, const QString& url) {
   // Make sure that "default" icon is used as the default option for new
   // feed.
   m_actionUseDefaultIcon->trigger();
-  int default_encoding_index = ui.m_cmbEncoding->findText(DEFAULT_FEED_ENCODING);
+  int default_encoding_index = m_ui.m_cmbEncoding->findText(DEFAULT_FEED_ENCODING);
 
   if (default_encoding_index >= 0) {
-    ui.m_cmbEncoding->setCurrentIndex(default_encoding_index);
+    m_ui.m_cmbEncoding->setCurrentIndex(default_encoding_index);
   }
 
   if (parent_to_select != nullptr) {
     if (parent_to_select->kind() == RootItem::Kind::Category) {
-      ui.m_cmbParentCategory->setCurrentIndex(ui.m_cmbParentCategory->findData(QVariant::fromValue((void*)parent_to_select)));
+      m_ui.m_cmbParentCategory->setCurrentIndex(m_ui.m_cmbParentCategory->findData(QVariant::fromValue((void*)parent_to_select)));
     }
     else if (parent_to_select->kind() == RootItem::Kind::Feed) {
-      int target_item = ui.m_cmbParentCategory->findData(QVariant::fromValue((void*)parent_to_select->parent()));
+      int target_item = m_ui.m_cmbParentCategory->findData(QVariant::fromValue((void*)parent_to_select->parent()));
 
       if (target_item >= 0) {
-        ui.m_cmbParentCategory->setCurrentIndex(target_item);
+        m_ui.m_cmbParentCategory->setCurrentIndex(target_item);
       }
     }
   }
 
   if (!url.isEmpty()) {
-    ui.m_txtUrl->lineEdit()->setText(url);
+    m_ui.m_txtSource->lineEdit()->setText(url);
   }
   else if (Application::clipboard()->mimeData()->hasText()) {
-    ui.m_txtUrl->lineEdit()->setText(Application::clipboard()->text());
+    m_ui.m_txtSource->lineEdit()->setText(Application::clipboard()->text());
   }
 
-  ui.m_txtUrl->setFocus();
+  m_ui.m_txtSource->setFocus();
 }
 
 void StandardFeedDetails::setExistingFeed(StandardFeed* feed) {
-  ui.m_cmbParentCategory->setCurrentIndex(ui.m_cmbParentCategory->findData(QVariant::fromValue((void*)feed->parent())));
+  m_ui.m_cmbSourceType->setCurrentIndex(m_ui.m_cmbSourceType->findData(QVariant::fromValue(feed->sourceType())));
-  ui.m_txtTitle->lineEdit()->setText(feed->title());
+  m_ui.m_cmbParentCategory->setCurrentIndex(m_ui.m_cmbParentCategory->findData(QVariant::fromValue((void*)feed->parent())));
-  ui.m_txtDescription->lineEdit()->setText(feed->description());
+  m_ui.m_txtTitle->lineEdit()->setText(feed->title());
-  ui.m_btnIcon->setIcon(feed->icon());
+  m_ui.m_txtDescription->lineEdit()->setText(feed->description());
-  ui.m_txtUrl->lineEdit()->setText(feed->url());
+  m_ui.m_btnIcon->setIcon(feed->icon());
-  ui.m_cmbType->setCurrentIndex(ui.m_cmbType->findData(QVariant::fromValue(int(feed->type()))));
+  m_ui.m_txtSource->lineEdit()->setText(feed->url());
-  ui.m_cmbEncoding->setCurrentIndex(ui.m_cmbEncoding->findData(feed->encoding(),
+  m_ui.m_txtPostProcessScript->lineEdit()->setText(feed->postProcessScript());
-                                                               Qt::ItemDataRole::DisplayRole,
+  m_ui.m_cmbType->setCurrentIndex(m_ui.m_cmbType->findData(QVariant::fromValue(int(feed->type()))));
-                                                               Qt::MatchFlag::MatchFixedString));
+  m_ui.m_cmbEncoding->setCurrentIndex(m_ui.m_cmbEncoding->findData(feed->encoding(),
+                                                                   Qt::ItemDataRole::DisplayRole,
+                                                                   Qt::MatchFlag::MatchFixedString));
 }
 
 void StandardFeedDetails::loadCategories(const QList<Category*>& categories, RootItem* root_item) {
-  ui.m_cmbParentCategory->addItem(root_item->fullIcon(), root_item->title(), QVariant::fromValue((void*) root_item));
+  m_ui.m_cmbParentCategory->addItem(root_item->fullIcon(), root_item->title(), QVariant::fromValue((void*) root_item));
 
   for (Category* category : categories) {
-    ui.m_cmbParentCategory->addItem(category->fullIcon(), category->title(), QVariant::fromValue((void*) category));
+    m_ui.m_cmbParentCategory->addItem(category->fullIcon(), category->title(), QVariant::fromValue((void*) category));
   }
 }

src/librssguard/services/standard/gui/standardfeeddetails.h

@@ -7,11 +7,12 @@
 
 #include "ui_standardfeeddetails.h"
 
+#include "services/standard/standardfeed.h"
+
 #include <QNetworkProxy>
 
 class Category;
 class RootItem;
-class StandardFeed;
 
 class StandardFeedDetails : public QWidget {
   Q_OBJECT
@@ -37,13 +38,15 @@ class StandardFeedDetails : public QWidget {
     void onLoadIconFromFile();
     void onUseDefaultIcon();
 
+    StandardFeed::SourceType sourceType() const;
+
   private:
     void prepareForNewFeed(RootItem* parent_to_select, const QString& url);
     void setExistingFeed(StandardFeed* feed);
     void loadCategories(const QList<Category*>& categories, RootItem* root_item);
 
   private:
-    Ui::StandardFeedDetails ui;
+    Ui::StandardFeedDetails m_ui;
     QMenu* m_iconMenu{};
     QAction* m_actionLoadIconFromFile{};
     QAction* m_actionUseDefaultIcon{};

src/librssguard/services/standard/gui/standardfeeddetails.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>429</width>
-    <height>260</height>
+    <height>321</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -103,17 +103,31 @@
    <item row="5" column="0">
     <widget class="QLabel" name="label">
      <property name="text">
-      <string>URL</string>
+      <string>Source</string>
      </property>
      <property name="buddy">
-      <cstring>m_txtUrl</cstring>
+      <cstring>m_txtSource</cstring>
      </property>
     </widget>
    </item>
    <item row="5" column="1">
-    <widget class="LineEditWithStatus" name="m_txtUrl" native="true"/>
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <widget class="QComboBox" name="m_cmbSourceType"/>
+     </item>
+     <item>
+      <widget class="LineEditWithStatus" name="m_txtSource" native="true">
+       <property name="sizePolicy">
+        <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+         <horstretch>1</horstretch>
+         <verstretch>0</verstretch>
+        </sizepolicy>
+       </property>
+      </widget>
+     </item>
+    </layout>
    </item>
-   <item row="6" column="0">
+   <item row="8" column="0">
     <widget class="QLabel" name="label_7">
      <property name="text">
       <string>Fetch metadata</string>
@@ -123,7 +137,7 @@
      </property>
     </widget>
    </item>
-   <item row="6" column="1">
+   <item row="8" column="1">
     <layout class="QHBoxLayout" name="horizontalLayout_2">
      <item>
       <widget class="QPushButton" name="m_btnFetchMetadata">
@@ -147,7 +161,7 @@
      </item>
     </layout>
    </item>
-   <item row="7" column="0">
+   <item row="9" column="0">
     <widget class="QLabel" name="m_lblIcon">
      <property name="text">
       <string>Icon</string>
@@ -157,7 +171,7 @@
      </property>
     </widget>
    </item>
-   <item row="7" column="1">
+   <item row="9" column="1">
     <widget class="QToolButton" name="m_btnIcon">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
@@ -194,6 +208,29 @@
      </property>
     </widget>
    </item>
+   <item row="6" column="1">
+    <widget class="LineEditWithStatus" name="m_txtPostProcessScript" native="true"/>
+   </item>
+   <item row="6" column="0">
+    <widget class="QLabel" name="label_4">
+     <property name="text">
+      <string>Post-process script</string>
+     </property>
+    </widget>
+   </item>
+   <item row="7" column="0" colspan="2">
+    <widget class="QLabel" name="m_lblScriptInfo">
+     <property name="text">
+      <string>You can use URL as a source of your feed or you can produce your feed with custom script. Also, you can post-process generated feed data with yet another script if you wish. These are advanced features and make sure to read the documentation before your use them.</string>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignCenter</set>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <customwidgets>
@@ -210,13 +247,6 @@
    <container>1</container>
   </customwidget>
  </customwidgets>
- <tabstops>
-  <tabstop>m_cmbParentCategory</tabstop>
-  <tabstop>m_cmbType</tabstop>
-  <tabstop>m_cmbEncoding</tabstop>
-  <tabstop>m_btnFetchMetadata</tabstop>
-  <tabstop>m_btnIcon</tabstop>
- </tabstops>
  <resources/>
  <connections/>
 </ui>

src/librssguard/services/standard/standardfeed.cpp

@@ -35,13 +35,14 @@ StandardFeed::StandardFeed(RootItem* parent_item)
   m_networkError = QNetworkReply::NetworkError::NoError;
   m_type = Type::Rss0X;
   m_sourceType = SourceType::Url;
-  m_encoding = QString();
+  m_encoding = m_postProcessScript = QString();
 }
 
 StandardFeed::StandardFeed(const StandardFeed& other)
   : Feed(other) {
   m_networkError = other.networkError();
   m_type = other.type();
+  m_postProcessScript = other.postProcessScript();
   m_sourceType = other.sourceType();
   m_encoding = other.encoding();
 }
@@ -112,6 +113,22 @@ QString StandardFeed::typeToString(StandardFeed::Type type) {
   }
 }
 
+QString StandardFeed::sourceTypeToString(StandardFeed::SourceType type) {
+  switch (type) {
+    case StandardFeed::SourceType::Url:
+      return QSL("URL");
+
+    case StandardFeed::SourceType::Script:
+      return tr("Script");
+
+    case StandardFeed::SourceType::LocalFile:
+      return tr("Local file");
+
+    default:
+      return tr("Unknown");
+  }
+}
+
 void StandardFeed::fetchMetadataForItself() {
   QPair<StandardFeed*, QNetworkReply::NetworkError> metadata = guessFeed(url(),
                                                                          username(),
@@ -141,6 +158,14 @@ void StandardFeed::fetchMetadataForItself() {
   }
 }
 
+QString StandardFeed::postProcessScript() const {
+  return m_postProcessScript;
+}
+
+void StandardFeed::setPostProcessScript(const QString& post_process_script) {
+  m_postProcessScript = post_process_script;
+}
+
 StandardFeed::SourceType StandardFeed::sourceType() const {
   return m_sourceType;
 }
@@ -370,9 +395,11 @@ bool StandardFeed::addItself(RootItem* parent) {
   // Now, add feed to persistent storage.
   QSqlDatabase database = qApp->database()->connection(metaObject()->className());
   bool ok;
-  int new_id = DatabaseQueries::addStandardFeed(database, parent->id(), parent->getParentServiceRoot()->accountId(), title(),
+  int new_id = DatabaseQueries::addStandardFeed(database, parent->id(), parent->getParentServiceRoot()->accountId(),
-                                                description(), creationDate(), icon(), encoding(), url(), passwordProtected(),
+                                                title(), description(), creationDate(), icon(), encoding(), url(),
-                                                username(), password(), autoUpdateType(), autoUpdateInitialInterval(), type(), &ok);
+                                                passwordProtected(), username(), password(), autoUpdateType(),
+                                                autoUpdateInitialInterval(), sourceType(), postProcessScript(),
+                                                type(), &ok);
 
   if (!ok) {
     // Query failed.
@@ -396,6 +423,7 @@ bool StandardFeed::editItself(StandardFeed* new_feed_data) {
                                          new_feed_data->encoding(), new_feed_data->url(), new_feed_data->passwordProtected(),
                                          new_feed_data->username(), new_feed_data->password(),
                                          new_feed_data->autoUpdateType(), new_feed_data->autoUpdateInitialInterval(),
+                                         new_feed_data->sourceType(), new_feed_data->postProcessScript(),
                                          new_feed_data->type())) {
     // Persistent storage update failed, no way to continue now.
     qWarningNN << LOGSEC_CORE
@@ -417,6 +445,7 @@ bool StandardFeed::editItself(StandardFeed* new_feed_data) {
   original_feed->setAutoUpdateInitialInterval(new_feed_data->autoUpdateInitialInterval());
   original_feed->setType(new_feed_data->type());
   original_feed->setSourceType(new_feed_data->sourceType());
+  original_feed->setPostProcessScript(new_feed_data->postProcessScript());
 
   // Editing is done.
   return true;

src/librssguard/services/standard/standardfeed.h

@@ -71,6 +71,9 @@ class StandardFeed : public Feed {
     QString encoding() const;
     void setEncoding(const QString& encoding);
 
+    QString postProcessScript() const;
+    void setPostProcessScript(const QString& post_process_script);
+
     QNetworkReply::NetworkError networkError() const;
 
     QList<Message> obtainNewMessages(bool* error_during_obtaining);
@@ -87,6 +90,7 @@ class StandardFeed : public Feed {
 
     // Converts particular feed type to string.
     static QString typeToString(Type type);
+    static QString sourceTypeToString(SourceType type);
 
   public slots:
     void fetchMetadataForItself();
@@ -94,11 +98,13 @@ class StandardFeed : public Feed {
   private:
     SourceType m_sourceType;
     Type m_type;
+    QString m_postProcessScript;
 
     QNetworkReply::NetworkError m_networkError;
     QString m_encoding;
 };
 
+Q_DECLARE_METATYPE(StandardFeed::SourceType)
 Q_DECLARE_METATYPE(StandardFeed::Type)
 
 #endif // FEEDSMODELFEED_H