diff --git a/resources/docs/Documentation.md b/resources/docs/Documentation.md index 224ff7f95..1e70d5299 100644 --- a/resources/docs/Documentation.md +++ b/resources/docs/Documentation.md @@ -9,6 +9,7 @@ * [Message filtering](#message-filtering) * [Misc](#misc) * [Portable user data](#portable-user-data) + * [Downloading new messages](#downloading-new-messages) * [Generating debug log file](#generating-debug-log-file) # Foreword @@ -163,6 +164,16 @@ This is _fully-portable mode_. Otherwise, standard "config" folder is used. Check `About RSS Guard` dialog to find more info on significant paths used. +## Downloading new messages +Here is the rough workflow which is done when you hit `Feeds & categories -> Update all items` or `Feeds & categories -> Update selected items`. At that point of time this happens: +1. RSS Guard creates a list of all/selected feeds. +2. Sequentially, for each feed do: + a. Download all available messages from online source. + b. Sequentially, for each message do: + 1. Sanitize title of the message. This includes replacing all non-breaking spaces with normal spaces, removing all leading spaces, replacing all multiple consecutive spaces with single space. Contents of message are converted from [percent-encoding](https://en.wikipedia.org/wiki/Percent-encoding). + 2. Run all [message filters](#message-filtering), one by one, one the message. Cache read/important message attributes changed by filters to queue which is later synchronized back to online feed service. + 3. Store the message into RSS Guard's database, creating completely new DB entry for it, or replacing existing message. + ## Generating debug log file If you run into problems with RSS Guard and you need your problems fixed, you should provide log file from the time when problem occurred. RSS Guard writes all important information to standard output, which is usually calling terminal. diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 7d7032bd6..c9cadc6d3 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -97,19 +97,8 @@ void FeedDownloader::updateOneFeed(Feed* feed) { // Now, sanitize messages (tweak encoding etc.). for (auto& msg : msgs) { - // Also, make sure that HTML encoding, encoding of special characters, etc., is fixed. - msg.m_contents = QUrl::fromPercentEncoding(msg.m_contents.toUtf8()); - msg.m_author = msg.m_author.toUtf8(); msg.m_accountId = acc_id; - - // Sanitize title. - msg.m_title = msg.m_title - - // Shrink consecutive whitespaces. - .replace(QRegularExpression(QSL("[\\s]{2,}")), QSL(" ")) - - // Remove all newlines and leading white space. - .remove(QRegularExpression(QSL("([\\n\\r])|(^\\s)"))); + msg.sanitize(); } if (!feed->messageFilters().isEmpty()) { diff --git a/src/librssguard/core/message.cpp b/src/librssguard/core/message.cpp index a9c601f41..b90ada677 100644 --- a/src/librssguard/core/message.cpp +++ b/src/librssguard/core/message.cpp @@ -6,10 +6,14 @@ #include "services/abstract/label.h" #include +#include +#include #include #include #include +#include #include +#include Enclosure::Enclosure(QString url, QString mime) : m_url(std::move(url)), m_mimeType(std::move(mime)) {} @@ -65,6 +69,21 @@ Message::Message() { m_assignedLabels = QList(); } +void Message::sanitize() { + // Also, make sure that HTML encoding, encoding of special characters, etc., is fixed. + m_contents = QUrl::fromPercentEncoding(m_contents.toUtf8()); + m_author = m_author.toUtf8(); + + // Sanitize title. + m_title = m_title + + // Shrink consecutive whitespaces. + .replace(QRegularExpression(QSL("[\\s\\u202F\\u00A0]{2,}")), QSL(" ")) + + // Remove all newlines and leading white space. + .remove(QRegularExpression(QSL("([\\n\\r])|(^\\s)"))); +} + Message Message::fromSqlRecord(const QSqlRecord& record, bool* result) { if (record.count() != MSG_DB_HAS_ENCLOSURES + 1) { if (result != nullptr) { diff --git a/src/librssguard/core/message.h b/src/librssguard/core/message.h index 0854d4146..81ac8841a 100644 --- a/src/librssguard/core/message.h +++ b/src/librssguard/core/message.h @@ -35,6 +35,8 @@ class Message { public: explicit Message(); + void sanitize(); + // Creates Message from given record, which contains // row from query SELECT * FROM Messages WHERE ....; static Message fromSqlRecord(const QSqlRecord& record, bool* result = nullptr);