From bc739ca9fa5a733fdb226b18420a51348bb455d3 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Fri, 28 May 2021 10:19:40 +0200 Subject: [PATCH] fix #420 and add some more fixes for XML -> json utility function, also make filtering dialog work with "rawContents" to enable basic testing --- resources/docs/Message-filters.md | 2 +- src/librssguard/core/filterutils.cpp | 33 +++++++++------ src/librssguard/core/message.cpp | 17 ++++++++ src/librssguard/core/message.h | 1 + .../core/messagesforfiltersmodel.cpp | 1 + .../gui/dialogs/formmessagefiltersmanager.cpp | 42 +++++++++++++------ 6 files changed, 70 insertions(+), 26 deletions(-) diff --git a/resources/docs/Message-filters.md b/resources/docs/Message-filters.md index e3e982bbe..7e0cafd23 100755 --- a/resources/docs/Message-filters.md +++ b/resources/docs/Message-filters.md @@ -50,7 +50,7 @@ Here is the reference of methods and properties of some types available in your | `String url` | URL of the message. | | `String author` | Author of the message. | | `String contents` | Contents of the message. | -| `String rawContents` | This is RAW contents of the message as it was obtained from remote service/feed. You can expect raw `XML` or `JSON` element data here. Note that this attribute has some value only if `alreadyStoredInDb` returns `false`. In other words, this attribute is not persistently stored inside RSS Guard's DB. Also, this attribute might not be filled when testing the filter, it is only filled during live filter execution on real incoming messages. | +| `String rawContents` | This is RAW contents of the message as it was obtained from remote service/feed. You can expect raw `XML` or `JSON` element data here. Note that this attribute has some value only if `alreadyStoredInDb` returns `false`. In other words, this attribute is not persistently stored inside RSS Guard's DB. Also, this attribute is artificially filled with ATOM-like data when testing the filter. | | `Number score` | Arbitrary number in range <0.0, 100.0>. You can use this number to sort messages in a custom fashion as this attribute also has its own column in messages list. | | `Date created` | Date/time of the message. | | `Boolean isRead` | Is message read? | diff --git a/src/librssguard/core/filterutils.cpp b/src/librssguard/core/filterutils.cpp index 1c9cc38e2..76bfce161 100755 --- a/src/librssguard/core/filterutils.cpp +++ b/src/librssguard/core/filterutils.cpp @@ -33,8 +33,13 @@ QString jsonProcessXmlElement(const QDomElement& elem) { } QStringList elems; + QString elem_text; for (int i = 0; i < elem.childNodes().size(); i++) { + if (elem.childNodes().at(i).isText()) { + elem_text = jsonEscapeString(elem.childNodes().at(i).nodeValue()); + } + if (!elem.childNodes().at(i).isElement()) { continue; } @@ -43,22 +48,26 @@ QString jsonProcessXmlElement(const QDomElement& elem) { jsonProcessXmlElement(elem.childNodes().at(i).toElement())); } - if (attrs.isEmpty()) { - if (elems.isEmpty()) { - return QSL("\"%1\"").arg(jsonEscapeString(elem.text())); - } - else { - return QSL("{%1}").arg(elems.join(QSL(",\n"))); - } + QString str; + + if (!elems.isEmpty() && !attrs.isEmpty()) { + str = QSL("{%1, %2, %3}").arg(attrs.join(QSL(",\n")), + elems.join(QSL(",\n")), + QSL("\"__text\": \"%1\"").arg(elem_text)); } - else if (elems.isEmpty()) { - return QSL("{%1, \"__text\": \"%2\"}").arg(attrs.join(QSL(",\n")), - jsonEscapeString(elem.text())); + else if (!elems.isEmpty()) { + str = QSL("{%1, %2}").arg(elems.join(QSL(",\n")), + QSL("\"__text\": \"%1\"").arg(elem_text)); + } + else if (!attrs.isEmpty()) { + str = QSL("{%1, %2}").arg(attrs.join(QSL(",\n")), + QSL("\"__text\": \"%1\"").arg(elem_text)); } else { - return QSL("{%1, %2}").arg(attrs.join(QSL(",\n")), - elems.join(QSL(",\n"))); + str = QSL("{%1}").arg(QSL("\"__text\": \"%1\"").arg(elem_text)); } + + return str; } QString FilterUtils::fromXmlToJson(const QString& xml) const { diff --git a/src/librssguard/core/message.cpp b/src/librssguard/core/message.cpp index 078da16ba..566d06cb1 100644 --- a/src/librssguard/core/message.cpp +++ b/src/librssguard/core/message.cpp @@ -120,6 +120,23 @@ Message Message::fromSqlRecord(const QSqlRecord& record, bool* result) { return message; } +QString Message::generateRawAtomContents(const Message& msg) { + return QSL("" + "%1" + "" + "%3" + "%6" + "%3" + "%4" + "%5" + "").arg(msg.m_title, + msg.m_url, + msg.m_created.toUTC().toString(QSL("yyyy-MM-ddThh:mm:ss")), + msg.m_url, + msg.m_contents.toHtmlEscaped(), + msg.m_author); +} + QDataStream& operator<<(QDataStream& out, const Message& my_obj) { out << my_obj.m_accountId << my_obj.m_customHash diff --git a/src/librssguard/core/message.h b/src/librssguard/core/message.h index 96e1a22a0..42d2d664a 100644 --- a/src/librssguard/core/message.h +++ b/src/librssguard/core/message.h @@ -39,6 +39,7 @@ class RSSGUARD_DLLSPEC Message { // Creates Message from given record, which contains // row from query SELECT * FROM Messages WHERE ....; static Message fromSqlRecord(const QSqlRecord& record, bool* result = nullptr); + static QString generateRawAtomContents(const Message& msg); public: QString m_title; diff --git a/src/librssguard/core/messagesforfiltersmodel.cpp b/src/librssguard/core/messagesforfiltersmodel.cpp index 59c232cbb..e2237c2a2 100755 --- a/src/librssguard/core/messagesforfiltersmodel.cpp +++ b/src/librssguard/core/messagesforfiltersmodel.cpp @@ -117,6 +117,7 @@ void MessagesForFiltersModel::testFilter(MessageFilter* filter, QJSEngine* engin for (int i = 0; i < m_messages.size(); i++) { Message* msg = messageForRow(i); + msg->m_rawContents = Message::generateRawAtomContents(*msg); msg_proxy->setMessage(msg); try { diff --git a/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp b/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp index b50c92aae..056ab59a5 100644 --- a/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp +++ b/src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp @@ -263,11 +263,13 @@ void FormMessageFiltersManager::testFilter() { " Author = '%3'\n" " Is read/important = '%4/%5'\n" " Created on = '%6'\n" - " Contents = '%7'").arg(msg.m_title, msg.m_url, msg.m_author, - msg.m_isRead ? tr("yes") : tr("no"), - msg.m_isImportant ? tr("yes") : tr("no"), - QString::number(msg.m_created.toMSecsSinceEpoch()), - msg.m_contents); + " Contents = '%7'\n" + " RAW contents = '%8'").arg(msg.m_title, msg.m_url, msg.m_author, + msg.m_isRead ? tr("yes") : tr("no"), + msg.m_isImportant ? tr("yes") : tr("no"), + QString::number(msg.m_created.toMSecsSinceEpoch()), + msg.m_contents, + msg.m_rawContents); m_ui.m_txtErrors->insertPlainText(answer); } @@ -315,22 +317,35 @@ void FormMessageFiltersManager::processCheckedFeeds() { // Create backup of message. Message* msg = &msgs[i]; msg->m_assignedLabels = labels_in_message; + + msg->m_rawContents = Message::generateRawAtomContents(*msg); + Message msg_backup(*msg); msg_obj.setMessage(msg); - MessageObject::FilteringAction result = fltr->filterMessage(&filter_engine); bool remove_from_list = false; - if (result == MessageObject::FilteringAction::Purge) { - remove_from_list = true; + try { + MessageObject::FilteringAction result = fltr->filterMessage(&filter_engine); - // Purge the message completely and remove leftovers. - DatabaseQueries::purgeMessage(database, msg->m_id); - DatabaseQueries::purgeLeftoverLabelAssignments(database, msg->m_accountId); + if (result == MessageObject::FilteringAction::Purge) { + remove_from_list = true; + + // Purge the message completely and remove leftovers. + DatabaseQueries::purgeMessage(database, msg->m_id); + DatabaseQueries::purgeLeftoverLabelAssignments(database, msg->m_accountId); + } + else if (result == MessageObject::FilteringAction::Ignore) { + remove_from_list = true; + } } - else if (result == MessageObject::FilteringAction::Ignore) { - remove_from_list = true; + catch (const FilteringException& ex) { + qCriticalNN << LOGSEC_CORE + << "Error when running script when processing existing messages:" + << QUOTE_W_SPACE_DOT(ex.message()); + + continue; } if (!msg_backup.m_isRead && msg->m_isRead) { @@ -585,6 +600,7 @@ Message FormMessageFiltersManager::testingMessage() const { msg.m_isImportant = m_ui.m_cbSampleImportant->isChecked(); msg.m_created = QDateTime::fromMSecsSinceEpoch(m_ui.m_txtSampleCreatedOn->text().toLongLong()); msg.m_contents = m_ui.m_txtSampleContents->toPlainText(); + msg.m_rawContents = Message::generateRawAtomContents(msg); return msg; }