gmail: refined e-mail downloading, now supports ISA - faster synchronization, particularly subsequent

localization/CMakeLists.txt

@@ -19,6 +19,7 @@ endif()
 FILE(GLOB TS_FILES  ${CMAKE_CURRENT_SOURCE_DIR}/*.ts)
 
 set_source_files_properties(${TS_FILES} PROPERTIES OUTPUT_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}")
+
 qt_add_translation(QM_FILES
   ${TS_FILES}
   OPTIONS "-compress"

resources/docs/Documentation.md

@@ -61,7 +61,7 @@ I organized the supported web-based feed readers into an elegant table:
 | Service | Two-way Synchronization | [Intelligent Synchronization Algorithm](#intel) (ISA) <sup>1</sup> | Synchronized Labels <sup>2</sup> <a id="sfrl"></a> | OAuth <sup>4</sup> |
 | :---              | :---:  | :---: | :---: | :---:
 | Feedly            | ✅ | ❌ | ✅ | ✅ (only for official binaries)
-| Gmail             | ✅ | ❌ | ❌ | ✅
+| Gmail             | ✅ | ✅ | ❌ | ✅
 | Google Reader API <sup>3</sup> | ✅ | ✅ | ✅ | ✅ (only for Inoreader)
 | Nextcloud News    | ✅ | ❌ | ❌ | ❌
 | Tiny Tiny RSS     | ✅ | ❌ | ✅ | ❌

resources/icons.qrc

@@ -49,6 +49,7 @@
     <file>./graphics/Breeze/actions/22/list-add.svg</file>
     <file>./graphics/Breeze/actions/22/list-remove.svg</file>
     <file>./graphics/Breeze/actions/32/mail-attachment.svg</file>
+    <file>./graphics/Breeze/actions/symbolic/mail-inbox-symbolic.svg</file>
     <file>./graphics/Breeze/actions/32/mail-mark-important.svg</file>
     <file>./graphics/Breeze/actions/32/mail-mark-junk.svg</file>
     <file>./graphics/Breeze/actions/32/mail-mark-read.svg</file>
@@ -122,6 +123,7 @@
     <file>./graphics/Breeze Dark/actions/22/list-add.svg</file>
     <file>./graphics/Breeze Dark/actions/22/list-remove.svg</file>
     <file>./graphics/Breeze Dark/actions/32/mail-attachment.svg</file>
+    <file>./graphics/Breeze Dark/actions/symbolic/mail-inbox-symbolic.svg</file>
     <file>./graphics/Breeze Dark/actions/32/mail-mark-important.svg</file>
     <file>./graphics/Breeze Dark/actions/32/mail-mark-junk.svg</file>
     <file>./graphics/Breeze Dark/actions/32/mail-mark-read.svg</file>

src/librssguard/core/feeddownloader.cpp

@@ -387,7 +387,6 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
 
     feed->setStatus(feed_ex.feedStatus(), feed_ex.message());
   }
-
   catch (const ApplicationException& app_ex) {
     qCriticalNN << LOGSEC_NETWORK
                 << "Unknown error when fetching feed:"

src/librssguard/services/gmail/gmailnetworkfactory.cpp

@@ -179,81 +179,80 @@ Downloader* GmailNetworkFactory::downloadAttachment(const QString& msg_id,
 }
 
 QList<Message> GmailNetworkFactory::messages(const QString& stream_id,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
                                              Feed::Status& error,
                                              const QNetworkProxy& custom_proxy) {
   QString bearer = m_oauth2->bearer().toLocal8Bit();
-  QString next_page_token;
-  QList<Message> messages;
-  ulong msecs_wait_between_batches = 1500;
 
   if (bearer.isEmpty()) {
     error = Feed::Status::AuthError;
-    return QList<Message>();
+    return {};
   }
 
-  // We need to quit event loop when the download finishes.
+  // 1. Get unread IDs for a feed.
-  QString target_url;
+  // 2. Get read IDs for a feed.
-  int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
+  // 3. Get starred IDs for a feed.
+  // 4. Download messages/contents for missing or changed IDs.
+  QStringList remote_read_ids_list, remote_unread_ids_list, remote_starred_ids_list;
 
-  do {
+  try {
-    target_url = QSL(GMAIL_API_MSGS_LIST);
+    remote_starred_ids_list = list(stream_id, {}, 0, QSL("is:starred"), custom_proxy);
-    target_url += QSL("?labelIds=%1").arg(stream_id);
+    remote_unread_ids_list = list(stream_id, {}, batchSize(), QSL("is:unread"), custom_proxy);
 
-    if (downloadOnlyUnreadMessages()) {
+    if (!downloadOnlyUnreadMessages()) {
-      target_url += QSL("&labelIds=%1").arg(QSL(GMAIL_SYSTEM_LABEL_UNREAD));
+      remote_read_ids_list = list(stream_id, {}, batchSize(), QSL("is:read"), custom_proxy);
-    }
-
-    if (batchSize() > 0) {
-      target_url += QSL("&maxResults=%1").arg(batchSize());
-    }
-
-    if (!next_page_token.isEmpty()) {
-      target_url += QString("&pageToken=%1").arg(next_page_token);
-    }
-
-    QByteArray messages_raw_data;
-    auto netw = NetworkFactory::performNetworkOperation(target_url,
-                                                        timeout,
-                                                        {},
-                                                        messages_raw_data,
-                                                        QNetworkAccessManager::Operation::GetOperation,
-                                                        { { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(),
-                                                          bearer.toLocal8Bit() } },
-                                                        false,
-                                                        {},
-                                                        {},
-                                                        custom_proxy);
-
-    if (netw.m_networkError == QNetworkReply::NetworkError::NoError) {
-      // We parse this chunk.
-      QString messages_data = QString::fromUtf8(messages_raw_data);
-      QList<Message> more_messages = decodeLiteMessages(messages_data, stream_id, next_page_token);
-
-      if (!more_messages.isEmpty()) {
-        // Now, we via batch HTTP request obtain full data for each message.
-        bool obtained = obtainAndDecodeFullMessages(more_messages, stream_id, custom_proxy);
-
-        if (obtained) {
-          messages.append(more_messages);
-          QThread::msleep(msecs_wait_between_batches);
-
-          // New batch of messages was obtained, check if we have enough.
-          if (batchSize() > 0 && batchSize() <= messages.size()) {
-            // We have enough messages.
-            break;
     }
   }
-        else {
+  catch (const NetworkException& net_ex) {
-          error = Feed::Status::NetworkError;
+    qCriticalNN << LOGSEC_GMAIL
-          return messages;
+                << "Failed to get list of e-mail IDs:" << QUOTE_W_SPACE_DOT(net_ex.message());
+    return {};
   }
+
+  // 1.
+  auto local_unread_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Unread);
+  QSet<QString> remote_unread_ids = FROM_LIST_TO_SET(QSet<QString>, remote_unread_ids_list);
+  QSet<QString> local_unread_ids = FROM_LIST_TO_SET(QSet<QString>, local_unread_ids_list);
+
+  // 2.
+  auto local_read_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Read);
+  QSet<QString> remote_read_ids = FROM_LIST_TO_SET(QSet<QString>, remote_read_ids_list);
+  QSet<QString> local_read_ids = FROM_LIST_TO_SET(QSet<QString>, local_read_ids_list);
+
+  // 3.
+  auto local_starred_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Starred);
+  QSet<QString> remote_starred_ids = FROM_LIST_TO_SET(QSet<QString>, remote_starred_ids_list);
+  QSet<QString> local_starred_ids = FROM_LIST_TO_SET(QSet<QString>, local_starred_ids_list);
+
+  // 4.
+  QSet<QString> to_download;
+
+  // Undownloaded unread e-mails.
+  to_download += remote_unread_ids - local_unread_ids;
+
+  // Undownloaded read e-mails.
+  if (!m_downloadOnlyUnreadMessages) {
+    to_download += remote_read_ids - local_read_ids;
   }
+
+  // Undownloaded starred e-mails.
+  to_download += remote_starred_ids - local_starred_ids;
+
+  // Read e-mails newly marked as unread in service.
+  auto moved_read = local_read_ids.intersect(remote_unread_ids);
+
+  to_download += moved_read;
+
+  // Unread e-mails newly marked as read in service.
+  if (!m_downloadOnlyUnreadMessages) {
+    auto moved_unread = local_unread_ids.intersect(remote_read_ids);
+
+    to_download += moved_unread;
   }
-    else {
+
-      error = Feed::Status::NetworkError;
+  qDebugNN << LOGSEC_GMAIL << "Will download" << QUOTE_W_SPACE(to_download.size()) << "e-mails.";
-      return messages;
+
-    }
+  auto messages = obtainAndDecodeFullMessages(QList<QString>(to_download.values()), stream_id, custom_proxy);
-  } while (!next_page_token.isEmpty());
 
   error = Feed::Status::Normal;
   return messages;
@@ -377,6 +376,72 @@ QNetworkReply::NetworkError GmailNetworkFactory::markMessagesStarred(RootItem::I
   return QNetworkReply::NetworkError::NoError;
 }
 
+QStringList GmailNetworkFactory::list(const QString& stream_id,
+                                      const QStringList& label_ids,
+                                      int max_results,
+                                      const QString& query,
+                                      const QNetworkProxy& custom_proxy) {
+  QList<QString> message_ids;
+  QString next_page_token;
+  QString bearer = m_oauth2->bearer().toLocal8Bit();
+  int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
+
+  do {
+    QString target_url = QSL(GMAIL_API_MSGS_LIST);
+
+    target_url += QSL("?labelIds=%1").arg(stream_id);
+
+    if (!label_ids.isEmpty()) {
+      for (const QString& label_id : label_ids) {
+        target_url += QSL("&labelIds=%1").arg(label_id);
+      }
+    }
+
+    if (!query.isEmpty()) {
+      target_url += QSL("&q=%1").arg(query);
+    }
+
+    int remaining = max_results - message_ids.size();
+
+    if (max_results <= 0 || remaining > 500) {
+      target_url += QSL("&maxResults=500");
+    }
+    else {
+      target_url += QSL("&maxResults=%1").arg(remaining);
+    }
+
+    if (!next_page_token.isEmpty()) {
+      target_url += QString("&pageToken=%1").arg(next_page_token);
+    }
+
+    QByteArray messages_raw_data;
+    auto netw = NetworkFactory::performNetworkOperation(target_url,
+                                                        timeout,
+                                                        {},
+                                                        messages_raw_data,
+                                                        QNetworkAccessManager::Operation::GetOperation,
+                                                        { { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(),
+                                                          bearer.toLocal8Bit() } },
+                                                        false,
+                                                        {},
+                                                        {},
+                                                        custom_proxy);
+
+    if (netw.m_networkError == QNetworkReply::NetworkError::NoError) {
+      // We parse this chunk.
+      QString messages_data = QString::fromUtf8(messages_raw_data);
+
+      message_ids << decodeLiteMessages(messages_data, next_page_token);
+
+    }
+    else {
+      throw NetworkException(netw.m_networkError, tr("failed to download IDs of e-mail messages"));
+    }
+  } while (!next_page_token.isEmpty() && (max_results <= 0 || message_ids.size() < max_results));
+
+  return message_ids;
+}
+
 QVariantHash GmailNetworkFactory::getProfile(const QNetworkProxy& custom_proxy) {
   QString bearer = m_oauth2->bearer().toLocal8Bit();
 
@@ -485,8 +550,6 @@ bool GmailNetworkFactory::fillFullMessage(Message& msg, const QJsonObject& json,
   msg.m_createdFromFeed = true;
   msg.m_created = TextFactory::parseDateTime(headers[QSL("Date")]);
 
-  QString aa = msg.m_rawContents;
-
   if (msg.m_title.isEmpty()) {
     msg.m_title = tr("No subject");
   }
@@ -607,15 +670,15 @@ QMap<QString, QString> GmailNetworkFactory::getMessageMetadata(const QString& ms
   }
 }
 
-bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages,
+QList<Message> GmailNetworkFactory::obtainAndDecodeFullMessages(const QStringList& message_ids,
                                                                 const QString& feed_id,
                                                                 const QNetworkProxy& custom_proxy) {
-  QHash<QString, int> msgs;
+  QHash<QString, Message> msgs;
   int next_message = 0;
   QString bearer = m_oauth2->bearer();
 
   if (bearer.isEmpty()) {
-    return false;
+    return {};
   }
 
   do {
@@ -623,16 +686,19 @@ bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages,
 
     multi->setContentType(QHttpMultiPart::ContentType::MixedType);
 
-    for (int window = next_message + 100; next_message < window && next_message < messages.size(); next_message++ ) {
+    for (int window = next_message + 100; next_message < window && next_message < message_ids.size(); next_message++ ) {
-      Message msg = messages[next_message];
+      QString msg_id = message_ids[next_message];
+      Message msg;
       QHttpPart part;
 
+      msg.m_customId = msg_id;
+
       part.setRawHeader(HTTP_HEADERS_CONTENT_TYPE, GMAIL_CONTENT_TYPE_HTTP);
-      QString full_msg_endpoint = QSL("GET /gmail/v1/users/me/messages/%1\r\n").arg(msg.m_customId);
+      QString full_msg_endpoint = QSL("GET /gmail/v1/users/me/messages/%1\r\n").arg(msg_id);
 
       part.setBody(full_msg_endpoint.toUtf8());
       multi->append(part);
-      msgs.insert(msg.m_customId, next_message);
+      msgs.insert(msg_id, msg);
     }
 
     QList<QPair<QByteArray, QByteArray>> headers;
@@ -660,42 +726,46 @@ bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages,
         QString msg_id = msg_doc[QSL("id")].toString();
 
         if (msgs.contains(msg_id)) {
-          Message& msg = messages[msgs.value(msg_id)];
+          Message& msg = msgs[msg_id];
+
+          if (msg.m_customId == "17f9bff0f98a868e") {
+            int a = 5;
+          }
 
           if (!fillFullMessage(msg, msg_doc, feed_id)) {
-            qWarningNN << LOGSEC_GMAIL << "Failed to get full message for custom ID:" << QUOTE_W_SPACE_DOT(msg.m_customId);
+            qWarningNN << LOGSEC_GMAIL << "Failed to get (or deliberately skipped) full message for custom ID:"
+                       << QUOTE_W_SPACE_DOT(msg.m_customId);
+
+            msgs.remove(msg_id);
           }
         }
       }
+
+      multi->deleteLater();
     }
     else {
-      return false;
+      multi->deleteLater();
+      return {};
     }
   }
-  while (next_message < messages.size());
+  while (next_message < message_ids.size());
 
-  return true;
+  return msgs.values();
 }
 
-QList<Message> GmailNetworkFactory::decodeLiteMessages(const QString& messages_json_data,
+QStringList GmailNetworkFactory::decodeLiteMessages(const QString& messages_json_data, QString& next_page_token) {
-                                                       const QString& stream_id,
+  QList<QString> message_ids;
-                                                       QString& next_page_token) {
-  QList<Message> messages;
   QJsonObject top_object = QJsonDocument::fromJson(messages_json_data.toUtf8()).object();
   QJsonArray json_msgs = top_object[QSL("messages")].toArray();
 
   next_page_token = top_object[QSL("nextPageToken")].toString();
-  messages.reserve(json_msgs.count());
+  message_ids.reserve(json_msgs.count());
 
   for (const QJsonValue& obj : json_msgs) {
     auto message_obj = obj.toObject();
-    Message message;
 
-    message.m_customId = message_obj[QSL("id")].toString();
+    message_ids << message_obj[QSL("id")].toString();
-    message.m_feedId = stream_id;
-
-    messages.append(message);
   }
 
-  return messages;
+  return message_ids;
 }

src/librssguard/services/gmail/gmailnetworkfactory.h

@@ -10,6 +10,7 @@
 #include "3rd-party/mimesis/mimesis.hpp"
 #include "services/abstract/feed.h"
 #include "services/abstract/rootitem.h"
+#include "services/abstract/serviceroot.h"
 
 #include <QNetworkReply>
 
@@ -41,13 +42,19 @@ class GmailNetworkFactory : public QObject {
     // API methods.
     QString sendEmail(Mimesis::Message msg, const QNetworkProxy& custom_proxy, Message* reply_to_message = nullptr);
     Downloader* downloadAttachment(const QString& msg_id, const QString& attachment_id, const QNetworkProxy& custom_proxy);
-    QList<Message> messages(const QString& stream_id, Feed::Status& error, const QNetworkProxy& custom_proxy);
+    QList<Message> messages(const QString& stream_id, const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                            Feed::Status& error, const QNetworkProxy& custom_proxy);
     QNetworkReply::NetworkError markMessagesRead(RootItem::ReadStatus status,
                                                  const QStringList& custom_ids,
                                                  const QNetworkProxy& custom_proxy);
     QNetworkReply::NetworkError markMessagesStarred(RootItem::Importance importance,
                                                     const QStringList& custom_ids,
                                                     const QNetworkProxy& custom_proxy);
+    QStringList list(const QString& stream_id,
+                     const QStringList& label_ids,
+                     int max_results,
+                     const QString& query,
+                     const QNetworkProxy& custom_proxy);
     QVariantHash getProfile(const QNetworkProxy& custom_proxy);
 
   private slots:
@@ -59,10 +66,10 @@ class GmailNetworkFactory : public QObject {
     QMap<QString, QString> getMessageMetadata(const QString& msg_id,
                                               const QStringList& metadata,
                                               const QNetworkProxy& custom_proxy);
-    bool obtainAndDecodeFullMessages(QList<Message>& lite_messages,
+    QList<Message> obtainAndDecodeFullMessages(const QStringList& message_ids,
                                                const QString& feed_id,
                                                const QNetworkProxy& custom_proxy);
-    QList<Message> decodeLiteMessages(const QString& messages_json_data, const QString& stream_id, QString& next_page_token);
+    QStringList decodeLiteMessages(const QString& messages_json_data, QString& next_page_token);
 
     void initializeOauth();
 

src/librssguard/services/gmail/gmailserviceroot.cpp

@@ -34,7 +34,10 @@ void GmailServiceRoot::replyToEmail() {
 
 RootItem* GmailServiceRoot::obtainNewTreeForSyncIn() const {
   auto* root = new RootItem();
-  Feed* inbox = new Feed(tr("Inbox"), QSL(GMAIL_SYSTEM_LABEL_INBOX), qApp->icons()->fromTheme(QSL("mail-inbox")), root);
+  Feed* inbox = new Feed(tr("Inbox"),
+                         QSL(GMAIL_SYSTEM_LABEL_INBOX),
+                         qApp->icons()->fromTheme(QSL("mail-inbox"), QSL("mail-inbox-symbolic")),
+                         root);
 
   inbox->setKeepOnTop(true);
 
@@ -81,7 +84,7 @@ QList<Message> GmailServiceRoot::obtainNewMessages(Feed* feed,
   Q_UNUSED(tagged_messages)
 
   Feed::Status error = Feed::Status::Normal;
-  QList<Message> messages = network()->messages(feed->customId(), error, networkProxy());
+  QList<Message> messages = network()->messages(feed->customId(), stated_messages, error, networkProxy());
 
   if (error != Feed::Status::NewMessages && error != Feed::Status::Normal) {
     throw FeedFetchException(error);
@@ -90,6 +93,10 @@ QList<Message> GmailServiceRoot::obtainNewMessages(Feed* feed,
   return messages;
 }
 
+bool GmailServiceRoot::wantsBaggedIdsOfExistingMessages() const {
+  return true;
+}
+
 bool GmailServiceRoot::downloadAttachmentOnMyOwn(const QUrl& url) const {
   QString str_url = url.toString();
   QString attachment_id = str_url.mid(str_url.indexOf(QL1C('?')) + 1);

src/librssguard/services/gmail/gmailserviceroot.h

@@ -34,6 +34,7 @@ class GmailServiceRoot : public ServiceRoot, public CacheForServiceRoot {
     virtual QList<Message> obtainNewMessages(Feed* feed,
                                              const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
                                              const QHash<QString, QStringList>& tagged_messages);
+    virtual bool wantsBaggedIdsOfExistingMessages() const;
 
   protected:
     virtual RootItem* obtainNewTreeForSyncIn() const;