gmail: refined e-mail downloading, now supports ISA - faster synchronization, particularly subsequent
This commit is contained in:
Martin Rotter
parent
28ec8e642b
commit
4539fcc777
8 changed files with 180 additions and 93 deletions
|
|
@ -19,6 +19,7 @@ endif()
|
|||
FILE(GLOB TS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.ts)
|
||||
|
||||
set_source_files_properties(${TS_FILES} PROPERTIES OUTPUT_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
|
||||
qt_add_translation(QM_FILES
|
||||
${TS_FILES}
|
||||
OPTIONS "-compress"
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ I organized the supported web-based feed readers into an elegant table:
|
|||
| Service | Two-way Synchronization | [Intelligent Synchronization Algorithm](#intel) (ISA) <sup>1</sup> | Synchronized Labels <sup>2</sup> <a id="sfrl"></a> | OAuth <sup>4</sup> |
|
||||
| :--- | :---: | :---: | :---: | :---:
|
||||
| Feedly | ✅ | ❌ | ✅ | ✅ (only for official binaries)
|
||||
| Gmail | ✅ | ❌ | ❌ | ✅
|
||||
| Gmail | ✅ | ✅ | ❌ | ✅
|
||||
| Google Reader API <sup>3</sup> | ✅ | ✅ | ✅ | ✅ (only for Inoreader)
|
||||
| Nextcloud News | ✅ | ❌ | ❌ | ❌
|
||||
| Tiny Tiny RSS | ✅ | ❌ | ✅ | ❌
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@
|
|||
<file>./graphics/Breeze/actions/22/list-add.svg</file>
|
||||
<file>./graphics/Breeze/actions/22/list-remove.svg</file>
|
||||
<file>./graphics/Breeze/actions/32/mail-attachment.svg</file>
|
||||
<file>./graphics/Breeze/actions/symbolic/mail-inbox-symbolic.svg</file>
|
||||
<file>./graphics/Breeze/actions/32/mail-mark-important.svg</file>
|
||||
<file>./graphics/Breeze/actions/32/mail-mark-junk.svg</file>
|
||||
<file>./graphics/Breeze/actions/32/mail-mark-read.svg</file>
|
||||
|
|
@ -122,6 +123,7 @@
|
|||
<file>./graphics/Breeze Dark/actions/22/list-add.svg</file>
|
||||
<file>./graphics/Breeze Dark/actions/22/list-remove.svg</file>
|
||||
<file>./graphics/Breeze Dark/actions/32/mail-attachment.svg</file>
|
||||
<file>./graphics/Breeze Dark/actions/symbolic/mail-inbox-symbolic.svg</file>
|
||||
<file>./graphics/Breeze Dark/actions/32/mail-mark-important.svg</file>
|
||||
<file>./graphics/Breeze Dark/actions/32/mail-mark-junk.svg</file>
|
||||
<file>./graphics/Breeze Dark/actions/32/mail-mark-read.svg</file>
|
||||
|
|
|
|||
|
|
@ -387,7 +387,6 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
|
|||
|
||||
feed->setStatus(feed_ex.feedStatus(), feed_ex.message());
|
||||
}
|
||||
|
||||
catch (const ApplicationException& app_ex) {
|
||||
qCriticalNN << LOGSEC_NETWORK
|
||||
<< "Unknown error when fetching feed:"
|
||||
|
|
|
|||
|
|
@ -179,81 +179,80 @@ Downloader* GmailNetworkFactory::downloadAttachment(const QString& msg_id,
|
|||
}
|
||||
|
||||
QList<Message> GmailNetworkFactory::messages(const QString& stream_id,
|
||||
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
|
||||
Feed::Status& error,
|
||||
const QNetworkProxy& custom_proxy) {
|
||||
QString bearer = m_oauth2->bearer().toLocal8Bit();
|
||||
QString next_page_token;
|
||||
QList<Message> messages;
|
||||
ulong msecs_wait_between_batches = 1500;
|
||||
|
||||
if (bearer.isEmpty()) {
|
||||
error = Feed::Status::AuthError;
|
||||
return QList<Message>();
|
||||
return {};
|
||||
}
|
||||
|
||||
// We need to quit event loop when the download finishes.
|
||||
QString target_url;
|
||||
int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
|
||||
// 1. Get unread IDs for a feed.
|
||||
// 2. Get read IDs for a feed.
|
||||
// 3. Get starred IDs for a feed.
|
||||
// 4. Download messages/contents for missing or changed IDs.
|
||||
QStringList remote_read_ids_list, remote_unread_ids_list, remote_starred_ids_list;
|
||||
|
||||
do {
|
||||
target_url = QSL(GMAIL_API_MSGS_LIST);
|
||||
target_url += QSL("?labelIds=%1").arg(stream_id);
|
||||
try {
|
||||
remote_starred_ids_list = list(stream_id, {}, 0, QSL("is:starred"), custom_proxy);
|
||||
remote_unread_ids_list = list(stream_id, {}, batchSize(), QSL("is:unread"), custom_proxy);
|
||||
|
||||
if (downloadOnlyUnreadMessages()) {
|
||||
target_url += QSL("&labelIds=%1").arg(QSL(GMAIL_SYSTEM_LABEL_UNREAD));
|
||||
}
|
||||
|
||||
if (batchSize() > 0) {
|
||||
target_url += QSL("&maxResults=%1").arg(batchSize());
|
||||
}
|
||||
|
||||
if (!next_page_token.isEmpty()) {
|
||||
target_url += QString("&pageToken=%1").arg(next_page_token);
|
||||
}
|
||||
|
||||
QByteArray messages_raw_data;
|
||||
auto netw = NetworkFactory::performNetworkOperation(target_url,
|
||||
timeout,
|
||||
{},
|
||||
messages_raw_data,
|
||||
QNetworkAccessManager::Operation::GetOperation,
|
||||
{ { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(),
|
||||
bearer.toLocal8Bit() } },
|
||||
false,
|
||||
{},
|
||||
{},
|
||||
custom_proxy);
|
||||
|
||||
if (netw.m_networkError == QNetworkReply::NetworkError::NoError) {
|
||||
// We parse this chunk.
|
||||
QString messages_data = QString::fromUtf8(messages_raw_data);
|
||||
QList<Message> more_messages = decodeLiteMessages(messages_data, stream_id, next_page_token);
|
||||
|
||||
if (!more_messages.isEmpty()) {
|
||||
// Now, we via batch HTTP request obtain full data for each message.
|
||||
bool obtained = obtainAndDecodeFullMessages(more_messages, stream_id, custom_proxy);
|
||||
|
||||
if (obtained) {
|
||||
messages.append(more_messages);
|
||||
QThread::msleep(msecs_wait_between_batches);
|
||||
|
||||
// New batch of messages was obtained, check if we have enough.
|
||||
if (batchSize() > 0 && batchSize() <= messages.size()) {
|
||||
// We have enough messages.
|
||||
break;
|
||||
if (!downloadOnlyUnreadMessages()) {
|
||||
remote_read_ids_list = list(stream_id, {}, batchSize(), QSL("is:read"), custom_proxy);
|
||||
}
|
||||
}
|
||||
else {
|
||||
error = Feed::Status::NetworkError;
|
||||
return messages;
|
||||
catch (const NetworkException& net_ex) {
|
||||
qCriticalNN << LOGSEC_GMAIL
|
||||
<< "Failed to get list of e-mail IDs:" << QUOTE_W_SPACE_DOT(net_ex.message());
|
||||
return {};
|
||||
}
|
||||
|
||||
// 1.
|
||||
auto local_unread_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Unread);
|
||||
QSet<QString> remote_unread_ids = FROM_LIST_TO_SET(QSet<QString>, remote_unread_ids_list);
|
||||
QSet<QString> local_unread_ids = FROM_LIST_TO_SET(QSet<QString>, local_unread_ids_list);
|
||||
|
||||
// 2.
|
||||
auto local_read_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Read);
|
||||
QSet<QString> remote_read_ids = FROM_LIST_TO_SET(QSet<QString>, remote_read_ids_list);
|
||||
QSet<QString> local_read_ids = FROM_LIST_TO_SET(QSet<QString>, local_read_ids_list);
|
||||
|
||||
// 3.
|
||||
auto local_starred_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Starred);
|
||||
QSet<QString> remote_starred_ids = FROM_LIST_TO_SET(QSet<QString>, remote_starred_ids_list);
|
||||
QSet<QString> local_starred_ids = FROM_LIST_TO_SET(QSet<QString>, local_starred_ids_list);
|
||||
|
||||
// 4.
|
||||
QSet<QString> to_download;
|
||||
|
||||
// Undownloaded unread e-mails.
|
||||
to_download += remote_unread_ids - local_unread_ids;
|
||||
|
||||
// Undownloaded read e-mails.
|
||||
if (!m_downloadOnlyUnreadMessages) {
|
||||
to_download += remote_read_ids - local_read_ids;
|
||||
}
|
||||
|
||||
// Undownloaded starred e-mails.
|
||||
to_download += remote_starred_ids - local_starred_ids;
|
||||
|
||||
// Read e-mails newly marked as unread in service.
|
||||
auto moved_read = local_read_ids.intersect(remote_unread_ids);
|
||||
|
||||
to_download += moved_read;
|
||||
|
||||
// Unread e-mails newly marked as read in service.
|
||||
if (!m_downloadOnlyUnreadMessages) {
|
||||
auto moved_unread = local_unread_ids.intersect(remote_read_ids);
|
||||
|
||||
to_download += moved_unread;
|
||||
}
|
||||
else {
|
||||
error = Feed::Status::NetworkError;
|
||||
return messages;
|
||||
}
|
||||
} while (!next_page_token.isEmpty());
|
||||
|
||||
qDebugNN << LOGSEC_GMAIL << "Will download" << QUOTE_W_SPACE(to_download.size()) << "e-mails.";
|
||||
|
||||
auto messages = obtainAndDecodeFullMessages(QList<QString>(to_download.values()), stream_id, custom_proxy);
|
||||
|
||||
error = Feed::Status::Normal;
|
||||
return messages;
|
||||
|
|
@ -377,6 +376,72 @@ QNetworkReply::NetworkError GmailNetworkFactory::markMessagesStarred(RootItem::I
|
|||
return QNetworkReply::NetworkError::NoError;
|
||||
}
|
||||
|
||||
QStringList GmailNetworkFactory::list(const QString& stream_id,
|
||||
const QStringList& label_ids,
|
||||
int max_results,
|
||||
const QString& query,
|
||||
const QNetworkProxy& custom_proxy) {
|
||||
QList<QString> message_ids;
|
||||
QString next_page_token;
|
||||
QString bearer = m_oauth2->bearer().toLocal8Bit();
|
||||
int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
|
||||
|
||||
do {
|
||||
QString target_url = QSL(GMAIL_API_MSGS_LIST);
|
||||
|
||||
target_url += QSL("?labelIds=%1").arg(stream_id);
|
||||
|
||||
if (!label_ids.isEmpty()) {
|
||||
for (const QString& label_id : label_ids) {
|
||||
target_url += QSL("&labelIds=%1").arg(label_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (!query.isEmpty()) {
|
||||
target_url += QSL("&q=%1").arg(query);
|
||||
}
|
||||
|
||||
int remaining = max_results - message_ids.size();
|
||||
|
||||
if (max_results <= 0 || remaining > 500) {
|
||||
target_url += QSL("&maxResults=500");
|
||||
}
|
||||
else {
|
||||
target_url += QSL("&maxResults=%1").arg(remaining);
|
||||
}
|
||||
|
||||
if (!next_page_token.isEmpty()) {
|
||||
target_url += QString("&pageToken=%1").arg(next_page_token);
|
||||
}
|
||||
|
||||
QByteArray messages_raw_data;
|
||||
auto netw = NetworkFactory::performNetworkOperation(target_url,
|
||||
timeout,
|
||||
{},
|
||||
messages_raw_data,
|
||||
QNetworkAccessManager::Operation::GetOperation,
|
||||
{ { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(),
|
||||
bearer.toLocal8Bit() } },
|
||||
false,
|
||||
{},
|
||||
{},
|
||||
custom_proxy);
|
||||
|
||||
if (netw.m_networkError == QNetworkReply::NetworkError::NoError) {
|
||||
// We parse this chunk.
|
||||
QString messages_data = QString::fromUtf8(messages_raw_data);
|
||||
|
||||
message_ids << decodeLiteMessages(messages_data, next_page_token);
|
||||
|
||||
}
|
||||
else {
|
||||
throw NetworkException(netw.m_networkError, tr("failed to download IDs of e-mail messages"));
|
||||
}
|
||||
} while (!next_page_token.isEmpty() && (max_results <= 0 || message_ids.size() < max_results));
|
||||
|
||||
return message_ids;
|
||||
}
|
||||
|
||||
QVariantHash GmailNetworkFactory::getProfile(const QNetworkProxy& custom_proxy) {
|
||||
QString bearer = m_oauth2->bearer().toLocal8Bit();
|
||||
|
||||
|
|
@ -485,8 +550,6 @@ bool GmailNetworkFactory::fillFullMessage(Message& msg, const QJsonObject& json,
|
|||
msg.m_createdFromFeed = true;
|
||||
msg.m_created = TextFactory::parseDateTime(headers[QSL("Date")]);
|
||||
|
||||
QString aa = msg.m_rawContents;
|
||||
|
||||
if (msg.m_title.isEmpty()) {
|
||||
msg.m_title = tr("No subject");
|
||||
}
|
||||
|
|
@ -607,15 +670,15 @@ QMap<QString, QString> GmailNetworkFactory::getMessageMetadata(const QString& ms
|
|||
}
|
||||
}
|
||||
|
||||
bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages,
|
||||
QList<Message> GmailNetworkFactory::obtainAndDecodeFullMessages(const QStringList& message_ids,
|
||||
const QString& feed_id,
|
||||
const QNetworkProxy& custom_proxy) {
|
||||
QHash<QString, int> msgs;
|
||||
QHash<QString, Message> msgs;
|
||||
int next_message = 0;
|
||||
QString bearer = m_oauth2->bearer();
|
||||
|
||||
if (bearer.isEmpty()) {
|
||||
return false;
|
||||
return {};
|
||||
}
|
||||
|
||||
do {
|
||||
|
|
@ -623,16 +686,19 @@ bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages,
|
|||
|
||||
multi->setContentType(QHttpMultiPart::ContentType::MixedType);
|
||||
|
||||
for (int window = next_message + 100; next_message < window && next_message < messages.size(); next_message++ ) {
|
||||
Message msg = messages[next_message];
|
||||
for (int window = next_message + 100; next_message < window && next_message < message_ids.size(); next_message++ ) {
|
||||
QString msg_id = message_ids[next_message];
|
||||
Message msg;
|
||||
QHttpPart part;
|
||||
|
||||
msg.m_customId = msg_id;
|
||||
|
||||
part.setRawHeader(HTTP_HEADERS_CONTENT_TYPE, GMAIL_CONTENT_TYPE_HTTP);
|
||||
QString full_msg_endpoint = QSL("GET /gmail/v1/users/me/messages/%1\r\n").arg(msg.m_customId);
|
||||
QString full_msg_endpoint = QSL("GET /gmail/v1/users/me/messages/%1\r\n").arg(msg_id);
|
||||
|
||||
part.setBody(full_msg_endpoint.toUtf8());
|
||||
multi->append(part);
|
||||
msgs.insert(msg.m_customId, next_message);
|
||||
msgs.insert(msg_id, msg);
|
||||
}
|
||||
|
||||
QList<QPair<QByteArray, QByteArray>> headers;
|
||||
|
|
@ -660,42 +726,46 @@ bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages,
|
|||
QString msg_id = msg_doc[QSL("id")].toString();
|
||||
|
||||
if (msgs.contains(msg_id)) {
|
||||
Message& msg = messages[msgs.value(msg_id)];
|
||||
Message& msg = msgs[msg_id];
|
||||
|
||||
if (msg.m_customId == "17f9bff0f98a868e") {
|
||||
int a = 5;
|
||||
}
|
||||
|
||||
if (!fillFullMessage(msg, msg_doc, feed_id)) {
|
||||
qWarningNN << LOGSEC_GMAIL << "Failed to get full message for custom ID:" << QUOTE_W_SPACE_DOT(msg.m_customId);
|
||||
qWarningNN << LOGSEC_GMAIL << "Failed to get (or deliberately skipped) full message for custom ID:"
|
||||
<< QUOTE_W_SPACE_DOT(msg.m_customId);
|
||||
|
||||
msgs.remove(msg_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
multi->deleteLater();
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
multi->deleteLater();
|
||||
return {};
|
||||
}
|
||||
}
|
||||
while (next_message < messages.size());
|
||||
while (next_message < message_ids.size());
|
||||
|
||||
return true;
|
||||
return msgs.values();
|
||||
}
|
||||
|
||||
QList<Message> GmailNetworkFactory::decodeLiteMessages(const QString& messages_json_data,
|
||||
const QString& stream_id,
|
||||
QString& next_page_token) {
|
||||
QList<Message> messages;
|
||||
QStringList GmailNetworkFactory::decodeLiteMessages(const QString& messages_json_data, QString& next_page_token) {
|
||||
QList<QString> message_ids;
|
||||
QJsonObject top_object = QJsonDocument::fromJson(messages_json_data.toUtf8()).object();
|
||||
QJsonArray json_msgs = top_object[QSL("messages")].toArray();
|
||||
|
||||
next_page_token = top_object[QSL("nextPageToken")].toString();
|
||||
messages.reserve(json_msgs.count());
|
||||
message_ids.reserve(json_msgs.count());
|
||||
|
||||
for (const QJsonValue& obj : json_msgs) {
|
||||
auto message_obj = obj.toObject();
|
||||
Message message;
|
||||
|
||||
message.m_customId = message_obj[QSL("id")].toString();
|
||||
message.m_feedId = stream_id;
|
||||
|
||||
messages.append(message);
|
||||
message_ids << message_obj[QSL("id")].toString();
|
||||
}
|
||||
|
||||
return messages;
|
||||
return message_ids;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include "3rd-party/mimesis/mimesis.hpp"
|
||||
#include "services/abstract/feed.h"
|
||||
#include "services/abstract/rootitem.h"
|
||||
#include "services/abstract/serviceroot.h"
|
||||
|
||||
#include <QNetworkReply>
|
||||
|
||||
|
|
@ -41,13 +42,19 @@ class GmailNetworkFactory : public QObject {
|
|||
// API methods.
|
||||
QString sendEmail(Mimesis::Message msg, const QNetworkProxy& custom_proxy, Message* reply_to_message = nullptr);
|
||||
Downloader* downloadAttachment(const QString& msg_id, const QString& attachment_id, const QNetworkProxy& custom_proxy);
|
||||
QList<Message> messages(const QString& stream_id, Feed::Status& error, const QNetworkProxy& custom_proxy);
|
||||
QList<Message> messages(const QString& stream_id, const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
|
||||
Feed::Status& error, const QNetworkProxy& custom_proxy);
|
||||
QNetworkReply::NetworkError markMessagesRead(RootItem::ReadStatus status,
|
||||
const QStringList& custom_ids,
|
||||
const QNetworkProxy& custom_proxy);
|
||||
QNetworkReply::NetworkError markMessagesStarred(RootItem::Importance importance,
|
||||
const QStringList& custom_ids,
|
||||
const QNetworkProxy& custom_proxy);
|
||||
QStringList list(const QString& stream_id,
|
||||
const QStringList& label_ids,
|
||||
int max_results,
|
||||
const QString& query,
|
||||
const QNetworkProxy& custom_proxy);
|
||||
QVariantHash getProfile(const QNetworkProxy& custom_proxy);
|
||||
|
||||
private slots:
|
||||
|
|
@ -59,10 +66,10 @@ class GmailNetworkFactory : public QObject {
|
|||
QMap<QString, QString> getMessageMetadata(const QString& msg_id,
|
||||
const QStringList& metadata,
|
||||
const QNetworkProxy& custom_proxy);
|
||||
bool obtainAndDecodeFullMessages(QList<Message>& lite_messages,
|
||||
QList<Message> obtainAndDecodeFullMessages(const QStringList& message_ids,
|
||||
const QString& feed_id,
|
||||
const QNetworkProxy& custom_proxy);
|
||||
QList<Message> decodeLiteMessages(const QString& messages_json_data, const QString& stream_id, QString& next_page_token);
|
||||
QStringList decodeLiteMessages(const QString& messages_json_data, QString& next_page_token);
|
||||
|
||||
void initializeOauth();
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,10 @@ void GmailServiceRoot::replyToEmail() {
|
|||
|
||||
RootItem* GmailServiceRoot::obtainNewTreeForSyncIn() const {
|
||||
auto* root = new RootItem();
|
||||
Feed* inbox = new Feed(tr("Inbox"), QSL(GMAIL_SYSTEM_LABEL_INBOX), qApp->icons()->fromTheme(QSL("mail-inbox")), root);
|
||||
Feed* inbox = new Feed(tr("Inbox"),
|
||||
QSL(GMAIL_SYSTEM_LABEL_INBOX),
|
||||
qApp->icons()->fromTheme(QSL("mail-inbox"), QSL("mail-inbox-symbolic")),
|
||||
root);
|
||||
|
||||
inbox->setKeepOnTop(true);
|
||||
|
||||
|
|
@ -81,7 +84,7 @@ QList<Message> GmailServiceRoot::obtainNewMessages(Feed* feed,
|
|||
Q_UNUSED(tagged_messages)
|
||||
|
||||
Feed::Status error = Feed::Status::Normal;
|
||||
QList<Message> messages = network()->messages(feed->customId(), error, networkProxy());
|
||||
QList<Message> messages = network()->messages(feed->customId(), stated_messages, error, networkProxy());
|
||||
|
||||
if (error != Feed::Status::NewMessages && error != Feed::Status::Normal) {
|
||||
throw FeedFetchException(error);
|
||||
|
|
@ -90,6 +93,10 @@ QList<Message> GmailServiceRoot::obtainNewMessages(Feed* feed,
|
|||
return messages;
|
||||
}
|
||||
|
||||
bool GmailServiceRoot::wantsBaggedIdsOfExistingMessages() const {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GmailServiceRoot::downloadAttachmentOnMyOwn(const QUrl& url) const {
|
||||
QString str_url = url.toString();
|
||||
QString attachment_id = str_url.mid(str_url.indexOf(QL1C('?')) + 1);
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ class GmailServiceRoot : public ServiceRoot, public CacheForServiceRoot {
|
|||
virtual QList<Message> obtainNewMessages(Feed* feed,
|
||||
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
|
||||
const QHash<QString, QStringList>& tagged_messages);
|
||||
virtual bool wantsBaggedIdsOfExistingMessages() const;
|
||||
|
||||
protected:
|
||||
virtual RootItem* obtainNewTreeForSyncIn() const;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue