pogmommy/rssguard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix gmail message downloading with batch size > 100.

Browse source
This commit is contained in:
Martin Rotter 2020-11-10 07:47:44 +01:00
parent 86ab890865
commit 47d2eb05c4
2 changed files with 51 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
src/librssguard/services/gmail/network/gmailnetworkfactory.cpp
View file

@ -189,14 +189,13 @@ QList<Message> GmailNetworkFactory::messages(const QString& stream_id, Feed::Sta
// We parse this chunk. // We parse this chunk.
QString messages_data = downloader.lastOutputData(); QString messages_data = downloader.lastOutputData();
QList<Message> more_messages = decodeLiteMessages(messages_data, stream_id, next_page_token); QList<Message> more_messages = decodeLiteMessages(messages_data, stream_id, next_page_token);
QList<Message> full_messages;
if (!more_messages.isEmpty()) { if (!more_messages.isEmpty()) {
// Now, we via batch HTTP request obtain full data for each message. // Now, we via batch HTTP request obtain full data for each message.
bool obtained = obtainAndDecodeFullMessages(more_messages, stream_id, full_messages); bool obtained = obtainAndDecodeFullMessages(more_messages, stream_id);
if (obtained) { if (obtained) {
messages.append(full_messages); messages.append(more_messages);
// New batch of messages was obtained, check if we have enough. // New batch of messages was obtained, check if we have enough.
if (batchSize() > 0 && batchSize() <= messages.size()) { if (batchSize() > 0 && batchSize() <= messages.size()) {
@ -479,67 +478,69 @@ QMap<QString, QString> GmailNetworkFactory::getMessageMetadata(const QString& ms
} }
} }
bool GmailNetworkFactory::obtainAndDecodeFullMessages(const QList<Message>& lite_messages, bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages, const QString& feed_id) {
const QString& feed_id, QHash<QString, int> msgs;
QList<Message>& full_messages) { int next_message = 0;
auto* multi = new QHttpMultiPart();
multi->setContentType(QHttpMultiPart::ContentType::MixedType);
QHash<QString, Message> msgs;
for (const Message& msg : lite_messages) {
QHttpPart part;
part.setRawHeader(HTTP_HEADERS_CONTENT_TYPE, GMAIL_CONTENT_TYPE_HTTP);
QString full_msg_endpoint = QString("GET /gmail/v1/users/me/messages/%1\r\n").arg(msg.m_customId);
part.setBody(full_msg_endpoint.toUtf8());
multi->append(part);
msgs.insert(msg.m_customId, msg);
}
QString bearer = m_oauth2->bearer(); QString bearer = m_oauth2->bearer();
if (bearer.isEmpty()) { if (bearer.isEmpty()) {
return false; return false;
} }
QList<QPair<QByteArray, QByteArray>> headers; do {
QList<HttpResponse> output; auto* multi = new QHttpMultiPart();
int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
headers.append(QPair<QByteArray, QByteArray>(QString(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(), multi->setContentType(QHttpMultiPart::ContentType::MixedType);
bearer.toLocal8Bit()));
NetworkResult res = NetworkFactory::performNetworkOperation(GMAIL_API_BATCH, for (int window = next_message + 100; next_message < window && next_message < messages.size(); next_message++ ) {
timeout, Message msg = messages[next_message];
multi, QHttpPart part;
output,
QNetworkAccessManager::Operation::PostOperation,
headers);
if (res.first == QNetworkReply::NetworkError::NoError) { part.setRawHeader(HTTP_HEADERS_CONTENT_TYPE, GMAIL_CONTENT_TYPE_HTTP);
// We parse each part of HTTP response (it contains HTTP headers and payload with msg full data). QString full_msg_endpoint = QString("GET /gmail/v1/users/me/messages/%1\r\n").arg(msg.m_customId);
for (const HttpResponse& part : output) {
auto xx = part.body();
QJsonObject msg_doc = QJsonDocument::fromJson(part.body().toUtf8()).object();
QString msg_id = msg_doc["id"].toString();
if (msgs.contains(msg_id)) { part.setBody(full_msg_endpoint.toUtf8());
Message& msg = msgs[msg_id]; multi->append(part);
msgs.insert(msg.m_customId, next_message);
}
if (fillFullMessage(msg, msg_doc, feed_id)) { QList<QPair<QByteArray, QByteArray>> headers;
full_messages.append(msg); QList<HttpResponse> output;
int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
headers.append(QPair<QByteArray, QByteArray>(QString(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(),
bearer.toLocal8Bit()));
NetworkResult res = NetworkFactory::performNetworkOperation(GMAIL_API_BATCH,
timeout,
multi,
output,
QNetworkAccessManager::Operation::PostOperation,
headers);
if (res.first == QNetworkReply::NetworkError::NoError) {
// We parse each part of HTTP response (it contains HTTP headers and payload with msg full data).
for (const HttpResponse& part : output) {
auto xx = part.body();
QJsonObject msg_doc = QJsonDocument::fromJson(part.body().toUtf8()).object();
QString msg_id = msg_doc["id"].toString();
if (msgs.contains(msg_id)) {
Message& msg = messages[msgs.value(msg_id)];
if (fillFullMessage(msg, msg_doc, feed_id)) {
// TODO: report error;
}
} }
} }
} }
else {
return false;
}
}
while (next_message < messages.size());
return true; return true;
}
else {
return false;
}
} }
QList<Message> GmailNetworkFactory::decodeLiteMessages(const QString& messages_json_data, QList<Message> GmailNetworkFactory::decodeLiteMessages(const QString& messages_json_data,

2
src/librssguard/services/gmail/network/gmailnetworkfactory.h
View file

@ -51,7 +51,7 @@ class GmailNetworkFactory : public QObject {
private: private:
bool fillFullMessage(Message& msg, const QJsonObject& json, const QString& feed_id); bool fillFullMessage(Message& msg, const QJsonObject& json, const QString& feed_id);
QMap<QString, QString> getMessageMetadata(const QString& msg_id, const QStringList& metadata); QMap<QString, QString> getMessageMetadata(const QString& msg_id, const QStringList& metadata);
bool obtainAndDecodeFullMessages(const QList<Message>& lite_messages, const QString& feed_id, QList<Message>& full_messages); bool obtainAndDecodeFullMessages(QList<Message>& lite_messages, const QString& feed_id);
QList<Message> decodeLiteMessages(const QString& messages_json_data, const QString& stream_id, QString& next_page_token); QList<Message> decodeLiteMessages(const QString& messages_json_data, const QString& stream_id, QString& next_page_token);
//RootItem* decodeFeedCategoriesData(const QString& categories); //RootItem* decodeFeedCategoriesData(const QString& categories);