pogmommy/rssguard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix gmail message downloading with batch size > 100.

Browse source
This commit is contained in:
Martin Rotter 2020-11-10 07:47:44 +01:00
parent 86ab890865
commit 47d2eb05c4
2 changed files with 51 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
src/librssguard/services/gmail/network/gmailnetworkfactory.cpp
View file

@ -189,14 +189,13 @@ QList<Message> GmailNetworkFactory::messages(const QString& stream_id, Feed::Sta
// We parse this chunk.
QString messages_data = downloader.lastOutputData();
QList<Message> more_messages = decodeLiteMessages(messages_data, stream_id, next_page_token);
QList<Message> full_messages;
if (!more_messages.isEmpty()) {
// Now, we via batch HTTP request obtain full data for each message.
bool obtained = obtainAndDecodeFullMessages(more_messages, stream_id, full_messages);
bool obtained = obtainAndDecodeFullMessages(more_messages, stream_id);
if (obtained) {
messages.append(full_messages);
messages.append(more_messages);
// New batch of messages was obtained, check if we have enough.
if (batchSize() > 0 && batchSize() <= messages.size()) {
@ -479,16 +478,22 @@ QMap<QString, QString> GmailNetworkFactory::getMessageMetadata(const QString& ms
}
}
bool GmailNetworkFactory::obtainAndDecodeFullMessages(const QList<Message>& lite_messages,
const QString& feed_id,
QList<Message>& full_messages) {
bool GmailNetworkFactory::obtainAndDecodeFullMessages(QList<Message>& messages, const QString& feed_id) {
QHash<QString, int> msgs;
int next_message = 0;
QString bearer = m_oauth2->bearer();
if (bearer.isEmpty()) {
return false;
}
do {
auto* multi = new QHttpMultiPart();
multi->setContentType(QHttpMultiPart::ContentType::MixedType);
QHash<QString, Message> msgs;
for (const Message& msg : lite_messages) {
for (int window = next_message + 100; next_message < window && next_message < messages.size(); next_message++ ) {
Message msg = messages[next_message];
QHttpPart part;
part.setRawHeader(HTTP_HEADERS_CONTENT_TYPE, GMAIL_CONTENT_TYPE_HTTP);
@ -496,13 +501,7 @@ bool GmailNetworkFactory::obtainAndDecodeFullMessages(const QList<Message>& lite
part.setBody(full_msg_endpoint.toUtf8());
multi->append(part);
msgs.insert(msg.m_customId, msg);
}
QString bearer = m_oauth2->bearer();
if (bearer.isEmpty()) {
return false;
msgs.insert(msg.m_customId, next_message);
}
QList<QPair<QByteArray, QByteArray>> headers;
@ -527,20 +526,22 @@ bool GmailNetworkFactory::obtainAndDecodeFullMessages(const QList<Message>& lite
QString msg_id = msg_doc["id"].toString();
if (msgs.contains(msg_id)) {
Message& msg = msgs[msg_id];
Message& msg = messages[msgs.value(msg_id)];
if (fillFullMessage(msg, msg_doc, feed_id)) {
full_messages.append(msg);
// TODO: report error;
}
}
}
return true;
}
else {
return false;
}
}
while (next_message < messages.size());
return true;
}
QList<Message> GmailNetworkFactory::decodeLiteMessages(const QString& messages_json_data,
const QString& stream_id,

2
src/librssguard/services/gmail/network/gmailnetworkfactory.h
View file

@ -51,7 +51,7 @@ class GmailNetworkFactory : public QObject {
private:
bool fillFullMessage(Message& msg, const QJsonObject& json, const QString& feed_id);
QMap<QString, QString> getMessageMetadata(const QString& msg_id, const QStringList& metadata);
bool obtainAndDecodeFullMessages(const QList<Message>& lite_messages, const QString& feed_id, QList<Message>& full_messages);
bool obtainAndDecodeFullMessages(QList<Message>& lite_messages, const QString& feed_id);
QList<Message> decodeLiteMessages(const QString& messages_json_data, const QString& stream_id, QString& next_page_token);
//RootItem* decodeFeedCategoriesData(const QString& categories);