From a579684b342186ad22d000dab7767fb660672af7 Mon Sep 17 00:00:00 2001 From: Pranav Raj S Date: Fri, 19 Jan 2024 02:29:04 -0800 Subject: [PATCH] fix: Update the message-id check to use messageId parsing from `mail` gem (#8743) - Update the message ID parsing logic to use the mail gem - Update the code to improve readability --- app/jobs/inboxes/fetch_imap_emails_job.rb | 108 ++++++++++++++-------- 1 file changed, 69 insertions(+), 39 deletions(-) diff --git a/app/jobs/inboxes/fetch_imap_emails_job.rb b/app/jobs/inboxes/fetch_imap_emails_job.rb index d42e5969b..c59ae2787 100644 --- a/app/jobs/inboxes/fetch_imap_emails_job.rb +++ b/app/jobs/inboxes/fetch_imap_emails_job.rb @@ -27,7 +27,6 @@ class Inboxes::FetchImapEmailsJob < MutexApplicationJob end def process_email_for_channel(channel) - # fetching email for microsoft provider if channel.microsoft? fetch_mail_for_ms_provider(channel) else @@ -38,42 +37,66 @@ class Inboxes::FetchImapEmailsJob < MutexApplicationJob end def fetch_mail_for_channel(channel) - imap_inbox = authenticated_imap_inbox(channel, channel.imap_password, 'PLAIN') + imap_client = build_imap_client(channel, channel.imap_password, 'PLAIN') - fetch_message_ids(imap_inbox, channel).each do |message_id_uid_pair| - uid, message_id = message_id_uid_pair - next if email_already_present?(channel, message_id) - - inbound_mail = Mail.read_from_string(imap_inbox.fetch(uid, 'RFC822')[0].attr['RFC822']) - mail_info_logger(channel, inbound_mail, uid) - - process_mail(inbound_mail, channel) + message_ids_with_seq = fetch_message_ids_with_sequence(imap_client, channel) + message_ids_with_seq.each do |message_id_with_seq| + process_message_id(channel, imap_client, message_id_with_seq) end end - def fetch_message_ids(imap_inbox, channel) - uids = fetch_uids(imap_inbox) - Rails.logger.info "FETCH_EMAILS_FROM: #{channel.email} - Found #{uids.length} emails \n\n\n\n" + def process_message_id(channel, imap_client, message_id_with_seq) + seq_no, message_id = message_id_with_seq - message_ids = [] - uids.each_slice(10).each do |batch| - uid_fetched = imap_inbox.fetch(batch, 'BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)]') - # print uid_fetched - uid_fetched.each do |data| - message_id = data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]'].to_s.scan(/<(.+?)>/).flatten.first - message_ids.push([data.seqno, message_id]) + return if email_already_present?(channel, message_id) + + # Fetch the original mail content using the sequence no + mail_str = imap_client.fetch(seq_no, 'RFC822')[0].attr['RFC822'] + + if mail_str.blank? + Rails.logger.info "[IMAP::FETCH_EMAIL_SERVICE] Fetch failed for #{channel.email} with message-id <#{message_id}>." + return + end + + inbound_mail = build_mail_from_string(mail_str) + mail_info_logger(channel, inbound_mail, seq_no) + process_mail(inbound_mail, channel) + end + + # Sends a FETCH command to retrieve data associated with a message in the mailbox. + # You can send batches of message sequence number in `.fetch` method. + def fetch_message_ids_with_sequence(imap_client, channel) + seq_nums = fetch_available_mail_sequence_numbers(imap_client) + + Rails.logger.info "[IMAP::FETCH_EMAIL_SERVICE] Fetching mails from #{channel.email}, found #{seq_nums.length}." + + message_ids_with_seq = [] + seq_nums.each_slice(10).each do |batch| + # Fetch only message-id only without mail body or contents. + batch_message_ids = imap_client.fetch(batch, 'BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)]') + + # .fetch returns an array of Net::IMAP::FetchData or nil + # (instead of an empty array) if there is no matching message. + # Check + if batch_message_ids.blank? + Rails.logger.info "[IMAP::FETCH_EMAIL_SERVICE] Fetching the batch failed for #{channel.email}." + next + end + + batch_message_ids.each do |data| + message_id = build_mail_from_string(data.attr['BODY[HEADER.FIELDS (MESSAGE-ID)]']).message_id + message_ids_with_seq.push([data.seqno, message_id]) end end - message_ids + message_ids_with_seq end - def email_already_present?(channel, message_id) - channel.inbox.messages.find_by(source_id: message_id).present? - end - - def fetch_uids(imap_inbox) - imap_inbox.search(['BEFORE', tomorrow, 'SINCE', yesterday]) + # Sends a SEARCH command to search the mailbox for messages that were + # created between yesterday and today and returns message sequence numbers. + # Return + def fetch_available_mail_sequence_numbers(imap_client) + imap_client.search(['BEFORE', tomorrow, 'SINCE', yesterday]) end def fetch_mail_for_ms_provider(channel) @@ -83,16 +106,15 @@ class Inboxes::FetchImapEmailsJob < MutexApplicationJob return unless access_token - imap_inbox = authenticated_imap_inbox(channel, access_token, 'XOAUTH2') - - process_mails(imap_inbox, channel) + imap_client = build_imap_client(channel, access_token, 'XOAUTH2') + process_mails(imap_client, channel) end - def process_mails(imap_inbox, channel) - fetch_uids(imap_inbox).each do |message_id| - inbound_mail = Mail.read_from_string imap_inbox.fetch(message_id, 'RFC822')[0].attr['RFC822'] + def process_mails(imap_client, channel) + fetch_available_mail_sequence_numbers(imap_client).each do |seq_no| + inbound_mail = Mail.read_from_string imap_client.fetch(seq_no, 'RFC822')[0].attr['RFC822'] - mail_info_logger(channel, inbound_mail, message_id) + mail_info_logger(channel, inbound_mail, seq_no) next if channel.inbox.messages.find_by(source_id: inbound_mail.message_id).present? @@ -107,19 +129,19 @@ class Inboxes::FetchImapEmailsJob < MutexApplicationJob #{channel.provider} Email id: #{inbound_mail.from} - message_source_id: #{inbound_mail.message_id} - sequence id: #{uid}") end - def authenticated_imap_inbox(channel, access_token, auth_method) + def build_imap_client(channel, access_token, auth_method) imap = Net::IMAP.new(channel.imap_address, channel.imap_port, true) imap.authenticate(auth_method, channel.imap_login, access_token) imap.select('INBOX') imap end - def yesterday - (Time.zone.today - 1).strftime('%d-%b-%Y') + def email_already_present?(channel, message_id) + channel.inbox.messages.find_by(source_id: message_id).present? end - def tomorrow - (Time.zone.today + 1).strftime('%d-%b-%Y') + def build_mail_from_string(raw_email_content) + Mail.read_from_string(raw_email_content) end def process_mail(inbound_mail, channel) @@ -134,4 +156,12 @@ class Inboxes::FetchImapEmailsJob < MutexApplicationJob def valid_access_token(channel) Microsoft::RefreshOauthTokenService.new(channel: channel).access_token end + + def yesterday + (Time.zone.today - 1).strftime('%d-%b-%Y') + end + + def tomorrow + (Time.zone.today + 1).strftime('%d-%b-%Y') + end end