Files
Home-Assistant/custom_components/mail_and_packages/utils/imap.py
T
2026-07-09 13:30:12 -04:00

723 lines
24 KiB
Python

"""IMAP connection and search utilities for Mail and Packages."""
import asyncio
import binascii
import logging
import re
import unicodedata
import aioimaplib
from aioimaplib import (
AUTH,
IMAP4,
IMAP4_SSL,
NONAUTH,
SELECTED,
AioImapException,
Cmd,
Command,
Exec,
)
from homeassistant.core import HomeAssistant
from homeassistant.exceptions import HomeAssistantError
from homeassistant.util import ssl
from custom_components.mail_and_packages.const import DEFAULT_IMAP_TIMEOUT
_LOGGER = logging.getLogger(__name__)
# Register ESEARCH command if not already present in aioimaplib
if "ESEARCH" not in aioimaplib.Commands:
aioimaplib.Commands["ESEARCH"] = Cmd("ESEARCH", (AUTH, SELECTED), Exec.is_async)
def encode_imap_utf7(s: str) -> str:
"""Encode a string into IMAP modified UTF-7."""
res = []
unicode_buffer = []
def flush_unicode():
if unicode_buffer:
u_str = "".join(unicode_buffer)
encoded_bytes = u_str.encode("utf-16be")
b64 = (
binascii.b2a_base64(encoded_bytes)
.decode("ascii")
.rstrip("\n=")
.replace("/", ",")
)
res.append(f"&{b64}-")
unicode_buffer.clear()
for char in s:
ord_c = ord(char)
if 0x20 <= ord_c <= 0x7E:
if char == "&":
flush_unicode()
res.append("&-")
else:
if unicode_buffer:
flush_unicode()
res.append(char)
else:
unicode_buffer.append(char)
flush_unicode()
return "".join(res)
def decode_imap_utf7(s: str) -> str:
"""Decode a string from IMAP modified UTF-7."""
res = []
i = 0
n = len(s)
while i < n:
char = s[i]
if char == "&":
end = s.find("-", i + 1)
if end == -1:
res.append("&")
i += 1
elif end == i + 1:
res.append("&")
i += 2
else:
b64_part = s[i + 1 : end]
b64_part = b64_part.replace(",", "/")
pad = len(b64_part) % 4
if pad:
b64_part += "=" * (4 - pad)
try:
decoded_bytes = binascii.a2b_base64(b64_part)
res.append(decoded_bytes.decode("utf-16be"))
except (binascii.Error, UnicodeDecodeError, ValueError):
res.append(s[i : end + 1])
i = end + 1
else:
res.append(char)
i += 1
return "".join(res)
_ATOM_SPECIALS = frozenset('(){%*"\\] ')
def _is_imap_atom(s: str) -> bool:
"""Check if the string is a valid IMAP atom."""
return bool(s) and all(0x20 < ord(c) < 0x7F and c not in _ATOM_SPECIALS for c in s)
def quote_folder(folder: str) -> str:
"""Ensure folder name is properly quoted for IMAP commands."""
if folder.startswith('"') and folder.endswith('"'):
return folder
return folder if _is_imap_atom(folder) else f'"{folder}"'
class InvalidAuth(HomeAssistantError):
"""Raise exception for invalid credentials."""
async def login(
hass: HomeAssistant,
host: str,
port: int,
user: str,
pwd: str,
security: str,
verify: bool = True,
oauth_token: str | None = None,
timeout: float = DEFAULT_IMAP_TIMEOUT,
) -> IMAP4_SSL | IMAP4:
"""Login to IMAP server asynchronously.
Supports both password and OAuth2 (XOAUTH2) authentication.
If oauth_token is provided, uses XOAUTH2 SASL mechanism.
Otherwise falls back to standard LOGIN command.
"""
ssl_context = (
ssl.client_context(ssl.SSLCipherList.PYTHON_DEFAULT)
if verify
else ssl.create_no_verify_ssl_context()
)
if security == "SSL":
account = IMAP4_SSL(
host=host, port=port, ssl_context=ssl_context, timeout=timeout
)
else:
account = IMAP4(host=host, port=port, timeout=timeout)
await account.wait_hello_from_server()
if account.protocol.state == NONAUTH:
try:
if oauth_token:
await account.xoauth2(user, oauth_token)
else:
await account.login(user, pwd)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error logging in to IMAP Server: %s", err)
raise InvalidAuth from err
if account.protocol.state not in {AUTH, SELECTED}:
_LOGGER.error("Error logging in to IMAP Server")
raise InvalidAuth
return account
async def selectfolder(account: IMAP4_SSL, folder: str) -> bool:
"""Select folder inside the mailbox asynchronously."""
if getattr(account, "_current_folder", None) == folder:
return True
encoded_folder = encode_imap_utf7(folder)
quoted_folder = quote_folder(encoded_folder)
try:
await account.select(quoted_folder)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error selecting folder %s: %s", folder, err)
return False
else:
account._current_folder = folder # noqa: SLF001
return True
def clean_search_string(val: str) -> str:
"""Clean search string for IMAP search compatibility.
Normalizes Unicode characters to NFKD decomposed form, strips non-ASCII
characters to ensure compatibility with US-ASCII only IMAP servers,
and removes any double quotes to prevent syntax corruption.
"""
if not val:
return ""
normalized = unicodedata.normalize("NFKD", val)
cleaned = normalized.encode("ascii", "ignore").decode("ascii")
return cleaned.replace('"', "")
def build_search( # noqa: C901
address: list,
date: str,
subject: str | list[str] = "",
body: str | list[str] = "",
header: str = "",
is_yahoo: bool = False,
) -> tuple:
"""Build IMAP search query.
Return tuple of utf8 flag and search query.
Non-ASCII characters are stripped from subject to ensure compatibility
with servers that only support US-ASCII charset (e.g. Microsoft Exchange).
IMAP SUBJECT performs substring matching, so stripping non-ASCII chars
still matches the original subject (e.g. 'Livr' matches 'Livré').
When `header` is provided, each address is matched as either a forwarded
email (via HEADER substring match) OR a direct email (via FROM), so the
same config works for carriers that are forwarded through a service like
SimpleLogin AND carriers whose emails arrive directly in the mailbox.
IMAP HEADER does substring matching, so "mcinfo@ups.com" will match a
header value of "UPS <mcinfo@ups.com>".
"""
the_date = f"SINCE {date}"
if not address:
raise ValueError("address list must not be empty")
# Build the address/header clause
if header:
# Each address matches via header (forwarded) OR FROM (direct), so
# users with mixed setups (some carriers forwarded, others direct)
# don't need separate configurations.
parts = [f'OR HEADER "{header}" "{a}" FROM "{a}"' for a in address]
if len(parts) == 1:
addr_clause = f"({parts[0]})" if is_yahoo else parts[0]
else:
or_prefix = " ".join(["OR"] * (len(parts) - 1))
addr_clause = (
f"({or_prefix} {' '.join(parts)})"
if is_yahoo
else f"{or_prefix} {' '.join(parts)}"
)
elif len(address) == 1:
addr_clause = f'FROM "{address[0]}"'
else:
joined = '" FROM "'.join(address)
or_prefix = " ".join(["OR"] * (len(address) - 1))
addr_clause = (
f'({or_prefix} FROM "{joined}")'
if is_yahoo
else f'{or_prefix} FROM "{joined}"'
)
# Handle multiple subjects
subject_part = ""
if subject:
subjects = [subject] if isinstance(subject, str) else subject
safe_subjects = [clean_search_string(s) for s in subjects]
safe_subjects = [s for s in safe_subjects if s]
if len(safe_subjects) == 1:
subject_part = f'SUBJECT "{safe_subjects[0]}"'
elif len(safe_subjects) > 1:
subject_prefix = " ".join(["OR"] * (len(safe_subjects) - 1))
subject_joined = '" SUBJECT "'.join(safe_subjects)
subject_part = (
f'({subject_prefix} SUBJECT "{subject_joined}")'
if is_yahoo
else f'{subject_prefix} SUBJECT "{subject_joined}"'
)
# Handle multiple bodies
body_part = ""
if body:
bodies = [body] if isinstance(body, str) else body
safe_bodies = [clean_search_string(b) for b in bodies]
safe_bodies = [b for b in safe_bodies if b]
if len(safe_bodies) == 1:
body_part = f'BODY "{safe_bodies[0]}"'
elif len(safe_bodies) > 1:
body_prefix = " ".join(["OR"] * (len(safe_bodies) - 1))
body_joined = '" BODY "'.join(safe_bodies)
body_part = (
f'({body_prefix} BODY "{body_joined}")'
if is_yahoo
else f'{body_prefix} BODY "{body_joined}"'
)
if is_yahoo:
if subject_part or body_part:
search_criteria = f"{subject_part} {body_part}".strip()
imap_search = f"({addr_clause} {search_criteria} {the_date})"
else:
imap_search = f"({addr_clause} {the_date})"
elif subject_part or body_part:
search_criteria = f"{subject_part} {body_part}".strip()
imap_search = f"{addr_clause} {search_criteria} {the_date}"
else:
imap_search = f"{addr_clause} {the_date}"
_LOGGER.debug("DEBUG imap_search: %s", imap_search)
return (False, imap_search)
def parse_search_response(lines: list[bytes]) -> list[bytes]:
"""Parse IMAP SEARCH response lines and return list of UID/ID bytes.
Handles both standard server responses (prefixed with b"SEARCH")
and mocked test inputs (which often contain raw UIDs directly).
Filters out the SEARCH keyword, tagged OK/status responses,
and any non-numeric tokens.
"""
uids = []
for line in lines:
if not line:
continue
parts = line.split()
if not parts:
continue
if parts[0] == b"SEARCH":
# Check if this is a search result line, e.g. b"SEARCH 1001 1002"
# (as opposed to b"SEARCH completed")
if len(parts) > 1 and parts[1].isdigit():
uids.extend(parts[1:])
# Check if this line is just a list of numeric UIDs (mock/test compatibility)
# and ignore status/existence responses like b"23 EXISTS"
elif all(p.isdigit() for p in parts):
uids.extend(parts)
return uids
def _parse_esearch_line(line_bytes: bytes) -> list[bytes]:
"""Parse a single ESEARCH line and return list of formatted UID bytes: b'folder/uid'."""
line_str = line_bytes.decode("utf-8", "ignore")
# Extract the correlator inside parentheses
start_paren = line_str.find("(")
end_paren = line_str.find(")", start_paren) if start_paren != -1 else -1
if start_paren == -1 or end_paren == -1:
return []
correlator = line_str[start_paren + 1 : end_paren]
# Extract mailbox name (could be quoted or unquoted)
mailbox_match = re.search(r'MAILBOX\s+"([^"]+)"', correlator)
if not mailbox_match:
mailbox_match = re.search(r"MAILBOX\s+(\S+)", correlator)
if not mailbox_match:
return []
mailbox = mailbox_match.group(1)
mailbox = decode_imap_utf7(mailbox)
# Extract the sequence set after 'UID ALL' anywhere in the line
seq_match = re.search(r"UID\s+ALL\s+(\S+)", line_str)
if not seq_match:
return []
seq_set = seq_match.group(1)
uids = []
for part in seq_set.split(","):
part = part.strip()
if not part:
continue
if ":" in part:
try:
start_str, end_str = part.split(":", 1)
start, end = int(start_str), int(end_str)
if start <= end:
uids.extend(str(x) for x in range(start, end + 1))
else:
uids.extend(str(x) for x in range(end, start + 1))
except ValueError:
pass
else:
uids.append(part)
return [f"{mailbox}/{uid}".encode() for uid in uids]
async def _execute_single_search(account: IMAP4_SSL, search_query: str) -> list[bytes]: # noqa: C901
"""Execute search query. If single folder, use standard search. If multiple, use hybrid ESEARCH/fallback."""
folders = getattr(account, "_folders", ["INBOX"])
if len(folders) <= 1:
res = await account.search(search_query, charset=None)
if res.result == "OK" and res.lines:
return parse_search_response(res.lines)
return []
all_uids = []
# Check for MULTISEARCH capability safely (handling mock/AsyncMock in tests)
is_multisearch = False
if hasattr(account, "has_capability"):
try:
res = account.has_capability("MULTISEARCH")
if asyncio.iscoroutine(res):
res.close()
is_multisearch = False
else:
is_multisearch = bool(res)
except Exception: # noqa: BLE001
pass
if is_multisearch:
# ESEARCH IN ("folder1" "folder2") query - encode and quote folders
folder_list = " ".join([quote_folder(encode_imap_utf7(f)) for f in folders])
args = ("IN", f"({folder_list})", search_query)
try:
timeout = getattr(account, "timeout", None)
if not isinstance(timeout, (int, float)):
timeout = None
res = await account.protocol.execute(
Command(
"ESEARCH",
account.protocol.new_tag(),
*args,
loop=account.protocol.loop,
timeout=timeout,
)
)
if res.result == "OK":
for line in res.lines:
if line:
all_uids.extend(_parse_esearch_line(line))
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error executing ESEARCH: %s", err)
else:
# Sequential select and search fallback - no limits on configured folders
for folder in folders:
select_ok = await selectfolder(account, folder)
if not select_ok:
continue
try:
res = await account.uid_search(search_query, charset=None)
if res.result == "OK" and res.lines:
parsed = parse_search_response(res.lines)
all_uids.extend(
f"{folder}/{uid.decode()}".encode() for uid in parsed
)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error searching folder %s: %s", folder, err)
return all_uids
async def email_search( # noqa: C901
account: IMAP4_SSL,
address: list,
date: str,
subject: str | list[str] = "",
body: str | list[str] = "",
header: str = "",
) -> tuple:
"""Search emails with from/header, subject, and date asynchronously.
Always uses charset=None to avoid sending CHARSET in the IMAP SEARCH
command, ensuring compatibility with servers like Microsoft Exchange
that only support US-ASCII.
When `header` is provided, searches via HEADER criterion instead of FROM,
matching the original sender in forwarding-service headers.
If multiple subjects are provided, they are searched in batches of 10
to keep the search query length safe.
"""
folders = getattr(account, "_folders", ["INBOX"])
is_yahoo = False
if hasattr(account, "host") and isinstance(account.host, str):
host_lower = account.host.lower()
is_yahoo = "yahoo" in host_lower or "aol" in host_lower
# If there are more than 2 body patterns, do not search them server-side
# to prevent slow query execution and timeouts on standard IMAP servers.
# Instead, we let the shipper's client-side text filtering handle it.
body_search = body
if body:
bodies = [body] if isinstance(body, str) else body
if len(bodies) > 2:
body_search = ""
if len(folders) <= 1:
if not isinstance(subject, list) or len(subject) <= 10:
_unused, search = build_search(
address, date, subject, body_search, header, is_yahoo=is_yahoo
)
try:
res = await account.search(search, charset=None)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error searching emails: %s", err)
return ("BAD", str(err))
else:
parsed = parse_search_response(res.lines)
return (res.result, [b" ".join(parsed)])
# Batch subjects in groups of 10
all_matched_ids = []
for i in range(0, len(subject), 10):
batch = subject[i : i + 10]
_unused, search = build_search(
address, date, batch, body_search, header, is_yahoo=is_yahoo
)
try:
res = await account.search(search, charset=None)
if res.result == "OK" and res.lines:
parsed = parse_search_response(res.lines)
all_matched_ids.extend(parsed)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error searching emails batch: %s", err)
# Deduplicate and return in same format as individual search
unique_ids = list(dict.fromkeys(all_matched_ids))
return ("OK", [b" ".join(unique_ids)])
# Multi-folder search logic
if not isinstance(subject, list) or len(subject) <= 10:
_unused, search = build_search(
address, date, subject, body_search, header, is_yahoo=is_yahoo
)
try:
uids = await _execute_single_search(account, search)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error searching emails: %s", err)
return ("BAD", str(err))
return ("OK", [b" ".join(uids)])
# Batch subjects in groups of 10
all_matched_ids = []
for i in range(0, len(subject), 10):
batch = subject[i : i + 10]
_unused, search = build_search(
address, date, batch, body_search, header, is_yahoo=is_yahoo
)
try:
uids = await _execute_single_search(account, search)
all_matched_ids.extend(uids)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error searching emails batch: %s", err)
# Deduplicate and return in same format as individual search
unique_ids = list(dict.fromkeys(all_matched_ids))
return ("OK", [b" ".join(unique_ids)])
async def email_fetch(account: IMAP4_SSL, num, parts: str = "(RFC822)") -> tuple:
"""Download specified email for parsing asynchronously."""
if account.host == "imap.mail.me.com":
parts = "BODY[]"
num_str = num.decode() if isinstance(num, bytes) else str(num)
if "/" in num_str:
folder, num_str = num_str.rsplit("/", 1)
await selectfolder(account, folder)
try:
res = await account.uid("FETCH", num_str, parts)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching email %s: %s", num_str, err)
return ("BAD", str(err))
else:
return (res.result, res.lines)
try:
res = await account.fetch(num_str, parts)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching email %s: %s", num_str, err)
return ("BAD", str(err))
else:
return (res.result, res.lines)
async def email_fetch_headers(account: IMAP4_SSL, num) -> tuple:
"""Download only the subject header of an email asynchronously."""
num_str = num.decode() if isinstance(num, bytes) else str(num)
if "/" in num_str:
folder, num_str = num_str.rsplit("/", 1)
await selectfolder(account, folder)
try:
res = await account.uid("FETCH", num_str, "(BODY[HEADER.FIELDS (SUBJECT)])")
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching email headers %s: %s", num_str, err)
return ("BAD", str(err))
else:
return (res.result, res.lines)
try:
res = await account.fetch(num_str, "(BODY[HEADER.FIELDS (SUBJECT)])")
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching email headers %s: %s", num_str, err)
return ("BAD", str(err))
else:
return (res.result, res.lines)
async def email_fetch_text(account: IMAP4_SSL, num, parts: str = "(BODY[1])") -> tuple:
"""Download the specific part of the email body asynchronously."""
if account.host == "imap.mail.me.com":
parts = "BODY[]"
num_str = num.decode() if isinstance(num, bytes) else str(num)
if "/" in num_str:
folder, num_str = num_str.rsplit("/", 1)
await selectfolder(account, folder)
try:
res = await account.uid("FETCH", num_str, parts)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching email text %s: %s", num_str, err)
return ("BAD", str(err))
else:
return (res.result, res.lines)
try:
res = await account.fetch(num_str, parts)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching email text %s: %s", num_str, err)
return ("BAD", str(err))
else:
return (res.result, res.lines)
async def email_fetch_batch( # noqa: C901
account: IMAP4_SSL, nums: list[str | bytes], parts: str = "(RFC822)"
) -> tuple:
"""Download specified emails for parsing asynchronously in a batch."""
if not nums:
return ("OK", [])
if account.host == "imap.mail.me.com":
parts = "BODY[]"
# Check if any ID contains a folder prefix
has_folder_prefix = False
for num in nums:
num_str = num.decode() if isinstance(num, bytes) else str(num)
if "/" in num_str:
has_folder_prefix = True
break
if not has_folder_prefix:
num_strs = [
num.decode() if isinstance(num, bytes) else str(num) for num in nums
]
num_list_str = ",".join(num_strs)
try:
res = await account.fetch(num_list_str, parts)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching emails batch %s: %s", num_list_str, err)
return ("BAD", str(err))
else:
return (res.result, res.lines)
# Group nums by their folder prefix
folder_to_nums = {}
for num in nums:
num_str = num.decode() if isinstance(num, bytes) else str(num)
if "/" in num_str:
folder, actual_num = num_str.rsplit("/", 1)
else:
folder, actual_num = None, num_str
folder_to_nums.setdefault(folder, []).append(actual_num)
all_results = []
overall_result = "OK"
for folder, folder_nums in folder_to_nums.items():
if folder is not None:
await selectfolder(account, folder)
num_list_str = ",".join(folder_nums)
try:
res = await account.uid("FETCH", num_list_str, parts)
if res.result != "OK":
overall_result = res.result
all_results.extend(res.lines)
except TimeoutError:
raise
except (AioImapException, OSError) as err:
_LOGGER.error("Error fetching emails batch %s: %s", num_list_str, err)
return ("BAD", str(err))
return (overall_result, all_results)
async def logout(account: IMAP4_SSL | IMAP4) -> None:
"""Logout from IMAP server asynchronously."""
try:
await account.logout()
except (TimeoutError, AioImapException, OSError, asyncio.CancelledError) as err:
_LOGGER.debug("Error logging out of IMAP Server: %s", err)