Updated apps

custom_components/mail_and_packages/shippers/__init__.py

@@ -0,0 +1,42 @@
+"""Shippers for Mail and Packages."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .amazon import AmazonShipper
+from .generic import GenericShipper
+from .post_de import PostDEShipper
+from .usps import USPSShipper
+
+if TYPE_CHECKING:
+    from homeassistant.core import HomeAssistant
+
+    from .base import Shipper
+
+SHIPPER_REGISTRY = {
+    "amazon": AmazonShipper,
+    "generic": GenericShipper,
+    "post_de": PostDEShipper,
+    "usps": USPSShipper,
+}
+
+
+def get_shipper_for_sensor(
+    hass: HomeAssistant,
+    config: dict,
+    sensor_type: str,
+) -> Shipper | None:
+    """Return the appropriate shipper for the given sensor type."""
+    # Check specialized shippers first
+    for name, shipper_class in SHIPPER_REGISTRY.items():
+        if name == "generic":
+            continue
+        if shipper_class.handles_sensor(sensor_type):
+            return shipper_class(hass, config)
+
+    # Fallback to generic
+    if GenericShipper.handles_sensor(sensor_type):
+        return GenericShipper(hass, config)
+
+    return None

custom_components/mail_and_packages/shippers/amazon.py

@@ -0,0 +1,612 @@
+"""Base Shipper class."""
+
+from __future__ import annotations
+
+import datetime
+import email
+import logging
+import re
+from pathlib import Path
+from shutil import copyfile
+from typing import Any
+
+import anyio
+import dateparser
+import homeassistant.helpers.config_validation as cv
+from aioimaplib import IMAP4_SSL
+
+from custom_components.mail_and_packages import const
+from custom_components.mail_and_packages.const import (
+    AMAZON_DELIVERED,
+    AMAZON_DELIVERED_SUBJECT,
+    AMAZON_EXCEPTION,
+    AMAZON_EXCEPTION_BODY,
+    AMAZON_EXCEPTION_ORDER,
+    AMAZON_EXCEPTION_SUBJECT,
+    AMAZON_HUB,
+    AMAZON_HUB_BODY,
+    AMAZON_HUB_CODE,
+    AMAZON_HUB_SUBJECT,
+    AMAZON_HUB_SUBJECT_SEARCH,
+    AMAZON_ORDER,
+    AMAZON_ORDERED_SUBJECT,
+    AMAZON_OTP,
+    AMAZON_OTP_CODE,
+    AMAZON_OTP_REGEX,
+    AMAZON_OTP_SUBJECT,
+    AMAZON_PACKAGES,
+    AMAZON_SHIPMENT_SUBJECT,
+    ATTR_COUNT,
+    CONF_AMAZON_DAYS,
+    CONF_AMAZON_DOMAIN,
+    CONF_AMAZON_FWDS,
+    CONF_DURATION,
+    CONF_FORWARDING_HEADER,
+    DEFAULT_AMAZON_DAYS,
+)
+from custom_components.mail_and_packages.utils.amazon import (
+    _extract_hub_code,
+    amazon_email_addresses,
+    download_amazon_img,
+    extract_order_numbers,
+    filter_amazon_strings,
+    get_decoded_subject,
+    get_email_body,
+    parse_amazon_arrival_date,
+    search_amazon_emails,
+)
+from custom_components.mail_and_packages.utils.cache import EmailCache
+from custom_components.mail_and_packages.utils.date import get_today
+from custom_components.mail_and_packages.utils.image import (
+    cleanup_images,
+    generate_delivery_gif,
+    random_filename,
+    resize_images,
+)
+from custom_components.mail_and_packages.utils.imap import (
+    email_fetch,
+    email_search,
+)
+
+from .base import Shipper
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class AmazonShipper(Shipper):
+    """Amazon shipper implementation."""
+
+    @property
+    def name(self) -> str:
+        """Return shipper name."""
+        return "amazon"
+
+    @classmethod
+    def handles_sensor(cls, sensor_type: str) -> bool:
+        """Return True if this shipper handles the given sensor type."""
+        return sensor_type.startswith("amazon_") or sensor_type == AMAZON_PACKAGES
+
+    async def process(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensor_type: str,
+        cache: EmailCache | None = None,
+    ) -> dict[str, Any]:
+        """Process Amazon-specific emails."""
+        forwarding_header = self.config.get(CONF_FORWARDING_HEADER, "")
+        if forwarding_header and forwarding_header != "(none)":
+            # Header mode: use native Amazon addresses; fwds not needed
+            fwds = None
+        else:
+            forwarding_header = ""
+            fwds = cv.ensure_list_csv(self.config.get(CONF_AMAZON_FWDS))
+        days = self.config.get(CONF_AMAZON_DAYS, DEFAULT_AMAZON_DAYS)
+        domain = self.config.get(CONF_AMAZON_DOMAIN)
+
+        if sensor_type in [AMAZON_PACKAGES, AMAZON_ORDER]:
+            param = "count" if sensor_type == AMAZON_PACKAGES else "order"
+            result = await self._parse_amazon_emails(
+                account, param, fwds, days, domain, cache, forwarding_header
+            )
+            return {sensor_type: result}
+
+        if sensor_type == AMAZON_HUB:
+            return await self._amazon_hub(account, fwds, cache, forwarding_header)
+
+        if sensor_type == AMAZON_OTP:
+            result = await self._amazon_otp(account, fwds, cache, forwarding_header)
+            return {sensor_type: result}
+
+        if sensor_type == AMAZON_EXCEPTION:
+            return await self._amazon_exception(
+                account, fwds, domain, cache, forwarding_header
+            )
+
+        if sensor_type == AMAZON_DELIVERED:
+            image_path = self.config.get("image_path")
+            image_name = self.config.get("amazon_image")
+            result = await self._amazon_search(
+                account,
+                image_path,
+                image_name,
+                domain,
+                fwds,
+                cache,
+                forwarding_header,
+            )
+            return {
+                sensor_type: result,
+                const.ATTR_AMAZON_IMAGE: image_name,
+                const.ATTR_IMAGE_PATH: image_path,
+            }
+
+        return {ATTR_COUNT: 0}
+
+    async def process_batch(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensors: list[str],
+        cache: EmailCache,
+        since_date: str | None = None,
+    ) -> dict[str, Any]:
+        """Process multiple Amazon sensors in batch."""
+        res = {}
+        for sensor in sensors:
+            sensor_res = await self.process(account, date, sensor, cache)
+            res.update(sensor_res)
+            # Replicate coordinator dictionary logic
+            if sensor not in sensor_res:
+                if ATTR_COUNT in sensor_res:
+                    res[sensor] = sensor_res[ATTR_COUNT]
+        return res
+
+    # Internal helper methods (migrated from helpers.py)
+
+    async def _parse_amazon_emails(
+        self,
+        account: IMAP4_SSL,
+        param: str,
+        fwds: list[str] | None = None,
+        days: int = DEFAULT_AMAZON_DAYS,
+        domain: str | None = None,
+        cache: EmailCache | None = None,
+        forwarding_header: str = "",
+    ) -> list[str] | int:
+        """Parse Amazon emails for delivery date and order number."""
+        today_date = get_today()
+        address_list = amazon_email_addresses(fwds, domain)
+        unique_emails = await search_amazon_emails(
+            account, address_list, days, domain, cache, forwarding_header
+        )
+        order_pattern = re.compile(r"[0-9]{3}-[0-9]{7}-[0-9]{7}")
+
+        context = {
+            "today": today_date,
+            "packages_arriving_today": {},
+            "delivered_packages": {},
+            "amazon_delivered": [],
+            "deliveries_today": [],
+            "all_shipped_orders": set(),
+            "order_pattern": order_pattern,
+        }
+
+        for email_id in unique_emails:
+            await self._process_amazon_email(account, email_id, context, cache)
+
+        final_count = self._calculate_final_count(context)
+
+        if param == "count":
+            return final_count
+        return list(context["all_shipped_orders"])
+
+    async def _process_amazon_email(
+        self,
+        account: IMAP4_SSL,
+        email_id: bytes | str,
+        ctx: dict,
+        cache: EmailCache | None = None,
+    ):
+        """Process a single Amazon email."""
+        fetch_id = email_id.decode() if isinstance(email_id, bytes) else email_id
+        if cache:
+            data = (await cache.fetch(fetch_id, "(RFC822)"))[1]
+        else:
+            data = (await email_fetch(account, fetch_id, "(RFC822)"))[1]
+
+        for response_part in data:
+            if not isinstance(response_part, (bytes, bytearray)):
+                continue
+
+            msg = email.message_from_bytes(response_part)
+            email_date = await self._parse_email_date(msg)
+            email_subject = get_decoded_subject(msg)
+
+            if any(s.lower() in email_subject.lower() for s in AMAZON_ORDERED_SUBJECT):
+                continue
+
+            email_msg = get_email_body(msg)
+            if any(
+                s.lower() in email_subject.lower() for s in AMAZON_DELIVERED_SUBJECT
+            ):
+                self._handle_delivered_email(email_subject, email_msg, ctx)
+                continue
+
+            await self._handle_shipping_email(email_subject, email_msg, email_date, ctx)
+
+    async def _parse_email_date(
+        self,
+        msg: email.message.Message,
+    ) -> datetime.date | None:
+        """Parse the date from an email message."""
+        date_str = msg.get("Date")
+        if not date_str:
+            return None
+        parsed = await self.hass.async_add_executor_job(dateparser.parse, date_str)
+        return parsed.date() if parsed else None
+
+    def _handle_delivered_email(self, subject: str, body: str | None, ctx: dict):
+        """Handle an Amazon 'delivered' email."""
+        orders = extract_order_numbers(subject, ctx["order_pattern"])
+        if not orders and body:
+            orders = extract_order_numbers(body, ctx["order_pattern"])
+        for o in orders:
+            ctx["delivered_packages"][o] = ctx["delivered_packages"].get(o, 0) + 1
+            if o not in ctx["amazon_delivered"]:
+                ctx["amazon_delivered"].append(o)
+
+    async def _handle_shipping_email(
+        self,
+        subject: str,
+        body: str | None,
+        date: datetime.date | None,
+        ctx: dict,
+    ):
+        """Handle an Amazon 'shipping' or 'arriving' email."""
+        order_id = self._extract_first_order_id(subject, body, ctx["order_pattern"])
+        if order_id:
+            ctx["all_shipped_orders"].add(order_id)
+
+        if body:
+            parsed_arrival = await parse_amazon_arrival_date(self.hass, body, date)
+            if parsed_arrival == ctx["today"]:
+                if order_id:
+                    ctx["packages_arriving_today"][order_id] = (
+                        ctx["packages_arriving_today"].get(order_id, 0) + 1
+                    )
+                else:
+                    ctx["deliveries_today"].append("Amazon Order")
+
+    def _extract_first_order_id(
+        self,
+        subject: str,
+        body: str | None,
+        pattern: re.Pattern,
+    ) -> str | None:
+        """Extract the first order number found in subject or body."""
+        orders = extract_order_numbers(subject, pattern)
+        if orders:
+            return orders[0]
+        if body:
+            orders = extract_order_numbers(body, pattern)
+            if orders:
+                return orders[0]
+        return None
+
+    def _calculate_final_count(self, ctx: dict) -> int:
+        """Calculate the final count of packages arriving today."""
+        deliveries_today = [
+            item
+            for item in ctx["deliveries_today"]
+            if item not in ctx["amazon_delivered"]
+        ]
+        final_count = 0
+        for order_id, arriving_count in ctx["packages_arriving_today"].items():
+            delivered_count = ctx["delivered_packages"].get(order_id, 0)
+            final_count += max(0, arriving_count - delivered_count)
+        return final_count + len(deliveries_today)
+
+    async def _amazon_search(
+        self,
+        account: IMAP4_SSL,
+        image_path: str,
+        amazon_image_name: str,
+        amazon_domain: str,
+        fwds: list[str] | None = None,
+        cache: EmailCache | None = None,
+        forwarding_header: str = "",
+    ) -> int:
+        """Find Amazon Delivered email and handle images."""
+        _LOGGER.debug("=== AMAZON DELIVERED SEARCH START ===")
+        subjects = AMAZON_DELIVERED_SUBJECT
+        today = get_today().strftime("%d-%b-%Y")
+        count = 0
+        all_image_urls = []
+
+        await self.hass.async_add_executor_job(
+            cleanup_images,
+            f"{image_path or ''}amazon/",
+        )
+
+        address_list = amazon_email_addresses(fwds, amazon_domain)
+        _LOGGER.debug("Amazon email search addresses: %s", address_list)
+        if amazon_domain:
+            subjects = filter_amazon_strings(subjects, amazon_domain)
+
+        (server_response, data) = await email_search(
+            account=account,
+            address=address_list,
+            date=today,
+            subject=subjects,
+            header=forwarding_header,
+        )
+        if server_response == "OK" and data[0]:
+            for email_id in data[0].split():
+                fetch_id = (
+                    email_id.decode() if isinstance(email_id, bytes) else email_id
+                )
+                if cache:
+                    msg_data = (await cache.fetch(fetch_id, "(RFC822)"))[1]
+                else:
+                    msg_data = (await email_fetch(account, fetch_id, "(RFC822)"))[1]
+
+                is_delivered, urls = self._is_amazon_delivered(msg_data, subjects)
+                if is_delivered:
+                    count += 1
+                    for url in urls:
+                        if url not in all_image_urls:
+                            all_image_urls.append(url)
+
+        await self._process_amazon_images(
+            all_image_urls, image_path, amazon_image_name, count
+        )
+
+        return count
+
+    def _is_amazon_delivered(
+        self, msg_data: list, subjects: list[str]
+    ) -> tuple[bool, list[str]]:
+        """Verify if email is a delivered notification and return image URLs."""
+        for response_part in msg_data:
+            if not isinstance(response_part, (bytes, bytearray)):
+                continue
+            msg = email.message_from_bytes(response_part)
+            subject = get_decoded_subject(msg)
+            if not subject:
+                continue
+
+            # Check if subject contains any delivered keyword (case-insensitive)
+            has_delivered = any(s.lower() in subject.lower() for s in subjects)
+            # Check if subject contains ordered or shipped keywords (case-insensitive)
+            has_ordered = any(
+                s.lower() in subject.lower() for s in AMAZON_ORDERED_SUBJECT
+            )
+            has_shipped = any(
+                s.lower() in subject.lower() for s in AMAZON_SHIPMENT_SUBJECT
+            )
+
+            if has_delivered and not has_ordered and not has_shipped:
+                urls = self._extract_amazon_image_urls(msg)
+                return True, urls
+        return False, []
+
+    def _extract_amazon_image_urls(self, msg: email.message.Message) -> list[str]:
+        """Extract image URLs from Amazon email body."""
+        urls = []
+        pattern = re.compile(rf"{const.AMAZON_IMG_PATTERN}")
+        for part in msg.walk():
+            if part.get_content_type() != "text/html":
+                continue
+            part_payload = part.get_payload(decode=True)
+            if part_payload:
+                part_content = part_payload.decode("utf-8", "ignore")
+                found = pattern.findall(part_content)
+                for url in found:
+                    if url[1] not in const.AMAZON_IMG_LIST:
+                        continue
+                    full_url = url[0] + url[1] + url[2]
+                    if full_url not in urls:
+                        urls.append(full_url)
+        return urls
+
+    async def _process_amazon_images(
+        self,
+        image_urls: list[str],
+        image_base_path: str,
+        image_name: str,
+        email_count: int,
+    ) -> None:
+        """Process and save Amazon delivery images."""
+        if not image_base_path or not image_name:
+            return
+
+        amazon_path = Path(image_base_path) / "amazon"
+        image_files = await self._download_all_images(image_urls, image_base_path)
+
+        if len(image_files) > 1:
+            await self._create_amazon_gif(image_files, amazon_path, image_name)
+        elif len(image_files) == 1:
+            await self._save_single_amazon_image(
+                image_files[0], amazon_path, image_name
+            )
+        else:
+            await self._copy_amazon_placeholder(amazon_path, image_name)
+
+    async def _download_all_images(self, urls: list[str], base_path: str) -> list[str]:
+        """Download all image URLs to temporary files."""
+        image_files = []
+        amazon_path = Path(base_path) / "amazon"
+        for url in urls:
+            temp_filename = random_filename()
+            await download_amazon_img(url, base_path, temp_filename, self.hass)
+            full_temp_path = amazon_path / temp_filename
+            if await anyio.Path(full_temp_path).exists():
+                image_files.append(str(full_temp_path))
+        return image_files
+
+    async def _create_amazon_gif(
+        self, image_files: list[str], amazon_path: Path, image_name: str
+    ) -> None:
+        """Create animated GIF from multiple images."""
+        _LOGGER.debug("Combining %d Amazon images into GIF", len(image_files))
+        resized_images = await self.hass.async_add_executor_job(
+            resize_images, image_files, 724, 320
+        )
+        gif_path = str(amazon_path / image_name)
+        duration = self.config.get(CONF_DURATION, 5) * 1000
+        await self.hass.async_add_executor_job(
+            generate_delivery_gif, resized_images, gif_path, duration
+        )
+        # Cleanup
+        for img in image_files + resized_images:
+            if await anyio.Path(img).exists():
+                await self.hass.async_add_executor_job(
+                    cleanup_images, str(Path(img).parent) + "/", Path(img).name
+                )
+
+    async def _save_single_amazon_image(
+        self, image_file: str, amazon_path: Path, image_name: str
+    ) -> None:
+        """Save a single image by renaming it to the final name."""
+        final_path = amazon_path / image_name
+        if await anyio.Path(final_path).exists():
+            await anyio.Path(final_path).unlink()
+        await self.hass.async_add_executor_job(Path(image_file).rename, final_path)
+        _LOGGER.debug("Single Amazon image saved: %s", image_name)
+
+    async def _copy_amazon_placeholder(
+        self, amazon_path: Path, image_name: str
+    ) -> None:
+        """Copy the Amazon no-delivery placeholder."""
+        nomail = f"{Path(__file__).parent.parent}/no_deliveries_amazon.jpg"
+        _LOGGER.debug("No Amazon images found in emails, using placeholder")
+        try:
+            await self.hass.async_add_executor_job(
+                copyfile, nomail, str(amazon_path / image_name)
+            )
+        except OSError as err:
+            _LOGGER.error("Error attempting to copy image: %s", err)
+
+    async def _amazon_hub(
+        self,
+        account: IMAP4_SSL,
+        fwds: list[str] | None = None,
+        cache: EmailCache | None = None,
+        forwarding_header: str = "",
+    ) -> dict[str, Any]:
+        """Find Amazon Hub code."""
+        _LOGGER.debug("=== AMAZON HUB SEARCH START ===")
+        count = 0
+        code = []
+        processed_ids = []
+        today = get_today().strftime("%d-%b-%Y")
+        address_list = amazon_email_addresses(fwds, "amazon.com")
+        for search_subject in AMAZON_HUB_SUBJECT:
+            (server_response, data) = await email_search(
+                account,
+                address_list,
+                today,
+                search_subject,
+                body=AMAZON_HUB_BODY,
+                header=forwarding_header,
+            )
+            if server_response == "OK" and data[0] is not None:
+                for num in data[0].split():
+                    if num in processed_ids:
+                        continue
+                    processed_ids.append(num)
+                    if cache:
+                        msg_parts = (await cache.fetch(num, "(RFC822)"))[1]
+                    else:
+                        msg_parts = (await email_fetch(account, num, "(RFC822)"))[1]
+                    for response_part in msg_parts:
+                        if isinstance(response_part, (bytes, bytearray)):
+                            msg = email.message_from_bytes(response_part)
+                            actual_subject = get_decoded_subject(msg)
+                            body = get_email_body(msg)
+                            if hub_code := _extract_hub_code(
+                                body,
+                                AMAZON_HUB_BODY,
+                                actual_subject,
+                                AMAZON_HUB_SUBJECT_SEARCH,
+                            ):
+                                count += 1
+                                if hub_code not in code:
+                                    code.append(hub_code)
+        return {AMAZON_HUB: count, AMAZON_HUB_CODE: code}
+
+    async def _amazon_otp(
+        self,
+        account: IMAP4_SSL,
+        fwds: list[str] | None = None,
+        cache: EmailCache | None = None,
+        forwarding_header: str = "",
+    ) -> dict[str, Any]:
+        """Find Amazon OTP code."""
+        code = []
+        today = get_today().strftime("%d-%b-%Y")
+        address_list = amazon_email_addresses(fwds, "amazon.com")
+        (server_response, data) = await email_search(
+            account,
+            address_list,
+            today,
+            AMAZON_OTP_SUBJECT,
+            body=AMAZON_OTP_REGEX,
+            header=forwarding_header,
+        )
+        if server_response == "OK" and data[0] is not None:
+            for num in data[0].split():
+                if cache:
+                    msg_parts = (await cache.fetch(num, "(RFC822)"))[1]
+                else:
+                    msg_parts = (await email_fetch(account, num, "(RFC822)"))[1]
+                for response_part in msg_parts:
+                    if isinstance(response_part, (bytes, bytearray)):
+                        msg = email.message_from_bytes(response_part)
+                        body = get_email_body(msg)
+                        if (
+                            found := re.compile(AMAZON_OTP_REGEX).search(body)
+                        ) is not None:
+                            code.append(found.group(2))
+        return {AMAZON_OTP: len(code), AMAZON_OTP_CODE: code}
+
+    async def _amazon_exception(
+        self,
+        account: IMAP4_SSL,
+        fwds: list[str] | None = None,
+        domain: str | None = None,
+        cache: EmailCache | None = None,
+        forwarding_header: str = "",
+    ) -> dict[str, Any]:
+        """Find Amazon exception emails."""
+        count = 0
+        orders = []
+        today = get_today().strftime("%d-%b-%Y")
+        address_list = amazon_email_addresses(fwds, domain)
+        (server_response, data) = await email_search(
+            account=account,
+            address=address_list,
+            date=today,
+            subject=AMAZON_EXCEPTION_SUBJECT,
+            header=forwarding_header,
+        )
+        if server_response == "OK" and data[0] is not None:
+            order_pattern = re.compile(r"[0-9]{3}-[0-9]{7}-[0-9]{7}")
+            for num in data[0].split():
+                if cache:
+                    msg_parts = (await cache.fetch(num, "(RFC822)"))[1]
+                else:
+                    msg_parts = (await email_fetch(account, num, "(RFC822)"))[1]
+                for response_part in msg_parts:
+                    if isinstance(response_part, (bytes, bytearray)):
+                        msg = email.message_from_bytes(response_part)
+                        body = get_email_body(msg)
+                        subject = get_decoded_subject(msg)
+                        if AMAZON_EXCEPTION_BODY in body:
+                            count += 1
+                            if found := order_pattern.findall(body):
+                                orders.extend(found)
+                            if found := order_pattern.findall(subject):
+                                orders.extend(found)
+        return {AMAZON_EXCEPTION: count, AMAZON_EXCEPTION_ORDER: orders}

custom_components/mail_and_packages/shippers/base.py

@@ -0,0 +1,54 @@
+"""Base Shipper class."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from aioimaplib import IMAP4_SSL
+from homeassistant.core import HomeAssistant
+
+from custom_components.mail_and_packages.utils.cache import EmailCache
+
+
+class Shipper(ABC):
+    """Base class for shipper-specific parsing logic."""
+
+    def __init__(self, hass: HomeAssistant, config: dict[str, Any]) -> None:
+        """Initialize the shipper."""
+        self.hass = hass
+        self.config = config
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Return the internal name of the shipper."""
+
+    @classmethod
+    @abstractmethod
+    def handles_sensor(cls, sensor_type: str) -> bool:
+        """Return True if this shipper handles the given sensor type."""
+
+    @abstractmethod
+    async def process(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensor_type: str,
+    ) -> dict[str, Any]:
+        """Process emails for this shipper on the given date for a specific sensor."""
+
+    @abstractmethod
+    async def process_batch(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensors: list[str],
+        cache: EmailCache,
+        since_date: str | None = None,
+    ) -> dict[str, Any]:
+        """Process multiple sensors for this shipper using batched fetching/searching.
+
+        since_date: earliest IMAP SINCE date for _delivering/_exception/_delivered
+        sensors. Defaults to date (today) if not provided.
+        """

custom_components/mail_and_packages/shippers/generic.py

@@ -0,0 +1,697 @@
+"""Generic Shipper class."""
+
+from __future__ import annotations
+
+import email
+import logging
+from email.header import decode_header
+from pathlib import Path
+from shutil import copyfile
+from typing import Any
+
+import anyio
+from aioimaplib import IMAP4_SSL
+
+from custom_components.mail_and_packages.const import (
+    AMAZON_DELIEVERED_BY_OTHERS_SEARCH_TEXT,
+    AMAZON_DELIVERED,
+    ATTR_BODY,
+    ATTR_BODY_COUNT,
+    ATTR_COUNT,
+    ATTR_EMAIL,
+    ATTR_PATTERN,
+    ATTR_SUBJECT,
+    ATTR_TRACKING,
+    CAMERA_DATA,
+    CAMERA_EXTRACTION_CONFIG,
+    CONF_FORWARDING_HEADER,
+    SENSOR_DATA,
+)
+from custom_components.mail_and_packages.utils.cache import EmailCache
+from custom_components.mail_and_packages.utils.email import find_text, find_text_matches
+from custom_components.mail_and_packages.utils.imap import (
+    email_fetch,
+    email_fetch_headers,
+    email_search,
+)
+from custom_components.mail_and_packages.utils.shipper import (
+    generic_delivery_image_extraction,
+    get_tracking,
+)
+
+from .base import Shipper
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class GenericShipper(Shipper):
+    """Generic Shipper class for UPS, FedEx, Walmart, etc."""
+
+    @property
+    def name(self) -> str:
+        """Return the internal name of the shipper."""
+        return "generic"
+
+    @classmethod
+    def handles_sensor(cls, sensor_type: str) -> bool:
+        """Return True if this shipper handles the given sensor type."""
+        return sensor_type in SENSOR_DATA
+
+    async def process(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensor_type: str,
+        cache: EmailCache | None = None,
+        since_date: str | None = None,
+    ) -> dict[str, Any]:
+        """Process emails for this shipper on the given date.
+
+        since_date: if provided, used instead of date for _delivering and
+        _exception sensors so that emails from previous days are included.
+        """
+        _LOGGER.debug("Processing generic sensor: %s", sensor_type)
+
+        if sensor_type not in SENSOR_DATA:
+            _LOGGER.error("Sensor %s not found in SENSOR_DATA", sensor_type)
+            return {ATTR_COUNT: 0}
+
+        config = SENSOR_DATA[sensor_type]
+        email_addresses = config.get(ATTR_EMAIL, [])
+        subjects = config.get(ATTR_SUBJECT, [])
+
+        # _packages sensors with no email/subject are computed in process_batch
+        # as delivering + delivered; skip IMAP search here.
+        if sensor_type.endswith("_packages") and not email_addresses and not subjects:
+            _LOGGER.debug(
+                "Skipping email search for %s: no email addresses configured",
+                sensor_type,
+            )
+            return {ATTR_COUNT: 0, ATTR_TRACKING: []}
+
+        forwarding_header, email_addresses = self._resolve_forwarding(email_addresses)
+
+        # _delivering/_exception/_packages use the extended window so in-transit
+        # packages remain visible across the midnight boundary.
+        # _delivered uses today's date for the sensor count (resets at midnight)
+        # but also searches the extended window to obtain tracking numbers for
+        # deduplication — without those, a package delivered yesterday would still
+        # appear as "delivering" today because the delivering email is in the window
+        # but the delivered email is not.
+        is_delivered = sensor_type.endswith("_delivered")
+        search_date = date
+        if (
+            since_date
+            and sensor_type.endswith(
+                ("_delivering", "_exception", "_delivered", "_packages")
+            )
+            and sensor_type != "post_de_delivering"
+        ):
+            search_date = since_date
+
+        result = {ATTR_COUNT: 0, ATTR_TRACKING: []}
+
+        # Skip email search for sensors with no email addresses configured
+        # (e.g. *_packages sensors that are empty dicts in SENSOR_DATA)
+        if not email_addresses:
+            _LOGGER.debug(
+                "Skipping email search for %s: no email addresses configured",
+                sensor_type,
+            )
+            return result
+
+        image_path = self.config.get("image_path")
+        # Setup image extraction
+        shipper_cfg = await self._setup_image_extraction(sensor_type, image_path)
+        image_found = False
+
+        count, found_data, image_found = await self._search_for_emails(
+            account,
+            email_addresses,
+            search_date,
+            subjects,
+            config,
+            shipper_cfg,
+            sensor_type,
+            result,
+            cache,
+            forwarding_header,
+        )
+
+        # Process tracking numbers
+        result[ATTR_TRACKING] = await self._process_tracking_numbers(
+            sensor_type,
+            found_data,
+            account,
+            cache,
+        )
+        if result[ATTR_TRACKING]:
+            count = len(result[ATTR_TRACKING])
+
+        if is_delivered:
+            result["pre_filtered_tracking"] = result.get(ATTR_TRACKING, [])
+
+        # For _delivered sensors, the extended-window search gives us tracking
+        # numbers needed for deduplication (above), but the count must reflect
+        # only today's deliveries so the sensor resets at midnight.
+        if is_delivered and since_date and search_date != date:
+            today_result: dict[str, Any] = {ATTR_COUNT: 0, ATTR_TRACKING: []}
+            today_count, today_found, _ = await self._search_for_emails(
+                account,
+                email_addresses,
+                date,
+                subjects,
+                config,
+                shipper_cfg,
+                sensor_type,
+                today_result,
+                cache,
+                forwarding_header,
+            )
+            today_tracking = await self._process_tracking_numbers(
+                sensor_type, today_found, account, cache
+            )
+            count = len(today_tracking) if today_tracking else today_count
+            result[ATTR_TRACKING] = today_tracking
+
+        result[ATTR_COUNT] = count
+        if shipper_cfg:
+            image_attr = f"{shipper_cfg['name']}_image"
+            result[image_attr] = shipper_cfg["image_name"]
+            result["image_path"] = image_path
+
+            if not image_found:
+                await self._copy_generic_placeholder(shipper_cfg)
+
+        return result
+
+    def _resolve_forwarding(self, email_addresses: list[str]) -> tuple[str, list[str]]:
+        """Return (forwarding_header, resolved_email_addresses).
+
+        Header mode: uses original-sender header for matching; address list
+        is passed as-is so IMAP can match via HEADER substring.
+        Address-list mode: prepends the user's forwarded addresses so that
+        emails arriving through a forwarding service are also matched.
+        """
+        forwarding_header = self.config.get(CONF_FORWARDING_HEADER, "")
+        if forwarding_header and forwarding_header != "(none)":
+            return forwarding_header, email_addresses
+        forwarding_header = ""
+        forwarded_emails = self.config.get("forwarded_emails", [])
+        if isinstance(forwarded_emails, str):
+            forwarded_emails = [
+                e.strip() for e in forwarded_emails.split(",") if e.strip()
+            ]
+        if forwarded_emails:
+            email_addresses = forwarded_emails + email_addresses
+        return forwarding_header, email_addresses
+
+    async def process_batch(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensors: list[str],
+        cache: EmailCache,
+        since_date: str | None = None,
+    ) -> dict[str, Any]:
+        """Process multiple generic sensors in batch."""
+        batch_results, all_tracking = await self._process_individual_sensors(
+            account, date, sensors, cache, since_date
+        )
+
+        self._deduplicate_batch_tracking(batch_results)
+        self._compute_package_totals(batch_results)
+
+        # Merge results and aggregate global tracking
+        res = {}
+        for sensor, sensor_res in batch_results:
+            tracking = (
+                sensor_res.pop("pre_filtered_tracking", [])
+                if sensor.endswith("_delivered")
+                else sensor_res.get(ATTR_TRACKING)
+            )
+            res.update(sensor_res)
+            # Expose per-sensor raw tracking for coordinator state management.
+            # Keyed as "_tracking_details" to distinguish from the public data dict.
+            if tracking and sensor.endswith(
+                ("_delivering", "_delivered", "_exception")
+            ):
+                res.setdefault("_tracking_details", {})[sensor] = list(tracking)
+
+        if all_tracking:
+            res[ATTR_TRACKING] = list(all_tracking)
+
+        return res
+
+    async def _process_individual_sensors(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensors: list[str],
+        cache: EmailCache,
+        since_date: str | None = None,
+    ) -> tuple[list[tuple[str, dict[str, Any]]], set[str]]:
+        """Process each sensor independently and aggregate tracking."""
+        batch_results = []
+        all_tracking = set()
+
+        for sensor in sensors:
+            sensor_res = await self.process(
+                account, date, sensor, cache, since_date=since_date
+            )
+            # Replicate coordinator dictionary logic for local sensor counts
+            if sensor not in sensor_res and ATTR_COUNT in sensor_res:
+                sensor_res[sensor] = sensor_res[ATTR_COUNT]
+
+            # Capture today-only tracking for _delivered sensors BEFORE
+            # _deduplicate_batch_tracking runs (which currently only modifies
+            # _delivering and _packages sensor results).
+            if sensor_res.get(ATTR_TRACKING) and sensor.endswith("_delivered"):
+                sensor_res[f"{sensor}_tracking"] = sensor_res[ATTR_TRACKING]
+
+            # Record results for post-processing
+            batch_results.append((sensor, sensor_res))
+
+            # Aggregate all tracking numbers found
+            if sensor_res.get(ATTR_TRACKING):
+                all_tracking.update(sensor_res[ATTR_TRACKING])
+
+        return batch_results, all_tracking
+
+    def _deduplicate_batch_tracking(
+        self,
+        batch_results: list[tuple[str, dict[str, Any]]],
+    ) -> None:
+        """Deduplicate tracking numbers across sensors based on shipper prefix."""
+        shippers = {}
+        for sensor, sensor_res in batch_results:
+            # Prefix is everything before the last underscore (e.g., 'ups', 'fedex')
+            prefix = "_".join(sensor.split("_")[:-1])
+            if prefix not in shippers:
+                shippers[prefix] = {
+                    "delivered": set(),
+                    "delivering": set(),
+                    "update_targets": [],
+                    "package_targets": [],
+                }
+
+            tracking = set(sensor_res.get(ATTR_TRACKING, []))
+            if sensor.endswith("_delivered"):
+                shippers[prefix]["delivered"].update(tracking)
+            elif sensor.endswith(("_delivering", "_exception")):
+                shippers[prefix]["delivering"].update(tracking)
+                shippers[prefix]["update_targets"].append((sensor, sensor_res))
+            elif sensor.endswith("_packages"):
+                shippers[prefix]["package_targets"].append((sensor, sensor_res))
+
+        for data in shippers.values():
+            # Remove "delivered" tracking numbers from in-transit sensors
+            self._apply_deduplication(data["update_targets"], data["delivered"])
+            # Remove "delivering" and "delivered" tracking numbers from _packages
+            # so _packages only shows packages not yet out for delivery or delivered
+            in_pipeline = data["delivering"] | data["delivered"]
+            self._apply_deduplication(data["package_targets"], in_pipeline)
+
+    def _apply_deduplication(
+        self,
+        targets: list[tuple[str, dict[str, Any]]],
+        delivered_ids: set[str],
+    ) -> None:
+        """Apply deduplication logic to a list of target sensors."""
+        if not delivered_ids:
+            return
+
+        for sensor, sensor_res in targets:
+            original_tracking = sensor_res.get(ATTR_TRACKING, [])
+            new_tracking = [
+                tid for tid in original_tracking if tid not in delivered_ids
+            ]
+
+            if len(new_tracking) != len(original_tracking):
+                sensor_res[ATTR_TRACKING] = new_tracking
+                sensor_res[sensor] = len(new_tracking)
+                if ATTR_COUNT in sensor_res:
+                    sensor_res[ATTR_COUNT] = len(new_tracking)
+
+    def _compute_package_totals(
+        self,
+        batch_results: list[tuple[str, dict[str, Any]]],
+    ) -> None:
+        """Compute _packages sensors with empty config as delivering + delivered.
+
+        These sensors have no IMAP search of their own; their value is the
+        sum of the shipper's _delivering and _delivered counts (matching the
+        original pre-refactor behaviour in helpers.py).
+        """
+        sensor_counts = {
+            sensor: sensor_res.get(sensor, sensor_res.get(ATTR_COUNT, 0))
+            for sensor, sensor_res in batch_results
+        }
+
+        for sensor, sensor_res in batch_results:
+            if not sensor.endswith("_packages"):
+                continue
+            config = SENSOR_DATA.get(sensor, {})
+            if config.get(ATTR_EMAIL) or config.get(ATTR_SUBJECT):
+                continue  # sensor has its own IMAP search config
+            prefix = sensor.replace("_packages", "")
+            computed = sensor_counts.get(f"{prefix}_delivering", 0) + sensor_counts.get(
+                f"{prefix}_delivered", 0
+            )
+            sensor_res[sensor] = computed
+            sensor_res[ATTR_COUNT] = computed
+
+    async def _copy_generic_placeholder(self, shipper_cfg: dict[str, Any]) -> None:
+        """Copy the generic placeholder for the shipper."""
+        shipper_name = shipper_cfg["name"]
+        # Try to find courier-specific placeholder
+        placeholder = Path(__file__).parent.parent / f"no_deliveries_{shipper_name}.jpg"
+        if not await anyio.Path(placeholder).exists():
+            placeholder = Path(__file__).parent.parent / "mail_none.gif"
+
+        target = (
+            Path(shipper_cfg["image_path"]) / shipper_name / shipper_cfg["image_name"]
+        )
+        _LOGGER.debug(
+            "No %s images found in emails, using placeholder: %s",
+            shipper_name,
+            placeholder.name,
+        )
+        try:
+            await self.hass.async_add_executor_job(
+                copyfile, str(placeholder), str(target)
+            )
+        except OSError as err:
+            _LOGGER.error("Error attempting to copy placeholder: %s", err)
+
+    async def _search_for_emails(
+        self,
+        account: IMAP4_SSL,
+        email_addresses: list[str],
+        date: str,
+        subjects: list[str],
+        config: dict[str, Any],
+        shipper_cfg: dict[str, Any] | None,
+        sensor_type: str,
+        result: dict[str, Any],
+        cache: EmailCache | None = None,
+        forwarding_header: str = "",
+    ) -> tuple[int, list[bytes], bool]:
+        """Search for and process emails."""
+        count = 0
+        unique_email_ids = set()
+        found_data = []
+        image_found = False
+
+        (server_response, sdata) = await email_search(
+            account=account,
+            address=email_addresses,
+            date=date,
+            subject=subjects,
+            body=config.get(ATTR_BODY, ""),
+            header=forwarding_header,
+        )
+
+        if server_response == "OK" and sdata[0]:
+            raw_ids = sdata[0].split()
+            _LOGGER.debug(
+                "Found %d matching email IDs for %s: %s",
+                len(raw_ids),
+                sensor_type,
+                [eid.decode() if isinstance(eid, bytes) else eid for eid in raw_ids],
+            )
+            verified_ids = await self._verify_matched_subjects(
+                account, raw_ids, sensor_type, subjects, cache
+            )
+            filtered_new_ids = self._filter_unique_ids(verified_ids, unique_email_ids)
+
+            if filtered_new_ids:
+                count, img_found = await self._process_matched_emails(
+                    account,
+                    config,
+                    filtered_new_ids,
+                    count,
+                    cache,
+                    shipper_cfg,
+                    sensor_type,
+                    result,
+                    found_data,
+                )
+                if img_found:
+                    image_found = True
+
+        return count, found_data, image_found
+
+    def _decode_subject(self, header_part: bytes | bytearray) -> str | None:
+        """Decode MIME encoded subject from email header part."""
+        msg = email.message_from_bytes(header_part)
+        header_val = msg.get("subject")
+        if not header_val:
+            return None
+
+        decoded = decode_header(header_val)[0]
+        subject_bytes, encoding = decoded
+        if encoding:
+            try:
+                if isinstance(subject_bytes, bytes):
+                    return subject_bytes.decode(encoding, "ignore").strip()
+                return str(subject_bytes).strip()
+            except (LookupError, UnicodeError):
+                pass
+
+        if isinstance(subject_bytes, bytes):
+            return subject_bytes.decode("utf-8", "ignore").strip()
+        return str(subject_bytes).strip()
+
+    async def _verify_matched_subjects(
+        self,
+        account: IMAP4_SSL,
+        email_ids: list[bytes],
+        sensor_type: str,
+        expected_subjects: list[str],
+        cache: EmailCache | None = None,
+    ) -> list[bytes]:
+        """Verify the subject of each matched email locally and log for debugging."""
+        if not expected_subjects:
+            return email_ids
+
+        verified_ids = []
+        expected_subjects_lower = [s.lower() for s in expected_subjects]
+
+        for eid in email_ids:
+            try:
+                if cache:
+                    header_data = (
+                        await cache.fetch(eid, "(BODY[HEADER.FIELDS (SUBJECT)])")
+                    )[1]
+                else:
+                    header_data = (await email_fetch_headers(account, eid))[1]
+
+                subject_found = False
+                for part in header_data:
+                    if isinstance(part, (bytes, bytearray)):
+                        subject = self._decode_subject(part)
+                        if not subject:
+                            continue
+
+                        _LOGGER.debug(
+                            "Matched email for %s (ID %s): %s",
+                            sensor_type,
+                            eid.decode() if isinstance(eid, bytes) else eid,
+                            subject,
+                        )
+                        subject_lower = subject.lower()
+                        if any(
+                            expected in subject_lower
+                            for expected in expected_subjects_lower
+                        ):
+                            subject_found = True
+
+                if subject_found:
+                    verified_ids.append(eid)
+                else:
+                    _LOGGER.debug(
+                        "Email ID %s rejected for %s: Subject did not match any expected subjects.",
+                        eid.decode() if isinstance(eid, bytes) else eid,
+                        sensor_type,
+                    )
+            except (OSError, AttributeError) as err:
+                _LOGGER.debug("Could not fetch subject for email %s: %s", eid, err)
+
+        return verified_ids
+
+    def _filter_unique_ids(
+        self, email_ids: list[bytes], unique_email_ids: set
+    ) -> list[bytes]:
+        """Filter out already processed email IDs."""
+        new_ids = []
+        for eid in email_ids:
+            eid_str = eid.decode() if isinstance(eid, bytes) else str(eid)
+            if eid_str not in unique_email_ids:
+                unique_email_ids.add(eid_str)
+                new_ids.append(eid)
+        return new_ids
+
+    async def _process_matched_emails(
+        self,
+        account: IMAP4_SSL,
+        config: dict[str, Any],
+        new_ids: list[bytes],
+        current_count: int,
+        cache: EmailCache | None,
+        shipper_cfg: dict[str, Any] | None,
+        sensor_type: str,
+        result: dict[str, Any],
+        found_data: list[bytes],
+    ) -> tuple[int, bool]:
+        """Process a batch of matched unique emails."""
+        image_found = False
+        count, matched_ids = await self._process_emails_by_type(
+            account, config, new_ids, current_count, cache
+        )
+        if matched_ids:
+            found_data.append(b" ".join(matched_ids))
+
+            if shipper_cfg:
+                if await self._extract_images_for_shipper(
+                    account, matched_ids, shipper_cfg, cache
+                ):
+                    image_found = True
+
+            if sensor_type.endswith("_delivered") and sensor_type != AMAZON_DELIVERED:
+                await self._check_amazon_mentions(account, matched_ids, result, cache)
+
+        return count, image_found
+
+    async def _process_tracking_numbers(
+        self,
+        sensor_type: str,
+        found_data: list,
+        account: IMAP4_SSL,
+        cache: EmailCache | None = None,
+    ) -> list:
+        """Process tracking numbers for the sensor."""
+        tracking_key = f"{'_'.join(sensor_type.split('_')[:-1])}_tracking"
+        if (
+            tracking_key not in SENSOR_DATA
+            or ATTR_PATTERN not in SENSOR_DATA[tracking_key]
+        ):
+            return []
+
+        pattern = SENSOR_DATA[tracking_key][ATTR_PATTERN][0]
+        tracking_nums = []
+        for sdata in found_data:
+            tracking_nums.extend(
+                await get_tracking(sdata.decode(), account, pattern, cache)
+            )
+
+        return list(dict.fromkeys(tracking_nums))
+
+    async def _setup_image_extraction(
+        self,
+        sensor_type: str,
+        image_path: str,
+    ) -> dict | None:
+        """Set up image extraction configuration."""
+        if not sensor_type.endswith("_delivered"):
+            return None
+
+        shipper_name = sensor_type.replace("_delivered", "")
+        camera_key = f"{shipper_name}_camera"
+        if camera_key not in CAMERA_DATA or camera_key in (
+            "usps_camera",
+            "generic_camera",
+        ):
+            return None
+
+        extraction_config = CAMERA_EXTRACTION_CONFIG.get(shipper_name, {})
+        absolute_image_path = image_path.rstrip("/") + "/"
+
+        def _create_dir():
+            path = Path(absolute_image_path) / shipper_name
+            if not path.exists():
+                path.mkdir(parents=True, exist_ok=True)
+
+        await self.hass.async_add_executor_job(_create_dir)
+
+        return {
+            "name": shipper_name,
+            "image_path": absolute_image_path,
+            "image_name": self.config.get(f"{shipper_name}_image")
+            or f"{shipper_name}_delivery.jpg",
+            "image_type": extraction_config.get("image_type", "jpeg"),
+            "cid_name": extraction_config.get("cid_name"),
+            "pattern": extraction_config.get("attachment_filename_pattern"),
+        }
+
+    async def _process_emails_by_type(
+        self,
+        account: IMAP4_SSL,
+        config: dict,
+        ids: list,
+        current_count: int,
+        cache: EmailCache | None = None,
+    ) -> tuple[int, list]:
+        """Process emails based on body search or just count."""
+        if ATTR_BODY in config:
+            body_count = config.get(ATTR_BODY_COUNT, False)
+            mock_data = (b" ".join(ids),)
+            count, matched_ids = await find_text_matches(
+                mock_data,
+                account,
+                config[ATTR_BODY],
+                body_count,
+                cache,
+            )
+            return current_count + count, matched_ids
+        return current_count + len(ids), list(ids)
+
+    async def _extract_images_for_shipper(
+        self,
+        account: IMAP4_SSL,
+        ids: list,
+        s_config: dict,
+        cache: EmailCache | None = None,
+    ) -> bool:
+        """Extract delivery images from emails."""
+        image_found = False
+        for eid in ids:
+            if cache:
+                msg_parts = (await cache.fetch(eid, "(RFC822)"))[1]
+            else:
+                msg_parts = (await email_fetch(account, eid, "(RFC822)"))[1]
+            for response_part in msg_parts:
+                if isinstance(response_part, (bytes, bytearray)):
+                    if generic_delivery_image_extraction(
+                        response_part,
+                        s_config["image_path"],
+                        s_config["image_name"],
+                        s_config["name"],
+                        s_config["image_type"],
+                        s_config["cid_name"],
+                        s_config["pattern"],
+                    ):
+                        _LOGGER.debug("Extracted image for %s", s_config["name"])
+                        image_found = True
+        return image_found
+
+    async def _check_amazon_mentions(
+        self,
+        account: IMAP4_SSL,
+        ids: list,
+        result: dict,
+        cache: EmailCache | None = None,
+    ):
+        """Check for Amazon mentions in emails."""
+        mock_data = (b" ".join(ids),)
+        amazon_mentions = await find_text(
+            mock_data,
+            account,
+            AMAZON_DELIEVERED_BY_OTHERS_SEARCH_TEXT,
+            False,
+            cache,
+        )
+        if amazon_mentions > 0:
+            result["amazon_delivered_by_others"] = (
+                result.get("amazon_delivered_by_others", 0) + amazon_mentions
+            )

custom_components/mail_and_packages/shippers/post_de.py

@@ -0,0 +1,370 @@
+"""Post DE Shipper class."""
+
+from __future__ import annotations
+
+import email
+import io
+import logging
+import shutil
+from pathlib import Path
+from typing import Any
+
+import anyio
+from aioimaplib import IMAP4_SSL
+from PIL import Image, UnidentifiedImageError
+
+from custom_components.mail_and_packages.const import (
+    ATTR_IMAGE_PATH,
+    CONF_CUSTOM_IMG_FILE,
+    CONF_DURATION,
+    CONF_FORWARDING_HEADER,
+    CONF_GENERATE_GRID,
+    CONF_GENERATE_MP4,
+    CONF_POST_DE_CUSTOM_IMG_FILE,
+    DEFAULT_CUSTOM_IMG_FILE,
+    SENSOR_DATA,
+)
+from custom_components.mail_and_packages.utils.cache import EmailCache
+from custom_components.mail_and_packages.utils.date import get_formatted_date
+from custom_components.mail_and_packages.utils.image import (
+    _generate_mp4,
+    cleanup_images,
+    generate_delivery_gif,
+    generate_grid_img,
+    random_filename,
+    resize_images,
+)
+from custom_components.mail_and_packages.utils.imap import email_fetch, email_search
+
+from .base import Shipper
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class PostDEShipper(Shipper):
+    """Post DE Briefankündigung shipper."""
+
+    @property
+    def name(self) -> str:
+        """Return shipper name."""
+        return "post_de"
+
+    @classmethod
+    def handles_sensor(cls, sensor_type: str) -> bool:
+        """Return True if this shipper handles the given sensor type."""
+        return sensor_type == "post_de_mail"
+
+    async def process(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensor_type: str,
+        cache: EmailCache | None = None,
+    ) -> dict[str, Any]:
+        """Process Post DE Briefankündigung emails."""
+        if sensor_type != "post_de_mail":
+            return {sensor_type: 0}
+
+        config = self._get_config()
+        image_count = 0
+        images = []
+        images_delete = []
+
+        (server_response, data) = await self._search_emails(account)
+
+        # Bail out on error
+        if server_response != "OK" or data[0] is None:
+            return {sensor_type: image_count}
+
+        # Setup image directory
+        post_de_dir = Path(config["image_output_path"]) / "post_de"
+        if not await self._setup_image_directory(str(post_de_dir)):
+            return {sensor_type: image_count}
+
+        _LOGGER.debug("Post DE Briefankündigung email found processing...")
+        for num in data[0].split():
+            (image_count, images) = await self._process_post_de_email(
+                account,
+                num,
+                str(post_de_dir),
+                image_count,
+                images,
+                cache,
+            )
+
+        image_count = len(images)
+
+        if image_count > 0:
+            await self._generate_mail_image(
+                images,
+                str(post_de_dir),
+                config["image_name"],
+                config["gif_duration"],
+                images_delete,
+            )
+        elif image_count == 0:
+            await self._copy_nomail_image(
+                str(post_de_dir),
+                config["image_name"],
+                config["custom_img"],
+            )
+
+        if config["gen_mp4"]:
+            await self._generate_mp4_video(
+                str(post_de_dir),
+                config["image_name"],
+            )
+        if config["gen_grid"]:
+            await self._generate_grid_image(
+                str(post_de_dir),
+                config["image_name"],
+                image_count,
+            )
+
+        return {
+            sensor_type: image_count,
+            "post_de_image": config["image_name"],
+            ATTR_IMAGE_PATH: config["image_output_path"],
+            "post_de_grid_image_name": config["image_name"].replace(
+                ".gif", "_grid.png"
+            ),
+        }
+
+    async def process_batch(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensors: list[str],
+        cache: EmailCache,
+        since_date: str | None = None,
+    ) -> dict[str, Any]:
+        """Process multiple Post DE sensors in batch."""
+        res = {}
+        for sensor in sensors:
+            res.update(await self.process(account, date, sensor, cache))
+
+            # Replicate coordinator dict structure
+            if sensor not in res:
+                res[sensor] = res.get(sensor, 0)
+
+        return res
+
+    async def _generate_mp4_video(self, path: str, name: str):
+        """Generate MP4 video from images."""
+        await self.hass.async_add_executor_job(_generate_mp4, path + "/", name)
+
+    async def _generate_grid_image(self, path: str, name: str, count: int):
+        """Generate grid image from images."""
+        await self.hass.async_add_executor_job(
+            generate_grid_img, path + "/", name, count
+        )
+
+    async def _generate_mail_image(
+        self,
+        images: list,
+        path: str,
+        name: str,
+        duration: int,
+        delete_list: list,
+    ):
+        """Generate animated GIF from mail images."""
+        try:
+            _LOGGER.debug("Resizing Post DE images to 724x320...")
+            all_images = await self.hass.async_add_executor_job(
+                resize_images,
+                images,
+                724,
+                320,
+            )
+            delete_list.extend(all_images)
+
+            _LOGGER.debug("Generating animated GIF for Post DE")
+            gif_path = str(Path(path) / name)
+            await self.hass.async_add_executor_job(
+                generate_delivery_gif,
+                all_images,
+                gif_path,
+                duration * 1000,
+            )
+            _LOGGER.debug("Post DE mail image generated.")
+        except (OSError, ValueError) as err:
+            _LOGGER.error("Error attempting to generate Post DE image: %s", err)
+
+        for image in delete_list:
+            await self.hass.async_add_executor_job(
+                cleanup_images,
+                f"{Path(image).parent}/",
+                Path(image).name,
+            )
+
+    async def _copy_nomail_image(self, path: str, name: str, custom_img: str | None):
+        """Copy the 'no mail' placeholder image."""
+
+        def _prepare():
+            if not Path(path).exists():
+                Path(path).mkdir(parents=True, exist_ok=True)
+            target = Path(path) / name
+            if target.is_file():
+                cleanup_images(path + "/", name)
+            src = custom_img or str(Path(__file__).parent.parent / "mail_none.gif")
+            shutil.copyfile(src, str(target))
+
+        _LOGGER.debug("No Post DE mail found.")
+        try:
+            await self.hass.async_add_executor_job(_prepare)
+        except OSError as err:
+            _LOGGER.error("Error attempting to copy Post DE image: %s", err)
+
+    def _get_config(self) -> dict:
+        """Get Post DE specific configuration."""
+
+        image_path = self.config.get("image_path")
+        return {
+            "image_output_path": image_path,
+            "gif_duration": self.config.get(CONF_DURATION),
+            "image_name": self.config.get("post_de_image") or "post_de_deliveries.gif",
+            "gen_mp4": self.config.get(CONF_GENERATE_MP4),
+            "custom_img": self.config.get(CONF_POST_DE_CUSTOM_IMG_FILE)
+            or self.config.get(CONF_CUSTOM_IMG_FILE)
+            or DEFAULT_CUSTOM_IMG_FILE,
+            "gen_grid": self.config.get(CONF_GENERATE_GRID),
+        }
+
+    async def _search_emails(self, account: IMAP4_SSL) -> tuple:
+        """Search for Post DE Briefankündigung emails."""
+        _LOGGER.debug("Attempting to find Post DE Briefankündigung mail")
+        _LOGGER.debug("Post DE search date: %s", get_formatted_date())
+
+        config = SENSOR_DATA["post_de_mail"]
+        email_addresses = config.get("email", [])
+        subjects = config.get("subject", [])
+
+        forwarding_header = self.config.get(CONF_FORWARDING_HEADER, "")
+        if forwarding_header and forwarding_header != "(none)":
+            pass
+        else:
+            forwarding_header = ""
+            forwarded_emails = self.config.get("forwarded_emails", [])
+            if isinstance(forwarded_emails, str):
+                forwarded_emails = [
+                    e.strip() for e in forwarded_emails.split(",") if e.strip()
+                ]
+            if forwarded_emails:
+                email_addresses = forwarded_emails + email_addresses
+
+        return await email_search(
+            account=account,
+            address=email_addresses,
+            date=get_formatted_date(),
+            subject=subjects,
+            header=forwarding_header,
+        )
+
+    async def _setup_image_directory(self, path: str) -> bool:
+        """Ensure image directory exists and is prepared."""
+        if not await anyio.Path(path).is_dir():
+            try:
+                await anyio.Path(path).mkdir(parents=True, exist_ok=True)
+            except OSError as err:
+                _LOGGER.error("Error creating directory: %s", err)
+                return False
+
+        # Clean up
+        await self.hass.async_add_executor_job(cleanup_images, path + "/")
+        return True
+
+    async def _process_post_de_email(  # noqa: C901
+        self,
+        account: IMAP4_SSL,
+        num: str,
+        image_output_path: str,
+        image_count: int,
+        images: list,
+        cache: EmailCache | None = None,
+    ) -> tuple[int, list]:
+        """Process a single Post DE email and extract envelope scans.
+
+        Expected email payload format is HTML containing inline <img> tags
+        referencing base64-encoded PNG/JPEG images.
+        """
+        if cache:
+            msg_parts = (await cache.fetch(num, "(RFC822)"))[1]
+        else:
+            msg_parts = (await email_fetch(account, num, "(RFC822)"))[1]
+        _LOGGER.debug("Processing Post DE email number: %s", num)
+        for response_part in msg_parts:
+            if isinstance(response_part, (bytes, bytearray)):
+                msg = email.message_from_bytes(response_part)
+                for part in msg.walk():
+                    if part.get_content_type() in ("image/png", "image/jpeg"):
+                        payload = part.get_payload(decode=True)
+                        if not payload:
+                            continue
+
+                        # Check image dimensions to skip logos/icons
+                        def _check_and_save(
+                            img_bytes: bytes,
+                            out_path: str,
+                            content_type: str,
+                        ) -> str | None:
+                            try:
+                                img = Image.open(io.BytesIO(img_bytes))
+                                if img.format is None:
+                                    _LOGGER.debug(
+                                        "Post DE image format is unidentified (None)"
+                                    )
+                                    return None
+
+                                # Validate format against expected content type
+                                if content_type == "image/png" and img.format != "PNG":
+                                    _LOGGER.debug(
+                                        "Post DE image format mismatch: expected PNG, got %s",
+                                        img.format,
+                                    )
+                                    return None
+                                if (
+                                    content_type == "image/jpeg"
+                                    and img.format != "JPEG"
+                                ):
+                                    _LOGGER.debug(
+                                        "Post DE image format mismatch: expected JPEG, got %s",
+                                        img.format,
+                                    )
+                                    return None
+
+                                width, height = img.size
+                                if width > 150 and height > 100:
+                                    ext = (
+                                        ".png"
+                                        if content_type == "image/png"
+                                        else ".jpg"
+                                    )
+                                    filename = random_filename(ext=ext)
+                                    target = Path(out_path) / filename
+                                    with target.open("wb") as f:
+                                        f.write(img_bytes)
+                                    return str(target)
+                            except UnidentifiedImageError as err:
+                                _LOGGER.warning(
+                                    "Unidentified image found in Post DE email: %s", err
+                                )
+                            except (OSError, ValueError, TypeError) as err:
+                                _LOGGER.debug(
+                                    "Error checking/saving Post DE image: %s", err
+                                )
+                            return None
+
+                        saved_path = await self.hass.async_add_executor_job(
+                            _check_and_save,
+                            payload,
+                            image_output_path,
+                            part.get_content_type(),
+                        )
+                        if saved_path:
+                            images.append(saved_path)
+                            image_count += 1
+                            _LOGGER.debug(
+                                "Extracted Post DE mail image: %s", saved_path
+                            )
+
+        return image_count, images

custom_components/mail_and_packages/shippers/usps.py

@@ -0,0 +1,425 @@
+"""USPS Shipper class."""
+
+from __future__ import annotations
+
+import base64
+import email
+import logging
+import re
+import shutil
+from pathlib import Path
+from typing import Any
+
+import anyio
+from aioimaplib import IMAP4_SSL
+from bs4 import BeautifulSoup
+
+from custom_components.mail_and_packages.const import (
+    ATTR_COUNT,
+    ATTR_EMAIL,
+    ATTR_GRID_IMAGE_NAME,
+    ATTR_IMAGE_PATH,
+    ATTR_SUBJECT,
+    ATTR_USPS_IMAGE,
+    ATTR_USPS_MAIL,
+    CONF_CUSTOM_IMG_FILE,
+    CONF_DURATION,
+    CONF_FORWARDING_HEADER,
+    CONF_GENERATE_GRID,
+    CONF_GENERATE_MP4,
+    DEFAULT_CUSTOM_IMG_FILE,
+    SENSOR_DATA,
+)
+from custom_components.mail_and_packages.utils.cache import EmailCache
+from custom_components.mail_and_packages.utils.date import get_formatted_date
+from custom_components.mail_and_packages.utils.image import (
+    _generate_mp4,
+    cleanup_images,
+    copy_overlays,
+    generate_delivery_gif,
+    generate_grid_img,
+    io_save_file,
+    random_filename,
+    resize_images,
+)
+from custom_components.mail_and_packages.utils.imap import email_fetch, email_search
+
+from .base import Shipper
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class USPSShipper(Shipper):
+    """USPS Informed Delivery shipper."""
+
+    @property
+    def name(self) -> str:
+        """Return shipper name."""
+        return "usps"
+
+    @classmethod
+    def handles_sensor(cls, sensor_type: str) -> bool:
+        """Return True if this shipper handles the given sensor type."""
+        return sensor_type == "usps_mail"
+
+    async def process(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensor_type: str,
+        cache: EmailCache | None = None,
+    ) -> dict[str, Any]:
+        """Process USPS Informed Delivery emails."""
+        config = self._get_usps_config()
+        image_count = 0
+        images = []
+        images_delete = []
+
+        (server_response, data) = await self._search_informed_delivery(account)
+
+        # Bail out on error
+        if server_response != "OK" or data[0] is None:
+            return {ATTR_COUNT: image_count}
+
+        # Setup image directory and overlays
+        if not await self._setup_image_directory(config["image_output_path"]):
+            return {ATTR_COUNT: image_count}
+
+        all_msg_content = ""
+        if server_response == "OK":
+            _LOGGER.debug("Informed Delivery email found processing...")
+            for num in data[0].split():
+                (image_count, images, email_content) = await self._process_usps_email(
+                    account,
+                    num,
+                    config["image_output_path"],
+                    image_count,
+                    images,
+                    cache,
+                )
+                all_msg_content += email_content
+
+        # Process images
+        images = await self._process_usps_images(all_msg_content, images)
+        image_count = len(images)
+
+        if image_count > 0:
+            await self._generate_mail_image(
+                images,
+                config["image_output_path"],
+                config["image_name"],
+                config["gif_duration"],
+                images_delete,
+            )
+        elif image_count == 0:
+            await self._copy_nomail_image(
+                config["image_output_path"],
+                config["image_name"],
+                config["custom_img"],
+            )
+
+        if config["gen_mp4"]:
+            await self._generate_mp4_video(
+                config["image_output_path"],
+                config["image_name"],
+            )
+        if config["gen_grid"]:
+            await self._generate_grid_image(
+                config["image_output_path"],
+                config["image_name"],
+                image_count,
+            )
+
+        return {
+            ATTR_COUNT: image_count,
+            ATTR_USPS_IMAGE: config["image_name"],
+            ATTR_IMAGE_PATH: config["image_output_path"],
+            ATTR_GRID_IMAGE_NAME: config["image_name"].replace(".gif", "_grid.png"),
+        }
+
+    async def process_batch(
+        self,
+        account: IMAP4_SSL,
+        date: str,
+        sensors: list[str],
+        cache: EmailCache,
+        since_date: str | None = None,
+    ) -> dict[str, Any]:
+        """Process multiple USPS sensors in batch."""
+        res = {}
+        for sensor in sensors:
+            res.update(await self.process(account, date, sensor, cache))
+
+            # Replicatecoordinator dict structure
+            if sensor not in res:
+                if ATTR_COUNT in res:
+                    res[sensor] = res[ATTR_COUNT]
+                    # Don't pop ATTR_COUNT because other things might need it, actually
+                    # coordinator used to pop it via explicit assignment
+
+        return res
+
+    async def _generate_mp4_video(self, path: str, name: str):
+        """Generate MP4 video from images."""
+        await self.hass.async_add_executor_job(_generate_mp4, path, name)
+
+    async def _generate_grid_image(self, path: str, name: str, count: int):
+        """Generate grid image from images."""
+        await self.hass.async_add_executor_job(generate_grid_img, path, name, count)
+
+    async def _process_usps_images(self, content: str, images: list) -> list:
+        """Process USPS images (placeholder and filtering)."""
+        # Old USPS format: plain-text email body contained the filename as a reference.
+        # New format is handled in _extract_usps_images on properly decoded HTML.
+        if re.compile(r"\bimage-no-mailpieces?700\.jpg\b").search(content) is not None:
+            placeholder = Path(__file__).parent.parent / "image-no-mailpieces700.jpg"
+            placeholder_str = str(placeholder)
+            if placeholder.exists() and placeholder_str not in images:
+                images.append(placeholder_str)
+                _LOGGER.debug(
+                    "Placeholder image found using: image-no-mailpieces700.jpg.",
+                )
+
+        # Announcement images removal
+        return self._remove_announcement_images(images)
+
+    def _remove_announcement_images(self, images: list) -> list:
+        """Remove announcement images."""
+        return [
+            el
+            for el in images
+            if not any(
+                ignore in el
+                for ignore in ["mailerProvidedImage", "ra_0", "Mail Attachment.txt"]
+            )
+        ]
+
+    async def _generate_mail_image(
+        self,
+        images: list,
+        path: str,
+        name: str,
+        duration: int,
+        delete_list: list,
+    ):
+        """Generate animated GIF from mail images."""
+        _LOGGER.debug("Resizing images to 724x320...")
+        all_images = await self.hass.async_add_executor_job(
+            resize_images,
+            images,
+            724,
+            320,
+        )
+        delete_list.extend(all_images)
+
+        try:
+            _LOGGER.debug("Generating animated GIF")
+            gif_path = str(Path(path) / name)
+            await self.hass.async_add_executor_job(
+                generate_delivery_gif,
+                all_images,
+                gif_path,
+                duration * 1000,
+            )
+            _LOGGER.debug("Mail image generated.")
+        except (OSError, ValueError) as err:
+            _LOGGER.error("Error attempting to generate image: %s", err)
+
+        for image in delete_list:
+            await self.hass.async_add_executor_job(
+                cleanup_images,
+                f"{Path(image).parent}/",
+                Path(image).name,
+            )
+
+    async def _copy_nomail_image(self, path: str, name: str, custom_img: str | None):
+        """Copy the 'no mail' placeholder image."""
+
+        def _prepare():
+            if not Path(path).exists():
+                Path(path).mkdir(parents=True, exist_ok=True)
+            target = Path(path) / name
+            if target.is_file():
+                cleanup_images(path, name)
+            src = custom_img or str(Path(__file__).parent.parent / "mail_none.gif")
+            shutil.copyfile(src, str(target))
+
+        _LOGGER.debug("No mail found.")
+        try:
+            await self.hass.async_add_executor_job(_prepare)
+        except OSError as err:
+            _LOGGER.error("Error attempting to copy image: %s", err)
+
+    def _get_usps_config(self) -> dict:
+        """Get USPS specific configuration."""
+        return {
+            "image_output_path": self.config.get("image_path"),
+            "gif_duration": self.config.get(CONF_DURATION),
+            "image_name": self.config.get("usps_image"),
+            "gen_mp4": self.config.get(CONF_GENERATE_MP4),
+            "custom_img": self.config.get(CONF_CUSTOM_IMG_FILE)
+            or DEFAULT_CUSTOM_IMG_FILE,
+            "gen_grid": self.config.get(CONF_GENERATE_GRID),
+        }
+
+    async def _search_informed_delivery(self, account: IMAP4_SSL) -> tuple:
+        """Search for USPS Informed Delivery emails."""
+        _LOGGER.debug("Attempting to find Informed Delivery mail")
+        _LOGGER.debug("Informed delivery search date: %s", get_formatted_date())
+
+        forwarding_header = self.config.get(CONF_FORWARDING_HEADER, "")
+        if forwarding_header and forwarding_header != "(none)":
+            email_addresses = SENSOR_DATA[ATTR_USPS_MAIL][ATTR_EMAIL]
+        else:
+            forwarding_header = ""
+            forwarded_emails = self.config.get("forwarded_emails", [])
+            if isinstance(forwarded_emails, str):
+                forwarded_emails = [
+                    e.strip() for e in forwarded_emails.split(",") if e.strip()
+                ]
+            if forwarded_emails:
+                email_addresses = (
+                    forwarded_emails + SENSOR_DATA[ATTR_USPS_MAIL][ATTR_EMAIL]
+                )
+            else:
+                email_addresses = SENSOR_DATA[ATTR_USPS_MAIL][ATTR_EMAIL]
+
+        return await email_search(
+            account=account,
+            address=email_addresses,
+            date=get_formatted_date(),
+            subject=SENSOR_DATA[ATTR_USPS_MAIL][ATTR_SUBJECT][0],
+            header=forwarding_header,
+        )
+
+    async def _setup_image_directory(self, path: str) -> bool:
+        """Ensure image directory exists and is prepared."""
+        if not await anyio.Path(path).is_dir():
+            try:
+                await anyio.Path(path).mkdir(parents=True, exist_ok=True)
+            except OSError as err:
+                _LOGGER.error("Error creating directory: %s", err)
+                return False
+
+        # Clean up and setup overlays
+        await self.hass.async_add_executor_job(cleanup_images, path)
+        await self.hass.async_add_executor_job(copy_overlays, path)
+        return True
+
+    async def _process_usps_email(
+        self,
+        account: IMAP4_SSL,
+        num: str,
+        image_output_path: str,
+        image_count: int,
+        images: list,
+        cache: EmailCache | None = None,
+    ) -> tuple[int, list, str]:
+        """Process a single USPS Informed Delivery email."""
+        if cache:
+            msg_parts = (await cache.fetch(num, "(RFC822)"))[1]
+        else:
+            msg_parts = (await email_fetch(account, num, "(RFC822)"))[1]
+        _LOGGER.debug("Processing email number: %s", num)
+        all_content = ""
+        for response_part in msg_parts:
+            if isinstance(response_part, (bytes, bytearray)):
+                all_content += str(response_part, "utf-8", errors="ignore")
+                msg = email.message_from_bytes(response_part)
+                for part in msg.walk():
+                    if part.get_content_type() == "text/html":
+                        (image_count, images) = await self._extract_usps_images(
+                            part,
+                            image_output_path,
+                            image_count,
+                            images,
+                        )
+                    elif part.get_content_type() == "image/jpeg":
+                        (image_count, images) = await self._extract_jpeg_attachment(
+                            part,
+                            image_output_path,
+                            image_count,
+                            images,
+                        )
+        return image_count, images, all_content
+
+    async def _extract_usps_images(
+        self,
+        part: email.message.Message,
+        image_output_path: str,
+        image_count: int,
+        images: list,
+    ) -> tuple[int, list]:
+        """Extract images from an email part (HTML/Base64)."""
+        payload = part.get_payload(decode=True)
+        content = (
+            payload.decode("utf-8", "ignore")
+            if isinstance(payload, (bytes, bytearray))
+            else str(payload)
+        )
+
+        # New USPS format: unscanned mailpieces use a div with a specific id.
+        # Check here on properly decoded HTML — raw RFC822 content is
+        # quoted-printable encoded and soft line breaks could split the string.
+        if "mailpiece-with-no-image-id" in content:
+            placeholder = Path(__file__).parent.parent / "image-no-mailpieces700.jpg"
+            placeholder_str = str(placeholder)
+            if placeholder.exists() and placeholder_str not in images:
+                images.append(placeholder_str)
+                image_count += 1
+                _LOGGER.debug(
+                    "Placeholder image found using: image-no-mailpieces700.jpg.",
+                )
+
+        if "data:image/jpeg;base64" not in content:
+            return image_count, images
+
+        soup = BeautifulSoup(content, "html.parser")
+        found_images = soup.find_all(id="mailpiece-image-src-id")
+
+        for image in found_images:
+            filename = random_filename()
+            img_data = str(image["src"]).split(",")[1]
+            try:
+                target_path = Path(image_output_path) / filename
+                await self.hass.async_add_executor_job(
+                    io_save_file,
+                    target_path,
+                    base64.b64decode(img_data),
+                )
+                images.append(str(target_path))
+                image_count += 1
+            except (OSError, ValueError, TypeError) as err:
+                _LOGGER.error("Error extracting image: %s", err)
+
+        return image_count, images
+
+    async def _extract_jpeg_attachment(
+        self,
+        part: email.message.Message,
+        image_output_path: str,
+        image_count: int,
+        images: list,
+    ) -> tuple[int, list]:
+        """Extract image from JPEG attachment."""
+        _LOGGER.debug("Extracting image from email attachment")
+        filename = part.get_filename()
+        junkmail = ["mailer", "content", "package"]
+        if filename is None:
+            return image_count, images
+        if any(junk in filename for junk in junkmail):
+            return image_count, images
+
+        try:
+            target_path = Path(image_output_path) / filename
+            await self.hass.async_add_executor_job(
+                io_save_file,
+                target_path,
+                part.get_payload(decode=True),
+            )
+            images.append(str(target_path))
+            image_count += 1
+        except OSError as err:
+            _LOGGER.critical("Error opening filepath: %s", err)
+
+        return image_count, images