361 files

custom_components/album_slideshow/coordinator.py

@@ -6,11 +6,13 @@ from datetime import timedelta
 from hashlib import sha256
 import json
 import logging
+import math
 import os
 from pathlib import Path
 import re
 from typing import Any
 
+import aiohttp
 import async_timeout
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.core import HomeAssistant
@@ -22,6 +24,12 @@ from .const import (
     PUBLICALBUM_ENDPOINT,
     CONF_PROVIDER,
     CONF_ALBUM_URL,
+    CONF_GOOGLE_METADATA,
+    DEFAULT_GOOGLE_METADATA,
+    CONF_GOOGLE_LOCATION,
+    DEFAULT_GOOGLE_LOCATION,
+    DEFAULT_GOOGLE_REVERSE_GEOCODE,
+    CAMERA_METADATA_FIELDS,
     CONF_LOCAL_PATH,
     CONF_MEDIA_CONTENT_ID,
     CONF_RECURSIVE,
@@ -97,6 +105,7 @@ from .const import (
 from .store import SlideshowStore
 
 _LOGGER = logging.getLogger(__name__)
+_GOOGLE_METADATA_INTERVAL = 1.0
 
 
 @dataclass
@@ -127,12 +136,20 @@ class MediaItem:
     # Provider-specific source identifier (e.g. the Immich asset id) used by
     # background enrichment to fetch per-item metadata.
     source_id: str | None = None
+    # Detected faces as normalised [x1, y1, x2, y2, weight, selected] boxes (0..1 in
+    # displayed-image coordinates). Cover-mode cropping keeps as many whole
+    # faces as fit. None means no face data (non-Immich source, not scanned
+    # yet or the face lookup failed); an empty list means "no faces".
+    faces: list[list[float | bool]] | None = None
     # True once the local-folder EXIF reader has visited this file.
     # Prevents re-reading EXIF on every coordinator refresh and lets the
     # background enrichment task skip already-processed files even after
     # an HA restart (the flag round-trips through the items cache).
     exif_scanned: bool = False
     photo_id: str | None = None
+    face_scanned: bool = False
+    camera_metadata: dict[str, Any] | None = None
+    google_location_scanned: bool = False
 
     def __post_init__(self) -> None:
         if self.photo_id is None:
@@ -311,11 +328,11 @@ def _pick_timestamp_ms(d: dict[str, Any], *path: str) -> int | None:
         if not isinstance(cur, dict):
             return None
         cur = cur.get(p)
-    if cur is None:
+    if cur is None or isinstance(cur, bool):
         return None
     if isinstance(cur, int):
-        # Heuristic: < 1e12 means seconds, otherwise milliseconds.
-        return cur * 1000 if cur < 10**12 else cur
+        # Infer seconds only within the plausible 1800-2100 date range.
+        return cur * 1000 if -5_364_662_400 <= cur <= 4_102_444_800 else cur
     if isinstance(cur, str):
         try:
             iso = cur.replace("Z", "+00:00")
@@ -451,6 +468,7 @@ _EXIF_GPS_LAT_REF = 1                     # "N" / "S"
 _EXIF_GPS_LAT = 2                         # rational tuple
 _EXIF_GPS_LON_REF = 3                     # "E" / "W"
 _EXIF_GPS_LON = 4                         # rational tuple
+_HEIF_EXIF_RE = re.compile(rb"Exif\x00\x00(?:MM\x00\x2a|II\x2a\x00)")
 
 # Nominatim (OpenStreetMap) is free but rate-limited to 1 request/second
 # per IP and asks integrators to identify themselves with a descriptive
@@ -777,7 +795,7 @@ def _read_exif_from_image(img: Any, out: dict[str, Any]) -> None:
 
 
 def _read_exif_from_bytes(
-    data: bytes, mtime_fallback_ms: int | None
+    data: bytes, mtime_fallback_ms: int | None, *, strict: bool = False,
 ) -> dict[str, Any]:
     """Read EXIF metadata from already-downloaded image bytes.
 
@@ -802,11 +820,61 @@ def _read_exif_from_bytes(
         with Image.open(io.BytesIO(data)) as img:
             _read_exif_from_image(img, out)
     except Exception as err:
+        if strict:
+            raise
         _LOGGER.debug("EXIF: failed to read image bytes: %s", err)
 
     return out
 
 
+def _read_gps_from_header(data: bytes) -> dict[str, float]:
+    """Read GPS from an original-image prefix; raise ``ValueError`` if it is too short to tell."""
+    if data[:2] == b"\xff\xd8":
+        payload = _jpeg_exif_segment(data)
+    elif data[4:8] == b"ftyp":
+        match = _HEIF_EXIF_RE.search(data)
+        if match is None:
+            raise ValueError("HEIF EXIF not found in header")
+        payload = data[match.start():]
+    else:
+        metadata = _read_exif_from_bytes(data, None, strict=True)
+        return {key: metadata[key] for key in ("latitude", "longitude") if key in metadata}
+    if payload is None:
+        return {}
+
+    from PIL import Image
+
+    try:
+        exif = Image.Exif()
+        exif.load(payload)
+        gps = exif.get_ifd(_EXIF_TAG_GPS_IFD)
+    except Exception:
+        return {}
+    lat = _gps_to_decimal(gps.get(_EXIF_GPS_LAT), gps.get(_EXIF_GPS_LAT_REF))
+    lon = _gps_to_decimal(gps.get(_EXIF_GPS_LON), gps.get(_EXIF_GPS_LON_REF))
+    return {} if lat is None or lon is None else {"latitude": lat, "longitude": lon}
+
+
+def _jpeg_exif_segment(data: bytes) -> bytes | None:
+    pos = 2
+    while pos + 4 <= len(data):
+        if data[pos] != 0xFF:
+            return None
+        marker = data[pos + 1]
+        if marker == 0xFF:
+            pos += 1
+            continue
+        if marker in (0xD9, 0xDA):
+            return None
+        end = pos + 2 + int.from_bytes(data[pos + 2:pos + 4], "big")
+        if marker == 0xE1 and data[pos + 4:pos + 10] == b"Exif\x00\x00":
+            if end > len(data):
+                raise ValueError("JPEG EXIF segment is truncated")
+            return data[pos + 4:end]
+        pos = end
+    raise ValueError("JPEG header ends before image data")
+
+
 def _format_nominatim_location(payload: dict[str, Any]) -> str | None:
     """Turn a Nominatim reverse-geocode response into a short label.
 
@@ -988,8 +1056,12 @@ def _merge_prior_enrichment(
             item.location = prev.location
         if prev.description and not item.description:
             item.description = prev.description
+        if prev.faces is not None and item.faces is None:
+            item.faces = prev.faces
         if prev.exif_scanned:
             item.exif_scanned = True
+        if prev.face_scanned and prev.faces is not None:
+            item.face_scanned = True
 
 
 class AlbumCoordinator(DataUpdateCoordinator):
@@ -1008,6 +1080,13 @@ class AlbumCoordinator(DataUpdateCoordinator):
         self.store = store
 
         self.provider: str = entry.data.get(CONF_PROVIDER, PROVIDER_GOOGLE_SHARED)
+        self._immich_cache_source = None
+        if self.provider == PROVIDER_IMMICH:
+            source = [entry.data.get(key) for key in (
+                CONF_IMMICH_URL, CONF_IMMICH_API_KEY, CONF_IMMICH_SELECTION_TYPE,
+                CONF_IMMICH_SELECTION_ID, CONF_IMMICH_FILTER, CONF_IMMICH_IMAGE_SIZE,
+            )]
+            self._immich_cache_source = sha256(json.dumps(source).encode()).hexdigest()
         self.album_url: str | None = entry.data.get(CONF_ALBUM_URL)
         self.local_path: str | None = entry.data.get(CONF_LOCAL_PATH)
         self.recursive: bool = bool(entry.data.get(CONF_RECURSIVE, True))
@@ -1106,7 +1185,29 @@ class AlbumCoordinator(DataUpdateCoordinator):
             raise
 
         items = data.get("items") or []
-        if self.provider in ENRICHING_PROVIDERS and items:
+        self._apply_google_location_privacy(items)
+        if (self.google_metadata_enabled or self.google_location_enabled) and items:
+            prior = (self.data or {}).get("items", []) if isinstance(self.data, dict) else []
+            if not self._items_cache_loaded:
+                cached = await self._load_cached_items()
+                prior = (cached or {}).get("items", []) + prior
+                self._items_cache_loaded = True
+            by_id = {item.source_id: item for item in prior if item.source_id}
+            for item in items:
+                previous = by_id.get(item.source_id)
+                if previous is None:
+                    continue
+                if self.google_metadata_enabled and isinstance(previous.camera_metadata, dict):
+                    item.camera_metadata = dict(previous.camera_metadata)
+                    item.description = previous.description
+                if self.google_location_enabled and previous.google_location_scanned:
+                    item.latitude = previous.latitude
+                    item.longitude = previous.longitude
+                    item.location = previous.location
+                    item.google_location_scanned = True
+            self._apply_google_location_privacy(items)
+            self._schedule_enrichment(data)
+        elif self.provider in ENRICHING_PROVIDERS and items:
             # Carry forward EXIF/geocode metadata for items we've already
             # scanned this session; new items get filled in by the
             # background worker below.
@@ -1144,13 +1245,57 @@ class AlbumCoordinator(DataUpdateCoordinator):
         finally:
             self._enrichment_task = None
 
+    @property
+    def google_metadata_enabled(self) -> bool:
+        options = getattr(getattr(self, "entry", None), "options", None) or {}
+        return self.provider == PROVIDER_GOOGLE_SHARED and bool(
+            options.get(CONF_GOOGLE_METADATA, DEFAULT_GOOGLE_METADATA)
+        )
+
+    @property
+    def google_location_enabled(self) -> bool:
+        options = getattr(getattr(self, "entry", None), "options", None) or {}
+        return self.provider == PROVIDER_GOOGLE_SHARED and bool(
+            options.get(CONF_GOOGLE_LOCATION, DEFAULT_GOOGLE_LOCATION)
+        )
+
+    @property
+    def google_reverse_geocode_enabled(self) -> bool:
+        options = getattr(getattr(self, "entry", None), "options", None) or {}
+        return self.google_location_enabled and bool(
+            options.get(CONF_REVERSE_GEOCODE, DEFAULT_GOOGLE_REVERSE_GEOCODE)
+        )
+
+    def _apply_google_location_privacy(self, items: list[MediaItem]) -> None:
+        if getattr(self, "provider", None) != PROVIDER_GOOGLE_SHARED:
+            return
+        for item in items:
+            if not self.google_location_enabled:
+                item.latitude = None
+                item.longitude = None
+                item.location = None
+                item.google_location_scanned = False
+            elif not self.google_reverse_geocode_enabled:
+                item.location = None
+
+    def _needs_enrichment(self, item: MediaItem) -> bool:
+        if self.provider == PROVIDER_GOOGLE_SHARED:
+            return (
+                self.google_metadata_enabled and bool(item.source_id) and item.camera_metadata is None
+            ) or (self.google_location_enabled and not item.google_location_scanned)
+        if not item.exif_scanned:
+            return True
+        return self.provider == PROVIDER_IMMICH and not item.face_scanned
+
     def _schedule_enrichment(self, data: dict[str, Any]) -> None:
         """Kick off the background EXIF + geocode worker if there's work."""
         items: list[MediaItem] = data.get("items") or []
-        unscanned = [it for it in items if not it.exif_scanned]
+        unscanned = [it for it in items if self._needs_enrichment(it)]
         # Providers that decrypt/return GPS inline (Ente) have nothing to scan
         # but still need the coordinates turned into a place label.
-        needs_geocode = any(
+        needs_geocode = (
+            self.provider != PROVIDER_GOOGLE_SHARED or self.google_reverse_geocode_enabled
+        ) and any(
             it.latitude is not None and it.longitude is not None and not it.location
             for it in items
         )
@@ -1174,7 +1319,8 @@ class AlbumCoordinator(DataUpdateCoordinator):
             "geocode_done": 0,
         }
         self._enrichment_task = self.hass.async_create_background_task(
-            self._enrich_items_background(data),
+            self._enrich_google_items_background(data) if self.provider == PROVIDER_GOOGLE_SHARED
+            else self._enrich_items_background(data),
             name=f"album_slideshow_enrich_{self.entry.entry_id}",
         )
 
@@ -1187,6 +1333,10 @@ class AlbumCoordinator(DataUpdateCoordinator):
         if not isinstance(payload, dict):
             return None
 
+        cached_source = payload.get("immich_source")
+        if cached_source is not None and cached_source != getattr(self, "_immich_cache_source", None):
+            return None
+
         raw_items = payload.get("items")
         if not isinstance(raw_items, list):
             return None
@@ -1210,12 +1360,21 @@ class AlbumCoordinator(DataUpdateCoordinator):
                     location=raw.get("location"),
                     description=raw.get("description"),
                     source_id=raw.get("source_id"),
+                    faces=raw.get("faces"),
                     exif_scanned=bool(raw.get("exif_scanned", False)),
                     photo_id=raw.get("photo_id"),
+                    face_scanned=bool(raw.get("face_scanned", False))
+                    and isinstance(raw.get("faces"), list),
+                    camera_metadata={
+                        key: value for key, value in raw["camera_metadata"].items()
+                        if key in CAMERA_METADATA_FIELDS
+                    } if isinstance(raw.get("camera_metadata"), dict) else None,
+                    google_location_scanned=bool(raw.get("google_location_scanned", False)),
                 ))
             except Exception:
                 continue
 
+        self._apply_google_location_privacy(items)
         return {
             "title": payload.get("title"),
             "items": items,
@@ -1223,6 +1382,7 @@ class AlbumCoordinator(DataUpdateCoordinator):
 
     async def _save_cached_items(self, data: dict[str, Any]) -> None:
         items = data.get("items") or []
+        self._apply_google_location_privacy(items)
         # Local-folder URLs are absolute paths on the host; persisting them
         # is fine but they don't survive a host reformat. Persist anyway,
         # the URL check at load time will skip any stale entries.
@@ -1243,12 +1403,18 @@ class AlbumCoordinator(DataUpdateCoordinator):
                     "location": it.location,
                     "description": it.description,
                     "source_id": it.source_id,
+                    "faces": it.faces,
                     "exif_scanned": it.exif_scanned,
                     "photo_id": it.photo_id,
+                    "face_scanned": it.face_scanned,
+                    "camera_metadata": it.camera_metadata,
+                    "google_location_scanned": it.google_location_scanned,
                 }
                 for it in items
             ],
         }
+        if source := getattr(self, "_immich_cache_source", None):
+            payload["immich_source"] = source
         try:
             await self._items_cache_store.async_save(payload)
         except Exception as err:  # pragma: no cover - storage layer
@@ -2168,7 +2334,7 @@ class AlbumCoordinator(DataUpdateCoordinator):
         return None, {}
 
     async def _enrich_immich_item(self, item: MediaItem) -> None:
-        """Fetch one Immich asset's detail and fill location/description."""
+        """Fetch Immich metadata and the face boxes used for cropping."""
         from . import immich as immich_api
 
         url = self.entry.data.get(CONF_IMMICH_URL)
@@ -2177,23 +2343,164 @@ class AlbumCoordinator(DataUpdateCoordinator):
             item.exif_scanned = True
             return
         client = immich_api.ImmichClient(self.hass, url, api_key)
-        try:
-            asset = await client.async_get_asset(item.source_id)
-        except Exception as err:
-            _LOGGER.debug("Immich: failed to fetch asset %s: %s", item.source_id, err)
-            item.exif_scanned = True
+        person_ids = immich_api.selected_person_ids(
+            self.entry.data.get(CONF_IMMICH_SELECTION_TYPE),
+            self.entry.data.get(CONF_IMMICH_SELECTION_ID),
+        )
+
+        requests = {}
+        needs_faces = not item.face_scanned
+        if not item.exif_scanned or needs_faces:
+            requests["asset"] = client.async_get_asset(item.source_id)
+        if needs_faces:
+            requests["faces"] = client.async_get_faces(item.source_id)
+        results = await asyncio.gather(*requests.values(), return_exceptions=True)
+        asset = None
+
+        for kind, result in zip(requests, results):
+            if isinstance(result, asyncio.CancelledError):
+                raise result
+            if isinstance(result, Exception):
+                _LOGGER.debug(
+                    "Immich: failed to fetch %s for asset %s: %s",
+                    kind,
+                    item.source_id,
+                    result,
+                )
+                continue
+            if kind == "asset":
+                if not isinstance(result, dict):
+                    continue
+                asset = result
+                info = immich_api.parse_asset_exif(result)
+                if "captured_at" in info:
+                    item.captured_at = info["captured_at"]
+                if "latitude" in info and "longitude" in info:
+                    item.latitude = info["latitude"]
+                    item.longitude = info["longitude"]
+                if "location" in info:
+                    item.location = info["location"]
+                if "description" in info:
+                    item.description = info["description"]
+                item.exif_scanned = True
+            else:
+                if asset is None:
+                    continue
+                edits = []
+                if asset.get("isEdited"):
+                    try:
+                        edits = await client.async_get_asset_edits(item.source_id)
+                        if not edits:
+                            continue
+                    except asyncio.CancelledError:
+                        raise
+                    except Exception:
+                        _LOGGER.debug("Immich: edit metadata unavailable for asset %s", item.source_id)
+                        continue
+                try:
+                    boxes = immich_api.parse_face_boxes(
+                        result, person_ids, edits=edits,
+                        original_size=immich_api.original_image_size(asset),
+                    )
+                except (TypeError, ValueError):
+                    _LOGGER.debug("Immich: unsupported face geometry for asset %s", item.source_id)
+                    continue
+                item.faces = boxes
+                item.face_scanned = True
+
+    async def _enrich_google_items_background(self, data: dict[str, Any]) -> None:
+        """Cache optional Google metadata without delaying or failing the playlist."""
+        from . import google_scraper
+
+        jobs = []
+        for item in data.get("items", []):
+            if self.google_metadata_enabled and item.source_id and item.camera_metadata is None:
+                jobs.append((item, False))
+            if self.google_location_enabled and not item.google_location_scanned:
+                jobs.append((item, True))
+        if not jobs:
+            try:
+                if self.google_reverse_geocode_enabled:
+                    await self._geocode_items_background(data)
+            finally:
+                self._enrich_progress["phase"] = "done"
             return
-        info = immich_api.parse_asset_exif(asset)
-        if "captured_at" in info:
-            item.captured_at = info["captured_at"]
-        if "latitude" in info and "longitude" in info:
-            item.latitude = info["latitude"]
-            item.longitude = info["longitude"]
-        if "location" in info:
-            item.location = info["location"]
-        if "description" in info:
-            item.description = info["description"]
-        item.exif_scanned = True
+        failures = 0
+        completed = 0
+        next_request = 0.0
+        loop = asyncio.get_running_loop()
+        try:
+            async with aiohttp.ClientSession(
+                cookie_jar=aiohttp.DummyCookieJar(), trust_env=False,
+            ) as session:
+                keys = None
+                for item, location_job in jobs:
+                    delay = next_request - loop.time()
+                    if delay > 0:
+                        await asyncio.sleep(delay)
+                    try:
+                        if location_job:
+                            header = await google_scraper.fetch_photo_location_header(session, item.url)
+                            try:
+                                metadata = await self.hass.async_add_executor_job(
+                                    _read_gps_from_header, header,
+                                )
+                            except Exception as err:
+                                # A photo that cannot be parsed must not stop the scan for the rest.
+                                _LOGGER.debug("Google photo GPS unreadable (%s)", type(err).__name__)
+                                continue
+                            latitude = metadata.get("latitude")
+                            longitude = metadata.get("longitude")
+                            if (
+                                all(type(value) in (int, float) and math.isfinite(value)
+                                    for value in (latitude, longitude))
+                                and -90 <= latitude <= 90 and -180 <= longitude <= 180
+                                and (abs(latitude) > 1e-6 or abs(longitude) > 1e-6)
+                            ):
+                                item.latitude, item.longitude = latitude, longitude
+                            else:
+                                item.latitude = item.longitude = None
+                            item.location = None
+                            item.google_location_scanned = True
+                        else:
+                            if keys is None:
+                                keys = await google_scraper.fetch_metadata_keys(session, self.album_url)
+                            metadata = await google_scraper.fetch_photo_metadata(
+                                session, keys, item.source_id,
+                            )
+                            item.description = metadata.pop("description", None)
+                            item.camera_metadata = metadata
+                    except asyncio.CancelledError:
+                        raise
+                    except Exception as err:
+                        failures += 1
+                        _LOGGER.debug(
+                            "Google photo metadata unavailable (%s, status=%s)",
+                            type(err).__name__, getattr(err, "status", None),
+                        )
+                        if failures >= 3 or getattr(err, "status", None) in (401, 403, 429):
+                            break
+                    else:
+                        failures = 0
+                        completed += 1
+                        if not self._needs_enrichment(item):
+                            self._enrich_progress["exif_done"] += 1
+                        if completed == 1 or completed % 10 == 0:
+                            await self._save_cached_items(data)
+                            self.async_set_updated_data(data)
+                    finally:
+                        next_request = loop.time() + _GOOGLE_METADATA_INTERVAL
+            if self.google_reverse_geocode_enabled:
+                await self._geocode_items_background(data)
+        except asyncio.CancelledError:
+            raise
+        except Exception as err:
+            _LOGGER.debug("Google photo enrichment stopped (%s)", type(err).__name__)
+        finally:
+            self._enrich_progress["phase"] = "done"
+            await self._save_cached_items(data)
+            if completed:
+                self.async_set_updated_data(data)
 
     async def _enrich_items_background(self, data: dict[str, Any]) -> None:
         """Read EXIF for unscanned local files, then reverse-geocode.
@@ -2213,7 +2520,7 @@ class AlbumCoordinator(DataUpdateCoordinator):
         scanned_since_save = 0
         try:
             for idx, item in enumerate(items):
-                if item.exif_scanned:
+                if not self._needs_enrichment(item):
                     # Fast path: yield occasionally so we don't starve
                     # the event loop when (re-)visiting a long list of
                     # already-processed items.
@@ -2318,6 +2625,8 @@ class AlbumCoordinator(DataUpdateCoordinator):
         the GPS coordinates themselves. Successful labels are written
         back into the items in-place and persisted via the items cache.
         """
+        if getattr(self, "provider", None) == PROVIDER_GOOGLE_SHARED and not self.google_reverse_geocode_enabled:
+            return
         if not bool(
             self.entry.options.get(CONF_REVERSE_GEOCODE, DEFAULT_REVERSE_GEOCODE)
             if hasattr(self.entry, "options") and self.entry.options is not None