"""Documents archive (ZIP) — the one export that carries file *contents*. The JSON/YAML backup deliberately keeps document metadata only; the binary blobs ride the HA backup. That leaves a portable JSON export with dangling file docs on a fresh instance. This module adds a dedicated, self-contained documents archive: manifest.json {"version":1, "objects":[{object_id, object_name, documents:[]}]} blobs/ the raw file contents (content-addressed, dedup'd) Export gathers the selected objects' documents + their unique blobs. Import writes every blob back, then re-attaches metadata to the matching object (by id first, then by name for a cross-instance restore), skipping documents that already exist so a repeated import is idempotent. Weblinks travel too (0 bytes) so the archive is a complete documents backup on its own. """ from __future__ import annotations import io import json import logging import zipfile from typing import Any from homeassistant.core import HomeAssistant from ..const import DOMAIN, GLOBAL_UNIQUE_ID from .documents import KIND_FILE, KIND_WEBLINK _LOGGER = logging.getLogger(__name__) MANIFEST_NAME = "manifest.json" BLOB_DIR = "blobs/" ARCHIVE_VERSION = 1 # Cap a single archive import so a crafted ZIP can't exhaust memory/disk. A # real documents backup is dominated by the blobs, already capped at 25 MB # each × 100 docs/object — this is a coarse whole-archive ceiling on top. MAX_ARCHIVE_BYTES = 500 * 1024 * 1024 # 500 MB uncompressed-blob budget MAX_MANIFEST_BYTES = 16 * 1024 * 1024 # the manifest is metadata only MAX_ARCHIVE_MEMBERS = 20000 # ceiling on ZIP entry count (blobs cap at 100/obj) def _read_member_bounded(zf: zipfile.ZipFile, name: str, limit: int) -> bytes: """Read a ZIP member but never materialise more than ``limit`` bytes. ``ZipFile.read`` inflates the WHOLE member before returning, so a crafted member (small compressed, huge inflated — a "zip bomb") would exhaust memory before any post-hoc size check. Reading through ``open()`` with a hard byte ceiling bounds the decompression regardless of the declared/actual size. """ with zf.open(name) as fh: data = fh.read(limit + 1) if len(data) > limit: raise ValueError("archive_member_too_large") return data def _get_store(hass: HomeAssistant) -> Any: from .. import DOCUMENT_STORE_KEY return hass.data.get(DOMAIN, {}).get(DOCUMENT_STORE_KEY) def _object_name_map(hass: HomeAssistant) -> tuple[dict[str, str], dict[str, str]]: """(object_id → entry_id-object_id) is identity; return the maps the import needs: existing object_ids (set) and name → object_id for cross-instance matching.""" from ..const import CONF_OBJECT ids: dict[str, str] = {} by_name: dict[str, str] = {} for entry in hass.config_entries.async_entries(DOMAIN): if entry.unique_id == GLOBAL_UNIQUE_ID: continue obj = entry.data.get(CONF_OBJECT, {}) oid = obj.get("id") if not oid: continue ids[oid] = oid name = obj.get("name") if name: by_name.setdefault(name, oid) return ids, by_name def _object_task_ids(hass: HomeAssistant, object_id: str) -> set[str]: """The current task ids of the object whose id is ``object_id`` (empty if none). Used to keep a same-instance archive restore's task links valid.""" from ..const import CONF_OBJECT, CONF_TASKS for entry in hass.config_entries.async_entries(DOMAIN): if entry.unique_id == GLOBAL_UNIQUE_ID: continue if entry.data.get(CONF_OBJECT, {}).get("id") == object_id: return set(entry.data.get(CONF_TASKS, {})) return set() def _object_part_ids(hass: HomeAssistant, object_id: str) -> set[str]: """The current spare-part ids of the object (same role as ``_object_task_ids``, for a doc's part links).""" from ..const import CONF_OBJECT, CONF_PARTS for entry in hass.config_entries.async_entries(DOMAIN): if entry.unique_id == GLOBAL_UNIQUE_ID: continue if entry.data.get(CONF_OBJECT, {}).get("id") == object_id: return set(entry.data.get(CONF_PARTS) or {}) return set() def build_documents_archive(hass: HomeAssistant, entry_ids: set[str] | None = None) -> bytes: """Build a documents ZIP for the selected objects (None = all). Runs on the event loop for the metadata gather (reads config entries) but the heavy blob reads happen here synchronously — call via the executor. """ from ..const import CONF_OBJECT from ..export import object_entries store = _get_store(hass) entries = object_entries(hass, entry_ids) manifest_objects: list[dict[str, Any]] = [] blob_hashes: set[str] = set() for entry in entries: obj = entry.data.get(CONF_OBJECT, {}) object_id = obj.get("id", "") if not object_id or store is None: continue docs = [] for d in store.for_object(object_id): if d.get("kind") == KIND_WEBLINK: docs.append( { "kind": KIND_WEBLINK, "url": d.get("url"), "title": d.get("title"), "tags": d.get("tags") or [], "task_ids": d.get("task_ids") or [], "part_ids": d.get("part_ids") or [], } ) else: h = d.get("hash") docs.append( { "kind": KIND_FILE, "hash": h, "title": d.get("title"), "filename": d.get("filename"), "mime": d.get("mime"), "size": d.get("size"), "tags": d.get("tags") or [], "task_ids": d.get("task_ids") or [], "part_ids": d.get("part_ids") or [], } ) if isinstance(h, str): blob_hashes.add(h) if docs: manifest_objects.append({"object_id": object_id, "object_name": obj.get("name", ""), "documents": docs}) manifest = {"version": ARCHIVE_VERSION, "objects": manifest_objects} buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: zf.writestr(MANIFEST_NAME, json.dumps(manifest, ensure_ascii=False, indent=2)) for h in sorted(blob_hashes): if store is None: break try: path = store.blob_path(h) except ValueError: continue if path.is_file(): zf.writestr(f"{BLOB_DIR}{h}", path.read_bytes()) else: _LOGGER.warning("Documents archive: blob %s missing on disk, skipped", h[:12]) return buf.getvalue() async def import_documents_archive(hass: HomeAssistant, data: bytes) -> dict[str, Any]: """Restore a documents ZIP: write blobs back, re-attach metadata. Objects are matched by id first (same instance), then by name (a cross-instance restore after a JSON import created fresh ids). Documents already present on the target object are skipped so a repeat import is idempotent. Returns counts. """ store = _get_store(hass) if store is None: return {"error": "documents store unavailable"} def _read() -> tuple[dict[str, Any], dict[str, bytes]]: blobs: dict[str, bytes] = {} manifest: dict[str, Any] = {} total = 0 with zipfile.ZipFile(io.BytesIO(data)) as zf: names = zf.namelist() if len(names) > MAX_ARCHIVE_MEMBERS: raise ValueError("archive_too_many_members") for name in names: if name == MANIFEST_NAME: # Bound the metadata member too (was read uncapped). manifest = json.loads(_read_member_bounded(zf, name, MAX_MANIFEST_BYTES).decode("utf-8")) elif name.startswith(BLOB_DIR) and not name.endswith("/"): digest = name[len(BLOB_DIR) :] if len(digest) == 64 and all(c in "0123456789abcdef" for c in digest): remaining = MAX_ARCHIVE_BYTES - total if remaining <= 0: raise ValueError("archive_too_large") # Read bounded by the remaining budget so a bomb can't # inflate past the whole-archive ceiling (checked DURING # the read, not after materialising the full member). content = _read_member_bounded(zf, name, remaining) total += len(content) blobs[digest] = content return manifest, blobs try: manifest, blobs = await hass.async_add_executor_job(_read) except (zipfile.BadZipFile, ValueError, json.JSONDecodeError, KeyError) as err: return {"error": f"invalid archive: {err}"} # 1) Write back only blobs a manifest document actually references — an # archive carrying extra blobs must not litter /config with orphans that # ride every HA backup and are never refcounted (disk-fill hardening). referenced: set[str] = set() for obj in manifest.get("objects", []): if not isinstance(obj, dict): continue for m in obj.get("documents", []): if isinstance(m, dict) and isinstance(m.get("hash"), str): referenced.add(m["hash"]) import hashlib written = 0 for digest, content in blobs.items(): if digest not in referenced: _LOGGER.info("Documents archive: blob %s referenced by no document, skipped", digest[:12]) continue if hashlib.sha256(content).hexdigest() != digest: _LOGGER.warning("Documents archive: blob %s failed hash check, skipped", digest[:12]) continue _, wrote_new = await hass.async_add_executor_job(store._store_blob_sync, content) if wrote_new: written += 1 # 2) Re-attach metadata to the matching object (id, then name). ids, by_name = _object_name_map(hass) docs_created = 0 objects_matched = 0 for obj in manifest.get("objects", []): if not isinstance(obj, dict): continue target = ids.get(str(obj.get("object_id") or "")) or by_name.get(str(obj.get("object_name") or "")) if target is None: _LOGGER.info("Documents archive: no object matches %r, its docs skipped", obj.get("object_name")) continue objects_matched += 1 # Skip docs already present on the target (idempotent re-import). existing = store.for_object(target) existing_keys = {(d.get("kind"), d.get("hash") or d.get("url")) for d in existing} fresh = [ m for m in obj.get("documents", []) if isinstance(m, dict) and (m.get("kind"), m.get("hash") or m.get("url")) not in existing_keys ] if fresh: # Keep task links that still resolve on the target (a same-instance # restore) via an identity map over the object's current task ids; # a cross-instance restore has fresh task ids, so those links drop # here and are re-established by the JSON import's remap instead. valid_task_ids = _object_task_ids(hass, target) identity = {tid: tid for tid in valid_task_ids} part_identity = {pid: pid for pid in _object_part_ids(hass, target)} docs_created += await store.async_import_documents( target, fresh, task_id_map=identity, part_id_map=part_identity ) return { "blobs_written": written, "documents_created": docs_created, "objects_matched": objects_matched, }