updated apps
This commit is contained in:
@@ -0,0 +1,289 @@
|
||||
"""Documents archive (ZIP) — the one export that carries file *contents*.
|
||||
|
||||
The JSON/YAML backup deliberately keeps document metadata only; the binary
|
||||
blobs ride the HA backup. That leaves a portable JSON export with dangling
|
||||
file docs on a fresh instance. This module adds a dedicated, self-contained
|
||||
documents archive:
|
||||
|
||||
manifest.json {"version":1, "objects":[{object_id, object_name,
|
||||
documents:[<metadata>]}]}
|
||||
blobs/<sha256> the raw file contents (content-addressed, dedup'd)
|
||||
|
||||
Export gathers the selected objects' documents + their unique blobs. Import
|
||||
writes every blob back, then re-attaches metadata to the matching object
|
||||
(by id first, then by name for a cross-instance restore), skipping documents
|
||||
that already exist so a repeated import is idempotent. Weblinks travel too
|
||||
(0 bytes) so the archive is a complete documents backup on its own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import zipfile
|
||||
from typing import Any
|
||||
|
||||
from homeassistant.core import HomeAssistant
|
||||
|
||||
from ..const import DOMAIN, GLOBAL_UNIQUE_ID
|
||||
from .documents import KIND_FILE, KIND_WEBLINK
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
BLOB_DIR = "blobs/"
|
||||
ARCHIVE_VERSION = 1
|
||||
# Cap a single archive import so a crafted ZIP can't exhaust memory/disk. A
|
||||
# real documents backup is dominated by the blobs, already capped at 25 MB
|
||||
# each × 100 docs/object — this is a coarse whole-archive ceiling on top.
|
||||
MAX_ARCHIVE_BYTES = 500 * 1024 * 1024 # 500 MB uncompressed-blob budget
|
||||
MAX_MANIFEST_BYTES = 16 * 1024 * 1024 # the manifest is metadata only
|
||||
MAX_ARCHIVE_MEMBERS = 20000 # ceiling on ZIP entry count (blobs cap at 100/obj)
|
||||
|
||||
|
||||
def _read_member_bounded(zf: zipfile.ZipFile, name: str, limit: int) -> bytes:
|
||||
"""Read a ZIP member but never materialise more than ``limit`` bytes.
|
||||
|
||||
``ZipFile.read`` inflates the WHOLE member before returning, so a crafted
|
||||
member (small compressed, huge inflated — a "zip bomb") would exhaust memory
|
||||
before any post-hoc size check. Reading through ``open()`` with a hard byte
|
||||
ceiling bounds the decompression regardless of the declared/actual size.
|
||||
"""
|
||||
with zf.open(name) as fh:
|
||||
data = fh.read(limit + 1)
|
||||
if len(data) > limit:
|
||||
raise ValueError("archive_member_too_large")
|
||||
return data
|
||||
|
||||
|
||||
def _get_store(hass: HomeAssistant) -> Any:
|
||||
from .. import DOCUMENT_STORE_KEY
|
||||
|
||||
return hass.data.get(DOMAIN, {}).get(DOCUMENT_STORE_KEY)
|
||||
|
||||
|
||||
def _object_name_map(hass: HomeAssistant) -> tuple[dict[str, str], dict[str, str]]:
|
||||
"""(object_id → entry_id-object_id) is identity; return the maps the import
|
||||
needs: existing object_ids (set) and name → object_id for cross-instance
|
||||
matching."""
|
||||
from ..const import CONF_OBJECT
|
||||
|
||||
ids: dict[str, str] = {}
|
||||
by_name: dict[str, str] = {}
|
||||
for entry in hass.config_entries.async_entries(DOMAIN):
|
||||
if entry.unique_id == GLOBAL_UNIQUE_ID:
|
||||
continue
|
||||
obj = entry.data.get(CONF_OBJECT, {})
|
||||
oid = obj.get("id")
|
||||
if not oid:
|
||||
continue
|
||||
ids[oid] = oid
|
||||
name = obj.get("name")
|
||||
if name:
|
||||
by_name.setdefault(name, oid)
|
||||
return ids, by_name
|
||||
|
||||
|
||||
def _object_task_ids(hass: HomeAssistant, object_id: str) -> set[str]:
|
||||
"""The current task ids of the object whose id is ``object_id`` (empty if
|
||||
none). Used to keep a same-instance archive restore's task links valid."""
|
||||
from ..const import CONF_OBJECT, CONF_TASKS
|
||||
|
||||
for entry in hass.config_entries.async_entries(DOMAIN):
|
||||
if entry.unique_id == GLOBAL_UNIQUE_ID:
|
||||
continue
|
||||
if entry.data.get(CONF_OBJECT, {}).get("id") == object_id:
|
||||
return set(entry.data.get(CONF_TASKS, {}))
|
||||
return set()
|
||||
|
||||
|
||||
def _object_part_ids(hass: HomeAssistant, object_id: str) -> set[str]:
|
||||
"""The current spare-part ids of the object (same role as
|
||||
``_object_task_ids``, for a doc's part links)."""
|
||||
from ..const import CONF_OBJECT, CONF_PARTS
|
||||
|
||||
for entry in hass.config_entries.async_entries(DOMAIN):
|
||||
if entry.unique_id == GLOBAL_UNIQUE_ID:
|
||||
continue
|
||||
if entry.data.get(CONF_OBJECT, {}).get("id") == object_id:
|
||||
return set(entry.data.get(CONF_PARTS) or {})
|
||||
return set()
|
||||
|
||||
|
||||
def build_documents_archive(hass: HomeAssistant, entry_ids: set[str] | None = None) -> bytes:
|
||||
"""Build a documents ZIP for the selected objects (None = all).
|
||||
|
||||
Runs on the event loop for the metadata gather (reads config entries) but
|
||||
the heavy blob reads happen here synchronously — call via the executor.
|
||||
"""
|
||||
from ..const import CONF_OBJECT
|
||||
from ..export import object_entries
|
||||
|
||||
store = _get_store(hass)
|
||||
entries = object_entries(hass, entry_ids)
|
||||
|
||||
manifest_objects: list[dict[str, Any]] = []
|
||||
blob_hashes: set[str] = set()
|
||||
for entry in entries:
|
||||
obj = entry.data.get(CONF_OBJECT, {})
|
||||
object_id = obj.get("id", "")
|
||||
if not object_id or store is None:
|
||||
continue
|
||||
docs = []
|
||||
for d in store.for_object(object_id):
|
||||
if d.get("kind") == KIND_WEBLINK:
|
||||
docs.append(
|
||||
{
|
||||
"kind": KIND_WEBLINK,
|
||||
"url": d.get("url"),
|
||||
"title": d.get("title"),
|
||||
"tags": d.get("tags") or [],
|
||||
"task_ids": d.get("task_ids") or [],
|
||||
"part_ids": d.get("part_ids") or [],
|
||||
}
|
||||
)
|
||||
else:
|
||||
h = d.get("hash")
|
||||
docs.append(
|
||||
{
|
||||
"kind": KIND_FILE,
|
||||
"hash": h,
|
||||
"title": d.get("title"),
|
||||
"filename": d.get("filename"),
|
||||
"mime": d.get("mime"),
|
||||
"size": d.get("size"),
|
||||
"tags": d.get("tags") or [],
|
||||
"task_ids": d.get("task_ids") or [],
|
||||
"part_ids": d.get("part_ids") or [],
|
||||
}
|
||||
)
|
||||
if isinstance(h, str):
|
||||
blob_hashes.add(h)
|
||||
if docs:
|
||||
manifest_objects.append({"object_id": object_id, "object_name": obj.get("name", ""), "documents": docs})
|
||||
|
||||
manifest = {"version": ARCHIVE_VERSION, "objects": manifest_objects}
|
||||
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr(MANIFEST_NAME, json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
for h in sorted(blob_hashes):
|
||||
if store is None:
|
||||
break
|
||||
try:
|
||||
path = store.blob_path(h)
|
||||
except ValueError:
|
||||
continue
|
||||
if path.is_file():
|
||||
zf.writestr(f"{BLOB_DIR}{h}", path.read_bytes())
|
||||
else:
|
||||
_LOGGER.warning("Documents archive: blob %s missing on disk, skipped", h[:12])
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
async def import_documents_archive(hass: HomeAssistant, data: bytes) -> dict[str, Any]:
|
||||
"""Restore a documents ZIP: write blobs back, re-attach metadata.
|
||||
|
||||
Objects are matched by id first (same instance), then by name (a
|
||||
cross-instance restore after a JSON import created fresh ids). Documents
|
||||
already present on the target object are skipped so a repeat import is
|
||||
idempotent. Returns counts.
|
||||
"""
|
||||
store = _get_store(hass)
|
||||
if store is None:
|
||||
return {"error": "documents store unavailable"}
|
||||
|
||||
def _read() -> tuple[dict[str, Any], dict[str, bytes]]:
|
||||
blobs: dict[str, bytes] = {}
|
||||
manifest: dict[str, Any] = {}
|
||||
total = 0
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
||||
names = zf.namelist()
|
||||
if len(names) > MAX_ARCHIVE_MEMBERS:
|
||||
raise ValueError("archive_too_many_members")
|
||||
for name in names:
|
||||
if name == MANIFEST_NAME:
|
||||
# Bound the metadata member too (was read uncapped).
|
||||
manifest = json.loads(_read_member_bounded(zf, name, MAX_MANIFEST_BYTES).decode("utf-8"))
|
||||
elif name.startswith(BLOB_DIR) and not name.endswith("/"):
|
||||
digest = name[len(BLOB_DIR) :]
|
||||
if len(digest) == 64 and all(c in "0123456789abcdef" for c in digest):
|
||||
remaining = MAX_ARCHIVE_BYTES - total
|
||||
if remaining <= 0:
|
||||
raise ValueError("archive_too_large")
|
||||
# Read bounded by the remaining budget so a bomb can't
|
||||
# inflate past the whole-archive ceiling (checked DURING
|
||||
# the read, not after materialising the full member).
|
||||
content = _read_member_bounded(zf, name, remaining)
|
||||
total += len(content)
|
||||
blobs[digest] = content
|
||||
return manifest, blobs
|
||||
|
||||
try:
|
||||
manifest, blobs = await hass.async_add_executor_job(_read)
|
||||
except (zipfile.BadZipFile, ValueError, json.JSONDecodeError, KeyError) as err:
|
||||
return {"error": f"invalid archive: {err}"}
|
||||
|
||||
# 1) Write back only blobs a manifest document actually references — an
|
||||
# archive carrying extra blobs must not litter /config with orphans that
|
||||
# ride every HA backup and are never refcounted (disk-fill hardening).
|
||||
referenced: set[str] = set()
|
||||
for obj in manifest.get("objects", []):
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
for m in obj.get("documents", []):
|
||||
if isinstance(m, dict) and isinstance(m.get("hash"), str):
|
||||
referenced.add(m["hash"])
|
||||
|
||||
import hashlib
|
||||
|
||||
written = 0
|
||||
for digest, content in blobs.items():
|
||||
if digest not in referenced:
|
||||
_LOGGER.info("Documents archive: blob %s referenced by no document, skipped", digest[:12])
|
||||
continue
|
||||
if hashlib.sha256(content).hexdigest() != digest:
|
||||
_LOGGER.warning("Documents archive: blob %s failed hash check, skipped", digest[:12])
|
||||
continue
|
||||
_, wrote_new = await hass.async_add_executor_job(store._store_blob_sync, content)
|
||||
if wrote_new:
|
||||
written += 1
|
||||
|
||||
# 2) Re-attach metadata to the matching object (id, then name).
|
||||
ids, by_name = _object_name_map(hass)
|
||||
docs_created = 0
|
||||
objects_matched = 0
|
||||
for obj in manifest.get("objects", []):
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
target = ids.get(str(obj.get("object_id") or "")) or by_name.get(str(obj.get("object_name") or ""))
|
||||
if target is None:
|
||||
_LOGGER.info("Documents archive: no object matches %r, its docs skipped", obj.get("object_name"))
|
||||
continue
|
||||
objects_matched += 1
|
||||
# Skip docs already present on the target (idempotent re-import).
|
||||
existing = store.for_object(target)
|
||||
existing_keys = {(d.get("kind"), d.get("hash") or d.get("url")) for d in existing}
|
||||
fresh = [
|
||||
m
|
||||
for m in obj.get("documents", [])
|
||||
if isinstance(m, dict) and (m.get("kind"), m.get("hash") or m.get("url")) not in existing_keys
|
||||
]
|
||||
if fresh:
|
||||
# Keep task links that still resolve on the target (a same-instance
|
||||
# restore) via an identity map over the object's current task ids;
|
||||
# a cross-instance restore has fresh task ids, so those links drop
|
||||
# here and are re-established by the JSON import's remap instead.
|
||||
valid_task_ids = _object_task_ids(hass, target)
|
||||
identity = {tid: tid for tid in valid_task_ids}
|
||||
part_identity = {pid: pid for pid in _object_part_ids(hass, target)}
|
||||
docs_created += await store.async_import_documents(
|
||||
target, fresh, task_id_map=identity, part_id_map=part_identity
|
||||
)
|
||||
|
||||
return {
|
||||
"blobs_written": written,
|
||||
"documents_created": docs_created,
|
||||
"objects_matched": objects_matched,
|
||||
}
|
||||
Reference in New Issue
Block a user