472 lines
22 KiB
Python
472 lines
22 KiB
Python
# WashData - Home Assistant integration for appliance cycle monitoring via smart plugs.
|
||
# Copyright (C) 2026 Lukas Bandura
|
||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||
#
|
||
# This program is free software: you can redistribute it and/or modify
|
||
# it under the terms of the GNU Affero General Public License as published
|
||
# by the Free Software Foundation, either version 3 of the License, or
|
||
# (at your option) any later version.
|
||
#
|
||
# This program is distributed in the hope that it will be useful,
|
||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
# GNU Affero General Public License for more details.
|
||
#
|
||
# You should have received a copy of the GNU Affero General Public License
|
||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||
"""On-device tuning of the matcher's scoring weights (Stage 4/5, opt-in).
|
||
|
||
Mirrors the offline ``devtools/dtw_ab_eval.py`` methodology but as a shippable,
|
||
NumPy-only, executor-safe pure function: it does leave-one-out matching over the
|
||
device's own labelled cycles, sweeps a small grid of the highest-impact scoring
|
||
weights (corr/MAE split, duration agreement weight, energy agreement weight, and
|
||
DTW ensemble weight independently), and - only if a candidate beats the shipped
|
||
defaults on a HELD-OUT split by a margin - returns a per-device config override. The caller persists it; the matcher reads it live
|
||
and falls back to the const defaults otherwise.
|
||
|
||
Discipline (same as model promotion): tune on a train split, gate on a held-out
|
||
split, require a margin, cap the grid to bounded scoring weights (never
|
||
structural behaviour). This guards against over-fitting the small, partly
|
||
manually-labelled per-user cycle set.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
import numpy as np
|
||
|
||
from .. import analysis
|
||
from ..const import (
|
||
DEFAULT_DTW_BANDWIDTH,
|
||
DEFAULT_DTW_MODE,
|
||
DEFAULT_PROFILE_MATCH_MAX_DURATION_RATIO,
|
||
DEFAULT_PROFILE_MATCH_MIN_DURATION_RATIO,
|
||
)
|
||
from ..signal_processing import resample_adaptive, resample_uniform
|
||
|
||
#: Matches the production matcher (``profile_store.async_match_profile``).
|
||
_MIN_DT = 5.0
|
||
_GAP_S = 21600.0
|
||
|
||
|
||
def _series(cycle: dict[str, Any]) -> tuple[np.ndarray, np.ndarray] | None:
|
||
"""``(offsets_s, watts)`` from a stored cycle, or None if unusable.
|
||
|
||
The offsets matter: ``analysis.find_best_alignment`` compares the two curves
|
||
**index by index** (its ``dt`` argument is explicitly unused), so both sides
|
||
must be on the same seconds-per-sample grid or the MAE compares different
|
||
moments of the cycle. Dropping the offsets - as this module did until register
|
||
item 303 - makes that impossible to honour.
|
||
"""
|
||
pd = cycle.get("power_data") or []
|
||
ts: list[float] = []
|
||
pw: list[float] = []
|
||
for p in pd:
|
||
# Both values are converted BEFORE either is appended: a row whose offset
|
||
# parses and whose power does not would otherwise leave `ts` one element
|
||
# longer than `pw`, and `resample_uniform` raises inside `np.interp` on
|
||
# unequal arrays - a tuning run lost to one malformed sample.
|
||
# OverflowError too: `json` keeps an oversized integer literal as an
|
||
# unbounded int, and `float()` on one raises rather than returning inf.
|
||
try:
|
||
offset, watts = float(p[0]), float(p[1])
|
||
except (TypeError, ValueError, IndexError, OverflowError):
|
||
continue
|
||
ts.append(offset)
|
||
pw.append(watts)
|
||
if len(pw) < 4:
|
||
return None
|
||
return np.asarray(ts, dtype=float), np.asarray(pw, dtype=float)
|
||
|
||
|
||
def _longest(segments: list[Any]) -> Any | None:
|
||
return max(segments, key=lambda s: len(s.power)) if segments else None
|
||
|
||
|
||
def _prep(cycles: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
|
||
"""Group labelled cycles by profile, keeping the raw time series."""
|
||
by_profile: dict[str, list[dict[str, Any]]] = {}
|
||
for c in cycles:
|
||
name = c.get("profile_name")
|
||
series = _series(c)
|
||
if not name or series is None:
|
||
continue
|
||
try:
|
||
dur = float(c.get("duration"))
|
||
except (TypeError, ValueError):
|
||
dur = 0.0
|
||
if dur <= 0:
|
||
# No reliable wall-clock duration: skip rather than fabricate one from the
|
||
# sample count (len(pw)), which distorts duration scoring on devices that
|
||
# sample every 30-60 s. Real cycles always carry a 'duration', so this only
|
||
# drops degenerate entries.
|
||
continue
|
||
ts, pw = series
|
||
review = c.get("ml_review")
|
||
golden = bool(review.get("golden")) if isinstance(review, dict) else False
|
||
by_profile.setdefault(name, []).append(
|
||
{"ts": ts, "pw": pw, "dur": dur, "golden": golden}
|
||
)
|
||
return by_profile
|
||
|
||
|
||
def _regrid(item: dict[str, Any], dt: float, cache: dict) -> list[float] | None:
|
||
"""One cycle's curve on a ``dt``-second grid, cached per (cycle, dt)."""
|
||
key = (id(item), round(float(dt), 2))
|
||
if key in cache:
|
||
return cache[key]
|
||
seg = _longest(resample_uniform(item["ts"], item["pw"], dt_s=dt, gap_s=_GAP_S))
|
||
out = seg.power.tolist() if seg is not None and len(seg.power) >= 2 else None
|
||
cache[key] = out
|
||
return out
|
||
|
||
|
||
|
||
def _envelope_avg(
|
||
name: str, pool: list[dict[str, Any]], excluded: int | None, dt: float, cache: dict
|
||
) -> list[float] | None:
|
||
"""The pool's DTW-warped envelope average, re-gridded to the query's ``dt``.
|
||
|
||
This is what `ProfileStore.async_match_profile` scores against once a profile
|
||
has >= 2 confirmed cycles and no pinned golden cycle, i.e. the common case
|
||
(register item 347(e)).
|
||
|
||
Cheap enough to do honestly, which the deferral used to deny (item 354): an
|
||
envelope is built from traces, so it does not depend on the config being
|
||
tuned and survives the whole grid search, and leave-one-out changes only the
|
||
target's OWN profile - so the number of distinct envelopes is
|
||
``profiles + targets``, not ``folds x profiles``. Both caches below are keyed
|
||
accordingly.
|
||
"""
|
||
ekey = ("env", name, excluded)
|
||
built = cache.get(ekey)
|
||
if built is None:
|
||
raw = [
|
||
(it["ts"].tolist(), it["pw"].tolist(), float(it["dur"]))
|
||
for it in pool
|
||
]
|
||
try:
|
||
built = analysis.compute_envelope_worker(raw, _BASE_CFG["dtw_bandwidth"])
|
||
except Exception: # pylint: disable=broad-exception-caught
|
||
built = False # cache the failure; do not retry per target
|
||
cache[ekey] = built
|
||
if not built:
|
||
return None
|
||
grid, _lo, _hi, avg, _std, _target = built
|
||
if len(grid) < 2 or len(avg) != len(grid):
|
||
return None
|
||
gkey = ("envgrid", name, excluded, round(float(dt), 2))
|
||
if gkey in cache:
|
||
return cache[gkey]
|
||
seg = _longest(
|
||
resample_uniform(
|
||
np.asarray(grid, dtype=float), np.asarray(avg, dtype=float),
|
||
dt_s=dt, gap_s=_GAP_S,
|
||
)
|
||
)
|
||
out = seg.power.tolist() if seg is not None and len(seg.power) >= 2 else None
|
||
cache[gkey] = out
|
||
return out
|
||
|
||
def _snaps(
|
||
by_profile: dict[str, list[dict]],
|
||
exclude: tuple[str, int] | None,
|
||
dt: float,
|
||
cache: dict,
|
||
) -> list[dict[str, Any]]:
|
||
"""One snapshot per profile, re-gridded to the QUERY's ``dt``.
|
||
|
||
Mirrors production: ``profile_store`` resamples the current cycle with
|
||
``resample_adaptive`` and then re-grids every candidate to that same
|
||
``used_dt`` via ``_get_cached_sample_segment``, so index *i* is the same
|
||
elapsed time on both sides.
|
||
|
||
**The template follows production's three-way rule, in production's order**
|
||
(`ProfileStore.async_match_profile`), which is register item 356 closing item
|
||
347(e): a pinned golden cycle's own sharp trace, else the DTW-warped ENVELOPE
|
||
AVERAGE once the pool holds >= 2 cycles, else the single representative
|
||
sample - which is also the safety net when an envelope will not build. The
|
||
middle branch is the common case, and it is the one that used to be missing:
|
||
scoring one representative cycle where live matching scores the envelope
|
||
average can favour weights that win on a trace nothing is matched against.
|
||
|
||
**Cost, and the trap inside it.** An envelope is built from traces, so it does
|
||
not depend on the weights being tuned and survives the whole grid search plus
|
||
every holdout call; and leave-one-out excludes exactly ONE cycle, so only the
|
||
target's own profile gets a different template while every other profile
|
||
keeps the full-pool one, shared by every target. Distinct templates are
|
||
therefore `profiles + targets`, not `folds x profiles` - measured on the worst
|
||
real export in `cycle_data/` (12 profiles with >= 2 cycles, 63 targets), 75
|
||
templates at 79 ms a DTW warp, ~5.9 s one-off. **That figure only holds if the
|
||
cache outlives one `_top1` call.** It did not at first: `_top1` built a fresh
|
||
cache and is called ~59 times (1 base + 48 grid + 10 holdout), which measured
|
||
16.1 s -> 360.6 s on a real export. Hence the `cache` argument threaded
|
||
through the whole run - nothing in it depends on `cfg`. Do not re-scope it.
|
||
|
||
The cheap approximation (averaging the pool's regridded curves without the
|
||
DTW warp) is measured WORSE and is not what `_envelope_avg` does.
|
||
|
||
The exposure is bounded regardless: `tune_matching_config` can
|
||
only move the bounded scoring weights, never structural matching behaviour,
|
||
`revert_matching_config` undoes it, and promotion is gated on **held-out
|
||
top-1 accuracy**: `tune_matching_config` promotes only when the tuned config
|
||
beats the baseline by `margin` on at least `min_wins` of `n_splits` held-out
|
||
subsamples (4 of 5) AND the mean held-out top-1 gain is itself >= `margin`.
|
||
Not AUC - no AUC is computed anywhere in this module. The AUC gate in
|
||
CLAUDE.md is `ML_TRAINING_AUC_MARGIN`, which governs the ml/ CLASSIFIERS in
|
||
`training_task.py` and has nothing to do with the matcher's scoring weights.
|
||
"""
|
||
snaps = []
|
||
for name, items in by_profile.items():
|
||
excluded = exclude[1] if (exclude is not None and exclude[0] == name) else None
|
||
pool = [
|
||
it for idx, it in enumerate(items)
|
||
if exclude is None or (name, idx) != exclude
|
||
]
|
||
if not pool:
|
||
continue
|
||
durs = [it["dur"] for it in pool]
|
||
avg = float(np.mean(durs))
|
||
# Production's three-way template rule, in production's order
|
||
# (`ProfileStore.async_match_profile`). Getting this wrong is register
|
||
# item 347(e): scoring one representative cycle where live matching
|
||
# scores the envelope average can favour weights that lose on the curve
|
||
# that actually does the matching.
|
||
curve = None
|
||
golden = [it for it in pool if it.get("golden")]
|
||
if golden:
|
||
# 1. a pinned golden cycle keeps its own sharp trace; the average
|
||
# smears the wash-phase peaks, which is why production prefers it.
|
||
curve = _regrid(golden[0], dt, cache)
|
||
elif len(pool) >= 2:
|
||
# 2. the common case: the DTW-warped envelope average.
|
||
curve = _envelope_avg(name, pool, excluded, dt, cache)
|
||
if not curve:
|
||
# 3. the single-sample fallback, and the safety net for a profile
|
||
# whose envelope will not build (too few usable points, a trace
|
||
# that resamples to nothing). Representative rather than
|
||
# arbitrary: closest to the pool's mean duration.
|
||
rep = min(pool, key=lambda it: abs(it["dur"] - avg))
|
||
curve = _regrid(rep, dt, cache)
|
||
if not curve:
|
||
continue
|
||
snaps.append({
|
||
"name": name,
|
||
"avg_duration": avg,
|
||
"sample_power": curve,
|
||
})
|
||
return snaps
|
||
|
||
|
||
def _top1(
|
||
by_profile: dict[str, list[dict]],
|
||
targets: list[tuple[str, int]],
|
||
cfg: dict[str, Any],
|
||
cache: dict | None = None,
|
||
) -> float:
|
||
"""Fraction of the given (profile, idx) targets whose true profile ranks #1
|
||
under leave-one-out matching with the given config.
|
||
|
||
``cache`` is shared ACROSS calls on purpose. Nothing in it depends on
|
||
``cfg``: the re-gridded curves are keyed by (cycle, dt) and the envelopes by
|
||
(profile, excluded index), while ``cfg`` only changes the scoring weights.
|
||
Left per-call - as it was - the grid search rebuilds every envelope for each
|
||
of its ~59 configs, which measured 16 s -> 361 s on one real export. This is
|
||
what makes the `profiles + targets` cost in register item 354 real rather
|
||
than theoretical."""
|
||
if not targets:
|
||
return 0.0
|
||
correct = 0
|
||
total = 0
|
||
if cache is None:
|
||
cache = {}
|
||
for name, idx in targets:
|
||
it = by_profile[name][idx]
|
||
# The query defines the grid, exactly as in production.
|
||
segments, used_dt = resample_adaptive(
|
||
it["ts"], it["pw"], min_dt=_MIN_DT, gap_s=_GAP_S
|
||
)
|
||
seg = _longest(segments)
|
||
if seg is None or len(seg.power) < 4:
|
||
continue
|
||
snaps = _snaps(by_profile, (name, idx), used_dt, cache)
|
||
if len(snaps) < 2:
|
||
continue
|
||
cands = analysis.compute_matches_worker(
|
||
seg.power.tolist(), it["dur"], snaps, cfg
|
||
)
|
||
total += 1
|
||
if cands and cands[0]["name"] == name:
|
||
correct += 1
|
||
return correct / total if total else 0.0
|
||
|
||
|
||
# The full production matcher config. A PARTIAL config does not inherit the
|
||
# production defaults - `compute_matches_worker` has its own fallbacks - so
|
||
# omitting a key here would tune the weights against a pipeline that never
|
||
# ships. `energy_mode` is added per device type at the call site.
|
||
_BASE_CFG = {
|
||
"min_duration_ratio": DEFAULT_PROFILE_MATCH_MIN_DURATION_RATIO,
|
||
"max_duration_ratio": DEFAULT_PROFILE_MATCH_MAX_DURATION_RATIO,
|
||
"dtw_bandwidth": DEFAULT_DTW_BANDWIDTH,
|
||
"dtw_mode": DEFAULT_DTW_MODE,
|
||
}
|
||
|
||
#: Bounded scoring weights the tuner may promote. All live in [0, 1], so a tuned
|
||
#: config can only shift emphasis (shape vs level vs energy, and how much the DTW
|
||
#: ensemble leans on the derivative/DDTW component) - never structural behaviour.
|
||
OVERRIDE_KEYS = ("corr_weight", "duration_weight", "energy_weight", "dtw_ensemble_w")
|
||
|
||
|
||
def _grid() -> list[dict[str, Any]]:
|
||
"""Small, high-impact grid over four bounded scoring weights.
|
||
|
||
Axes: corr/MAE split × duration agreement weight × energy agreement weight
|
||
× DTW ensemble weight. The duration and energy axes are now independent so
|
||
the tuner can find asymmetric configurations (e.g. a device with highly
|
||
variable energy but stable duration benefits from a low energy_weight and a
|
||
high duration_weight). All values are bounded scoring weights (see
|
||
OVERRIDE_KEYS) so a promoted config can never change structural behaviour.
|
||
Grid size: 4 × 2 × 2 × 3 = 48 configurations (was 4 × 2 × 3 = 24).
|
||
"""
|
||
out = []
|
||
for cw in (0.40, 0.45, 0.50, 0.60):
|
||
for dur_w in (0.15, 0.22):
|
||
for en_w in (0.15, 0.22):
|
||
for ew in (0.55, 0.70, 0.85):
|
||
out.append({
|
||
"corr_weight": cw,
|
||
"duration_weight": dur_w,
|
||
"energy_weight": en_w,
|
||
"dtw_ensemble_w": ew,
|
||
})
|
||
return out
|
||
|
||
|
||
def tune_matching_config(
|
||
cycles: list[dict[str, Any]],
|
||
device_type: str | None = None,
|
||
*,
|
||
min_cycles: int = 25,
|
||
# Kept intentionally low so per-device tuning becomes useful early; the noise
|
||
# a small sample would introduce is controlled by the multi-split majority gate
|
||
# below (a lucky single split can't promote), not by a large ``min_targets``.
|
||
min_targets: int = 12,
|
||
margin: float = 0.03,
|
||
seed: int = 0,
|
||
) -> dict[str, Any]:
|
||
"""Leave-one-out per-device tuning of matcher scoring weights.
|
||
|
||
Methodology (no target leakage between selection and gating):
|
||
1. Partition the device's labelled cycles ONCE into a *search* pool and an
|
||
untouched *holdout* pool; no target is ever used for both.
|
||
2. **Select** the candidate config as the grid entry with the best
|
||
leave-one-out top-1 on the SEARCH pool only. (Reference snapshots are
|
||
built from all cycles — as in production, where a query is matched
|
||
against aggregates of the full profile library; only the *query* targets
|
||
are partitioned.)
|
||
3. **Gate** the fixed candidate on the HOLDOUT pool: it must beat the
|
||
shipped defaults by at least ``margin`` on a MAJORITY of reshuffled
|
||
holdout subsamples (a variance check that rejects a lucky single split)
|
||
AND on the holdout mean. ``min_targets`` is kept intentionally low so
|
||
per-device tuning becomes useful early; the majority gate — not a large
|
||
sample — controls the noise.
|
||
|
||
Returns a status dict; ``promoted`` is True only when both holdout gates pass.
|
||
When promoted, ``config`` holds the override to persist (bounded scoring
|
||
weights only — never structural matching behaviour). Never raises for data
|
||
reasons; returns {"promoted": False, "reason": ...}.
|
||
"""
|
||
by_profile = _prep(cycles)
|
||
multi = {n: items for n, items in by_profile.items() if len(items) >= 2}
|
||
n_cycles = sum(len(v) for v in by_profile.values())
|
||
if len(multi) < 2 or n_cycles < min_cycles:
|
||
return {"promoted": False, "reason": "insufficient data", "n_cycles": n_cycles, "n_profiles": len(by_profile)}
|
||
|
||
# Partition targets ONCE, up front, into a search pool (used to pick the
|
||
# candidate config) and an untouched holdout pool (used only to gate it). No
|
||
# target is ever used for both selection and gating -> no target leakage.
|
||
rng = np.random.default_rng(seed)
|
||
targets = [(n, i) for n, items in multi.items() for i in range(len(items))]
|
||
rng.shuffle(targets)
|
||
if len(targets) < min_targets:
|
||
return {"promoted": False, "reason": "too few targets", "n_targets": len(targets)}
|
||
cut = max(1, len(targets) // 2)
|
||
search_pool, holdout_pool = targets[:cut], targets[cut:]
|
||
if not holdout_pool:
|
||
return {"promoted": False, "reason": "too few targets", "n_targets": len(targets)}
|
||
|
||
# Tune under the same Stage-4 energy mode production uses for this device type,
|
||
# so promoted weights are consistent with the live matcher.
|
||
base = {**_BASE_CFG, "energy_mode": analysis.stage4_energy_mode(device_type)}
|
||
# Candidate: the grid config with the best top-1 on the SEARCH pool only.
|
||
# One cache for the entire run: the grid search and both holdout arms all
|
||
# reuse the same envelopes and re-gridded curves (see `_top1`).
|
||
shared: dict = {}
|
||
best_search = _top1(by_profile, search_pool, base, shared)
|
||
best_cfg = base
|
||
for extra in _grid():
|
||
acc = _top1(by_profile, search_pool, {**base, **extra}, shared)
|
||
if acc > best_search:
|
||
best_search, best_cfg = acc, {**base, **extra}
|
||
override = {k: best_cfg[k] for k in OVERRIDE_KEYS if k in best_cfg}
|
||
|
||
# Gate the FIXED candidate on the held-out pool: require it to beat the defaults
|
||
# by ``margin`` on a MAJORITY of reshuffled subsamples of the holdout (variance
|
||
# check), rejecting a lucky single split while keeping min_targets low.
|
||
n_splits, min_wins = 5, 4
|
||
base_tests: list[float] = []
|
||
tuned_tests: list[float] = []
|
||
wins = 0
|
||
for k in range(n_splits):
|
||
r = np.random.default_rng(seed + 1 + k)
|
||
pool = list(holdout_pool)
|
||
r.shuffle(pool)
|
||
held = pool[: max(1, len(pool) // 2)]
|
||
bt = _top1(by_profile, held, base, shared)
|
||
tt = _top1(by_profile, held, best_cfg, shared)
|
||
base_tests.append(bt)
|
||
tuned_tests.append(tt)
|
||
if tt - bt >= margin:
|
||
wins += 1
|
||
mean_base = float(np.mean(base_tests)) if base_tests else 0.0
|
||
mean_tuned = float(np.mean(tuned_tests)) if tuned_tests else 0.0
|
||
has_override = bool(override)
|
||
enough_wins = wins >= min_wins
|
||
enough_margin = (mean_tuned - mean_base) >= margin
|
||
promoted = has_override and enough_wins and enough_margin
|
||
if promoted:
|
||
reason = f"beat baseline on {wins}/{n_splits} held-out subsamples"
|
||
elif not has_override:
|
||
# AUDITED and correct (register item 356), because it fires on every real
|
||
# export in `cycle_data/` and that looks like a stuck mechanism. It is
|
||
# not. `_BASE_CFG` deliberately omits the four OVERRIDE_KEYS, so
|
||
# `compute_matches_worker` falls back to `MATCH_CORR_WEIGHT` /
|
||
# `MATCH_DURATION_WEIGHT` / `MATCH_ENERGY_WEIGHT` /
|
||
# `MATCH_DTW_ENSEMBLE_W` - the tuner's baseline IS production's default,
|
||
# not a partial config with different fallbacks. `override` is therefore
|
||
# empty exactly when no grid entry beat that baseline, which is what this
|
||
# string says. The weights do reach the scorer: on the least saturated
|
||
# real device (12 profiles, 63 targets, base top-1 0.635) the grid
|
||
# produces three distinct scores, all <= base. The grid also contains the
|
||
# exact default combination (0.45 / 0.22 / 0.22 / 0.70), so the defaults
|
||
# are evaluated on equal terms, and `acc > best_search` is strict so a tie
|
||
# keeps them. Consistent with the corpus result that matcher top-1 is
|
||
# near-saturated: there is nothing here for per-device tuning to win.
|
||
reason = "defaults already optimal (no override)"
|
||
elif not enough_wins:
|
||
reason = f"only {wins}/{n_splits} held-out subsamples beat baseline by margin"
|
||
else:
|
||
reason = f"mean held-out gain {mean_tuned - mean_base:+.3f} below margin {margin}"
|
||
return {
|
||
"promoted": promoted,
|
||
"config": override if promoted else None,
|
||
"baseline_test_top1": round(mean_base, 3),
|
||
"tuned_test_top1": round(mean_tuned, 3),
|
||
"train_top1": round(best_search, 3),
|
||
"holdout_wins": wins,
|
||
"holdout_splits": n_splits,
|
||
"n_targets": len(targets),
|
||
"reason": reason,
|
||
}
|