309 lines
16 KiB
Python
309 lines
16 KiB
Python
# #region Tooling.LabReset [C:5] [TYPE Module] [SEMANTICS laboratory,maintenance,reset,quiescence]
|
|
# @BRIEF Separate inert host receipt and explicitly invoked inventory/archive/reset for two exact owned laboratories.
|
|
# @PRE Root reviews every phase digest; caller supplies private connection and stopped-app receipt; no live operation is implied by preparation.
|
|
# @POST Any discrepancy STOPs without retry; DB commit and file reset have distinct durable journal boundaries.
|
|
# @INVARIANT No container start/stop/create, migration, app import, secret extraction, fallback selection or automatic recovery.
|
|
# @RATIONALE Root invocation separates authority review from destructive execution; current storage owner preserves DB→root order.
|
|
# @REJECTED A combined DB/files atomicity claim or a successful inventory as implicit reset authorization.
|
|
import argparse
|
|
from datetime import UTC, datetime
|
|
from hashlib import sha256
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
|
|
from dashboard_testing_lab_reset_archive import (
|
|
Journal, Stop, decode, digest, json_bytes, private_directory, read_json, require, write_bytes,
|
|
)
|
|
|
|
ALLOWLIST_SHA = "7c6b5140ff3ea42e9e56a6a7e20ee0cfec0b2ab8493410855925a161b90d7a08"
|
|
IMAGE = "sha256:13834a9b54d4dd4397aa713b56087e4a28170cb8d31deb1bd51ac425b2861933"
|
|
ROLES = {"stage1_candidate": "ss-tools-stage1-backend-durable-candidate",
|
|
"own_synthetic_candidate": "ss-tools-chart-response-verify-backend-20261010"}
|
|
TOOLS = [f"dashboard_testing_lab_reset{suffix}.py" for suffix in ("", "_db", "_files", "_archive")]
|
|
|
|
|
|
# #region Tooling.LabReset.Arguments [C:2] [TYPE Function]
|
|
def arguments(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
modes = parser.add_mutually_exclusive_group(required=True)
|
|
modes.add_argument("--host-check", action="store_true")
|
|
modes.add_argument("--phase", choices=("inventory", "archive", "reset"))
|
|
parser.add_argument("--role", choices=tuple(ROLES))
|
|
for flag in ("allowlist", "expected-allowlist-sha256", "source-sha", "source-root", "backend-root",
|
|
"specs-root", "evidence-dir", "evidence-owner-uid", "input-owner-uid"):
|
|
parser.add_argument(f"--{flag}", required=True)
|
|
for flag in ("quiescence-receipt", "expected-quiescence-sha256", "database-identity-sha256",
|
|
"inventory", "expected-inventory-sha256", "archive", "expected-archive-sha256"):
|
|
parser.add_argument(f"--{flag}")
|
|
return parser.parse_args(argv)
|
|
# #endregion Tooling.LabReset.Arguments
|
|
|
|
|
|
# #region Tooling.LabReset.SourceTree [C:2] [TYPE Function]
|
|
def tree_hash(root):
|
|
root = Path(root)
|
|
require(root.is_dir() and not root.is_symlink(), "LAB_RESET_SOURCE_CONFLICT")
|
|
result = []
|
|
for path in sorted(root.rglob("*")):
|
|
if "__pycache__" in path.parts or path.suffix == ".pyc":
|
|
continue
|
|
require(not path.is_symlink(), "LAB_RESET_SOURCE_CONFLICT")
|
|
if path.is_file():
|
|
result.append([str(path.relative_to(root)), sha256(path.read_bytes()).hexdigest()])
|
|
return digest(result)
|
|
# #endregion Tooling.LabReset.SourceTree
|
|
|
|
|
|
# #region Tooling.LabReset.Source [C:2] [TYPE Function]
|
|
def source_hashes(args):
|
|
scripts = Path(args.source_root) / "scripts"
|
|
tools = {name: sha256((scripts / name).read_bytes()).hexdigest() for name in TOOLS}
|
|
return {"tools": tools, "backend_src": tree_hash(Path(args.backend_root) / "src"),
|
|
"specs": tree_hash(args.specs_root)}
|
|
# #endregion Tooling.LabReset.Source
|
|
|
|
|
|
# #region Tooling.LabReset.Command [C:2] [TYPE Function]
|
|
# @INVARIANT Only fixed read-only git/docker commands are reachable; stderr stays private on failure.
|
|
def command(argv):
|
|
result = subprocess.run(argv, capture_output=True, text=True, timeout=15, check=False)
|
|
if result.returncode != 0:
|
|
raise Stop("LAB_RESET_HOST_CHECK_FAILED") from subprocess.CalledProcessError(
|
|
result.returncode, argv, output=result.stdout, stderr=result.stderr)
|
|
return result.stdout.strip()
|
|
# #endregion Tooling.LabReset.Command
|
|
|
|
|
|
# #region Tooling.LabReset.Mounts [C:2] [TYPE Function]
|
|
def canonical_mounts(mounts):
|
|
destinations = [mount["Destination"] for mount in mounts]
|
|
require(all(isinstance(value, str) and value.startswith("/") for value in destinations)
|
|
and len(set(destinations)) == len(destinations), "LAB_RESET_CONTAINER_MOUNTS_CHANGED")
|
|
return sorted(mounts, key=lambda mount: mount["Destination"])
|
|
# #endregion Tooling.LabReset.Mounts
|
|
|
|
|
|
# #region Tooling.LabReset.Container [C:3] [TYPE Function]
|
|
def validate_container(observed, scope):
|
|
require(observed["name"] == "/" + scope["container"] and observed["image"] == IMAGE
|
|
and re.fullmatch(r"[0-9a-f]{64}", observed["id"]), "LAB_RESET_CONTAINER_IDENTITY_CONFLICT")
|
|
state = observed["state"]
|
|
require(not any(state[key] for key in ("running", "paused", "restarting", "dead"))
|
|
and state["pid"] == 0 and state["status"] == "exited", "LAB_RESET_APPLICATION_NOT_STOPPED")
|
|
require(not state["finished_at"].startswith("0001-"), "LAB_RESET_STOP_TIME_REQUIRED")
|
|
require(digest(canonical_mounts(observed["mounts"])) == digest(canonical_mounts(scope["mounts"])),
|
|
"LAB_RESET_CONTAINER_MOUNTS_CHANGED")
|
|
# #endregion Tooling.LabReset.Container
|
|
|
|
|
|
# #region Tooling.LabReset.Host [C:3] [TYPE Function]
|
|
# @POST Receipt contains actual narrowly inspected stopped-container fields and source hashes; no worker action follows.
|
|
def host_check(args, allowlist):
|
|
require(command(["git", "-C", args.source_root, "rev-parse", "HEAD"]) == args.source_sha,
|
|
"LAB_RESET_SOURCE_CONFLICT")
|
|
require(not command(["git", "-C", args.source_root, "status", "--porcelain"]), "LAB_RESET_SOURCE_DIRTY")
|
|
template = ('{"id":{{json .Id}},"name":{{json .Name}},"image":{{json .Image}},'
|
|
'"mounts":{{json .Mounts}},"state":{"running":{{json .State.Running}},'
|
|
'"paused":{{json .State.Paused}},"restarting":{{json .State.Restarting}},'
|
|
'"dead":{{json .State.Dead}},"pid":{{json .State.Pid}},"status":{{json .State.Status}},'
|
|
'"finished_at":{{json .State.FinishedAt}}}}')
|
|
import json
|
|
containers = {}
|
|
for role, name in ROLES.items():
|
|
observed = json.loads(command(["docker", "inspect", "--format", template, name]))
|
|
validate_container(observed, allowlist["roles"][role])
|
|
observed["mounts"] = canonical_mounts(observed["mounts"])
|
|
containers[role] = observed
|
|
return {"schema_version": 1, "created_at": datetime.now(UTC).isoformat(),
|
|
"allowlist_sha256": ALLOWLIST_SHA, "source_sha": args.source_sha,
|
|
"source_hashes": source_hashes(args), "containers": containers, "inert": True}
|
|
# #endregion Tooling.LabReset.Host
|
|
|
|
|
|
# #region Tooling.LabReset.Receipt [C:3] [TYPE Function]
|
|
def validate_receipt(args, allowlist, receipt, now=None):
|
|
require(receipt["schema_version"] == 1 and receipt["inert"] is True
|
|
and receipt["source_sha"] == args.source_sha
|
|
and receipt["allowlist_sha256"] == ALLOWLIST_SHA, "LAB_RESET_RECEIPT_CONFLICT")
|
|
age = ((now or datetime.now(UTC)) - datetime.fromisoformat(receipt["created_at"])).total_seconds()
|
|
require(0 <= age <= 900, "LAB_RESET_RECEIPT_EXPIRED")
|
|
require(set(receipt["containers"]) == set(ROLES), "LAB_RESET_RECEIPT_CONFLICT")
|
|
for role in ROLES:
|
|
validate_container(receipt["containers"][role], allowlist["roles"][role])
|
|
stopped = datetime.fromisoformat(receipt["containers"][role]["state"]["finished_at"])
|
|
require(stopped <= datetime.fromisoformat(receipt["created_at"]), "LAB_RESET_RECEIPT_CONFLICT")
|
|
require(receipt["source_hashes"] == source_hashes(args), "LAB_RESET_SOURCE_CONFLICT")
|
|
# #endregion Tooling.LabReset.Receipt
|
|
|
|
|
|
# #region Tooling.LabReset.Allowlist [C:2] [TYPE Function]
|
|
def allowlist_input(args):
|
|
require(args.expected_allowlist_sha256 == ALLOWLIST_SHA, "LAB_RESET_ALLOWLIST_NOT_APPROVED")
|
|
value = read_json(args.allowlist, ALLOWLIST_SHA, private=True, owner_uid=int(args.input_owner_uid))
|
|
require(set(value["roles"]) == set(ROLES), "LAB_RESET_ALLOWLIST_INVALID")
|
|
for role, name in ROLES.items():
|
|
require(value["roles"][role]["container"] == name, "LAB_RESET_ALLOWLIST_INVALID")
|
|
return value
|
|
# #endregion Tooling.LabReset.Allowlist
|
|
|
|
|
|
# #region Tooling.LabReset.PrivateInput [C:3] [TYPE Function]
|
|
def private_input(args, path, expected, *, host=False):
|
|
require(path and expected, "LAB_RESET_PHASE_INPUT_REQUIRED")
|
|
path = Path(path)
|
|
require(path.parent == Path(args.evidence_dir), "LAB_RESET_PRIVATE_INPUT_LOCATION")
|
|
# Exact source classes: reviewed host receipt vs prior-phase worker output.
|
|
owner = int(args.input_owner_uid) if host else os.geteuid()
|
|
return read_json(path, expected, private=True, owner_uid=owner)
|
|
# #endregion Tooling.LabReset.PrivateInput
|
|
|
|
|
|
# #region Tooling.LabReset.Binding [C:1] [TYPE Function]
|
|
def binding(args, receipt):
|
|
return {"role": args.role, "allowlist_sha256": ALLOWLIST_SHA, "source_sha": args.source_sha,
|
|
"source_hashes": receipt["source_hashes"], "database_identity_sha256": args.database_identity_sha256,
|
|
"containers": receipt["containers"]}
|
|
# #endregion Tooling.LabReset.Binding
|
|
|
|
|
|
# #region Tooling.LabReset.Inventory [C:2] [TYPE Function]
|
|
def inventory_record(args, receipt, database, files, holds):
|
|
return {"schema_version": 1, "binding": binding(args, receipt), "database": database,
|
|
"files": files, "holds_sha256": holds}
|
|
# #endregion Tooling.LabReset.Inventory
|
|
|
|
|
|
# #region Tooling.LabReset.ArchiveCheck [C:3] [TYPE Function]
|
|
def archive_check(args, scope, archive, inventory, state):
|
|
from dashboard_testing_lab_reset_db import archive_rows
|
|
from dashboard_testing_lab_reset_files import verify_archive_files
|
|
require(archive["inventory_sha256"] == args.expected_inventory_sha256
|
|
and archive["binding"] == inventory["binding"], "LAB_RESET_ARCHIVE_SCOPE_CONFLICT")
|
|
require(archive["database_rows"] == archive_rows(state), "LAB_RESET_ARCHIVE_SCOPE_CONFLICT")
|
|
for table in archive["database_rows"].values():
|
|
for row in table["rows"]:
|
|
decode(row)
|
|
verify_archive_files(scope, args.evidence_dir, archive["files"])
|
|
# #endregion Tooling.LabReset.ArchiveCheck
|
|
|
|
|
|
# #region Tooling.LabReset.LockDeadline [C:2] [TYPE Function]
|
|
def lock_deadline(signum, frame):
|
|
raise Stop("LAB_RESET_ROOT_LOCK_TIMEOUT")
|
|
# #endregion Tooling.LabReset.LockDeadline
|
|
|
|
|
|
# #region Tooling.LabReset.LockedPhase [C:4] [TYPE Function]
|
|
# @PRE Rows/schema/ref-probes and selected row locks acquired already; root-held path takes no new DB locks.
|
|
def locked_phase(args, receipt, scope, connection, database, state, journal):
|
|
from dashboard_testing_lab_reset_db import archive_rows, delete_rows, verify_deleted
|
|
from dashboard_testing_lab_reset_files import (
|
|
archive_files, checkpoint, reset_files, validate_holds, verify_reset,
|
|
)
|
|
files = checkpoint(scope)
|
|
current = inventory_record(args, receipt, database, files, validate_holds(scope, state, journal))
|
|
journal.event("REFLECT", "Exact laboratory inventory verified under database and root locks",
|
|
selected_rows=len(state[3]), tables=len(database["tables"]), holds_sha256=current["holds_sha256"])
|
|
if args.phase == "inventory":
|
|
return write_bytes(args.evidence_dir, "inventory.json", json_bytes(current))
|
|
reviewed = private_input(args, args.inventory, args.expected_inventory_sha256)
|
|
require(current == reviewed, "LAB_RESET_INVENTORY_CHANGED")
|
|
if args.phase == "archive":
|
|
archive = {"schema_version": 1, "inventory_sha256": args.expected_inventory_sha256,
|
|
"binding": current["binding"], "database_rows": archive_rows(state),
|
|
"files": archive_files(scope, args.evidence_dir)}
|
|
archive_check(args, scope, archive, reviewed, state)
|
|
return write_bytes(args.evidence_dir, "archive.json", json_bytes(archive))
|
|
archive = private_input(args, args.archive, args.expected_archive_sha256)
|
|
archive_check(args, scope, archive, reviewed, state)
|
|
journal.event("REASON", "Exact archived primary-key deletion begins")
|
|
delete_rows(connection, state)
|
|
verify_deleted(connection, scope, database, state)
|
|
journal.event("REFLECT", "Protected database contents verified before commit")
|
|
journal.event("REASON", "Database commit requested; exception leaves outcome unknown",
|
|
database_commit_status="requested_outcome_unknown")
|
|
journal.database_commit_status = "requested_outcome_unknown"
|
|
connection.commit()
|
|
journal.database_commit_status = "confirmed"
|
|
journal.event("REFLECT", "Database committed; filesystem boundary remains separate", database_committed=True)
|
|
reset_files(scope, journal)
|
|
verify_reset(scope, files)
|
|
return args.expected_archive_sha256
|
|
# #endregion Tooling.LabReset.LockedPhase
|
|
|
|
|
|
# #region Tooling.LabReset.Worker [C:4] [TYPE Function]
|
|
def worker(args, allowlist, journal):
|
|
from dashboard_testing_lab_reset_db import locked_database, lock_selected, snapshot, verify_deleted
|
|
from dashboard_testing_lab_reset_files import root_lock, verify_reset
|
|
require(args.role in ROLES and args.database_identity_sha256, "LAB_RESET_PHASE_INPUT_REQUIRED")
|
|
require(os.environ["STORAGE_ROOT_PATH"] == "/app/storage", "LAB_RESET_ROOT_INVALID")
|
|
receipt = private_input(args, args.quiescence_receipt, args.expected_quiescence_sha256, host=True)
|
|
validate_receipt(args, allowlist, receipt)
|
|
if args.phase in ("archive", "reset"):
|
|
private_input(args, args.inventory, args.expected_inventory_sha256)
|
|
if args.phase == "reset":
|
|
private_input(args, args.archive, args.expected_archive_sha256)
|
|
scope = allowlist["roles"][args.role]
|
|
with locked_database(os.environ["DATABASE_URL"], scope, args.database_identity_sha256) as connection:
|
|
database, state = snapshot(connection, scope)
|
|
lock_selected(connection, state)
|
|
lock = root_lock(scope)
|
|
previous = signal.signal(signal.SIGALRM, lock_deadline)
|
|
try:
|
|
signal.alarm(5)
|
|
lock.__enter__()
|
|
finally:
|
|
signal.alarm(0)
|
|
signal.signal(signal.SIGALRM, previous)
|
|
try:
|
|
result = locked_phase(args, receipt, scope, connection, database, state, journal)
|
|
finally:
|
|
lock.__exit__(None, None, None)
|
|
if args.phase == "reset":
|
|
# A new verification transaction acquires all DB locks before reentering root.
|
|
with locked_database(os.environ["DATABASE_URL"], scope, args.database_identity_sha256) as connection:
|
|
verify_deleted(connection, scope, database, state)
|
|
with root_lock(scope):
|
|
reviewed = private_input(args, args.inventory, args.expected_inventory_sha256)
|
|
verify_reset(scope, reviewed["files"])
|
|
return result
|
|
# #endregion Tooling.LabReset.Worker
|
|
|
|
|
|
# #region Tooling.LabReset.Main [C:3] [TYPE Function]
|
|
def main(argv=None):
|
|
args = arguments(argv)
|
|
for value in (args.evidence_dir, args.source_root, args.backend_root, args.specs_root):
|
|
require(Path(value).is_absolute(), "LAB_RESET_PATH_INVALID")
|
|
for root in ("/app/storage", "/app/backend/git_repos", "/app/backend/data/dashboard_testing"):
|
|
require(not Path(args.evidence_dir).is_relative_to(root), "LAB_RESET_PRIVATE_INPUT_LOCATION")
|
|
private_directory(args.evidence_dir, int(args.evidence_owner_uid))
|
|
require(re.fullmatch(r"[0-9a-f]{40}", args.source_sha), "LAB_RESET_SOURCE_CONFLICT")
|
|
sys.path.insert(0, str(Path(args.backend_root)))
|
|
journal = Journal(args.evidence_dir, "host-check" if args.host_check else args.phase)
|
|
try:
|
|
journal.event("REASON", "Reviewed laboratory phase preconditions begin")
|
|
allowlist = allowlist_input(args)
|
|
if args.host_check:
|
|
result = write_bytes(args.evidence_dir, "quiescence.json", json_bytes(host_check(args, allowlist)))
|
|
else:
|
|
result = worker(args, allowlist, journal)
|
|
journal.event("REFLECT", "Laboratory phase completed; next phase requires review", result_sha256=result)
|
|
return 0
|
|
except Exception as exc:
|
|
journal.failure(exc)
|
|
return 2
|
|
finally:
|
|
journal.close()
|
|
# #endregion Tooling.LabReset.Main
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|
|
# #endregion Tooling.LabReset
|