diff --git a/reconcile_contracts.py b/reconcile_contracts.py index e6d6bbf50..a02904e5a 100644 --- a/reconcile_contracts.py +++ b/reconcile_contracts.py @@ -47,13 +47,18 @@ REJECTED_DECISION_PATTERNS = { "resolved recurrence suppressed": r"(resolved|known-issue|accepted).*not re-alerted", "save silently activates revision": r"(?:save|saved|saving).*current_revision\s+(?:advanced|advances)", "current pointer advances on edit": r"current_revision\s+pointer\s+advances\s+on\s+edit", + "runtime agent blanket exclusion": r"agent is not in the hot path", + "required authoring runtime parameter": r"unresolved required parameter.*preview_only", } # Final closure invariants are presence checks for the canonical, normative # clauses. They deliberately complement (not replace) schema/OpenAPI parsing. REQUIRED_CONTRACT_FRAGMENTS = { "038-dashboard-scenario-model/data-model.md": ( - "AuthoringValidation", "RunPreflight", "ActionRegistry(version)", "PROD permits read-only execution only", + "AuthoringValidation", "RunPreflight", "ActionRegistry(version)", "VerificationProgram", "SqlEvidenceSpec", "TransformSpec", "AgentEvaluationSpec", "mutating browser steps in PROD are prohibited", + ), + "038-dashboard-scenario-model/contracts/verification-program.md": ( + "SqlEvidenceSpec", "TransformSpec", "AgentEvaluationSpec", "DecisionPolicy", "Superset SQL Lab adapter", "SQL compilation gate", ), "036-agent-test-stabilization/data-model.md": ( "owner_type", "scenario_run", "load_run", "DelegatedAuthorityPolicy", "InvestigationSignal", @@ -68,16 +73,16 @@ REQUIRED_CONTRACT_FRAGMENTS = { "/metadata", "If-Match", "agent_action_id", "/migration-proposals/{proposal_id}/resolve", ), "044-dashboard-scenario-execution/data-model.md": ( - "pending_approval", "Idempotency uses canonical execution-request hash", "BrowserExecutor", "A PROD mutation is allowed only", "checkpoint_type", "manual_run_only=true", "InvestigationSignal", "AnalyticsContextKey", + "pending_approval", "Idempotency uses canonical execution-request hash", "BrowserExecutor", "SqlEvidenceExecutor", "AgentEvaluation", "DecisionPolicy", "checkpoint_type", "manual_run_only=true", "InvestigationSignal", "AnalyticsContextKey", "Mutating browser steps in PROD are prohibited", ), "044-dashboard-scenario-execution/contracts/openapi.yaml": ( - "ScenarioExecutionResult", "RunComparison", "ScenarioRunEvent", "requested_target_reference", + "ScenarioExecutionResult", "RunComparison", "ScenarioRunEvent", "requested_target_reference", "StepOutcome", "AgentEvaluationSummary", "agent_evaluation_completed", ), "046-dashboard-scenario-automation/data-model.md": ( "revision_id (required when pinned)", "canonical_execution_request_hash", "Concurrency bucket", "AUTOMATION_INELIGIBLE_HUMAN_STEP", "candidate", "atomic activation", ), "047-dashboard-scenario-analytics/data-model.md": ( - "InvestigationQueueItem", "InvestigationCase", "compatibility_family", "FailureEpisode", "product_health", "does not start AgentRun", "Closure policy", "AnalyticsContextKey", "InvestigationSignal", + "InvestigationQueueItem", "InvestigationCase", "compatibility_family", "FailureEpisode", "product_health", "agent_evaluation_health", "does not start AgentRun", "Closure policy", "AnalyticsContextKey", "InvestigationSignal", ), "036-agent-test-stabilization/contracts/investigation-cases.md": ( "InvestigationQueueItem", "InvestigationCase", "AgentAction", "MUST NOT use a modal", "DelegatedAuthorityPolicy", "InvestigationSignal", "Process Boundary Matrix", @@ -200,6 +205,56 @@ def check_fixtures_vs_schema(spec_id: str = "038-dashboard-scenario-model") -> l return out +def check_verification_program_boundaries(paths: list[Path]) -> list[str]: + """Assert the agent-authored / immutable-runtime Verification Program boundary. + + These checks intentionally inspect canonical machine contracts rather than prose: + 038 must model the program and keep runtime values out; 044 must expose typed + outcome/evaluation events without a request surface that can mutate SQL/code. + """ + out = [] + by_name = {p.relative_to(SPECS_DIR).as_posix(): p for p in paths} + schema_path = by_name.get("038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json") + if schema_path: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + required = set(schema.get("required") or []) + props = schema.get("properties") or {} + defs = schema.get("$defs") or {} + if "verification_program" not in required or "verification_program" not in props: + out.append("[PROGRAM] 038 schema: verification_program must be required canonical content") + for name in ("verificationProgram", "sqlEvidenceSpec", "transformSpec", "assertionSpec", "agentEvaluationSpec"): + if name not in defs: + out.append(f"[PROGRAM] 038 schema: missing $defs/{name}") + parameter_props = (defs.get("parameter") or {}).get("properties") or {} + forbidden = sorted({"value", "status"} & set(parameter_props)) + if forbidden: + out.append(f"[PROGRAM] 038 schema: ParameterDefinition contains runtime field(s) {', '.join(forbidden)}") + registry_path = by_name.get("038-dashboard-scenario-model/contracts/action-registry.yaml") + valid_fixture = by_name.get("038-dashboard-scenario-model/fixtures/api/scenario_valid.json") + if not registry_path: + out.append("[PROGRAM] 038 contract: missing versioned action-registry.yaml") + elif valid_fixture: + registry = yaml.safe_load(registry_path.read_text(encoding="utf-8")) or {} + registered = {(item.get("tool"), item.get("action")) for item in registry.get("actions", []) if isinstance(item, dict)} + fixture = json.loads(valid_fixture.read_text(encoding="utf-8")) + for step in fixture.get("steps", []): + pair = (step.get("tool"), step.get("action")) + if pair not in registered: + out.append(f"[PROGRAM] 038 registry: valid fixture action not registered: {pair[0]}/{pair[1]}") + run_api = by_name.get("044-dashboard-scenario-execution/contracts/openapi.yaml") + if run_api: + document = yaml.safe_load(run_api.read_text(encoding="utf-8")) or {} + schemas = ((document.get("components") or {}).get("schemas") or {}) + for name in ("ScenarioExecutionResult", "RunComparison", "ScenarioRunEvent", "StepOutcome", "AgentEvaluationSummary"): + if name not in schemas: + out.append(f"[PROGRAM] 044 OpenAPI: missing typed schema {name}") + text = run_api.read_text(encoding="utf-8") + forbidden_runtime_sql = ("sql_template" in text and "/api/scenario-runs" in text) + if forbidden_runtime_sql: + out.append("[PROGRAM] 044 OpenAPI: runtime API must not accept sql_template mutation") + return out + + def check_revision_vs_content_hash(paths: list[Path]) -> list[str]: """flag 'revision_id' where the field is used to mean content (hash) vs identity (uuid).""" out = [] @@ -316,6 +371,7 @@ def gate(spec_ids: list[str]) -> int: findings += check_rejected_decision_drift(paths) findings += check_final_closure_invariants(paths) findings += check_fixtures_vs_schema(d.name) + findings += check_verification_program_boundaries(paths) if not findings: print(" PASS — machine contracts consistent") else: diff --git a/specs/036-agent-test-stabilization/contracts/agent-pipeline.md b/specs/036-agent-test-stabilization/contracts/agent-pipeline.md index a39ddf909..a101cfa18 100644 --- a/specs/036-agent-test-stabilization/contracts/agent-pipeline.md +++ b/specs/036-agent-test-stabilization/contracts/agent-pipeline.md @@ -24,7 +24,7 @@ # @PRE RBAC filtering has already run. # @POST Tool list excludes arbitrary SQL operations. # @SIDE_EFFECT Emits pipeline_result audit metadata. -# @INVARIANT No direct SQL tool reaches build_dashboard_test_scenario. +# @INVARIANT No raw credentialed/arbitrary SQL tool reaches build_dashboard_test_scenario; bounded authoring SqlEvidenceSpec reaches 038 compiler validation only. # @TEST_INVARIANT No_Direct_SQL -> VERIFIED_BY: scenario_tool_list, replayed_sql_call. # @TEST_EDGE hallucinated_sql_tool_call -> invocation guard rejects. # @REJECTED Relying only on system prompt to avoid SQL — tool availability is enforceable. diff --git a/specs/036-agent-test-stabilization/contracts/investigation-cases.md b/specs/036-agent-test-stabilization/contracts/investigation-cases.md index b05a27f7d..df8e3ff69 100644 --- a/specs/036-agent-test-stabilization/contracts/investigation-cases.md +++ b/specs/036-agent-test-stabilization/contracts/investigation-cases.md @@ -49,6 +49,8 @@ Producers publish one idempotent `InvestigationSignal { source_type, source_id, | Process class | Agent role | Deterministic owner | Approval mode | |---|---|---|---| | reasoning, evidence search, hypothesis, plan | leads | case/event storage | delegated read policy | +| authoring verification logic (SQL/DSL/assertion/evaluation proposal) | compiles/proposes | 038 validation + 042 immutable revision | delegated policy; save never activates | +| declared runtime semantic evaluation | may reason inside bounded spec | 044 deterministic orchestration + DecisionPolicy | no mutation, no lifecycle ownership | | query/compare/validate/index/health/fingerprint | consumes output | 037/038/040/041/044/046/047 engines | never agent-decided | | diagnostic run and fixture experiment | proposes/executes | runner, capacity, mutation contract | delegated only in exact policy scope | | revision save | may execute | 038 validation + 042 immutable revision | delegated policy | diff --git a/specs/037-superset-baseline-engine/spec.md b/specs/037-superset-baseline-engine/spec.md index 94af91fad..d0b4a2201 100644 --- a/specs/037-superset-baseline-engine/spec.md +++ b/specs/037-superset-baseline-engine/spec.md @@ -5,7 +5,7 @@ @RELATION DEPENDS_ON -> [Doc.Adr.ADR0005] @RELATION DEPENDS_ON -> [AgentTestStabilization.Spec] @RATIONALE Dashboard test assertions must validate the same Superset-side chart or dataset execution path that powers dashboards, not a parallel SQL path that can diverge from Superset filter/query semantics. -@REJECTED Direct SQL execution for dashboard test truth — rejected because the target requirement is stable Superset dataset/chart execution and filter fidelity, not separate database querying. +@REJECTED Direct SQL execution for **chart/baseline truth** — rejected because this feature requires stable Superset dataset/chart execution and filter fidelity. This does not prohibit 038 validated SqlEvidenceSpec for an independent source-mart oracle. ## Navigation (DSA Indexer keywords) @SEMANTICS: spec, requirements, feature, superset, baseline, chart-data, dataset, filters, normalization, dashboard-testing @@ -89,7 +89,7 @@ - **AGBASE-FR-008**: Comparison output MUST include source, actual value, expected value, diff, status, tolerance rule, and warnings. Status enum MUST include `immutability_violation` as a distinct, critical category separate from `stale_baseline` and `stale_visual_baseline`. - **AGBASE-FR-009**: Non-pass comparison outcomes MUST emit an idempotent 036 InvestigationSignal with immutable evidence provenance; 047 owns Queue creation/update. They MUST NOT automatically begin an agent conversation, alter a baseline, or reclassify comparison truth. - **AGBASE-FR-010**: An opened InvestigationCase MAY use baseline-engine tools for read-only inspection, diagnostic comparison and candidate creation. Any baseline publication remains governed by deterministic catalog validation and its ActionApprovalGate policy. -- **AGBASE-FR-009**: Direct SQL execution, generated SQL assertions, and SQL-based baseline provenance are explicitly out of scope. +- **AGBASE-FR-009**: Direct SQL execution, generated SQL assertions and SQL-based provenance are out of scope for **chart/baseline truth**. 038 validated immutable SqlEvidenceSpec remains the independent source-mart oracle contract and does not alter this feature's Superset-native baseline semantics. - **AGBASE-FR-010**: The baseline catalog MUST support visual baselines for screenshot comparison, with the same release-pinning and immutability rules as metric baselines. - **AGBASE-FR-011**: The system MUST compute a `StructureDiff` between two releases' `DashboardQueryModel` snapshots. The diff MUST classify changes by target (chart, filter, column), kind (scope_change, column_reorder, chart_removed, etc.), severity (critical, warning, info), and affected artifacts. StructureDiff is orthogonal to metric comparison. - **AGBASE-FR-012**: Baseline entries for closed periods MUST carry an `immutability` block: enabled flag, period identifier, frozen_at timestamp, and policy (alert, block_publish, require_investigation). Immutability violations MUST be surfaced at the PROD publish gate and scheduled checks, not silently ignored. diff --git a/specs/038-dashboard-scenario-model/contracts/action-registry.yaml b/specs/038-dashboard-scenario-model/contracts/action-registry.yaml new file mode 100644 index 000000000..06fd8c68e --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/action-registry.yaml @@ -0,0 +1,31 @@ +version: v1 +description: Canonical version-pinned executable action catalog. Runtime dispatches no action outside this file. +actions: + - { tool: browser, action: open_dashboard, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: browser, action: navigate_tab, risk: UI_INTERACTION, timeout_ms: 15000, retry_safe: true, mutates: false } + - { tool: browser, action: apply_native_filter, risk: UI_INTERACTION, timeout_ms: 15000, retry_safe: true, mutates: false } + - { tool: browser, action: inspect_filter_state, risk: READ_ONLY, timeout_ms: 10000, retry_safe: true, mutates: false } + - { tool: browser, action: apply_table_filter, risk: UI_INTERACTION, timeout_ms: 15000, retry_safe: true, mutates: false } + - { tool: browser, action: extract_table, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: browser, action: scroll_to, risk: UI_INTERACTION, timeout_ms: 10000, retry_safe: true, mutates: false } + - { tool: browser, action: inspect_columns, risk: READ_ONLY, timeout_ms: 10000, retry_safe: true, mutates: false } + - { tool: browser, action: click, risk: UI_INTERACTION, timeout_ms: 10000, retry_safe: false, mutates: false } + - { tool: browser, action: select_rows, risk: UI_INTERACTION, timeout_ms: 15000, retry_safe: true, mutates: false } + - { tool: browser, action: edit_row, risk: TEST_DATA_MUTATION, timeout_ms: 30000, retry_safe: false, mutates: true } + - { tool: browser, action: bulk_edit, risk: TEST_DATA_MUTATION, timeout_ms: 60000, retry_safe: false, mutates: true } + - { tool: browser, action: download, risk: UI_INTERACTION, timeout_ms: 60000, retry_safe: false, mutates: false } + - { tool: browser, action: refresh, risk: UI_INTERACTION, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: browser, action: wait_for_state, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: browser, action: apply_filters, risk: UI_INTERACTION, timeout_ms: 15000, retry_safe: true, mutates: false } + - { tool: browser, action: download_xlsx, risk: UI_INTERACTION, timeout_ms: 60000, retry_safe: false, mutates: false } + - { tool: superset_api, action: execute_metric, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: superset_api, action: dataset_field_assert, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: sql_evidence, action: execute_pinned_sql, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: transform, action: execute_dsl, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: assertion, action: compare_to_baseline, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: assertion, action: evaluate_comparison_spec, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: agent_evaluation, action: evaluate_declared_spec, risk: READ_ONLY, timeout_ms: 60000, retry_safe: false, mutates: false } + - { tool: xlsx, action: parse_xlsx, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: screenshot, action: capture, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: report, action: generate_report, risk: READ_ONLY, timeout_ms: 30000, retry_safe: true, mutates: false } + - { tool: artifact, action: register, risk: READ_ONLY, timeout_ms: 10000, retry_safe: true, mutates: false } diff --git a/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json b/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json index cb204a9cf..dc2dd0e54 100644 --- a/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json +++ b/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json @@ -15,6 +15,7 @@ "objective", "input_fingerprints", "parameters", + "verification_program", "phases", "steps", "outputs", @@ -130,6 +131,10 @@ "$ref": "#/$defs/parameter" } }, + "verification_program": { + "$ref": "#/$defs/verificationProgram", + "description": "Canonical immutable Verification Program IR; included in content_hash." + }, "phases": { "type": "array", "items": { @@ -254,6 +259,31 @@ } } }, + "verificationProgram": { + "type": "object", + "additionalProperties": false, + "required": ["navigation_program", "evidence_program", "transformation_program", "assertion_program", "semantic_evaluation_program"], + "properties": { + "navigation_program": { "type": "array", "items": { "$ref": "#/$defs/programRef" } }, + "evidence_program": { "type": "array", "items": { "oneOf": [{ "$ref": "#/$defs/programRef" }, { "$ref": "#/$defs/sqlEvidenceSpec" }] } }, + "transformation_program": { "type": "array", "items": { "$ref": "#/$defs/transformSpec" } }, + "assertion_program": { "type": "array", "items": { "$ref": "#/$defs/assertionSpec" } }, + "semantic_evaluation_program": { "type": "array", "items": { "$ref": "#/$defs/agentEvaluationSpec" } } + } + }, + "programRef": { "type": "object", "additionalProperties": false, "required": ["logical_step_id"], "properties": { "logical_step_id": { "type": "string", "format": "uuid" } } }, + "sqlEvidenceSpec": { + "type": "object", "additionalProperties": false, + "required": ["snippet_id", "logical_step_id", "connection_ref", "database_identity", "sql_template", "sql_hash", "parameter_definitions", "expected_output_schema", "relation_refs", "execution_limits", "generation_provenance", "validation_result"], + "properties": { + "snippet_id": { "type": "string" }, "logical_step_id": { "type": "string", "format": "uuid" }, "connection_ref": { "type": "string" }, "database_identity": { "type": "string" }, "sql_template": { "type": "string", "minLength": 1 }, "sql_hash": { "$ref": "#/$defs/sha256" }, + "parameter_definitions": { "type": "array", "items": { "$ref": "#/$defs/parameter" } }, "expected_output_schema": { "type": "object" }, "relation_refs": { "type": "array", "items": { "type": "string" } }, "execution_limits": { "$ref": "#/$defs/executionLimits" }, "generation_provenance": { "type": "object" }, "validation_result": { "type": "object", "required": ["valid"] } + } + }, + "executionLimits": { "type": "object", "additionalProperties": false, "required": ["timeout_ms", "max_rows", "max_bytes", "max_complexity"], "properties": { "timeout_ms": { "type": "integer", "minimum": 1 }, "max_rows": { "type": "integer", "minimum": 1 }, "max_bytes": { "type": "integer", "minimum": 1 }, "max_complexity": { "type": "integer", "minimum": 1 } } }, + "transformSpec": { "type": "object", "additionalProperties": false, "required": ["logical_step_id", "version", "operations"], "properties": { "logical_step_id": { "type": "string", "format": "uuid" }, "version": { "type": "string" }, "operations": { "type": "array", "items": { "type": "object", "required": ["op"], "properties": { "op": { "enum": ["select", "filter", "rename", "cast", "join", "group_by", "sum", "count", "distinct", "coalesce", "normalize_string", "normalize_date", "difference", "ratio", "tolerance_compare"] } } } } } }, + "assertionSpec": { "type": "object", "additionalProperties": false, "required": ["logical_step_id", "kind", "left_ref", "right_ref"], "properties": { "logical_step_id": { "type": "string", "format": "uuid" }, "kind": { "enum": ["numeric_equality", "numeric_tolerance", "row_comparison", "column_comparison", "aggregate_comparison", "set_equality", "field_mapping", "null_fill_check", "cross_dashboard"] }, "left_ref": { "type": "string" }, "right_ref": { "type": "string" }, "tolerance": { "type": ["number", "null"] }, "field_mapping": { "type": "object" } } }, + "agentEvaluationSpec": { "type": "object", "additionalProperties": false, "required": ["spec_id", "logical_step_id", "provider_id", "model_id", "prompt_template_id", "prompt_template_version", "prompt_template_hash", "input_manifest", "evidence_refs", "tool_allowlist", "output_schema", "decision_policy_id"], "properties": { "spec_id": { "type": "string" }, "logical_step_id": { "type": "string", "format": "uuid" }, "provider_id": { "type": "string" }, "model_id": { "type": "string" }, "prompt_template_id": { "type": "string" }, "prompt_template_version": { "type": "string" }, "prompt_template_hash": { "$ref": "#/$defs/sha256" }, "input_manifest": { "type": "object" }, "evidence_refs": { "type": "array", "items": { "type": "string" } }, "tool_allowlist": { "type": "array", "items": { "type": "string" } }, "output_schema": { "type": "object" }, "decision_policy_id": { "type": "string" } } }, "ref": { "type": "object", "additionalProperties": false, @@ -383,8 +413,11 @@ "enum": [ "browser", "superset_api", + "sql_evidence", + "transform", "xlsx", "assertion", + "agent_evaluation", "screenshot", "report", "artifact", @@ -439,12 +472,11 @@ }, "risk": { "enum": [ - "read", - "browser_interaction", - "test_data_mutation", - "external_mutation", - "dangerous_mutation", - "draft_write", + "READ_ONLY", + "UI_INTERACTION", + "TEST_DATA_MUTATION", + "EXTERNAL_MUTATION", + "DANGEROUS_MUTATION", "human" ] }, @@ -593,7 +625,7 @@ "null" ] }, - "step_id": { + "logical_step_id": { "type": [ "string", "null" diff --git a/specs/038-dashboard-scenario-model/contracts/modules.md b/specs/038-dashboard-scenario-model/contracts/modules.md index 435814f8d..8da55ed00 100644 --- a/specs/038-dashboard-scenario-model/contracts/modules.md +++ b/specs/038-dashboard-scenario-model/contracts/modules.md @@ -16,8 +16,8 @@ # @RELATION DEPENDS_ON -> [ScenarioGraph.Resolver] # @RELATION DEPENDS_ON -> [ScenarioGraph.PackCompiler] # @RATIONALE A thin REST surface exposes the deterministic scenario operations to 039 and agent tools without leaking compiler internals. -# @REJECTED Exposing raw Pydantic models over the API — request schemas must forbid code/SQL/paths to keep the boundary safe. -# @INVARIANT Request schemas forbid executable code, SQL, raw baseline values, and local paths. +# @REJECTED Exposing raw Pydantic models over the API — request schemas must forbid arbitrary code/unsafe paths and require SqlEvidenceSpec compilation. +# @INVARIANT Request schemas permit only validated immutable SqlEvidenceSpec; executable code, raw baseline values and local paths are forbidden. # #endregion ScenarioGraph.Api # #region ScenarioGraph.Catalog.Load [C:4] [TYPE Function] [SEMANTICS scenario,checklist,catalog,version] @@ -31,7 +31,7 @@ # @RATIONALE Versioned declarative catalog keeps the 19 PDF cases reusable and auditable across dashboards. # @REJECTED Embedding the checklist as Python conditionals — mixed intent/data makes coverage unverifiable. # @TEST_EDGE missing_case -> startup/catalog validation failure. -# @TEST_EDGE sql_template_for_technical_case -> rejected. +# @TEST_EDGE source-mart evidence case -> validated SqlEvidenceSpec or needs_context; unsafe SQL -> rejected. # #endregion ScenarioGraph.Catalog.Load # #region ScenarioGraph.CapabilityMapper.Map [C:5] [TYPE Function] [SEMANTICS scenario,capability,mapping,coverage] @@ -54,7 +54,7 @@ # @PRE Intent, query model, catalog, baseline summary, and parameters have valid fingerprints; provenance passed separately (no mandatory agent_run_id). # @POST Same canonical inputs/compiler version yield byte-identical graph and stable keys/order; emits scenario_key + content_hash (NO scenario_id/revision_id). # @SIDE_EFFECT None. -# @DATA_CONTRACT CompileInput (scenario_key basis + objective + query model + checklist + parameters + capabilities + baselines) + CompileProvenance -> DashboardTestScenario (scenario_key + content_hash) +# @DATA_CONTRACT CompileInput (scenario_key basis + ChangeRequestContext + objective + query model + checklist + ParameterDefinitions + capabilities + baselines) + CompileProvenance -> DashboardTestScenario (scenario_key + content_hash + VerificationProgram) # @INVARIANT Steps consume only context/parameter/baseline/earlier-step refs; initial graph creation mints logical_step_id and edits/migrations carry it forward. # @TEST_INVARIANT Deterministic_Graph -> VERIFIED_BY: repeated_compile, shuffled_input_order. # @TEST_EDGE missing_selector -> NEEDS_SELECTOR step and save blocker. @@ -73,7 +73,7 @@ # @DATA_CONTRACT DashboardTestScenario -> ScenarioValidationResult # @RATIONALE Validation is a hard safety boundary between agent-produced intent and artifact generation; deterministic findings give the user a recoverable explanation instead of a runtime surprise. # @REJECTED Silent graph repair or best-effort artifact generation — rejected because auto-fixing refs, cycles, or unsafe actions can change business intent without review. -# @INVARIANT Cycles, missing/duplicate refs, unregistered tools, SQL, raw metric truth, and path traversal block compilation. +# @INVARIANT Cycles, missing/duplicate refs, unregistered tools/actions, arbitrary or unsafe SQL, raw metric truth and path traversal block compilation. SqlEvidenceSpec passes only its AST/policy/preview compilation gate. # @TEST_EDGE cycle -> error contains cycle path. # @TEST_EDGE duplicate_output -> both producer ids reported. # @TEST_EDGE raw_metric_expected -> forbidden baseline literal error. @@ -82,7 +82,7 @@ # #region ScenarioGraph.Resolver.Resolve [C:4] [TYPE Function] [SEMANTICS scenario,resolve,parameter,revision] # @ingroup ScenarioGraph -# @BRIEF Apply typed parameter/selector/manual resolutions and emit a new compiled definition (content_hash changes; identity unchanged). +# @BRIEF Apply typed ParameterDefinition/selector/manual resolutions and emit a new compiled definition (content_hash changes; identity unchanged). # @PRE Base content_hash matches; changes target declared unresolved items. # @POST Unrelated step keys/order remain unchanged; new content_hash links to base content_hash; logical_step_id stable. # @SIDE_EFFECT None. @@ -152,7 +152,7 @@ # @RELATION DEPENDS_ON -> [ScenarioGraph.Api] # @RATIONALE Agent tools stay thin: the agent explains intent, the deterministic compiler owns the graph. # @REJECTED Agent-side graph construction with free-form tool selection — bypasses validation and determinism. -# @INVARIANT Agent cannot submit executable code, custom tool categories, raw expected metrics, or artifact paths. +# @INVARIANT Agent cannot submit executable code, custom tool categories, raw expected metrics, artifact paths or runtime rewrites; it may author bounded SqlEvidenceSpec/DSL only through compiler validation. # #endregion AgentChat.Tools.ScenarioGraph #endregion DashboardScenarioModel.Modules diff --git a/specs/038-dashboard-scenario-model/contracts/openapi.yaml b/specs/038-dashboard-scenario-model/contracts/openapi.yaml index 3d504ef3c..4dece900f 100644 --- a/specs/038-dashboard-scenario-model/contracts/openapi.yaml +++ b/specs/038-dashboard-scenario-model/contracts/openapi.yaml @@ -66,8 +66,8 @@ paths: summary: Validate a candidate ScenarioGraph and return all findings description: | Pure validation of a candidate graph: schema, DAG, refs, parameters, - baselines, tools, safety (no SQL / no raw metric truth / no path - traversal), and checklist coverage. Returns deterministic findings. + baselines, tools, SQL compilation, safety and checklist coverage. + SQL is allowed only as validated immutable SqlEvidenceSpec; arbitrary code remains forbidden. security: - BearerAuth: [scenario.compile] requestBody: @@ -269,7 +269,7 @@ components: CompileRequest: type: object additionalProperties: false - required: [objective, query_model, checklist_catalog_version, baseline_version, capabilities, parameters] + required: [objective, query_model, checklist_catalog_version, baseline_version, capabilities, parameters, change_request_context] properties: provenance: type: object @@ -289,17 +289,39 @@ components: baseline_version: { type: string } capabilities: { type: object } parameters: { type: object } + change_request_context: { $ref: "#/components/schemas/ChangeRequestContext" } + + ChangeRequestContext: + type: object + additionalProperties: false + required: [request_id, objective, acceptance_criteria] + properties: + request_id: { type: string } + objective: { type: string } + affected_dashboards: { type: array, items: { type: string } } + added_fields: { type: array, items: { type: string } } + removed_fields: { type: array, items: { type: string } } + renamed_fields: { type: array, items: { type: object } } + technical_details: { type: string } + requested_tables_charts: { type: array, items: { type: string } } + expected_business_behavior: { type: string } + related_dashboards: { type: array, items: { type: string } } + source_relations: { type: array, items: { type: string } } + acceptance_criteria: { type: array, minItems: 1, items: { type: string } } + control_totals: { type: object } ScenarioGraphInput: type: object - description: Candidate graph for validation. Forbids SQL, code bodies, raw metric truth, paths. + description: Candidate immutable Verification Program. SQL is allowed only as a validated SqlEvidenceSpec; arbitrary code, raw metric truth and unsafe paths are forbidden. additionalProperties: false - required: [schema_version, scenario_key, dashboard_context, objective, phases, steps] + required: [schema_version, scenario_key, dashboard_context, objective, verification_program, phases, steps] properties: schema_version: { type: integer } scenario_key: { type: string } dashboard_context: { type: object } objective: { type: object } + change_request_context: { $ref: "#/components/schemas/ChangeRequestContext" } + verification_program: { $ref: "#/components/schemas/VerificationProgram" } phases: { type: array, items: { type: string } } steps: { type: array, items: { type: object } } @@ -310,6 +332,65 @@ components: scenario: { type: object } validation: { $ref: "#/components/schemas/ValidationResult" } + VerificationProgram: + type: object + required: [navigation_program, evidence_program, transformation_program, assertion_program, semantic_evaluation_program] + properties: + navigation_program: { type: array, items: { type: object } } + evidence_program: { type: array, items: { oneOf: [{ type: object }, { $ref: "#/components/schemas/SqlEvidenceSpec" }] } } + transformation_program: { type: array, items: { $ref: "#/components/schemas/TransformSpec" } } + assertion_program: { type: array, items: { $ref: "#/components/schemas/ComparisonSpec" } } + semantic_evaluation_program: { type: array, items: { $ref: "#/components/schemas/AgentEvaluationSpec" } } + SqlEvidenceSpec: + type: object + required: [snippet_id, logical_step_id, connection_ref, database_identity, sql_template, sql_hash, parameter_definitions, expected_output_schema, relation_refs, execution_limits, generation_provenance, validation_result] + properties: + snippet_id: { type: string } + logical_step_id: { type: string } + connection_ref: { type: string } + database_identity: { type: string } + sql_template: { type: string, description: "Read-only saved template; immutable after revision save" } + sql_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + parameter_definitions: { type: array, items: { type: object } } + expected_output_schema: { type: object } + relation_refs: { type: array, items: { type: string } } + execution_limits: { type: object, required: [timeout_ms, max_rows, max_bytes, max_complexity] } + generation_provenance: { type: object } + validation_result: { type: object, required: [valid] } + TransformSpec: + type: object + required: [logical_step_id, version, operations] + properties: + logical_step_id: { type: string } + version: { type: string } + operations: { type: array, items: { type: object, properties: { op: { type: string, enum: [select, filter, rename, cast, join, group_by, sum, count, distinct, coalesce, normalize_string, normalize_date, difference, ratio, tolerance_compare] } } } } + ComparisonSpec: + type: object + required: [logical_step_id, kind, left_ref, right_ref] + properties: + logical_step_id: { type: string } + kind: { type: string, enum: [numeric_equality, numeric_tolerance, row_comparison, column_comparison, aggregate_comparison, set_equality, field_mapping, null_fill_check, cross_dashboard] } + left_ref: { type: string } + right_ref: { type: string } + tolerance: { type: number } + field_mapping: { type: object } + AgentEvaluationSpec: + type: object + required: [spec_id, logical_step_id, provider_id, model_id, prompt_template_id, prompt_template_version, prompt_template_hash, input_manifest, evidence_refs, tool_allowlist, output_schema, decision_policy_id] + properties: + spec_id: { type: string } + logical_step_id: { type: string } + provider_id: { type: string } + model_id: { type: string } + prompt_template_id: { type: string } + prompt_template_version: { type: string } + prompt_template_hash: { type: string, pattern: "^[a-f0-9]{64}$" } + input_manifest: { type: object } + evidence_refs: { type: array, items: { type: string } } + tool_allowlist: { type: array, items: { type: string } } + output_schema: { type: object } + decision_policy_id: { type: string } + ValidationResult: type: object required: [valid, errors, warnings, blockers, coverage, topological_order, graph_hash] @@ -324,6 +405,8 @@ components: unresolved_selectors: { type: array, items: { type: string } } unresolved_baselines: { type: array, items: { type: string } } graph_hash: { type: string } + sql_compilation_results: { type: array, items: { type: object } } + verification_program_valid: { type: boolean } ResolveRequest: type: object @@ -337,7 +420,7 @@ components: type: object required: [kind, target, value] properties: - kind: { enum: [parameter, selector, manual_conversion, remove_step] } + kind: { enum: [parameter_definition, selector, manual_conversion, remove_step] } target: { type: string } value: {} reason: { type: [string, "null"] } @@ -588,7 +671,7 @@ components: validation: valid: true errors: [] - warnings: [{ code: "STALE_BASELINE", detail: "baseline 2026-06-15 superseded by 2026-07-01", step_id: "phase-4-C05-compare_to_baseline" }] + warnings: [{ code: "STALE_BASELINE", detail: "baseline 2026-06-15 superseded by 2026-07-01", logical_step_id: "4b7c0000-0000-0000-0000-000000000005" }] blockers: [] coverage: [{ case_id: "B01", classification: "automated" }, { case_id: "C04", classification: "automated" }, { case_id: "C05", classification: "automated" }] topological_order: ["phase-1-B01-open_dashboard", "phase-2-B01-apply_filters"] diff --git a/specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md b/specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md index cd8e1ef2f..047cc77c5 100644 --- a/specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md +++ b/specs/038-dashboard-scenario-model/contracts/ux/scenario-graph-ux.md @@ -74,7 +74,7 @@ ready, needs_context, needs_selector, needs_baseline, manual, unsupported, block | Fifteen-plus steps render | Valid 18-step scenario | Load preview | No dependency information loss; lanes/table consistent | | Cycle not rendered as executable | Cycle fixture | Load preview | Cycle path shown; no executable graph | | Technical cases never show SQL | T01–T03 dashboard | Load preview | Superset API or human checkpoint shown | -| Parameter resolution updates affected steps only | Resolve one parameter | Apply change | Unrelated step ids/order unchanged; new revision hash | +| ParameterDefinition resolution updates affected steps only | Apply definition change | Apply change | Unrelated logical_step_id/order unchanged; new content_hash | | Preview-only pack explains blockers | Invalid/unresolved graph | Generate pack | All save blockers listed; preview_only state explicit | | 409 stale revision recovery | Stale base revision | Resolve | Persistent panel with recompile guidance; no silent merge | | VLM finding disposition | Unresolved finding | Confirm/dismiss/inconclusive | Typed status; audit event; graph unchanged | diff --git a/specs/038-dashboard-scenario-model/contracts/verification-program.md b/specs/038-dashboard-scenario-model/contracts/verification-program.md new file mode 100644 index 000000000..d4b02d375 --- /dev/null +++ b/specs/038-dashboard-scenario-model/contracts/verification-program.md @@ -0,0 +1,36 @@ +#region VerificationProgram.Contract [C:5] [TYPE ADR] [SEMANTICS verification-program,ir,sql,evidence,transform,assertion,agent-evaluation] +@BRIEF Canonical immutable IR compiled by the authoring agent and deterministically executed by 044. +@RELATION DEPENDS_ON -> [DashboardScenarioModel.DataModel] +@RELATION BINDS_TO -> [ScenarioExecution.DataModel] +@RATIONALE A scenario is a verification program, not merely a browser-click DAG: business checks often require source-mart evidence, deterministic transformations and explicit semantic evaluation. +@REJECTED Runtime SQL/code generation or rewrite — rejected because it destroys reproducibility, security review and content-addressed revision identity. + +## Program shape + +`VerificationProgram { navigation_program, evidence_program, transformation_program, assertion_program, semantic_evaluation_program }` is required executable content of `DashboardTestScenario` and is included, canonically, in `content_hash` and every `step_content_hash` that references it. Programs use only declared refs and version-pinned action/DSL/prompt registries. + +## SqlEvidenceSpec + +`SqlEvidenceSpec { snippet_id, logical_step_id, connection_ref, database_identity, sql_template, sql_hash, parameter_definitions[], expected_output_schema, relation_refs[], execution_limits, generation_provenance, validation_result }` is a small, read-only source-evidence query. It is authored only during creation, edit, migration/revalidation or investigation proposal; runtime executes exactly the pinned `sql_template` through the Superset SQL Lab adapter with typed `ParameterBinding` substitution. It cannot add WHERE/JOIN/projection/relation text, mutate SQL, or accept credentials from an agent/client. + +## TransformSpec and ComparisonSpec + +`TransformSpec` is a bounded, versioned DSL — `select|filter|rename|cast|join|group_by|sum|count|distinct|coalesce|normalize_string|normalize_date|difference|ratio|tolerance_compare` — over declared evidence refs. No Python, shell, SQL or arbitrary code is permitted. + +`ComparisonSpec` / `AssertionSpec` declares numeric equality/tolerance, row/column/set equality, aggregate checks, field mappings, null/fill checks and cross-dashboard comparisons. Its expected values are evidence/baseline/typed-control refs, never runtime prose. + +## AgentEvaluationSpec and DecisionPolicy + +`AgentEvaluationSpec { spec_id, logical_step_id, provider_id, model_id, prompt_template_id, prompt_template_version, prompt_template_hash, input_manifest, evidence_refs, tool_allowlist, output_schema, decision_policy_id }` is permitted only for semantic/visual/ambiguous checks that cannot reasonably compile to SqlEvidenceSpec + TransformSpec + AssertionSpec. It has bounded declared evidence/tool access and no mutation authority. + +`DecisionPolicy { policy_id, version, deterministic_hard_failure, high_confidence_failure, low_confidence, disagreement, missing_evidence }` deterministically maps evidence plus a typed AgentEvaluation to `StepOutcome`. A bare model verdict is never ScenarioResult authority. + +## SQL compilation gate + +Before a SqlEvidenceSpec can be saved, compiler validation MUST parse its AST; require one SELECT/WITH statement; prohibit DDL/DML, mutation SETTINGS, unsafe/external/file functions; enforce connection/relation/schema allowlists, typed binds, timeout/row/byte/complexity limits, expected-output schema, and bounded preview/test execution. Exploratory authoring SQL is a separate bounded policy class and must be minimized to the columns/filters actually used before persistence. + +## Runtime boundary + +044 orchestration stays deterministic. It may dispatch an explicitly declared AgentEvaluationSpec, but that step cannot change the ScenarioGraph, SQL/DSL, run lifecycle, executor order or mutations. Bad runtime evidence becomes a finding → 043 proposal → validated new revision → later run. + +#endregion VerificationProgram.Contract diff --git a/specs/038-dashboard-scenario-model/data-model.md b/specs/038-dashboard-scenario-model/data-model.md index 9bd064c62..785adbe82 100644 --- a/specs/038-dashboard-scenario-model/data-model.md +++ b/specs/038-dashboard-scenario-model/data-model.md @@ -1,11 +1,13 @@ #region DashboardScenarioModel.DataModel [C:5] [TYPE ADR] [SEMANTICS data-model,scenario,graph,step,validation] -@BRIEF Canonical scenario definition: scenario_key, content_hash, steps with logical_step_id, parameter, ref, capability, validation, and draft-pack (authoring-only) models. Runtime state (VlmFinding, HumanCheckpoint, RunArtifact) is owned by 044 — this module defines WHAT to execute, not what happened. +@BRIEF Canonical immutable Verification Program IR: scenario_key, content_hash, navigation/evidence/transform/assertion/semantic programs, typed steps, parameters, capability and authoring-validation models. Runtime state is owned by 044. @RELATION DEPENDS_ON -> [DashboardScenarioModel.Research] @RELATION DEPENDS_ON -> [ScenarioExecution.DataModel] @RATIONALE 038 is the clean IR/compiler layer; identity/revision/runtime are reconciled with 042-047. Semantic identity (scenario_key) is separated from entity identity (scenario_id UUID, assigned by 042) and content identity (content_hash). @REJECTED scenario_id as a deterministic slug — rejected because clones of the same dashboard+objective would collide; entity identity MUST be a UUID assigned at Registry persistence. @REJECTED revision_hash/parent_revision_hash — rejected in favor of revision_id (UUID) + content_hash; the compiler does not fabricate identity. @REJECTED Runtime state (VlmFinding dispositions, HumanCheckpoint) inside the canonical scenario — rejected because an immutable revision must not embed runtime observations; VlmAnalysisSpec describes WHAT, VlmFinding/Disposition are 044 runtime. +@REJECTED Direct SQL forbidden as an absolute principle — rejected because source-mart evidence is necessary for real checks; only validated, immutable, read-only SqlEvidenceSpec is permitted. +@REJECTED Runtime LLM SQL/DSL rewrite — rejected because a ScenarioRun must execute precisely the content-addressed program saved in its revision. ## DashboardTestScenario — compiled definition (no persistence identity) @@ -16,7 +18,7 @@ Required: - **content_hash** (SHA-256 of the canonical executable graph only — timestamps/display-only excluded); - dashboard_context and objective; - input_fingerprints: query model, checklist catalog, baseline_version, parameters; -- parameters, phases, steps; +- parameters, phases, steps, and required `verification_program` (`navigation_program`, `evidence_program`, `transformation_program`, `assertion_program`, `semantic_evaluation_program`); - outputs and artifact_plan; - checklist_coverage; - warnings, blockers, risk_summary. @@ -35,6 +37,16 @@ Fields: name, label, type, required, default (optional), source, validation, aff This immutable definition contains no resolved `value` or runtime `status`. Those belong exclusively to 044 `ParameterBinding`, so a value such as `test_date=2026-08-10` never changes a scenario `content_hash`. +## VerificationProgram — first-class executable IR + +`VerificationProgram` is the canonical runtime program embedded in every saved scenario: navigation declares registered browser/API actions; evidence declares source/chart/XLSX observations; transformations declare bounded deterministic `TransformSpec` DSL; assertions declare `ComparisonSpec`/`AssertionSpec`; semantic evaluation declares only explicitly necessary `AgentEvaluationSpec`. All program entries are ref-addressed by `logical_step_id`, version-pinned and canonicalized into `content_hash`. + +`SqlEvidenceSpec` is a small source-evidence statement: `snippet_id, logical_step_id, connection_ref, database_identity, sql_template, sql_hash, parameter_definitions[], expected_output_schema, relation_refs[], execution_limits, generation_provenance, validation_result`. It is an authoring artifact, saved only after the SQL compilation gate. Runtime may bind declared parameters but MUST execute exactly the pinned template through the Superset SQL Lab adapter; it cannot rewrite SQL, relations, joins, projections or filters. + +`TransformSpec` is a bounded DSL only: `select, filter, rename, cast, join, group_by, sum, count, distinct, coalesce, normalize_string, normalize_date, difference, ratio, tolerance_compare`. `ComparisonSpec`/`AssertionSpec` model numeric tolerance, rows/columns/sets, aggregates, maps, null/fill and cross-dashboard checks. Arbitrary Python, shell and executable code are forbidden. + +`AgentEvaluationSpec` is the exception for declared semantic/visual/ambiguous evaluation. It pins provider/model/prompt/input manifest/evidence/tool allowlist/output schema and DecisionPolicy. It is not an orchestration instruction and cannot mutate program content. + ## ScenarioStep — with logical identity | Field | Rule | @@ -45,7 +57,7 @@ This immutable definition contains no resolved `value` or runtime `status`. Thos | **step_content_hash** | SHA-256 of the step's executable content (mutable across edits) | | phase | setup/interact/observe/assert/evidence/report | | title/description | Bounded display text | -| tool | browser, superset_api, xlsx, assertion, screenshot, report, artifact, human | +| tool | browser, superset_api, sql_evidence, transform, assertion, agent_evaluation, xlsx, screenshot, report, artifact, human | | action | Must be allowed by the version-pinned `ActionRegistry` for its tool; never free-form runtime dispatch | | inputs | Typed refs only | | outputs | Unique ScenarioRef declarations | @@ -53,7 +65,7 @@ This immutable definition contains no resolved `value` or runtime `status`. Thos | depends_on | Existing logical_step_ids; DAG | | automation_status | ready, needs_context, needs_selector, needs_baseline, manual, unsupported, blocked | | checklist_case_ids | Known catalog ids | -| risk | read, browser_interaction, test_data_mutation, external_mutation, dangerous_mutation, draft_write, human | +| risk | READ_ONLY, UI_INTERACTION, TEST_DATA_MUTATION, EXTERNAL_MUTATION, DANGEROUS_MUTATION, human | | mutation_contract | Mandatory for every mutating action: safe_test_fixture_id, scope, allowed environment, affected keys, cleanup/reconciliation, side-effect key | | capture_spec | Required when tool=screenshot; null otherwise | | vlm_analysis_spec | Required when tool=assertion and input is screenshot; null otherwise | @@ -62,9 +74,9 @@ The compiler assigns a UUID when it creates an initial graph. An editor/migratio ## ActionRegistry and mutation safety -`ActionRegistry(version)` is the canonical 038 catalog of every allowed `{tool, action}`. Each entry declares typed inputs/outputs, allowed risk, timeout, idempotency, retry safety, and whether it mutates state. 044 BrowserExecutor resolves actions only from this pinned registry. +`ActionRegistry(version)` is the canonical 038 catalog of every allowed `{tool, action}`. Each entry declares typed inputs/outputs, allowed risk, timeout, idempotency, retry safety, mutation policy and version. 044 resolves every executor action only from this same pinned registry; unknown `{tool, action}` is a validation error, not a fallback dispatch. -Mutating actions require `mutation_contract`. MVP policy: `dangerous_mutation` is never automated; PROD permits read-only execution only; `test_data_mutation` is permitted only in an explicitly listed non-PROD environment against a named safe fixture, with bounded affected record keys and rollback/reconciliation. Missing safety context maps the checklist case to `needs_context` or HumanCheckpoint, never automatic dispatch. +Mutating actions require `mutation_contract`. Policy: `DANGEROUS_MUTATION` is never automated; mutating browser steps in PROD are prohibited; `TEST_DATA_MUTATION` is permitted only in an explicitly listed non-PROD environment against a named safe fixture, with bounded affected record keys, side-effect idempotency and rollback/reconciliation. Missing safety context maps the checklist case to `needs_context` or HumanCheckpoint, never automatic dispatch. ### ScreenshotCaptureSpec @@ -114,7 +126,7 @@ CapabilityMapping holds case_id, classification, matched/missing capabilities, s ## AuthoringValidation and RunPreflight -`AuthoringValidation` determines whether an immutable ScenarioRevision/DraftPack may be saved. Fields: valid, errors, warnings, blockers, coverage, topological_order, unresolved_selectors, unresolved_baselines, parameter_definitions_valid, graph_hash. Required ParameterDefinitions without a default are valid authoring inputs and appear only as `unbound_parameter_names` warnings. +`AuthoringValidation` determines whether an immutable ScenarioRevision/DraftPack may be saved. Fields: valid, errors, warnings, blockers, coverage, topological_order, unresolved_selectors, unresolved_baselines, parameter_definitions_valid, verification_program_valid, sql_compilation_results, graph_hash. Required ParameterDefinitions without a default are valid authoring inputs and appear only as `unbound_parameter_names` warnings. `RunPreflight` is owned by 044: it resolves ParameterBinding, target, baselines, RLS/principal and runtime policy for one launch. An unbound required parameter makes only that launch `run_ineligible`; it never changes `content_hash` or blocks saving a reusable scenario. @@ -142,7 +154,7 @@ DraftPack is the **authoring** output (generated files previewed before Save): c The schema/validator rejects: -- sql, query text, shell command, executable code body; +- arbitrary/unvalidated SQL, query text outside SqlEvidenceSpec, shell command or executable code body; - raw numeric expected value for a metric assertion; - absolute or parent-traversing artifact paths; - unregistered tool/action; diff --git a/specs/038-dashboard-scenario-model/fixtures/api/scenario_cycle.json b/specs/038-dashboard-scenario-model/fixtures/api/scenario_cycle.json index 9d4e867fd..bbe969f55 100644 --- a/specs/038-dashboard-scenario-model/fixtures/api/scenario_cycle.json +++ b/specs/038-dashboard-scenario-model/fixtures/api/scenario_cycle.json @@ -21,6 +21,7 @@ "parameters": "0000000000000000000000000000000000000000000000000000000000000000" }, "parameters": [], + "verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []}, "phases": [ "setup", "interact" @@ -47,7 +48,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "59f95912-1d19-bb85-caca-ea3b373ccd5e", "step_key": "step-a", "position": 0, @@ -82,7 +83,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "27421842-d740-8d18-44d0-9c01ba22d063", "step_key": "step-b", "position": 1, diff --git a/specs/038-dashboard-scenario-model/fixtures/api/scenario_duplicate_output.json b/specs/038-dashboard-scenario-model/fixtures/api/scenario_duplicate_output.json index 6dbae7a8d..83388c867 100644 --- a/specs/038-dashboard-scenario-model/fixtures/api/scenario_duplicate_output.json +++ b/specs/038-dashboard-scenario-model/fixtures/api/scenario_duplicate_output.json @@ -49,6 +49,7 @@ ] } ], + "verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []}, "phases": [ "setup", "interact", @@ -86,7 +87,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "42f40cf4-2cac-9377-a7d6-b782a3444aed", "step_key": "phase-1-B01-open_dashboard", "position": 0, @@ -132,7 +133,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "b4219430-b9b7-a57d-ead9-20c05061e61d", "step_key": "phase-2-B01-apply_filters", "position": 1, @@ -170,7 +171,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "31e09885-d86c-1f26-035a-b20d6612a8aa", "step_key": "phase-3-B01-execute_metric", "position": 2, @@ -206,7 +207,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "9d94e3bf-72b9-43d9-404f-c9f68a76ae88", "step_key": "phase-4-C04-download_xlsx", "position": 3, @@ -242,7 +243,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "cc423c40-39bb-11cf-02b3-dbfe17ad08d8", "step_key": "phase-5-C05-parse_xlsx_metric", "position": 4, @@ -278,7 +279,7 @@ "checklist_case_ids": [ "T01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "8c7a79d2-561d-4b91-434e-31584c5db2e5", "step_key": "phase-6-T01-verify_fields", "position": 5, @@ -322,7 +323,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "12f1b598-d06e-e2ae-12b2-2f1e17d61cc0", "step_key": "phase-7-C05-compare_to_baseline", "position": 6, @@ -364,7 +365,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "draft_write", + "risk": "READ_ONLY", "logical_step_id": "66b6393a-64e7-a2b7-c5ad-69e6ce78ad84", "step_key": "phase-8-C04-generate_report", "position": 7, @@ -391,7 +392,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "d91edbd6-43a8-3295-8782-584b8122fc3d", "step_key": "phase-9-dup-output", "position": 8, @@ -466,7 +467,7 @@ "warnings": [], "blockers": [], "risk_summary": { - "max_risk": "draft_write", + "max_risk": "READ_ONLY", "mutation_steps": 0, "needs_test_data": false }, diff --git a/specs/038-dashboard-scenario-model/fixtures/api/scenario_missing_ref.json b/specs/038-dashboard-scenario-model/fixtures/api/scenario_missing_ref.json index f50cdb585..2e87f02da 100644 --- a/specs/038-dashboard-scenario-model/fixtures/api/scenario_missing_ref.json +++ b/specs/038-dashboard-scenario-model/fixtures/api/scenario_missing_ref.json @@ -49,6 +49,7 @@ ] } ], + "verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []}, "phases": [ "setup", "interact", @@ -86,7 +87,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "ed28a9df-f2db-6a4c-1f7b-f602ddea3354", "step_key": "phase-1-B01-open_dashboard", "position": 0, @@ -132,7 +133,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "a2c76cb9-3305-8d42-d189-933ea909fb71", "step_key": "phase-2-B01-apply_filters", "position": 1, @@ -170,7 +171,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "5c59437d-78cc-717d-c548-e68cca1a4faa", "step_key": "phase-3-B01-execute_metric", "position": 2, @@ -206,7 +207,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "ae7a5d83-8f31-17a4-7951-61243c4738c0", "step_key": "phase-4-C04-download_xlsx", "position": 3, @@ -242,7 +243,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "fee5901f-bdbc-a7dc-5856-03df5d39e2b2", "step_key": "phase-5-C05-parse_xlsx_metric", "position": 4, @@ -278,7 +279,7 @@ "checklist_case_ids": [ "T01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "27dcb041-7506-32d4-bdbd-b06d0e40e498", "step_key": "phase-6-T01-verify_fields", "position": 5, @@ -322,7 +323,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "bce6c798-32db-9eb6-ba12-7c779b9957eb", "step_key": "phase-7-C05-compare_to_baseline", "position": 6, @@ -364,7 +365,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "draft_write", + "risk": "READ_ONLY", "logical_step_id": "471d2a23-6b87-f18d-a391-1100c0bc7171", "step_key": "phase-8-C04-generate_report", "position": 7, @@ -392,7 +393,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "2fb63a53-3e28-ff33-0f07-539c69a81ca8", "step_key": "phase-9-missing-ref", "position": 8, @@ -467,7 +468,7 @@ "warnings": [], "blockers": [], "risk_summary": { - "max_risk": "draft_write", + "max_risk": "READ_ONLY", "mutation_steps": 0, "needs_test_data": false }, diff --git a/specs/038-dashboard-scenario-model/fixtures/api/scenario_raw_baseline.json b/specs/038-dashboard-scenario-model/fixtures/api/scenario_raw_baseline.json index 5e2babe80..729c1e6fd 100644 --- a/specs/038-dashboard-scenario-model/fixtures/api/scenario_raw_baseline.json +++ b/specs/038-dashboard-scenario-model/fixtures/api/scenario_raw_baseline.json @@ -49,6 +49,7 @@ ] } ], + "verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []}, "phases": [ "setup", "interact", @@ -86,7 +87,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "350cc442-9dc4-8a87-8c05-10f9061bc333", "step_key": "phase-1-B01-open_dashboard", "position": 0, @@ -132,7 +133,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "d57c118b-2cec-78fd-521b-40f417b9cb9f", "step_key": "phase-2-B01-apply_filters", "position": 1, @@ -170,7 +171,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "e6b53b50-88ea-9c24-361a-217c9fa70f8a", "step_key": "phase-3-B01-execute_metric", "position": 2, @@ -206,7 +207,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "059ec187-5e46-d174-cdfd-525b7e766437", "step_key": "phase-4-C04-download_xlsx", "position": 3, @@ -242,7 +243,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "d090622c-26f4-0e97-e31c-b38e375b933e", "step_key": "phase-5-C05-parse_xlsx_metric", "position": 4, @@ -280,7 +281,7 @@ "checklist_case_ids": [ "T01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "4eada846-3288-774a-ed0e-4909571a04d5", "step_key": "phase-6-T01-verify_fields", "position": 5, @@ -324,7 +325,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "faf07f7f-5a39-a2d8-56a4-258acc13ef22", "step_key": "phase-7-C05-compare_to_baseline", "position": 6, @@ -366,7 +367,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "draft_write", + "risk": "READ_ONLY", "logical_step_id": "97df0515-e508-ed05-4944-9c10302baa1b", "step_key": "phase-8-C04-generate_report", "position": 7, @@ -441,7 +442,7 @@ "warnings": [], "blockers": [], "risk_summary": { - "max_risk": "draft_write", + "max_risk": "READ_ONLY", "mutation_steps": 0, "needs_test_data": false }, diff --git a/specs/038-dashboard-scenario-model/fixtures/api/scenario_sql_injection.json b/specs/038-dashboard-scenario-model/fixtures/api/scenario_sql_injection.json index ff7cb72ac..9a23b20b2 100644 --- a/specs/038-dashboard-scenario-model/fixtures/api/scenario_sql_injection.json +++ b/specs/038-dashboard-scenario-model/fixtures/api/scenario_sql_injection.json @@ -49,6 +49,7 @@ ] } ], + "verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []}, "phases": [ "setup", "interact", @@ -86,7 +87,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "5effc389-922f-2b67-077f-fd6d14b1d6d4", "step_key": "phase-1-B01-open_dashboard", "position": 0, @@ -132,7 +133,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "79512b6c-4f55-95aa-c266-42029595a544", "step_key": "phase-2-B01-apply_filters", "position": 1, @@ -170,7 +171,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "d91845fd-ba40-016d-4007-111dc1ee5aee", "step_key": "phase-3-B01-execute_metric", "position": 2, @@ -206,7 +207,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "c656a892-7ae9-0ca6-4a87-f6acd3d79516", "step_key": "phase-4-C04-download_xlsx", "position": 3, @@ -242,7 +243,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "6e685a71-3eb8-786c-9e35-0a8835ced1fc", "step_key": "phase-5-C05-parse_xlsx_metric", "position": 4, @@ -278,7 +279,7 @@ "checklist_case_ids": [ "T01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "acb0a443-0777-cfed-f6b0-aa56655fd07a", "step_key": "phase-6-T01-verify_fields", "position": 5, @@ -322,7 +323,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "2efc15a9-77d0-2ced-a44e-02811416ba5a", "step_key": "phase-7-C05-compare_to_baseline", "position": 6, @@ -364,7 +365,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "draft_write", + "risk": "READ_ONLY", "logical_step_id": "27d97074-280d-675c-2a00-57d8e8b499fd", "step_key": "phase-8-C04-generate_report", "position": 7, @@ -385,7 +386,7 @@ "checklist_case_ids": [ "T02" ], - "risk": "read", + "risk": "READ_ONLY", "description": "SELECT * FROM dashboard_80 WHERE comment IS NOT NULL", "logical_step_id": "ec961dc8-505d-6ce1-1536-4103ad336071", "step_key": "phase-9-sql-step", @@ -461,7 +462,7 @@ "warnings": [], "blockers": [], "risk_summary": { - "max_risk": "draft_write", + "max_risk": "READ_ONLY", "mutation_steps": 0, "needs_test_data": false }, diff --git a/specs/038-dashboard-scenario-model/fixtures/api/scenario_valid.json b/specs/038-dashboard-scenario-model/fixtures/api/scenario_valid.json index 790480223..19dee5f15 100644 --- a/specs/038-dashboard-scenario-model/fixtures/api/scenario_valid.json +++ b/specs/038-dashboard-scenario-model/fixtures/api/scenario_valid.json @@ -49,6 +49,7 @@ ] } ], + "verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []}, "phases": [ "setup", "interact", @@ -86,7 +87,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "dcf0a1db-63e2-79bb-83cd-d255cc74a2c5", "step_key": "phase-1-B01-open_dashboard", "position": 0, @@ -132,7 +133,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "13dbf34f-0257-3a27-6da0-91da450a368f", "step_key": "phase-2-B01-apply_filters", "position": 1, @@ -170,7 +171,7 @@ "checklist_case_ids": [ "B01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "8c110790-308a-6837-d85f-c3f9ceb7790a", "step_key": "phase-3-B01-execute_metric", "position": 2, @@ -206,7 +207,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "browser_interaction", + "risk": "UI_INTERACTION", "logical_step_id": "164bce75-34cc-47cc-1d40-d188a1b1bc8c", "step_key": "phase-4-C04-download_xlsx", "position": 3, @@ -242,7 +243,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "ac6898ca-86a6-594f-d3f4-a2c81ecb4820", "step_key": "phase-5-C05-parse_xlsx_metric", "position": 4, @@ -278,7 +279,7 @@ "checklist_case_ids": [ "T01" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "66256133-6ff5-6ce5-a1f3-f4530fe4a953", "step_key": "phase-6-T01-verify_fields", "position": 5, @@ -322,7 +323,7 @@ "checklist_case_ids": [ "C05" ], - "risk": "read", + "risk": "READ_ONLY", "logical_step_id": "5b10947f-7795-d7d3-4f68-9ff36e8bf585", "step_key": "phase-7-C05-compare_to_baseline", "position": 6, @@ -364,7 +365,7 @@ "checklist_case_ids": [ "C04" ], - "risk": "draft_write", + "risk": "READ_ONLY", "logical_step_id": "4d64101d-9342-9636-1604-a866c24ff5b9", "step_key": "phase-8-C04-generate_report", "position": 7, @@ -439,7 +440,7 @@ "warnings": [], "blockers": [], "risk_summary": { - "max_risk": "draft_write", + "max_risk": "READ_ONLY", "mutation_steps": 0, "needs_test_data": false }, diff --git a/specs/038-dashboard-scenario-model/research.md b/specs/038-dashboard-scenario-model/research.md index dbce8d9eb..cf0beea00 100644 --- a/specs/038-dashboard-scenario-model/research.md +++ b/specs/038-dashboard-scenario-model/research.md @@ -13,7 +13,7 @@ The source research/Чеклист 29.05 (1).pdf is 21 pages and contains 19 cas - complex C01–C07; - technical T01–T03. -The PDF mixes reusable behavior, FI-0080-specific data, historic pass/fail notes, screenshots, and direct SQL instructions. Historic result text is evidence, not a reusable expected result. SQL instructions are not executable in this feature and are mapped to Superset API verification or human checkpoints. +The PDF mixes reusable behavior, FI-0080-specific data, historic pass/fail notes, screenshots, and source-evidence SQL instructions. Historic result text is evidence, not a reusable expected result. SQL instructions become a validated immutable SqlEvidenceSpec when source-mart evidence is required; otherwise they map to Superset API verification or human checkpoints. ## 2. Deterministic Compiler Boundary diff --git a/specs/038-dashboard-scenario-model/spec.md b/specs/038-dashboard-scenario-model/spec.md index 897a1c1b6..9992fcc35 100644 --- a/specs/038-dashboard-scenario-model/spec.md +++ b/specs/038-dashboard-scenario-model/spec.md @@ -1,11 +1,11 @@ #region DashboardScenarioModel.Spec [C:3] [TYPE ADR] [SEMANTICS spec,requirements,scenario,dashboard-testing] -@BRIEF Define a validated ScenarioGraph model for unique dashboard test flows generated by agents. +@BRIEF Define the validated immutable Verification Program IR for dashboard test flows authored by agents and executed by 044. @RELATION DEPENDS_ON -> [Doc.Adr.ADR0001] @RELATION DEPENDS_ON -> [Doc.Adr.ADR0002] @RELATION DEPENDS_ON -> [AgentTestStabilization.Spec] @RELATION DEPENDS_ON -> [SupersetBaselineEngine.Spec] @RATIONALE Unique dashboard tests need a stable intermediate model between agent reasoning and generated artifacts; direct LLM-to-code generation is not reviewable or safely composable. -@REJECTED Asking users to choose low-level outputs such as Playwright vs SQL vs XLSX — rejected because each dashboard scenario is a goal-oriented flow whose steps select tools automatically. +@REJECTED Asking users to choose low-level outputs such as Playwright vs SQL vs XLSX — rejected because each dashboard scenario is goal-oriented; the agent proposes inspectable compiled evidence/transform/assertion steps. @REJECTED Direct generation of executable scripts without a validated scenario graph — rejected because it hides missing selectors, baseline refs, and unsafe steps until runtime. ## Navigation (DSA Indexer keywords) @@ -32,7 +32,7 @@ **Independent Test**: Provide dashboard query model and checklist fixture input and verify a deterministic `DashboardTestScenario` graph is produced. **Acceptance**: -1. **Given** a dashboard context, query model, and testing objective **When** the agent builds a scenario **Then** the output contains scenario id, dashboard context, parameters, steps, dependencies, expected outputs, warnings, and risk summary. +1. **Given** a dashboard context, ChangeRequestContext, query model, and testing objective **When** the agent builds a scenario **Then** the output contains an inspectable Verification Program, parameters, steps, dependencies, evidence sources, warnings, and risk summary (038 does not emit persistence ids). 2. **Given** a scenario needs browser, Superset API, XLSX parsing, screenshots, assertions, and report steps **When** graph is rendered **Then** every step declares its tool and input/output refs. 3. **Given** required metadata is missing **When** graph is generated **Then** missing data is represented as `NEEDS_CONTEXT` or `NEEDS_SELECTOR`, not invented. @@ -60,7 +60,7 @@ **Acceptance**: 1. **Given** the normalized checklist template contains basic, complex, and technical cases **When** a dashboard capability model is supplied **Then** applicable cases are mapped to scenario steps or human checkpoints. 2. **Given** a checklist case requires capabilities absent from the dashboard **When** mapping runs **Then** the case is marked unsupported or manual-only with rationale. -3. **Given** multiple tool choices could verify a case **When** mapping runs **Then** the scenario selects the tool chain that best matches the business goal and available capabilities. +3. **Given** multiple tool choices could verify a case **When** mapping runs **Then** the scenario selects inspectable navigation/evidence/transform/assertion steps; source-mart evidence may be a validated SqlEvidenceSpec. --- @@ -104,7 +104,7 @@ | E10 | 5xx backend failure on compile | server-error | Error section + retry; partial graph not persisted | User retries; L2 UX test | | E11 | Malformed VLM response / empty findings | integration | Findings array empty; step inconclusive with reason; stale prompt blocked (422 STALE_PROMPT) | Re-run analysis; L1 VLM test | | E12 | Missing runtime parameter at launch | data-quality | Saved pack remains save_eligible; 044 RunPreflight rejects only that launch | Supply a typed binding; L1 preflight test | -| E13 | Unsafe path / executable code / SQL injection into pack | security | Template/path/code validation blocks before draft registration | L1 security test; injected-code fixture | +| E13 | Unsafe path / executable code / SQL injection into pack | security | SQL compilation gate blocks unsafe SQL; arbitrary code/path validation blocks before draft registration | L1 security test; injected-code fixture | | E14 | Duplicate submit of draft-pack | idempotency | Idempotency key / revision hash prevents double registration | L1 API test; 409 on changed revision | ## Requirements @@ -114,6 +114,7 @@ - **AGSCN-FR-001**: The system MUST define a `DashboardTestScenario` model with dashboard context, objective, parameters, steps, dependencies, outputs, artifacts, risks, and warnings. - **AGSCN-FR-002**: Every scenario step MUST declare tool category, action, inputs, outputs, expected result, dependencies, and automation status. - **AGSCN-FR-003**: Supported tool categories MUST include browser automation, Superset API execution, XLSX parsing, assertion, screenshot/evidence, report generation, artifact generation, and human checkpoint. +- **AGSCN-FR-003a**: `DashboardTestScenario` MUST contain a first-class, content-hashed Verification Program with navigation, evidence, transformation, assertion and semantic-evaluation programs. It is immutable runtime input, not a runtime planning hint. - **AGSCN-FR-004**: Assertions against reference values MUST use baseline references or baseline candidate references; raw expected numbers MUST NOT be embedded directly in executable steps. - **AGSCN-FR-005**: Authoring validation MUST detect missing refs, cycles, duplicate outputs, missing/stale baselines, unknown selectors, unsupported tools, and invalid ParameterDefinitions. Required runtime bindings are enforced only by 044 RunPreflight. - **AGSCN-FR-006**: Checklist mapping MUST use normalized checklist cases derived from the research PDF and capability tags, not hardcoded one-size-fits-all scripts. Mapping accepts the target release_version for baseline lookup. @@ -122,6 +123,10 @@ - **AGSCN-FR-009**: The scenario model MUST remain implementation-neutral and must not require the user to choose low-level artifacts such as Playwright, XLSX, or API output upfront. - **AGSCN-FR-010**: Screenshot steps MUST carry a capture SPECIFICATION: target (tab/viewport), viewport dimensions, readiness strategy, masking selectors, and max wait. **Execution of capture is owned by 044** (ScenarioExecution CaptureService delegating to `Plugin.Service.ScreenshotService`), with artifacts `owner_type=scenario_run`; 038 defines the spec, not the runtime path. - **AGSCN-FR-011**: Visual-analysis steps MUST carry a typed `VlmAnalysisSpec` (profile/provider/model/prompt template/hash/confidence). **Runtime VLM submission and `VlmFinding` production are owned by 044**, reusing `Plugin.Service.LLMClient` resolved through `Services.LlmProvider.LLMProviderService` (multimodal-required, encrypted-key handling, JSON mode) and redacting raw responses via `Plugin.Service.RedactionService`. A stub/default submit returning empty findings without a real provider call is incomplete. +- **AGSCN-FR-011a**: Read-only `SqlEvidenceSpec` MAY be authored during creation/edit/revalidation or investigation proposal only. Save MUST require AST/policy/schema/preview validation. A ScenarioRun executes exactly the saved SQL template via the Superset SQL Lab adapter with typed bindings; runtime LLM SQL rewrite, relation/projection/join/filter mutation and credential handoff are forbidden. +- **AGSCN-FR-011b**: `TransformSpec` MUST use only the bounded versioned DSL and `ComparisonSpec`/`AssertionSpec` MUST compare declared evidence refs. Arbitrary Python/code is forbidden. +- **AGSCN-FR-011c**: `AgentEvaluationSpec` MAY cover only declared semantic/visual/ambiguous checks. It MUST pin model/prompt/evidence/input/tool access/output schema and DecisionPolicy; it cannot alter graph, SQL/DSL, orchestration, lifecycle or mutations. +- **AGSCN-FR-011d**: Authoring MUST accept first-class `ChangeRequestContext`; the compiler must mark missing needed context as `needs_context`, never guess it. - **AGSCN-FR-012**: Human checkpoint steps MAY reference specific VLM finding ids. Resolution options (confirm, **false_positive**, inconclusive) MUST be typed and auditable — this is a 044 `HumanCheckpoint`, **distinct from** the 036 authorization `ActionApprovalGate`. Disposition changes finding status, not graph structure. - **AGSCN-FR-013**: The agent MAY plan scenario coverage, resolve ambiguity, and create a validated WorkingDraft or executable revision when delegated policy permits. Every resulting graph MUST pass the deterministic 038 validator and retain canonical immutable provenance. - **AGSCN-FR-014**: Graph authoring and revision actions MUST be available in the persistent scenario workspace or agent thread; modal/dialog interaction MUST NOT be required for authoring, review, conflict recovery, or approval. @@ -130,7 +135,9 @@ - **DashboardTestScenario**: Reviewable graph representing a dashboard-specific testing objective and all steps required to validate it. - **ScenarioStep**: One executable, generated, assertion, evidence, or human checkpoint node in the graph. -- **ScenarioParameter**: User- or context-provided input used by one or more scenario steps. +- **ParameterDefinition**: Immutable name/type/default/validation/affected-step declaration; launch values are 044 ParameterBindings and never affect content_hash. +- **VerificationProgram**: Content-hashed navigation/evidence/transform/assertion/semantic-evaluation IR. +- **SqlEvidenceSpec**: Validated immutable read-only source-mart evidence query. - **ScenarioRef**: Named output produced by one step and consumed by later steps. - **ChecklistCase**: Normalized item from the research checklist with capability tags and expected verification semantics. - **CapabilityMapping**: Decision record mapping dashboard capabilities to applicable checklist cases and step templates. diff --git a/specs/038-dashboard-scenario-model/tasks.md b/specs/038-dashboard-scenario-model/tasks.md index 82c9ffca5..67b56c5c2 100644 --- a/specs/038-dashboard-scenario-model/tasks.md +++ b/specs/038-dashboard-scenario-model/tasks.md @@ -96,10 +96,10 @@ @TEST_EDGE: 401→UNAUTHORIZED, 403→FORBIDDEN (per scope), 409 stale→STALE_REVISION, 422→VALIDATION_ERROR, 429→RATE_LIMITED - [x] T033 [P] Add thin compile/validate/resolve/generate tools in `agent/src/ss_tools/agent/tools_038.py` (4 tools registered; 36 total) - [x] T034 Verify agent schemas cannot carry code, SQL, raw expected metrics, custom tools, or artifact paths - (`extra="forbid"` + bounded JSON fields; 4 tools in `_SCENARIO_TOOL_ALLOWLIST`; no SQL tools in scenario mode) + (`extra="forbid"` + bounded JSON fields; versioned ActionRegistry; SqlEvidenceSpec permitted only through compilation gate) - [x] T035 Run quickstart, JSON/OpenAPI schema validation, scoped/full backend tests, and ruff (68 backend + 20 agent tests green; ruff clean backend + agent) -- [x] T036 Audit all 19 cases, direct SQL/code bans, contract anchors, ATTN_1–4, and unresolved relations +- [x] T036 Audit all 19 cases, arbitrary SQL/code bans, SqlEvidenceSpec compilation gate, contract anchors, ATTN_1–4, and unresolved relations (belief audit 0 errors; regions balanced) **Checkpoint**: API + agent tools match openapi.yaml; RBAC enforced. ✅ (88 tests green) diff --git a/specs/038-dashboard-scenario-model/validation.md b/specs/038-dashboard-scenario-model/validation.md index 8cf11ca9e..14ed17741 100644 --- a/specs/038-dashboard-scenario-model/validation.md +++ b/specs/038-dashboard-scenario-model/validation.md @@ -1,29 +1,31 @@ #region DashboardScenarioModel.ValidationReport [C:3] [TYPE ADR] [SEMANTICS validation,gate,scenario] @defgroup Validation Pre-implementation validation gate for the Dashboard Scenario Model (compiler-layer scope). -## Status: VALID (compiler-layer scope) — regenerated 2026-08-09 +## Status: VALID (compiler-layer scope) — generated 2026-08-11 -**Date**: 2026-08-09 (regenerated from current artifacts) +**Date**: 2026-08-11 (generated from current artifacts by `reconcile_contracts.py 036-047`) **Feature**: 038 Dashboard Scenario Model **Scope**: Compiler/IR layer ONLY. Runtime capture/VLM/disposition/evidence is owned by 044. > **History**: Prior versions of this file self-contradicted (claimed PASS covering full implementation while runtime T057–T059 were open, and asserted YAML validity that the parser rejected). This revision is regenerated from the machine-readable artifacts and is compiler-scope only. -## Machine-Contract Verification (2026-08-09) +## Machine-Contract Verification (2026-08-11) | Check | Result | |-------|--------| | OpenAPI YAML parse (`contracts/openapi.yaml`) | ✅ parses (fix: quoted flow-scalar with `:` on disposition description) | -| OpenAPI identity (scenario_key/content_hash, runtime endpoints deprecated) | ✅ verified | +| OpenAPI semantic paths + identity (scenario_key/content_hash, runtime endpoints deprecated) | ✅ verified | | JSON Schema parse (`dashboard-test-scenario.schema.json`) | ✅ valid JSON | -| JSON Schema identity (scenario_key, content_hash, logical_step_id, step_key, position, step_content_hash, affected_logical_step_ids, logical_step_ids; no scenario_id/revision_hash) | ✅ verified | +| JSON Schema Verification Program (`SqlEvidenceSpec`, Transform/Comparison/AgentEvaluation specs) | ✅ required canonical content, included in content hash contract | +| ParameterDefinition runtime-state exclusion | ✅ no `value` or `status`; 044 owns ParameterBinding | +| Versioned ActionRegistry | ✅ current valid fixture actions registered; unknown runtime action blocked | | Fixtures validate against JSON Schema | ✅ 6/6 (`fixtures/api/*.json`), after migration to canonical identity | | Fixture fingerprint lengths (input_fingerprints.* 64-hex) | ✅ fixed (was 66-char `parameters`) | -| Disposition enum (false_positive, no `dismiss`) | ✅ aligned with 044 HumanCheckpoint | +| 044 runtime boundary | ✅ typed StepOutcome/AgentEvaluation events; no runtime SQL mutation request surface | ## Compiler-Layer Gate Decision -**Verdict**: ✅ PASS (compiler/IR scope) — 038 compiler/validator/resolver/canonicalizer + JSON Schema + fixtures are internally consistent and machine-validated. +**Verdict**: ✅ PASS (compiler/IR scope) — 038 compiler/validator/resolver/canonicalizer + Verification Program schema/action registry + fixtures are internally consistent and machine-validated. **NOT** a full-feature PASS: execution (real VLM submit, real capture, evidence owner_type=scenario_run, HumanCheckpoint) is owned by 044 and gated there. Do not read this as authorizing 044 runtime implementation. @@ -37,7 +39,7 @@ None for the compiler-layer scope. The prior "runtime closure" findings are relo |----------|--------| | spec.md, plan.md, research.md, data-model.md, tasks.md, traceability.md, quickstart.md, ux_reference.md | ✅ | | checklists/requirements.md | ✅ | -| contracts/modules.md, contracts/openapi.yaml | ✅ | +| contracts/modules.md, contracts/openapi.yaml, contracts/verification-program.md, contracts/action-registry.yaml | ✅ | | contracts/dashboard-test-scenario.schema.json, contracts/capture-profile.schema.json | ✅ | | contracts/ux/{scenario-graph-ux,alternatives,decisions,api-ux}.md | ✅ | | contracts/openapi-traceability.md | ✅ | @@ -50,8 +52,8 @@ None for the compiler-layer scope. The prior "runtime closure" findings are relo - W03: Frontend rendering is DTO-only in 038; 039 owns the create UI, 045 owns run monitor. Confirm downstream UI tasks exist (already in 039/045). - Runtime endpoints in `contracts/openapi.yaml` are `deprecated` (moved to 044); OpenAPI-traceability must reflect this when regenerated. -## Validated Inputs +## Generation rule -The `Validated Inputs` digest table from prior versions is stale (references 2026-07-31 SHA-256s). Digests MUST be re-recorded by `/speckit.validate` against current files before implementation; this report is a structural pass, not a digest lock. +This report is generated from the current package gate rather than a manually maintained digest table. Any contract change requires rerunning `backend/.venv/bin/python reconcile_contracts.py 036-047`; a non-zero result invalidates this verdict. #endregion DashboardScenarioModel.ValidationReport diff --git a/specs/039-dashboard-scenario-ui/contracts/modules.md b/specs/039-dashboard-scenario-ui/contracts/modules.md index 11b08eb2c..afdcbbdb9 100644 --- a/specs/039-dashboard-scenario-ui/contracts/modules.md +++ b/specs/039-dashboard-scenario-ui/contracts/modules.md @@ -19,7 +19,7 @@ // @ACTION initialize(context, agentRun) — enter scenario mode for valid v2 intent only. // @ACTION applyScenarioResponse(response) — atomically replace revision/validation. // @ACTION updateParameter(name, value) — local declared typed edit. -// @ACTION applyParameters() — resolve dirty parameters against base revision. +// @ACTION applyParameterDefinitions() — validate dirty ParameterDefinitions against base revision; launch values remain 045-owned. // @ACTION generateDraftPack() — request 038 safe template pack. // @ACTION selectArtifact(id) / loadPreview(id) — side-effect-free preview. // @ACTION requestSave(ids) / requestBaselineApproval(candidate) — delegate gate creation. @@ -91,7 +91,7 @@ - + diff --git a/specs/039-dashboard-scenario-ui/contracts/ux/decisions.md b/specs/039-dashboard-scenario-ui/contracts/ux/decisions.md index bc3ed9597..d8be2c4f6 100644 --- a/specs/039-dashboard-scenario-ui/contracts/ux/decisions.md +++ b/specs/039-dashboard-scenario-ui/contracts/ux/decisions.md @@ -5,11 +5,11 @@ 2. Scenario mode is determined only by valid UIContext v2 intent. 3. Reuse AgentRun progress/recovery and ConfirmationCard; no duplicate lifecycles. 4. Present business objective, phases, steps, coverage, then tool evidence. -5. Parameter changes create immutable revisions and do not rerun inspection. +5. ParameterDefinition changes create immutable revisions; runtime ParameterBindings do not and are collected only at launch. 6. Show all unsupported/manual/needs-context cases. 7. Preview/download remains side-effect free; save eligibility comes from DraftPack. 8. Baseline approval requires visible provenance/diff and reason. -9. “Superset API; no direct SQL” is persistent in relevant comparison/step views. +9. Evidence views distinguish Superset API evidence from validated immutable SQL Lab SqlEvidenceSpec; SQL has no runtime edit/rewrite control. 10. Accessible step table is complete even if graph visualization is unavailable. #endregion DashboardScenarioUi.UxDecisions diff --git a/specs/039-dashboard-scenario-ui/data-model.md b/specs/039-dashboard-scenario-ui/data-model.md index 76096f8dd..6f364e0cb 100644 --- a/specs/039-dashboard-scenario-ui/data-model.md +++ b/specs/039-dashboard-scenario-ui/data-model.md @@ -11,11 +11,13 @@ Owned by `DashboardTesting.WorkspaceModel`, composed under `AgentChat.Model`. ON |---|---|---| | mode | scenario or ordinary chat | UIContext v2 intent | | scenario | DashboardTestScenario/null | 038 response (via agent) | +| changeRequestContext | ChangeRequestContext/null | authoring input; never guessed by compiler | +| verificationProgram | VerificationProgram/null | 038 content-hashed program preview | | validation | ScenarioValidationResult/null | 038 response | | draftPack | DraftPack/null | 038 response | | selectedPhaseId/selectedStepId | string/null | local navigation | -| parameterDrafts | map name → typed draft | initialized from scenario parameters | -| parameterErrors | map name → message code | local validation | +| parameterDefinitionsDraft | map name → typed definition draft | initialized from scenario ParameterDefinitions; no runtime values | +| parameterDefinitionErrors | map name → message code | local validation | | baselineSummary | approved/stale/missing/candidate groups | 037 response | | selectedArtifactId | UUID/null | local preview | | preview | loading/content/error | 036 opaque preview | @@ -30,7 +32,7 @@ Owned by `DashboardTesting.WorkspaceModel`, composed under `AgentChat.Model`. ON ### Agent Workspace — Derived Values - workspaceState from intent, AgentRun status/stage, scenario, draft pack, and domain error; -- canApplyParameters when changed drafts are valid; +- canApplyParameterDefinitions when changed definition drafts are valid; - canGenerateDraft when scenario exists and authoring blockers (for example selectors/baselines) are resolved; runtime ParameterBindings are collected only in 045 launch preflight; - canRequestSave when draftPack.status=save_eligible, artifacts valid, and no blockers; - baselineApprovalReady when candidate selected and non-blank reason; @@ -40,10 +42,10 @@ Owned by `DashboardTesting.WorkspaceModel`, composed under `AgentChat.Model`. ON 1. Domain status never derives from assistant text. 2. Graph/validation are server-authoritative immutable revisions. -3. Parameter edits cannot modify undeclared fields. +3. ParameterDefinition edits cannot modify undeclared fields or create runtime bindings. 4. Durable actions delegate to 036 delegated-action policy; only actions requiring authorization render a pending ActionApprovalGate. 5. Download/preview never set persisted state. -6. Direct SQL is absent from actions and explanatory alternatives. +6. SQL is shown only as a read-only validated SqlEvidenceSpec preview (template hash, relation/connection refs, limits, output schema and validation); neither chat nor runtime controls can rewrite it. 7. Switching run/context resets all scenario-local selections and drafts. 8. Evidence and findings are bound to the owning AgentRun and never cross run boundaries. 9. Finding disposition is recorded once per finding; replay of same disposition is idempotent. diff --git a/specs/039-dashboard-scenario-ui/plan.md b/specs/039-dashboard-scenario-ui/plan.md index 04acb7cb9..3e7ae994e 100644 --- a/specs/039-dashboard-scenario-ui/plan.md +++ b/specs/039-dashboard-scenario-ui/plan.md @@ -13,7 +13,7 @@ Add a dashboard-level scenario action and a model-first workspace inside /agent. **Storage**: no frontend persistence of authoritative domain data; run id may be route/session recovery hint only **Testing**: L1 model no-render, L2 component UX, Playwright browser flow **Performance Goals**: parameter readiness projection under 200ms; 15–100 steps usable at 1366px; no duplicate stream re-render -**Constraints**: no Svelte 4 syntax, no component-owned business state, no prose parsing, no raw colors, no direct SQL copy/action +**Constraints**: no Svelte 4 syntax, no component-owned business state, no prose parsing, no raw colors, no runtime SQL copy/action; validated SqlEvidenceSpec is preview-only authoring content. **Scale**: 100 steps, 50 params, 50 drafts, 19 coverage rows ## Constitution Check diff --git a/specs/039-dashboard-scenario-ui/quickstart.md b/specs/039-dashboard-scenario-ui/quickstart.md index 250f0c703..6e80972d4 100644 --- a/specs/039-dashboard-scenario-ui/quickstart.md +++ b/specs/039-dashboard-scenario-ui/quickstart.md @@ -21,7 +21,7 @@ npm run build 2. Click “Создать сценарий тестирования”; verify exact dashboard/env/intent in /agent. 3. Confirm run id appears before analysis tool activity. 4. Review 18-step scenario: phases, step table, graph, all 19 coverage rows, warnings/blockers. -5. Verify relevant steps state Superset API/no direct SQL. +5. Verify relevant steps distinguish Superset API evidence from immutable validated SQL Lab evidence; no runtime SQL edit/rewrite control is present. 6. Resolve date, counterparty, and baseline choice; only affected steps update. 7. Generate draft; preview each text artifact and download without Git changes. 8. Verify preview-only/invalid draft blocks save. diff --git a/specs/039-dashboard-scenario-ui/research.md b/specs/039-dashboard-scenario-ui/research.md index 8c9387c46..e21781c5a 100644 --- a/specs/039-dashboard-scenario-ui/research.md +++ b/specs/039-dashboard-scenario-ui/research.md @@ -49,7 +49,7 @@ Transitions are driven by structured AgentRun events and authoritative ScenarioR - A visual dependency graph is supplementary; the table provides complete accessible semantics. - Tool categories are visible for trust but not user-selected. - All 19 checklist cases remain visible in coverage, including manual/unsupported/needs-context. -- Superset metric steps state “Superset API; no direct SQL”. +- Superset metric steps state “Superset chart API”; source-mart steps state “validated immutable SQL Lab evidence” with their hash and limits. ## 7. Draft and Approval diff --git a/specs/039-dashboard-scenario-ui/spec.md b/specs/039-dashboard-scenario-ui/spec.md index 858caa692..2c5734e49 100644 --- a/specs/039-dashboard-scenario-ui/spec.md +++ b/specs/039-dashboard-scenario-ui/spec.md @@ -7,7 +7,7 @@ @RELATION DEPENDS_ON -> [SupersetBaselineEngine.Spec] @RELATION DEPENDS_ON -> [DashboardScenarioModel.Spec] @RATIONALE Users should approve a business-level dashboard test scenario and required parameters; low-level tool chains are visible for trust but selected by the agent and scenario validator. -@REJECTED Dropdowns such as "Playwright UI tests" vs "SQL checks" vs "XLSX checks" — rejected because each dashboard requires a unique cross-tool scenario, and SQL is explicitly out of scope. +@REJECTED Dropdowns such as "Playwright UI tests" vs "SQL checks" vs "XLSX checks" — rejected because each dashboard requires a unique cross-tool program. Validated immutable SQL evidence is visible in the program, not chosen as a loose UI mode. ## Navigation (DSA Indexer keywords) @SEMANTICS: spec, requirements, feature, ux, agent, scenario, dashboard-testing, artifacts, baseline @@ -40,19 +40,19 @@ **Acceptance**: 1. **Given** the agent analyzes a dashboard **When** scenario graph is ready **Then** the UI shows goal, phases, step table, dependency graph, tool categories, expected results, coverage, warnings, and blockers. 2. **Given** some checklist cases are not automatable **When** scenario preview renders **Then** they appear as human checkpoints, unsupported, or needs-context with rationale. -3. **Given** the scenario includes Superset API metric validation **When** preview renders **Then** the UI clearly states Superset-native execution is used and direct SQL is not used. +3. **Given** the scenario includes source-mart evidence **When** preview renders **Then** the UI shows its validated SqlEvidenceSpec (relation/connection identity, hash, typed parameters, limits and output schema) and distinguishes it from Superset chart API evidence. --- -### Story 3 — Collect Business Parameters and Baseline Choices (P1) +### Story 3 — Define Parameters and Baseline Constraints (P1) -**Why P1**: Unique dashboard tests need values such as test date, counterparty, and baseline selection before generation is meaningful. +**Why P1**: Reusable dashboard tests need typed parameter definitions and baseline constraints; actual launch values belong to 045 RunPreflight. -**Independent Test**: Scenario preview requests parameters; user fills them; dependent step statuses update from unresolved to ready. +**Independent Test**: Scenario preview renders parameter definitions; user edits default/validation/source and sees affected steps, while a required no-default scenario remains save-eligible. **Acceptance**: -1. **Given** a scenario requires parameters **When** preview renders **Then** each parameter has label, type, validation message, default/source if available, and affected steps. -2. **Given** an approved baseline exists for metric+filters **When** the user selects matching parameters **Then** baseline match is shown and assertion steps become ready. +1. **Given** a scenario declares parameters **When** preview renders **Then** each ParameterDefinition has label, type, validation, default/source if available, and affected steps; it contains no runtime value/status. +2. **Given** an approved baseline exists for metric+filters **When** the analyst defines binding constraints **Then** baseline compatibility is shown without creating a runtime binding. 3. **Given** no approved baseline exists **When** the user proceeds **Then** the UI offers discovery candidate flow and marks baseline approval as separate HITL action. --- @@ -99,12 +99,13 @@ These requirements cover the persistent agent workspace where the analyst create - **AGUI-FR-001**: Dashboard pages MUST expose a single business-level action labeled "Создать сценарий тестирования"; UI MUST NOT present low-level artifact/tool choices as the primary entry point. - **AGUI-FR-002**: `/agent` MUST show Dashboard Test Scenario Agent mode when opened with scenario intent from 036. - **AGUI-FR-003**: The UI MUST render structured progress stages from agent metadata: context, inspect, scenario, parameters, generate, validate, save. -- **AGUI-FR-004**: The scenario preview MUST show business goal, phases, step list, tool category per step, dependency graph, coverage, warnings, blockers, and expected results. -- **AGUI-FR-005**: Parameter collection MUST support typed required parameters and update dependent scenario readiness without restarting the full flow. +- **AGUI-FR-004**: The scenario preview MUST show business goal, phases, step list, tool category per step, dependency graph, coverage, warnings, blockers, expected results, and the visible Verification Program (navigation, evidence, transforms, assertions and declared agentic evaluations). +- **AGUI-FR-005**: Authoring MUST support typed ParameterDefinition/default/validation/source edits and show affected steps without restarting the flow. Required parameters without defaults remain save-eligible; 045 alone collects runtime ParameterBindings. - **AGUI-FR-006**: Baseline UI MUST show approved baseline matches, stale warnings, missing baselines, and draft candidate approval paths. - **AGUI-FR-007**: Generated artifact preview MUST show file tree, file content preview, validation status, unresolved markers, and warnings before save. - **AGUI-FR-008**: The agent MAY save a validated executable scenario revision under delegated policy. Baseline approval and every non-delegated risky action MUST use a 036 inline ActionApprovalGate; baseline approval requires a reason. -- **AGUI-FR-009**: The UI MUST explicitly communicate that metric validation uses Superset-native execution and does not execute direct SQL. +- **AGUI-FR-009**: The UI MUST distinguish Superset chart API evidence from source-mart SqlEvidenceSpec executed through the Superset SQL Lab adapter. It MUST show SQL as an immutable validated program artifact, never as runtime/chat-editable text. +- **AGUI-FR-009a**: The UI MUST collect and display ChangeRequestContext (request, affected dashboards/fields, technical detail, relations, acceptance criteria and control totals). Missing needed context is explicit `needs_context`, not agent inference. - **AGUI-FR-010**: All new UI state MUST follow Svelte 5 runes/model-first conventions and remain accessible by keyboard for inline action cards, parameter forms, and previews; modal/dialog interaction MUST NOT be required to complete a workflow. - **AGUI-FR-011**: Screenshot evidence artifacts MUST render in a dedicated EvidencePanel showing the captured image, capture metadata (viewport, filters_hash, timestamp), and any linked VLM findings with severity, region, and confidence. - **AGUI-FR-012**: VLM findings MUST be reviewable with typed disposition controls: confirm (accepts finding as valid), dismiss (marks as false positive), or inconclusive (defers to human checkpoint). Disposition changes MUST be auditable and MUST NOT alter the scenario graph. @@ -175,7 +176,7 @@ These entities are independent of the agent workspace. They consume `Verificatio - **SC-003**: Parameter updates resolve dependent scenario readiness within 200ms in model tests. - **SC-004**: Artifact preview blocks save when unresolved markers are present in fixture data. - **SC-005**: 100% of durable actions have immutable delegated-policy provenance; baseline approval cannot submit without a reason and any required gate. -- **SC-006**: UX copy never presents direct SQL as a validation path for this feature. +- **SC-006**: UX exposes every SqlEvidenceSpec as a validated immutable authoring artifact and never as a runtime-editable or agent-generated-on-run validation path. - **SC-007**: Pipeline verification views render StructureDiff, metric results, and VerificationRun data without initializing an AgentRun or WorkspaceModel. - **SC-008**: VerificationStatusBadge appears on dashboard, release, and PREPROD pages within 200ms of data load. diff --git a/specs/039-dashboard-scenario-ui/ux_reference.md b/specs/039-dashboard-scenario-ui/ux_reference.md index d8e538494..65ae9c0bd 100644 --- a/specs/039-dashboard-scenario-ui/ux_reference.md +++ b/specs/039-dashboard-scenario-ui/ux_reference.md @@ -123,6 +123,6 @@ The user opens a dashboard and clicks "Создать сценарий тест ## 5. Tone & Voice * **Style**: Clear, operational, confidence-aware. -* **Terminology**: Use "scenario", "draft", "artifact", "baseline", "Superset API", "human checkpoint". Avoid "SQL check" for this feature. +* **Terminology**: Use "scenario", "verification program", "draft", "artifact", "baseline", "Superset API", "SQL Lab evidence", "human checkpoint". Never imply that SQL is editable or generated at run time. #endregion DashboardScenarioUi.UxReference diff --git a/specs/042-dashboard-scenario-registry/data-model.md b/specs/042-dashboard-scenario-registry/data-model.md index bcaa313cc..80aa409a6 100644 --- a/specs/042-dashboard-scenario-registry/data-model.md +++ b/specs/042-dashboard-scenario-registry/data-model.md @@ -14,7 +14,7 @@ Indexes: dashboard_id, owner_id, lifecycle_status, tags (GIN), name (trigram for ## ScenarioRevision — revision identity (revision for #7) -Fields: revision_id (UUID — unique immutable identity), scenario_id, content_hash (SHA-256 of the **executable** canonical graph only, per 038 rules — timestamps/display-only excluded), parent_revision_id (nullable), graph_snapshot (DashboardTestScenario JSON), execution_template_hash (hash of revision-derived execution template only; no environment/parameters/baselines), template_version, schema_version, compatibility_family, change_summary (added/changed/removed), created_by, created_at, activation_status (`candidate|current`), activated_by?, activated_at?, activation_agent_action_id?. `is_current` is a read-model convenience derived only from `activation_status=current`. +Fields: revision_id (UUID — unique immutable identity), scenario_id, content_hash (SHA-256 of the **executable** canonical Verification Program graph, including SQL templates/hashes, DSL, assertions and AgentEvaluationSpec; timestamps/display-only excluded), parent_revision_id (nullable), graph_snapshot (DashboardTestScenario JSON), execution_template_hash (hash of revision-derived execution template only; no environment/ParameterBinding/baselines), template_version, schema_version, compatibility_family, change_summary (added/changed/removed), created_by, created_at, activation_status (`candidate|current`), activated_by?, activated_at?, activation_agent_action_id?. `is_current` is a read-model convenience derived only from `activation_status=current`. **Metadata vs executable split (#8)**: entity-metadata edits (name/description/tags) update `ScenarioRegistryEntry` fields and DO NOT create a new executable `ScenarioRevision` (content_hash unchanged). Only executable-graph edits create a new revision. `revision_id` (UUID) is the unique identity; `content_hash` detects content change. diff --git a/specs/042-dashboard-scenario-registry/spec.md b/specs/042-dashboard-scenario-registry/spec.md index 5554bdccf..e5636d50c 100644 --- a/specs/042-dashboard-scenario-registry/spec.md +++ b/specs/042-dashboard-scenario-registry/spec.md @@ -103,7 +103,7 @@ - **SCREG-FR-001**: The system MUST persist scenarios as a first-class entity in a `scenario_registry` projection (scenario_id, revision, dashboard, environment compatibility, owner, tags, lifecycle status, validation status, last run, health). - **SCREG-FR-002**: The system MUST expose `GET /dashboard-testing/scenarios` (list/search/filter by name, tag, dashboard, status, owner) and `GET /dashboard-testing/scenarios/{id}` (detail). -- **SCREG-FR-003**: Every executable-graph edit MUST create a new immutable `candidate` revision (`revision_id` UUID + `content_hash` + `parent_revision_id`); runs pin an explicit revision snapshot. Entity-metadata edits (name/description/tags) MUST NOT create an executable revision (content_hash unchanged). +- **SCREG-FR-003**: Every executable-graph edit MUST create a new immutable `candidate` revision (`revision_id` UUID + `content_hash` + `parent_revision_id`); the content hash includes the 038 Verification Program (SQL/DSL/assertion/AgentEvaluationSpec) but excludes ParameterBindings and other runtime state. Runs pin an explicit revision snapshot. Entity-metadata edits (name/description/tags) MUST NOT create an executable revision (content_hash unchanged). - **SCREG-FR-003a**: `ActivateCurrentRevision` MUST be a separate atomic operation. It may advance `current_revision` only after deterministic eligibility checks and the recorded 036 delegated-authority decision or ActionApprovalGate; saving a revision never silently changes an automation target. - **SCREG-FR-004**: Scenario detail MUST be loadable by id independent of any agent session (not event-driven). - **SCREG-FR-005**: Lifecycle states MUST include DRAFT, READY, STALE, NEEDS_REVALIDATION, BLOCKED, DISABLED, DEPRECATED, ARCHIVED; only valid transitions apply and are audit-logged. diff --git a/specs/043-dashboard-scenario-editor/data-model.md b/specs/043-dashboard-scenario-editor/data-model.md index f0f290707..a0f4227e0 100644 --- a/specs/043-dashboard-scenario-editor/data-model.md +++ b/specs/043-dashboard-scenario-editor/data-model.md @@ -21,7 +21,7 @@ Invariant: op payloads only reference registered templates/operators/baseline re ## WorkingDraft — server-stored, no client-draft bypass (#7) -`apply` persists a WorkingDraft server-side and returns `{draft_id, digest}`. `save(draft_id, digest)` reloads it server-side, re-validates, canonicalizes, re-hashes, and compares the base revision. The client NEVER returns the full graph; `save` cannot accept an arbitrary `draft` object. A WorkingDraft is bound to a base revision and expires on conflict/staleness. +`apply` persists a WorkingDraft server-side and returns `{draft_id, digest}`. `save(draft_id, digest)` reloads it server-side, re-validates the Verification Program (including SqlEvidenceSpec compilation), canonicalizes, re-hashes, and compares the base revision. The client NEVER returns the full graph; `save` cannot accept an arbitrary `draft` object. A WorkingDraft is bound to a base revision and expires on conflict/staleness. Fields: draft_id, scenario_id, base_revision_id, ops[], applied_graph, digest, created_by, delegated_by?, agent_run_id?, investigation_case_id?, created_at, status (open|saved|awaiting_approval|expired). @@ -35,7 +35,7 @@ Fields: logical_step_id, target_logical_step_id, action (add|remove). Validated ## EditRevisionResult -Fields: new_revision_id, parent_revision_id, content_hash, activation_status=`candidate`, server_derived_change_summary {added, changed, removed}, validation (AuthoringValidation), diff_payload, policy_decision, agent_action_id?. Returned by save; never mutates prior revisions or advances `current_revision`. The caller submits only `{draft_id, digest}` plus its authenticated/delegated action identity; it never supplies audit/provenance summary. Promotion is the distinct 042 `ActivateCurrentRevision` operation, with its own deterministic eligibility and delegated-authority/gate decision. +Fields: new_revision_id, parent_revision_id, content_hash, activation_status=`candidate`, server_derived_change_summary {added, changed, removed, verification_program_diff}, validation (AuthoringValidation), diff_payload, policy_decision, agent_action_id?. Returned by save; never mutates prior revisions or advances `current_revision`. The caller submits only `{draft_id, digest}` plus its authenticated/delegated action identity; it never supplies audit/provenance summary. Promotion is the distinct 042 `ActivateCurrentRevision` operation, with its own deterministic eligibility and delegated-authority/gate decision. ## EditProposal and ScenarioMigration diff --git a/specs/043-dashboard-scenario-editor/spec.md b/specs/043-dashboard-scenario-editor/spec.md index 2f24572bb..92b929413 100644 --- a/specs/043-dashboard-scenario-editor/spec.md +++ b/specs/043-dashboard-scenario-editor/spec.md @@ -111,6 +111,7 @@ - **SCEDIT-FR-003**: Assertion editing MUST use a constrained editor (registered operators + baseline references); free-form SQL, shell, paths, and raw numeric baseline literals MUST be forbidden. - **SCEDIT-FR-004**: Dependency editing MUST be visual (graph), with 038 cycle/duplicate-output validation on every change. - **SCEDIT-FR-005**: Every durable executable graph edit MUST produce a new immutable revision; metadata uses metadata_version concurrency. Deterministic policy decides whether the actor/agent may save immediately or must obtain an inline ActionApprovalGate. +- **SCEDIT-FR-005a**: Every proposal/diff MUST expose Verification Program changes: SqlEvidenceSpec template/hash/relation refs, TransformSpec operations, assertions, AgentEvaluationSpec and DecisionPolicy. A runtime finding can only change this content through a new validated proposal and revision. - **SCEDIT-FR-006**: "Edit with agent" MUST create a server-stored validated proposal/draft with a diff. A delegated agent MAY save the revision; no agent edit may bypass 038 validation, 042 revision provenance, or a required gate. - **SCEDIT-FR-009**: Every save MUST use a server-stored WorkingDraft (draft_id + digest); the client MUST NOT return the full graph to save (no arbitrary-draft bypass). Save re-validates/canonicalizes/hashes server-side. - **SCEDIT-FR-010**: A stale scenario MUST be revalidatable: affected refs mapped against the current dashboard, conflicts surfaced manually, a proposed revision + diff shown for approval (Scenario Migration workflow). diff --git a/specs/044-dashboard-scenario-execution/contracts/openapi.yaml b/specs/044-dashboard-scenario-execution/contracts/openapi.yaml index aad1e6a2e..29cc4d354 100644 --- a/specs/044-dashboard-scenario-execution/contracts/openapi.yaml +++ b/specs/044-dashboard-scenario-execution/contracts/openapi.yaml @@ -146,6 +146,8 @@ components: target_snapshot: { $ref: "#/components/schemas/TargetSnapshot" } execution_principal_fingerprint: { type: string } analytics_context_key: { type: string, description: "Server-derived immutable analytics grouping key" } + verification_program_hash: { type: string } + action_registry_version: { type: string } step_runs: { type: array, items: { $ref: "#/components/schemas/ScenarioStepRun" } } ScenarioStepRun: type: object @@ -158,6 +160,8 @@ components: outputs: { type: object } artifact_refs: { type: array, items: { type: string } } error_code: { type: string, nullable: true } + step_outcome: { $ref: "#/components/schemas/StepOutcome" } + agent_evaluations: { type: array, items: { $ref: "#/components/schemas/AgentEvaluationSummary" } } ParameterBinding: type: object required: [parameter_name, resolved_value, source, resolved_at] @@ -185,6 +189,57 @@ components: failures: { type: array, items: { type: object } } provenance: { type: object } analytics_context_key: { type: string } + step_outcomes: { type: array, items: { $ref: "#/components/schemas/StepOutcome" } } + agent_evaluation_summary: { type: object } + StepOutcome: + type: object + required: [status, reason_codes, deterministic_evidence_refs] + properties: + status: { type: string, enum: [passed, failed, inconclusive, blocked, waiting_human] } + reason_codes: { type: array, items: { type: string } } + deterministic_evidence_refs: { type: array, items: { type: string } } + agent_evaluation_ids: { type: array, items: { type: string } } + decision_policy_id: { type: [string, "null"] } + decision_policy_version: { type: [string, "null"] } + AgentEvaluationSummary: + type: object + required: [evaluation_id, logical_step_id, verdict, confidence, model_id, prompt_template_version] + properties: + evaluation_id: { type: string } + logical_step_id: { type: string } + verdict: { type: string, enum: [pass, fail, inconclusive] } + confidence: { type: number, minimum: 0, maximum: 1 } + model_id: { type: string } + prompt_template_version: { type: string } + reason_codes: { type: array, items: { type: string } } + evidence_refs: { type: array, items: { type: string } } + AgentEvaluation: + allOf: + - $ref: "#/components/schemas/AgentEvaluationSummary" + - type: object + required: [scenario_run_id, attempt, provider_id, model_version, prompt_template_id, input_manifest_hash, findings, raw_response_artifact_ref, started_at, finished_at] + properties: + scenario_run_id: { type: string } + attempt: { type: integer, minimum: 1 } + provider_id: { type: string } + model_version: { type: string } + prompt_template_id: { type: string } + input_manifest_hash: { type: string } + findings: { type: array, items: { type: object } } + raw_response_artifact_ref: { type: string } + started_at: { type: string, format: date-time } + finished_at: { type: string, format: date-time } + DecisionPolicy: + type: object + required: [policy_id, version, deterministic_hard_failure, high_confidence_failure, low_confidence, disagreement, missing_evidence] + properties: + policy_id: { type: string } + version: { type: string } + deterministic_hard_failure: { type: string, enum: [failed] } + high_confidence_failure: { type: string, enum: [failed, inconclusive] } + low_confidence: { type: string, enum: [inconclusive] } + disagreement: { type: string, enum: [waiting_human, inconclusive] } + missing_evidence: { type: string, enum: [blocked, inconclusive] } RunComparison: type: object required: [run_a, run_b, step_deltas, compatibility] @@ -196,18 +251,50 @@ components: ScenarioRunEvent: oneOf: - { $ref: "#/components/schemas/RunStartedEvent" } + - { $ref: "#/components/schemas/ApprovalRequiredEvent" } + - { $ref: "#/components/schemas/RunQueuedEvent" } - { $ref: "#/components/schemas/StepStartedEvent" } + - { $ref: "#/components/schemas/StepProgressEvent" } + - { $ref: "#/components/schemas/EvidenceCreatedEvent" } + - { $ref: "#/components/schemas/AgentEvaluationStartedEvent" } + - { $ref: "#/components/schemas/AgentEvaluationCompletedEvent" } - { $ref: "#/components/schemas/StepCompletedEvent" } - { $ref: "#/components/schemas/CheckpointCreatedEvent" } - { $ref: "#/components/schemas/RunCompletedEvent" } + - { $ref: "#/components/schemas/RunFailedEvent" } + - { $ref: "#/components/schemas/RunCancelledEvent" } RunStartedEvent: type: object required: [id, sequence, event_type, run_id, occurred_at] properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: run_started }, run_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object } } + ApprovalRequiredEvent: + type: object + required: [id, sequence, event_type, run_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: approval_required }, run_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object, required: [approval_gate_id] } } + RunQueuedEvent: + type: object + required: [id, sequence, event_type, run_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: run_queued }, run_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object } } StepStartedEvent: type: object required: [id, sequence, event_type, run_id, logical_step_id, occurred_at] properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: step_started }, run_id: { type: string }, logical_step_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object } } + StepProgressEvent: + type: object + required: [id, sequence, event_type, run_id, logical_step_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: step_progress }, run_id: { type: string }, logical_step_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object, required: [progress] } } + EvidenceCreatedEvent: + type: object + required: [id, sequence, event_type, run_id, logical_step_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: evidence_created }, run_id: { type: string }, logical_step_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object, required: [evidence_ref] } } + AgentEvaluationStartedEvent: + type: object + required: [id, sequence, event_type, run_id, logical_step_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: agent_evaluation_started }, run_id: { type: string }, logical_step_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object } } + AgentEvaluationCompletedEvent: + type: object + required: [id, sequence, event_type, run_id, logical_step_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: agent_evaluation_completed }, run_id: { type: string }, logical_step_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { $ref: "#/components/schemas/AgentEvaluationSummary" } } StepCompletedEvent: type: object required: [id, sequence, event_type, run_id, logical_step_id, occurred_at] @@ -220,3 +307,11 @@ components: type: object required: [id, sequence, event_type, run_id, occurred_at] properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: run_completed }, run_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object } } + RunFailedEvent: + type: object + required: [id, sequence, event_type, run_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: run_failed }, run_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object } } + RunCancelledEvent: + type: object + required: [id, sequence, event_type, run_id, occurred_at] + properties: { id: { type: string }, sequence: { type: integer }, event_type: { const: run_cancelled }, run_id: { type: string }, occurred_at: { type: string, format: date-time }, payload: { type: object } } diff --git a/specs/044-dashboard-scenario-execution/data-model.md b/specs/044-dashboard-scenario-execution/data-model.md index 98ea4a902..6a16132a1 100644 --- a/specs/044-dashboard-scenario-execution/data-model.md +++ b/specs/044-dashboard-scenario-execution/data-model.md @@ -6,7 +6,7 @@ ## ScenarioRun and PROD approval lifecycle -`ScenarioRun` is created before dispatch. Fields: id, scenario_id, scenario_revision_id (revision_id UUID), scenario_content_hash, dashboard_id, environment_id, status (`pending_approval|queued|running|waiting_human|blocked|cancel_requested|cancelled|passed|failed|inconclusive`), phase (preflight|setup|executing|waiting_human|draining|terminal), parameter_bindings (immutable JSON), baselines_pinned (version), target_snapshot, execution_principal_fingerprint, execution_toggles (optional evidence only), trigger_source (server-owned), agent_run_id? (provenance), verification_run_id? (aggregation), idempotency_key (unique), started_at, finished_at, resume_token, error_code, runner_version. +`ScenarioRun` is created before dispatch. Fields: id, scenario_id, scenario_revision_id (revision_id UUID), scenario_content_hash, verification_program_hash, action_registry_version, dashboard_id, environment_id, status (`pending_approval|queued|running|waiting_human|blocked|cancel_requested|cancelled|passed|failed|inconclusive`), phase (preflight|setup|executing|waiting_human|draining|terminal), parameter_bindings (immutable JSON), baselines_pinned (version), target_snapshot, execution_principal_fingerprint, execution_toggles (optional evidence only), trigger_source (server-owned), agent_run_id? (provenance), verification_run_id? (aggregation), idempotency_key (unique), started_at, finished_at, resume_token, error_code, runner_version. If the selected revision contains a `human` step, it is derived as `manual_run_only=true`: it may start only from the authenticated analyst manual-run route. Scheduler, deploy/ETL/API trigger and any background runner are ineligible; no HumanCheckpoint may be skipped or defaulted. @@ -28,11 +28,19 @@ Terminal `failed`, `inconclusive` and `blocked` outcomes emit a 036 `Investigati ## ScenarioStepRun -Fields: id, run_id FK, logical_step_id (immutable UUID, from #8), step_position (mutable), step_content_hash (mutable), attempt, status (queued|running|waiting_human|passed|failed|inconclusive|blocked|skipped), started_at, finished_at, inputs_snapshot (JSON, **no secrets**), outputs (JSON), artifact_refs (JSON), error_code, progress, timeout_ms, side_effect_key (nullable). +Fields: id, run_id FK, logical_step_id (immutable UUID, from #8), step_position (mutable), step_content_hash (mutable), attempt, status (queued|running|waiting_human|passed|failed|inconclusive|blocked|skipped), started_at, finished_at, inputs_snapshot (JSON, **no secrets**), outputs (JSON), artifact_refs (JSON), error_code, progress, timeout_ms, side_effect_key (nullable), `step_outcome` (`StepOutcome`). + +`StepOutcome { status, reason_codes[], deterministic_evidence_refs[], agent_evaluation_ids[], decision_policy_id?, decision_policy_version?, decided_at }` is the authoritative result of one logical step. It is distinct from executor output and from a model verdict. + +## AgentEvaluation and DecisionPolicy + +`AgentEvaluation { evaluation_id, scenario_run_id, logical_step_id, attempt, provider_id, model_id, model_version, prompt_template_id, prompt_template_version, input_manifest_hash, evidence_refs, verdict, confidence, findings, reason_codes, raw_response_artifact_ref, started_at, finished_at }` is immutable evidence generated only for a declared 038 AgentEvaluationSpec. Its tool/evidence access is bounded by that spec; it cannot mutate program content, invoke mutation actions, change run state or choose downstream scheduling. + +`DecisionPolicy { policy_id, version, deterministic_hard_failure, high_confidence_failure, low_confidence, disagreement, missing_evidence }` is a versioned deterministic mapper. Defaults: deterministic hard failure→failed; high-confidence policy-qualified agent failure→failed; low confidence→inconclusive; evaluation/evidence disagreement→waiting_human via a HumanCheckpoint only for manual runs; missing required evidence→blocked or inconclusive. Scenario aggregation consumes StepOutcome, not AgentEvaluation verdicts directly. ## RunnerPlan — deterministic derivation from revision (#2) -RunnerPlan is **derived deterministically at run start** from the selected `ScenarioRevision`, NOT read from a stored `runner.plan.json`. Fields: scenario_revision_id, scenario_content_hash, env targets, resolved params, pinned baselines, topological order, executor mapping per step, retry/timeout policy, human-checkpoint list. Run refuses if `derived_runner_plan.scenario_revision_id != selected revision_id`. +RunnerPlan is **derived deterministically at run start** from the selected immutable `ScenarioRevision`/Verification Program, NOT read from a stored `runner.plan.json`. Fields: scenario_revision_id, scenario_content_hash, verification_program_hash, action_registry_version, env targets, resolved params, pinned baselines, topological order, executor mapping per step, retry/timeout policy, decision-policy map, human-checkpoint list. Run refuses if its revision/program/action-registry hashes differ from the selected revision. The materialized `runner.plan.json` in git is a **reference artifact**, never the runtime source of truth; it may be regenerated from any revision. @@ -76,18 +84,21 @@ pending_approval → queued → running → waiting_human | blocked → passed | Mapping `tool -> executor`: - browser -> `BrowserExecutor` → version-pinned 038 `ActionRegistry` → Playwright/session infrastructure - superset_api -> 037 metric_executor_async / SupersetClient.ChartData.Execute +- sql_evidence -> `SqlEvidenceExecutor` → Superset SQL Lab backend/API → configured database connection (no credentials exposed to agent) +- transform -> version-pinned bounded 038 TransformSpec DSL executor - xlsx -> xlsx parser + 037 normalization -- assertion -> 037 comparison.py (+ vlm via LLMClient) +- assertion -> 037 comparison.py + 038 ComparisonSpec/AssertionSpec executor +- agent_evaluation -> bounded provider adapter executing a declared 038 AgentEvaluationSpec and emitting `AgentEvaluation`; DecisionPolicy owns StepOutcome - screenshot -> 038 capture.py + ScreenshotService (owner_type=scenario_run) - report -> report-template render + artifact (owner_type=scenario_run) - artifact -> generic artifact register (owner_type=scenario_run) - human -> EXCLUDED (runner-lifecycle HumanCheckpoint control, not an executor) -`BrowserExecutor` implements only registered actions (for example `open_dashboard`, `apply_native_filter`, `apply_table_filter`, `click`, `fill`, `select_rows`, `edit_row`, `bulk_edit`, `refresh`, `download`, `navigate_tab`, `wait_for_state`). ScreenshotService is evidence infrastructure only, not the browser action executor. It validates the registry-declared inputs/outputs/risk/timeout/idempotency before dispatch. +`BrowserExecutor` implements only registered actions: `open_dashboard`, `navigate_tab`, `apply_native_filter`, `inspect_filter_state`, `apply_table_filter`, `extract_table`, `scroll_to`, `inspect_columns`, `click`, `select_rows`, `edit_row`, `bulk_edit`, `download`, `refresh`, `wait_for_state`. ScreenshotService is evidence infrastructure only, not the browser action executor. It validates the registry-declared inputs/outputs/risk/timeout/idempotency before dispatch. ## Mutation authorization and execution policy -Authorization is evaluated per run and step as `environment_class + scenario risk profile + action mutation profile`. Read-only PROD steps require the ScenarioExecution approval policy. Every mutation requires the immutable 038 mutation contract, scoped target keys, precondition evidence, side-effect identity, cleanup/reconciliation outcome and `retry_safe=false` unless the registry proves an idempotent compensating action. A PROD mutation is allowed only by an explicit policy for the exact operation and a consumed ActionApprovalGate; non-PROD test-data mutation may be delegated inside an authorized fixture lease. +Authorization is evaluated per run and step as `environment_class + scenario risk profile + action mutation profile`. Read-only PROD steps require the ScenarioExecution approval policy. Every mutation requires the immutable 038 mutation contract, scoped target keys, precondition evidence, side-effect identity, cleanup/reconciliation outcome and `retry_safe=false` unless the registry proves an idempotent compensating action. Mutating browser steps in PROD are prohibited. Non-PROD test-data mutation may be delegated only inside an authorized fixture lease; this mutation policy is separate from PROD execution approval. ## Immutable Execution Snapshot diff --git a/specs/044-dashboard-scenario-execution/plan.md b/specs/044-dashboard-scenario-execution/plan.md index a2add64a7..b3cbfd505 100644 --- a/specs/044-dashboard-scenario-execution/plan.md +++ b/specs/044-dashboard-scenario-execution/plan.md @@ -4,7 +4,7 @@ ## Summary -Implement a deterministic ScenarioRun/ScenarioStepRun execution engine that walks the validated DashboardTestScenario DAG, dispatches each registered `{tool, action}` to a typed executor (reusing 037/038/036/040 services), and manages the full run lifecycle including human-checkpoint pause/resume, cancel, retry, and timeout. RunnerPlan is derived at launch; `runner.plan.json` is a diagnostic/reference artifact. Agent is not in the hot path. +Implement deterministic ScenarioRun/ScenarioStepRun orchestration for the validated immutable Verification Program, dispatching each registered `{tool, action}` to a typed executor (reusing 037/038/036/040 services) and handling lifecycle. RunnerPlan is derived at launch; `runner.plan.json` is diagnostic/reference only. Explicit AgentEvaluationSpec is bounded inside a typed step, never orchestration. ## Technical Context diff --git a/specs/044-dashboard-scenario-execution/research.md b/specs/044-dashboard-scenario-execution/research.md index 43cfa0b9d..0db14e3e7 100644 --- a/specs/044-dashboard-scenario-execution/research.md +++ b/specs/044-dashboard-scenario-execution/research.md @@ -4,7 +4,7 @@ ## R1. Execution model — deterministic backend runner -**Decision**: A deterministic `ScenarioRunner` walks the DAG in topological order and dispatches each step by tool to a typed executor. The agent is not in the hot path. +**Decision**: Deterministic `ScenarioRunner` orchestration walks the immutable Verification Program in topological order and dispatches registered executors. A declared bounded AgentEvaluationSpec may reason inside one typed step, but cannot own orchestration or rewrite program content. **Rationale**: Matches 038 determinism (direct LLM-to-code rejected) and 040's non-agent RunnerPool pattern. Execution must be reproducible; an LLM per step breaks that. diff --git a/specs/044-dashboard-scenario-execution/spec.md b/specs/044-dashboard-scenario-execution/spec.md index 8181f7677..e76b4d08e 100644 --- a/specs/044-dashboard-scenario-execution/spec.md +++ b/specs/044-dashboard-scenario-execution/spec.md @@ -6,8 +6,8 @@ @RELATION DEPENDS_ON -> [SupersetBaselineEngine.Spec] @RELATION DEPENDS_ON -> [ScenarioRegistry.Spec] @RELATION DEPENDS_ON -> [DashboardScenarioModel.Spec] -@RATIONALE 038 generates a compiled definition but no engine reads or executes the graph. A ScenarioRun is a first-class entity independent of AgentRun (creation) and VerificationRun (release pipeline). Deterministic execution reuses 037 executors and 040 RunnerPool patterns; the RunnerPlan is derived from the revision at run start (stored runner.plan.json is a reference artifact); the agent is NOT in the hot path — it participates only at HumanCheckpoints and result interpretation. -@REJECTED Agent-orchestrated step execution — rejected because each step would be an LLM call (nondeterminism/drift), exactly what 038 forbids; execution is deterministic. +@RATIONALE 038 generates an immutable Verification Program IR. A ScenarioRun is a first-class entity independent of AgentRun and VerificationRun. Orchestration and deterministic program executors remain deterministic; a revision may additionally declare bounded AgentEvaluation steps whose typed outputs are resolved by DecisionPolicy, never by free-form orchestration. +@REJECTED Agent-orchestrated step execution — rejected because each step would be an LLM call and could rewrite program flow. Explicit versioned AgentEvaluationSpec inside a deterministic step boundary is allowed. @REJECTED Reusing AgentRun as the execution run — rejected because AgentRun is the creation-process run; ScenarioRun is the created-test execution. Reusing VerificationRun — rejected because it is release-pipeline category verification, not arbitrary-DAG execution. @REJECTED `human` as a dispatched executor — rejected; it is a runner-lifecycle suspend/resume control primitive, not a side-effect executor. @@ -40,7 +40,7 @@ **Independent Test**: Execute a fixture scenario with browser/superset/xlsx/assertion/screenshot steps and verify each step dispatches to the correct executor and records a ScenarioStepRun. **Acceptance**: -1. **Given** the runner walks the DAG in topological order **When** each step is reached **Then** `ScenarioStep.Dispatch` routes by `step.tool` to the typed executor (browser→ScreenshotService, superset_api→037, xlsx→parser+norm, assertion→037 comparison, screenshot→038 capture, report/artifact→036). +1. **Given** the runner walks the DAG in topological order **When** each step is reached **Then** `ScenarioStep.Dispatch` routes by the version-pinned 038 ActionRegistry to a typed executor (browser→BrowserExecutor/Playwright, superset_api→037, sql_evidence→Superset SQL Lab adapter, transform/assertion→bounded 038 program engines, agent_evaluation→bounded evaluation adapter, xlsx→parser+norm, screenshot→ScreenshotService). 2. **Given** a step produces an output ref **When** a dependent step consumes it **Then** the ref is bound from the producer's outputs. 3. **Given** a step fails **When** execution continues **Then** dependents are blocked per failure policy and the run records the failure. @@ -102,8 +102,8 @@ ### Functional - **SCEX-FR-001**: A ScenarioRun MUST be a first-class entity pinned to `scenario_id` (UUID) + `revision_id` (UUID) + `content_hash`, environment, parameter snapshot, and optional `agent_run_id`/`verification_run_id` provenance. -- **SCEX-FR-002**: The runner MUST walk the DAG in topological order and dispatch each step by `step.tool` to a typed executor (browser, superset_api, xlsx, assertion, screenshot, report, artifact). -- **SCEX-FR-003**: Execution MUST be deterministic and MUST NOT require an LLM call per step; the agent is not in the hot path. +- **SCEX-FR-002**: The runner MUST walk the DAG in topological order and dispatch each step only through the version-pinned 038 ActionRegistry. Browser, Superset API/SQL Lab, XLSX, bounded transform/assertion, screenshot, report and artifact executors are typed; unknown `{tool,action}` is rejected. +- **SCEX-FR-003**: Scenario orchestration MUST be deterministic and MUST NOT generate/rewrite SQL, DSL, assertions, graph or executor order at runtime. A declared AgentEvaluationSpec MAY run inside its bounded step contract; it is not agent-per-step orchestration. - **SCEX-FR-004**: A `human` step MUST suspend the run (status `waiting_human`), persist full HumanCheckpoint state (eligibility, evidence, expiry, CAS decision version), and internally resume dependents after its one-time disposition. HumanCheckpoint is distinct from ActionApprovalGate. - **SCEX-FR-004a**: A revision containing a `human` step MUST be `manual_run_only`. Scheduled, trigger, deploy, ETL and API execution are rejected before run creation; a HumanCheckpoint is never skipped to obtain an automatic PASS. - **SCEX-FR-005**: The lifecycle MUST include pending_approval, queued, running, waiting_human, blocked, cancel_requested, cancelled, passed, failed, inconclusive; cancel drains in-flight within a bounded window. @@ -113,8 +113,11 @@ - **SCEX-FR-009**: Executors MUST reuse 037 (metric_executor/comparison), 038 (capture), 036 (evidence/artifacts/HITL), and existing browser/xlsx infra; a second Playwright/LLM/SQL stack is forbidden. - **SCEX-FR-010**: `human` is a runner-lifecycle control, not a dispatched executor; the executor registry covers the other seven tools. - **SCEX-FR-011**: Failed, inconclusive and blocked runs MUST emit idempotent 036 InvestigationSignals carrying immutable run/evidence provenance; 047 creates/updates the Queue/Episode. They MUST NOT automatically start a chat, an AgentRun, or a remediation action. -- **SCEX-FR-012**: An opened InvestigationCase MAY use the agent to construct diagnostic runs and controlled experiments under delegated policy. The agent is never in the DAG hot path and cannot bypass executor contracts, runner lifecycle, capacity, mutation policy, or a required ActionApprovalGate. +- **SCEX-FR-012**: An opened InvestigationCase MAY use the agent to construct diagnostic runs and controlled experiments under delegated policy. Agent work cannot bypass executor contracts, runner lifecycle, capacity, mutation policy or a required ActionApprovalGate; declared AgentEvaluationSpec is the only permitted runtime reasoning boundary. - **SCEX-FR-013**: A HumanCheckpoint remains a manual-run-only analyst decision. The agent may summarize evidence but MUST NOT consume the checkpoint, choose its disposition, or turn it into scheduled automation. +- **SCEX-FR-014**: `SqlEvidenceExecutor` MUST execute exactly the immutable 038 SqlEvidenceSpec through Superset backend/SQL Lab with pinned database identity, ExecutionPrincipal and RLS/security fingerprint. Runtime only supplies typed ParameterBindings and may not alter SQL, relation, projection, JOIN or WHERE. +- **SCEX-FR-015**: `AgentEvaluation` MUST be a separate immutable runtime record and DecisionPolicy MUST deterministically map it plus deterministic evidence to StepOutcome. A bare model verdict never directly sets ScenarioResult. +- **SCEX-FR-016**: Browser actions and mutation safety MUST use the same versioned 038 ActionRegistry/mutation contract. Mutating browser steps in PROD are prohibited; test-data mutation needs fixture scope, record keys, side-effect/retry and cleanup policy independent of PROD approval. ### Key Entities @@ -137,7 +140,7 @@ ### Session 2026-08-07 -- Q: Is the agent the runner? → A: No. The backend runner is deterministic. The agent acts before/after runs through case-owned diagnostic/remediation actions, never in the executor hot path. +- Q: Is the agent the runner? → A: No. The backend runner orchestrates deterministically. The agent authors before/after runs and may reason only inside a declared bounded AgentEvaluationSpec. - Q: Is human an executor? → A: No. It is a runner-lifecycle suspend/resume control; the executor registry covers browser/superset_api/xlsx/assertion/screenshot/report/artifact. - Q: How does this differ from VerificationRun/AgentRun? → A: ScenarioRun executes the user-created DashboardTestScenario DAG; AgentRun is the creation run; VerificationRun is release-pipeline category verification. Three distinct run concepts. diff --git a/specs/045-dashboard-run-monitor/data-model.md b/specs/045-dashboard-run-monitor/data-model.md index 769279d5c..9d562f763 100644 --- a/specs/045-dashboard-run-monitor/data-model.md +++ b/specs/045-dashboard-run-monitor/data-model.md @@ -30,4 +30,6 @@ Human checkpoint evidence (screenshot, VLM finding) bound to a gate; disposition Failure/blocked/inconclusive results show a linked Investigation Queue item when one exists. `Investigate with agent` opens the persistent 047 case workspace; it is not a modal, does not change the run, and never consumes a HumanCheckpoint. +`AgentEvaluationPanel` is a result subprojection, not a chat surface: evaluation id, logical_step_id, model/prompt version, evidence manifest, typed verdict/confidence/reason codes and DecisionPolicy-derived StepOutcome. The UI reads this only from typed 044 result/SSE schemas. + #endregion ScenarioRunMonitor.DataModel diff --git a/specs/045-dashboard-run-monitor/spec.md b/specs/045-dashboard-run-monitor/spec.md index 987a94c65..dafcf3ca7 100644 --- a/specs/045-dashboard-run-monitor/spec.md +++ b/specs/045-dashboard-run-monitor/spec.md @@ -122,6 +122,8 @@ - **RUNMON-FR-009**: A Global Run Operations Center MUST list all runs (active/queued/waiting-human/failed/recent) with filters and a "Waiting for me" view for pending human checkpoints. - **RUNMON-FR-010**: Run Configuration MUST match the 044 start contract (environment, revision, release, baseline_set, parameters, execution_toggles for optional evidence only); mandatory graph steps MUST NOT be toggleable off. - **RUNMON-FR-011**: Failed, blocked and inconclusive results MUST expose their Investigation Queue item and an explicit "Investigate with agent" transition into the persistent 047 case workspace. Opening it never mutates run truth or auto-executes tools. +- **RUNMON-FR-012**: Live Monitor MUST render only typed 044 ScenarioRunEvent payloads (including approval, evidence, agent-evaluation, checkpoint and terminal events), never parse assistant prose. Result and step inspection MUST render StepOutcome, EvidenceReference and AgentEvaluationSummary separately. +- **RUNMON-FR-013**: An AgentEvaluation display MUST show declared prompt/model version, evidence manifest, verdict, confidence, reason codes and the DecisionPolicy-derived StepOutcome. It must never present the raw model verdict as ScenarioResult. - **RUNMON-FR-012**: Run configuration, gate decisions, conflict recovery and investigation entry MUST use persistent pages/panels or inline cards; modal/dialog interaction MUST NOT be required. ### Key Entities diff --git a/specs/046-dashboard-scenario-automation/data-model.md b/specs/046-dashboard-scenario-automation/data-model.md index 7ecbbc5de..396d2093e 100644 --- a/specs/046-dashboard-scenario-automation/data-model.md +++ b/specs/046-dashboard-scenario-automation/data-model.md @@ -34,6 +34,6 @@ Layered retention independent of the analytics minimum history window: run metad ## RunBinding -A triggered run pins scenario_id + revision_id + environment_id + target snapshot + execution principal + server-owned trigger source (from 044). Concurrency bucket (for capacity) may be environment+workload class; dedup identity is `canonical_execution_request_hash`, or for an event trigger `(source_type, source_event_id, scenario_id)`. External API triggers require an `Idempotency-Key`; same key/hash returns the same run, while a changed canonical request returns 409. +A triggered run pins scenario_id + revision_id + immutable Verification Program/content hash + environment_id + target snapshot + execution principal + server-owned trigger source (from 044). Concurrency bucket (for capacity) may be environment+workload class; dedup identity is `canonical_execution_request_hash`, or for an event trigger `(source_type, source_event_id, scenario_id)`. External API triggers require an `Idempotency-Key`; same key/hash returns the same run, while a changed canonical request returns 409. Automation cannot supply or rewrite runtime SQL/DSL/program content. #endregion ScenarioAutomation.DataModel diff --git a/specs/047-dashboard-scenario-analytics/contracts/openapi.yaml b/specs/047-dashboard-scenario-analytics/contracts/openapi.yaml index 8170bcfb3..265e7e9d6 100644 --- a/specs/047-dashboard-scenario-analytics/contracts/openapi.yaml +++ b/specs/047-dashboard-scenario-analytics/contracts/openapi.yaml @@ -197,11 +197,15 @@ components: product_health: { type: string, enum: [healthy, attention, unknown] } scenario_test_health: { type: string, enum: [healthy, attention, unknown] } infrastructure_health: { type: string, enum: [healthy, attention, unknown] } + agent_evaluation_health: { type: string, enum: [healthy, attention, unknown] } overall_attention: { type: string, enum: [healthy, attention, unknown] } success_rate: { type: number } flaky_ratio: { type: number } infra_failure_ratio: { type: number } inconclusive_ratio: { type: number } + agent_disagreement_ratio: { type: number } + low_confidence_ratio: { type: number } + model_instability_ratio: { type: number } generated_at: { type: string, format: date-time } confidence: { type: string, enum: [insufficient_history, low, high] } Trends: diff --git a/specs/047-dashboard-scenario-analytics/data-model.md b/specs/047-dashboard-scenario-analytics/data-model.md index 637e75b95..3e28a9e41 100644 --- a/specs/047-dashboard-scenario-analytics/data-model.md +++ b/specs/047-dashboard-scenario-analytics/data-model.md @@ -29,15 +29,15 @@ The projection is versioned/CAS and append-only audited. It is never a free-stan ## FlakinessSignal — deterministic rules -Window: last N eligible runs (default 30). Eligibility per step: same environment class, same `logical_step_id`, same `compatibility_family`, same baseline family, and comparable target/principal context. Infrastructure outcomes, cancelled and inconclusive runs are excluded from the pass/fail denominator. A step is flaky only when pass and fail are both observed, a pass occurs after the first failure, at least two pass/fail state transitions occur, and failure ratio is within `(X,Y)` (default 5%–50%). A one-way PASS→FAIL change is a regression signal, never flaky. +Window: last N eligible runs (default 30). Eligibility per deterministic step: same environment class, same `logical_step_id`, same `compatibility_family`, same baseline family, and comparable target/principal context. Infrastructure outcomes, cancelled and inconclusive runs are excluded from the deterministic pass/fail denominator. AgentEvaluation verdict variation is never counted as deterministic flakiness. It is separately aggregated as evaluation disagreement, low-confidence rate and model instability keyed by AgentEvaluationSpec/model/prompt version. A deterministic step is flaky only when pass and fail are both observed, a pass occurs after the first failure, at least two pass/fail state transitions occur, and failure ratio is within `(X,Y)` (default 5%–50%). A one-way PASS→FAIL change is a regression signal, never flaky. Fields: scenario_id, logical_step_id, compatibility_family, context_key, window, eligible_runs, total_runs, failures, flaky_runs, ratio, is_flaky, excluded_outcomes. ## ScenarioHealth — deterministic, contextual -`AnalyticsContextKey` is exactly the server-derived 044 SHA-256 over `environment_class + compatibility_family + baseline_family + dashboard_release_id + dashboard_fingerprint + dataset_lineage_fingerprint + execution_principal_fingerprint`. Fields: scenario_id, environment_class, context_key (=AnalyticsContextKey), window, product_health, scenario_test_health, infrastructure_health, overall_attention, success_rate, flaky_ratio, infra_failure_ratio, inconclusive_ratio, most_unstable_step, generated_at, confidence. +`AnalyticsContextKey` is exactly the server-derived 044 SHA-256 over `environment_class + compatibility_family + baseline_family + dashboard_release_id + dashboard_fingerprint + dataset_lineage_fingerprint + execution_principal_fingerprint`. Fields: scenario_id, environment_class, context_key (=AnalyticsContextKey), window, product_health, scenario_test_health, infrastructure_health, agent_evaluation_health, overall_attention, success_rate, flaky_ratio, infra_failure_ratio, inconclusive_ratio, agent_disagreement_ratio, low_confidence_ratio, model_instability_ratio, most_unstable_step, generated_at, confidence. -Product health consumes product/data regressions; scenario-test health consumes scenario bugs, stale baselines and flaky signals; infrastructure health consumes typed infrastructure outcomes. Untriaged failures raise `overall_attention=attention` with cause `unknown`, but do not fabricate a classification. 042 displays only `overall_attention`, or `unknown` if analytics is unavailable. +Product health consumes product/data regressions; scenario-test health consumes scenario bugs, stale baselines and deterministic flaky signals; infrastructure health consumes typed infrastructure outcomes; agent-evaluation health consumes disagreement/low-confidence/model-instability. Untriaged failures raise `overall_attention=attention` with cause `unknown`, but do not fabricate a classification. 042 displays only `overall_attention`, or `unknown` if analytics is unavailable. ## RecurringFailureGroup and FailureEpisode — immutable identity diff --git a/specs/047-dashboard-scenario-analytics/spec.md b/specs/047-dashboard-scenario-analytics/spec.md index 54f02f198..a79438a48 100644 --- a/specs/047-dashboard-scenario-analytics/spec.md +++ b/specs/047-dashboard-scenario-analytics/spec.md @@ -74,7 +74,7 @@ - **SCAN-FR-002**: An analyst MUST be able to open a queue item into a durable InvestigationCase with chat, immutable evidence snapshot, hypotheses, AgentAction timeline, linked AgentRuns, approvals, verification and final disposition. The agent may execute only delegated/policy-authorized actions. - **SCAN-FR-003**: Triage MUST be a CAS/audited compact projection of a case disposition; the historical RunResult stays immutable truth. - **SCAN-FR-004**: The system MUST detect flakiness per step using same environment class, logical_step_id, compatibility_family, baseline family and comparable context. It requires a post-failure pass and two result transitions; infra/cancelled/inconclusive outcomes are excluded from the pass/fail denominator. -- **SCAN-FR-005**: Health MUST be contextual and computed separately for product, scenario-test and infrastructure health plus overall attention. +- **SCAN-FR-005**: Health MUST be contextual and computed separately for product, scenario-test, infrastructure and agent-evaluation health plus overall attention. Agent verdict variance MUST NOT be classified as deterministic flakiness. - **SCAN-FR-006**: Health MUST feed the 042 registry health badge. - **SCAN-FR-007**: The system MUST render trends and group recurring failures by immutable compatibility-scoped fingerprint, never including triage classification. - **SCAN-FR-008**: A matching failure inside an active FailureEpisode is deduplicated; a matching failure after resolution MUST open a new alertable FailureEpisode and Queue item. @@ -83,6 +83,7 @@ - **SCAN-FR-011**: Investigation, approval, conflict recovery and closure MUST use persistent case workspaces and inline cards; modal/dialog interaction MUST NOT be required. - **SCAN-FR-012**: A case may become `resolved` only with verification evidence and completed reconciliation. `accepted` requires an analyst-recorded accepted-risk/won't-fix/duplicate rationale. A new matching active signal MUST reopen the work (or create a new case for a new FailureEpisode). - **SCAN-FR-013**: Health and flakiness eligibility MUST use the exact 044 `AnalyticsContextKey`; grouping by environment or revision alone is forbidden. +- **SCAN-FR-014**: Analytics MUST distinguish deterministic failure, agent-evaluation disagreement, low confidence, model instability, infrastructure failure, scenario-definition defect and product regression. Recurring fingerprints remain immutable raw evidence fingerprints and MUST NOT include a human/agent classification. ### Key Entities diff --git a/specs/REVIEW-042-047-CLOSURE.md b/specs/REVIEW-042-047-CLOSURE.md index 1c40dc47b..83ca6e20b 100644 --- a/specs/REVIEW-042-047-CLOSURE.md +++ b/specs/REVIEW-042-047-CLOSURE.md @@ -10,12 +10,12 @@ ### #1. Scenario never reaches the Registry after creation **Problem**: 042 has `GET /scenarios` but no `POST /scenarios`; 039 Save → 042 registration gap. -**Resolution**: Add `CreateScenario` transaction to 042: input `{validated_revision, draft_pack, owner, dashboard}`, atomic `ScenarioRegistryEntry + ScenarioRevision #1 + artifact materialization binding`, output `{scenario_id, revision_hash}`. 039 Save calls it. Add task + route + spec story. +**Resolution**: Add `CreateScenario` transaction to 042: input `{validated_revision, draft_pack, owner, dashboard}`, atomic `ScenarioRegistryEntry + ScenarioRevision #1 + artifact materialization binding`, output `{scenario_id, revision_id + content_hash}`. 039 Save calls it. Add task + route + spec story. **[FILE]** `042/contracts/openapi.yaml` (POST /scenarios), `042/data-model.md` (CreateScenario), `042/tasks.md` (T0xx), `042/spec.md` (US: Save→Register). ### #2. ScenarioRevision ↔ runner.plan.json can diverge **Problem**: r18 graph edited, runner.plan.json still r17 → run claims r18 but executes r17. Kills reproducibility. -**Resolution**: **RunnerPlan is a deterministic derivation from ScenarioRevision at run start**, never a stored runtime source of truth. Each revision stores `runner_plan_hash` (derived). Run refuses if `runner_plan.scenario_revision_hash != selected revision`. +**Resolution**: **RunnerPlan is a deterministic derivation from ScenarioRevision at run start**, never a stored runtime source of truth. The run compares its derived revision/program/action-registry hashes to the selected revision before execution. **[FILE]** `044/data-model.md`, `044/contracts/modules.md` (RunnerPlan derivation), `042/data-model.md` (revision carries runner_plan_hash). ### #3. Wrong reuse of 036 ApprovalGate for human checkpoint / ScenarioRun @@ -32,7 +32,7 @@ ### #5. Backend contract for 045 missing (history/result/compare/retry/SSE) **Problem**: 045 UI promises RunHistory/RunComparison but 044 has no `GET /scenarios/{id}/runs`, `GET /scenario-runs/{id}/result`, `GET /scenario-runs/compare`, step retry, or an SSE event schema. -**Resolution**: Add to 044 OpenAPI: `GET /scenarios/{id}/runs`, `GET /scenario-runs/{id}/result`, `GET /scenario-runs/compare?a=&b=`, `POST /scenario-runs/{id}/steps/{step}/retry`, and an **SSE event contract** (id, sequence, event_type, run_id, step_id?, attempt?, occurred_at, payload; Last-Event-ID replay, heartbeat, terminal close) — reuse the 036 AgentRunEvent pattern. +**Resolution**: Add to 044 OpenAPI: `GET /scenarios/{id}/runs`, `GET /scenario-runs/{id}/result`, `GET /scenario-runs/compare?a=&b=`, `POST /scenario-runs/{id}/steps/{logical_step_id}/retry`, and an **SSE event contract** (id, sequence, event_type, run_id, logical_step_id?, attempt?, occurred_at, payload; Last-Event-ID replay, heartbeat, terminal close) — reuse the 036 AgentRunEvent pattern. **[FILE]** `044/contracts/openapi.yaml`, `044/contracts/modules.md`, `044/spec.md`. ### #6. Durable worker semantics (lease/heartbeat/idempotency/crash recovery) @@ -46,7 +46,7 @@ **[FILE]** `043/data-model.md`, `043/contracts/modules.md`, `043/contracts/openapi.yaml`, `043/tasks.md`. ### #8. Stable logical step identity -**Problem**: step_id = phase+case+action+ordinal can shift on edit; breaks 045 compare + 047 flakiness across revisions. +**Problem**: an ordinal-derived step identity can shift on edit; breaks 045 compare + 047 flakiness across revisions. **Resolution**: `logical_step_id = UUID` (immutable) + `step_position` (mutable) + `step_content_hash` (mutable). Analytics key on logical_step_id. **[FILE]** `044/data-model.md`, `042/data-model.md`, `045/data-model.md`, `047/data-model.md`. @@ -99,7 +99,7 @@ **[FILE]** `047/contracts/openapi.yaml`. ### #25. Strict flakiness rules -**Resolution**: Define window + eligibility (same environment class, logical step, major revision, baseline family) + flaky iff pass AND fail observed AND failure ratio within (X,Y) AND infra failures excluded. +**Resolution**: Define window + eligibility (same environment class, logical step, compatibility_family, baseline family) + flaky iff pass AND fail observed AND failure ratio within (X,Y) AND infra failures excluded. **[FILE]** `047/data-model.md`. ### #26. Immutable recurring fingerprint @@ -128,7 +128,7 @@ Reconciled the stale 038 core with 042–047. Bindings applied to **038** and all **normative** (spec/research/checklists/ux/prototype): - **#1/#7** 038 identity: `scenario_id` slug → **`scenario_key`** (semantic); `scenario_id` (UUID) + `revision_id` (UUID) assigned by 042 at Save; compiler emits `content_hash` only. -- **#2/#8/#11** Replaced `revision_hash`/`parent_revision_hash`/`scenario_revision_hash` with `revision_id`/`content_hash`/`parent_revision_id` across 042/043/044/045 (openapi, data-model, modules, spec, research, checklists, tasks). 043 unified fully. +- **#2/#8/#11** Replaced `revision_id + content_hash`/`parent_revision_id + content_hash`/`scenario_revision_id + scenario_content_hash` with `revision_id`/`content_hash`/`parent_revision_id` across 042/043/044/045 (openapi, data-model, modules, spec, research, checklists, tasks). 043 unified fully. - **#3/#4/#13** 038 step schema: added `logical_step_id` (UUID, immutable) + `step_key`/`position`/`step_content_hash`; runtime `VlmFinding`/`HumanDisposition` moved to 044; `VlmAnalysisSpec`/`ScreenshotCaptureSpec` stay in 038. - **#5/#6/#12** 038 runtime capture/VLM/disposition endpoints marked `deprecated`→410 MOVED_TO_044; `agent_run_id` removed from compile/capture (provenance optional, source_type: agent_run|editor|migration|api). Runtime evidence = `Artifact(owner_type=scenario_run)`, never authoring DraftPack. - **#8** `false_positive` vocabulary unified; `dismiss` removed (037 dispositions split; 047 triage split investigation_status/classification/resolution). @@ -139,13 +139,13 @@ Reconciled the stale 038 core with 042–047. Bindings applied to **038** and al Closed the P0 "prose-fixed, contracts-stale" gap by validating the machine-readable source of truth: - **OpenAPI YAML**: fixed 3 syntax failures (`038` line 472 unquoted `:`, `043` line 32 flow `{...}`, `046` line 23 & 51 `Schedule[]`/`TriggerRule[]`). All 036–047 OpenAPI now parse. -- **038 JSON Schema** (`dashboard-test-scenario.schema.json`): migrated to canonical identity — `scenario_id`→`scenario_key`, `revision_hash`→`content_hash`; step `id`→`logical_step_id`+`step_key`+`position`+`step_content_hash`; `vlm_analysis`→`vlm_analysis_spec`; `affected_step_ids`→`affected_logical_step_ids`; `coverage.step_ids`→`logical_step_ids`; disposition `dismissed`→`false_positive`. +- **038 JSON Schema** (`dashboard-test-scenario.schema.json`): migrated to canonical identity — `scenario_id`→`scenario_key`, `revision_id + content_hash`→`content_hash`; step `id`→`logical_step_id`+`step_key`+`position`+`step_content_hash`; `vlm_analysis`→`vlm_analysis_spec`; `affected_step_ids`→`affected_logical_step_ids`; `coverage.step_ids`→`logical_step_ids`; disposition `dismissed`→`false_positive`. - **038 fixtures** (`fixtures/api/*.json`, 6): migrated to canonical identity via transform script; fixed 66-char fingerprint; **6/6 validate against the JSON Schema** (jsonschema). - **038 validation.md**: regenerated cleanly (was self-contradictory PASS). Compiler-scope PASS; runtime gated by 044. -- **039 prototype**: `scenario_id`→`scenario_key` in JSON display; **038 prototype** `revision_hash`→`content_hash`. +- **039 prototype**: `scenario_id`→`scenario_key` in JSON display; **038 prototype** `revision_id + content_hash`→`content_hash`. - **New tool** `reconcile_contracts.py`: parses all OpenAPI/JSON, checks forbidden old-identity tokens in machine files (with compiled-output scoping for `scenario_id`), validates fixtures vs schema. Gate: **PASS (0 findings)** on 038–047 and `all`. -**Follow-up items (documented, not silently dropped)**: #5 saga/outbox for CreateScenario atomicity, #6 runner_plan_hash→ExecutionTemplateHash vs per-run RunnerPlanHash, #7 036 gate generalisation, #8 PROD approval lifecycle, #9 logical_step_id stability (position in key), #10 ParameterDefinition vs ParameterBinding split, #12 execution principal/RLS, #13 browser replay on resume, #16 metadata-vs-executable revision, #20-32 operational/security semantics. These are prose-level design decisions for future passes; the machine contracts are now internally consistent. +**Follow-up items** listed here were resolved by the 2026-08-10/11 canonical contracts below; implementation must use those newer sections as normative source. ## Status - [x] Closure doc written @@ -184,3 +184,17 @@ The second HOLD review is resolved with these binding decisions: 10. 046 separates dedup identity from capacity concurrency; 042 consumes (not derives) 047 health; 047 uses compatibility family and alertable FailureEpisodes. `reconcile_contracts.py` now enforces 20 final-closure clauses in addition to parsability, schemas, semantic path parameters, fixtures, identity drift and rejected-decision patterns. The stale 041 task digest has also been corrected; future validation-table generation remains a separate tooling improvement. + +## Verification Program Reconciliation (2026-08-11) + +The canonical system model is **agent-authored, deterministically executed verification programs**: + +- 038 now makes `VerificationProgram` required, content-hashed IR: navigation, evidence, bounded transforms, assertions and explicitly declared semantic evaluation. +- Source-mart `SqlEvidenceSpec` is permitted only during authoring/edit/revalidation/investigation proposal, passes AST/policy/schema/preview validation and is executed at runtime unchanged through the Superset SQL Lab adapter with typed bindings and pinned security context. +- `TransformSpec` is bounded DSL; `ComparisonSpec` is first-class; arbitrary code and runtime SQL/DSL rewrite remain forbidden. +- 044 orchestration stays deterministic but supports bounded, versioned `AgentEvaluationSpec`; immutable AgentEvaluation evidence becomes StepOutcome only through DecisionPolicy. +- 039 previews program structure, evidence sources and SQL/DSL/assertion/evaluation diffs; ChangeRequestContext is explicit and required for compilation. +- 045 renders typed events/results, including DecisionPolicy outcomes and AgentEvaluation summaries, without prose parsing. +- 047 distinguishes deterministic failure from model disagreement, low confidence and model instability; only deterministic outcomes feed deterministic flakiness. + +**Generated gate command:** `backend/.venv/bin/python reconcile_contracts.py 036-047` validates YAML/JSON, OpenAPI paths, JSON fixtures, legacy drift, Verification Program schema/action registry and runtime mutation boundary. Current result: **PASS (0 findings)**.