Files
ss-tools/specs/038-dashboard-scenario-model/contracts/dashboard-test-scenario.schema.json

864 lines
23 KiB
JSON

{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://superset-tools.local/schemas/dashboard-test-scenario-v1.json",
"title": "DashboardTestScenario",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"compiler_version",
"template_version",
"action_registry_version",
"scenario_key",
"content_hash",
"dashboard_context",
"objective",
"input_fingerprints",
"parameters",
"verification_program",
"phases",
"steps",
"outputs",
"artifact_plan",
"checklist_coverage",
"warnings",
"blockers",
"risk_summary"
],
"properties": {
"schema_version": {
"const": 1
},
"compiler_version": {
"type": "string"
},
"template_version": {
"type": "string"
},
"action_registry_version": {
"type": "string",
"description": "Version-pinned 038 ActionRegistry used to validate every {tool, action}."
},
"scenario_key": {
"type": "string",
"pattern": "^[a-z0-9][A-Za-z0-9_-]{2,127}$",
"description": "Semantic identity (dashboard key + normalized objective slug). Entity scenario_id (UUID) is assigned by 042 at Save."
},
"content_hash": {
"$ref": "#/$defs/sha256",
"description": "SHA-256 of the canonical executable graph; timestamps/display-only excluded."
},
"dashboard_context": {
"type": "object",
"additionalProperties": false,
"required": [
"environment_id",
"dashboard_id",
"dashboard_name"
],
"properties": {
"environment_id": {
"type": "string"
},
"dashboard_id": {
"type": "integer",
"minimum": 1
},
"dashboard_name": {
"type": "string"
},
"route": {
"type": "string"
}
}
},
"objective": {
"type": "object",
"additionalProperties": false,
"required": [
"goal",
"selected_case_ids"
],
"properties": {
"goal": {
"type": "string",
"minLength": 1,
"maxLength": 2000
},
"selected_case_ids": {
"type": "array",
"uniqueItems": true,
"items": {
"$ref": "#/$defs/checklistId"
}
},
"rationale": {
"type": [
"string",
"null"
],
"maxLength": 4000
}
}
},
"input_fingerprints": {
"type": "object",
"additionalProperties": false,
"required": [
"query_model",
"checklist_catalog",
"baseline_version",
"parameters"
],
"properties": {
"query_model": {
"$ref": "#/$defs/sha256"
},
"checklist_catalog": {
"$ref": "#/$defs/sha256"
},
"baseline_version": {
"$ref": "#/$defs/sha256"
},
"parameters": {
"$ref": "#/$defs/sha256"
}
}
},
"parameters": {
"type": "array",
"items": {
"$ref": "#/$defs/parameter"
}
},
"verification_program": {
"$ref": "#/$defs/verificationProgram",
"description": "Canonical immutable Verification Program IR; included in content_hash."
},
"phases": {
"type": "array",
"items": {
"enum": [
"setup",
"interact",
"observe",
"assert",
"evidence",
"report"
]
}
},
"steps": {
"type": "array",
"maxItems": 100,
"items": {
"$ref": "#/$defs/step"
}
},
"outputs": {
"type": "array",
"items": {
"$ref": "#/$defs/ref"
}
},
"artifact_plan": {
"type": "array",
"items": {
"$ref": "#/$defs/artifactPlan"
}
},
"checklist_coverage": {
"type": "array",
"items": {
"$ref": "#/$defs/coverage"
}
},
"warnings": {
"type": "array",
"items": {
"$ref": "#/$defs/finding"
}
},
"blockers": {
"type": "array",
"items": {
"$ref": "#/$defs/finding"
}
},
"risk_summary": {
"type": "object",
"additionalProperties": true
}
},
"$defs": {
"sha256": {
"type": "string",
"pattern": "^[a-f0-9]{64}$"
},
"checklistId": {
"type": "string",
"pattern": "^(B0[1-9]|C0[1-7]|T0[1-3])$"
},
"parameter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"label",
"type",
"required",
"source",
"affected_logical_step_ids"
],
"properties": {
"name": {
"type": "string",
"pattern": "^[a-z][a-z0-9_]{1,63}$"
},
"label": {
"type": "string"
},
"type": {
"enum": [
"string",
"integer",
"decimal",
"boolean",
"date",
"datetime",
"enum",
"string_list",
"baseline_choice",
"selector_hint"
]
},
"required": {
"type": "boolean"
},
"default": {},
"source": {
"type": [
"string",
"null"
]
},
"validation": {
"type": [
"object",
"null"
]
},
"affected_logical_step_ids": {
"type": "array",
"uniqueItems": true,
"description": "logical_step_id values affected by this parameter.",
"items": {
"type": "string",
"format": "uuid"
}
}
}
},
"verificationProgram": {
"type": "object",
"additionalProperties": false,
"required": ["navigation_program", "evidence_program", "transformation_program", "assertion_program", "semantic_evaluation_program"],
"properties": {
"navigation_program": { "type": "array", "items": { "$ref": "#/$defs/programRef" } },
"evidence_program": { "type": "array", "items": { "oneOf": [{ "$ref": "#/$defs/programRef" }, { "$ref": "#/$defs/sqlEvidenceSpec" }] } },
"transformation_program": { "type": "array", "items": { "$ref": "#/$defs/transformSpec" } },
"assertion_program": { "type": "array", "items": { "$ref": "#/$defs/assertionSpec" } },
"semantic_evaluation_program": { "type": "array", "items": { "$ref": "#/$defs/agentEvaluationSpec" } }
}
},
"programRef": { "type": "object", "additionalProperties": false, "required": ["logical_step_id"], "properties": { "logical_step_id": { "type": "string", "format": "uuid" } } },
"sqlEvidenceSpec": {
"type": "object", "additionalProperties": false,
"required": ["snippet_id", "logical_step_id", "connection_ref", "database_identity", "sql_template", "sql_hash", "parameter_definitions", "expected_output_schema", "relation_refs", "execution_limits", "generation_provenance", "validation_result"],
"properties": {
"snippet_id": { "type": "string" }, "logical_step_id": { "type": "string", "format": "uuid" }, "connection_ref": { "type": "string" }, "database_identity": { "type": "string" }, "sql_template": { "type": "string", "minLength": 1 }, "sql_hash": { "$ref": "#/$defs/sha256" },
"parameter_definitions": { "type": "array", "items": { "$ref": "#/$defs/parameter" } }, "expected_output_schema": { "type": "object" }, "relation_refs": { "type": "array", "items": { "type": "string" } }, "execution_limits": { "$ref": "#/$defs/executionLimits" }, "generation_provenance": { "type": "object" }, "validation_result": { "type": "object", "required": ["valid"] }
}
},
"executionLimits": { "type": "object", "additionalProperties": false, "required": ["timeout_ms", "max_rows", "max_bytes", "max_complexity"], "properties": { "timeout_ms": { "type": "integer", "minimum": 1 }, "max_rows": { "type": "integer", "minimum": 1 }, "max_bytes": { "type": "integer", "minimum": 1 }, "max_complexity": { "type": "integer", "minimum": 1 } } },
"transformSpec": { "type": "object", "additionalProperties": false, "required": ["logical_step_id", "version", "operations"], "properties": { "logical_step_id": { "type": "string", "format": "uuid" }, "version": { "type": "string" }, "operations": { "type": "array", "items": { "type": "object", "required": ["op"], "properties": { "op": { "enum": ["select", "filter", "rename", "cast", "join", "group_by", "sum", "count", "distinct", "coalesce", "normalize_string", "normalize_date", "difference", "ratio", "tolerance_compare"] } } } } } },
"assertionSpec": { "type": "object", "additionalProperties": false, "required": ["logical_step_id", "kind", "left_ref", "right_ref"], "properties": { "logical_step_id": { "type": "string", "format": "uuid" }, "kind": { "enum": ["numeric_equality", "numeric_tolerance", "row_comparison", "column_comparison", "aggregate_comparison", "set_equality", "field_mapping", "null_fill_check", "cross_dashboard"] }, "left_ref": { "type": "string" }, "right_ref": { "type": "string" }, "tolerance": { "type": ["number", "null"] }, "field_mapping": { "type": "object" } } },
"agentEvaluationSpec": { "type": "object", "additionalProperties": false, "required": ["spec_id", "logical_step_id", "provider_id", "model_id", "prompt_template_id", "prompt_template_version", "prompt_template_hash", "input_manifest", "evidence_refs", "tool_allowlist", "output_schema", "decision_policy_id"], "properties": { "spec_id": { "type": "string" }, "logical_step_id": { "type": "string", "format": "uuid" }, "provider_id": { "type": "string" }, "model_id": { "type": "string" }, "prompt_template_id": { "type": "string" }, "prompt_template_version": { "type": "string" }, "prompt_template_hash": { "$ref": "#/$defs/sha256" }, "input_manifest": { "type": "object" }, "evidence_refs": { "type": "array", "items": { "type": "string" } }, "tool_allowlist": { "type": "array", "items": { "type": "string" } }, "output_schema": { "type": "object" }, "decision_policy_id": { "type": "string" } } },
"ref": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"kind"
],
"properties": {
"name": {
"type": "string",
"pattern": "^(context|param|step|baseline|candidate|artifact)\\.[A-Za-z0-9_.-]+$"
},
"kind": {
"enum": [
"context",
"parameter",
"step_output",
"baseline",
"candidate",
"artifact"
]
},
"value_type": {
"type": [
"string",
"null"
]
}
}
},
"expected": {
"type": "object",
"additionalProperties": false,
"required": [
"kind"
],
"properties": {
"kind": {
"enum": [
"structural",
"baseline_ref",
"candidate_ref",
"human_evidence"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"predicate": {
"type": [
"string",
"null"
]
},
"description": {
"type": [
"string",
"null"
],
"maxLength": 2000
}
}
},
"step": {
"type": "object",
"additionalProperties": false,
"required": [
"logical_step_id",
"step_key",
"phase",
"title",
"tool",
"action",
"inputs",
"outputs",
"expected",
"depends_on",
"automation_status",
"checklist_case_ids",
"risk"
],
"properties": {
"logical_step_id": {
"type": "string",
"format": "uuid",
"description": "Immutable stable identity (UUID); survives reorder/revision. Key for 045 compare and 047 flakiness."
},
"step_key": {
"type": "string",
"pattern": "^[a-z0-9][A-Za-z0-9_-]{2,127}$",
"description": "Readable semantic identifier (phase + case + action slug); NOT an identity (may shift on reorder)."
},
"position": {
"type": "integer",
"minimum": 0,
"description": "Derived ordering; mutable; NOT an identity."
},
"step_content_hash": {
"$ref": "#/$defs/sha256",
"description": "SHA-256 of this step's executable content; changes on edit."
},
"phase": {
"enum": [
"setup",
"interact",
"observe",
"assert",
"evidence",
"report"
]
},
"title": {
"type": "string",
"maxLength": 300
},
"description": {
"type": [
"string",
"null"
],
"maxLength": 2000
},
"tool": {
"enum": [
"browser",
"superset_api",
"sql_evidence",
"transform",
"xlsx",
"assertion",
"agent_evaluation",
"screenshot",
"report",
"artifact",
"human"
]
},
"action": {
"type": "string",
"pattern": "^[a-z][a-z0-9_]{1,63}$"
},
"inputs": {
"type": "array",
"items": {
"$ref": "#/$defs/ref"
}
},
"outputs": {
"type": "array",
"items": {
"$ref": "#/$defs/ref"
}
},
"expected": {
"$ref": "#/$defs/expected"
},
"depends_on": {
"type": "array",
"uniqueItems": true,
"description": "References to logical_step_id values of producer steps.",
"items": {
"type": "string",
"format": "uuid"
}
},
"automation_status": {
"enum": [
"ready",
"needs_context",
"needs_selector",
"needs_baseline",
"manual",
"unsupported",
"blocked"
]
},
"checklist_case_ids": {
"type": "array",
"uniqueItems": true,
"items": {
"$ref": "#/$defs/checklistId"
}
},
"risk": {
"enum": [
"READ_ONLY",
"UI_INTERACTION",
"TEST_DATA_MUTATION",
"EXTERNAL_MUTATION",
"DANGEROUS_MUTATION",
"human"
]
},
"mutation_contract": {
"type": ["object", "null"],
"additionalProperties": false,
"required": ["safe_test_fixture_id", "mutation_scope", "allowed_environment", "affected_record_keys", "cleanup_policy", "side_effect_key"],
"properties": {
"safe_test_fixture_id": { "type": "string" },
"mutation_scope": { "type": "string", "enum": ["controlled_test_data", "external_system"] },
"allowed_environment": { "type": "array", "minItems": 1, "items": { "type": "string" } },
"affected_record_keys": { "type": "array", "minItems": 1, "items": { "type": "string" } },
"cleanup_policy": { "type": "string", "enum": ["rollback", "reconcile", "none"] },
"side_effect_key": { "type": "string" }
}
},
"capture_spec": {
"oneOf": [
{
"$ref": "#/$defs/captureSpec"
},
{
"type": "null"
}
]
},
"vlm_analysis_spec": {
"oneOf": [
{
"$ref": "#/$defs/vlmAnalysis"
},
{
"type": "null"
}
]
}
}
},
"artifactPlan": {
"type": "object",
"additionalProperties": false,
"required": [
"artifact_key",
"kind",
"relative_path_template",
"template_id",
"input_refs",
"required"
],
"properties": {
"artifact_key": {
"type": "string"
},
"kind": {
"enum": [
"scenario",
"runner_plan",
"report_template",
"evidence_manifest",
"browser_module",
"xlsx_module",
"baseline_candidate"
]
},
"relative_path_template": {
"type": "string",
"pattern": "^(?!/)(?!.*\\.\\.).+$"
},
"template_id": {
"type": "string"
},
"input_refs": {
"type": "array",
"items": {
"type": "string"
}
},
"required": {
"type": "boolean"
},
"generation_blockers": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"coverage": {
"type": "object",
"required": [
"case_id",
"classification",
"rationale",
"logical_step_ids"
],
"properties": {
"case_id": {
"$ref": "#/$defs/checklistId"
},
"classification": {
"enum": [
"automated",
"human_checkpoint",
"unsupported",
"needs_context"
]
},
"rationale": {
"type": "string"
},
"logical_step_ids": {
"type": "array",
"items": {
"type": "string",
"format": "uuid"
}
}
}
},
"finding": {
"type": "object",
"required": [
"code",
"severity",
"message"
],
"properties": {
"code": {
"type": "string"
},
"severity": {
"enum": [
"info",
"warning",
"error",
"blocker"
]
},
"message": {
"type": "string"
},
"json_pointer": {
"type": [
"string",
"null"
]
},
"logical_step_id": {
"type": [
"string",
"null"
]
},
"case_id": {
"oneOf": [
{
"$ref": "#/$defs/checklistId"
},
{
"type": "null"
}
]
},
"recovery_options": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"captureSpec": {
"type": "object",
"additionalProperties": false,
"required": [
"target",
"viewport",
"readiness",
"method"
],
"properties": {
"target": {
"enum": [
"tab",
"viewport"
]
},
"tab_identifier": {
"type": [
"string",
"null"
]
},
"viewport": {
"type": "object",
"required": [
"width",
"height"
],
"properties": {
"width": {
"type": "integer",
"minimum": 320
},
"height": {
"type": "integer",
"minimum": 240
}
}
},
"readiness": {
"enum": [
"canvas_stabilized",
"network_idle",
"fixed_wait"
]
},
"readiness_timeout_ms": {
"type": "integer",
"default": 15000,
"minimum": 1000
},
"mask_selectors": {
"type": "array",
"items": {
"type": "string"
}
},
"method": {
"enum": [
"cdp",
"full_page",
"region"
]
}
}
},
"vlmAnalysis": {
"type": "object",
"additionalProperties": false,
"required": [
"provider_id",
"model_id",
"prompt_template_id",
"prompt_version",
"prompt_template_hash"
],
"properties": {
"provider_id": {
"type": "string"
},
"model_id": {
"type": "string"
},
"prompt_template_id": {
"type": "string"
},
"prompt_version": {
"type": "string",
"pattern": "^\\d+\\.\\d+\\.\\d+(-[a-z0-9.]+)?$"
},
"prompt_template_hash": {
"$ref": "#/$defs/sha256"
},
"confidence_threshold": {
"type": "number",
"minimum": 0,
"maximum": 1,
"default": 0.7
}
}
},
"vlmFinding": {
"type": "object",
"description": "DEPRECATED — runtime VlmFinding moved to 044 ScenarioExecution; retained only for reference. Disposition enum aligned to 044 HumanCheckpoint.",
"required": [
"finding_id",
"source_artifact_id",
"severity",
"code",
"confidence",
"description",
"disposition",
"model_provenance"
],
"properties": {
"finding_id": {
"type": "string"
},
"source_artifact_id": {
"type": "string",
"format": "uuid"
},
"severity": {
"enum": [
"info",
"warning",
"error"
]
},
"code": {
"type": "string"
},
"region": {
"type": [
"object",
"null"
],
"properties": {
"selector": {
"type": "string"
},
"bounds": {
"type": "object",
"properties": {
"x": {
"type": "integer"
},
"y": {
"type": "integer"
},
"w": {
"type": "integer"
},
"h": {
"type": "integer"
}
}
}
}
},
"confidence": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"description": {
"type": "string",
"maxLength": 1000
},
"disposition": {
"enum": [
"unresolved",
"confirmed",
"false_positive",
"inconclusive"
]
},
"disposition_comment": {
"type": [
"string",
"null"
],
"maxLength": 2000
},
"model_provenance": {
"type": "object",
"required": [
"model_id",
"prompt_version",
"prompt_template_hash",
"analyzed_at"
],
"properties": {
"model_id": {
"type": "string"
},
"prompt_version": {
"type": "string"
},
"prompt_template_hash": {
"$ref": "#/$defs/sha256"
},
"analyzed_at": {
"type": "string",
"format": "date-time"
}
}
}
}
}
}
}