472 lines
13 KiB
JSON
472 lines
13 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"compiler_version": "038.1.0",
|
|
"template_version": "v1",
|
|
"action_registry_version": "v1",
|
|
"dashboard_context": {
|
|
"environment_id": "env-prod-01",
|
|
"dashboard_id": 80,
|
|
"dashboard_name": "FI-0080 Operational Reporting",
|
|
"route": "/superset/dashboard/80"
|
|
},
|
|
"objective": {
|
|
"goal": "Verify filters, metric, XLSX export, and baseline",
|
|
"selected_case_ids": [
|
|
"B01",
|
|
"C04",
|
|
"C05",
|
|
"T01"
|
|
],
|
|
"rationale": "Release candidate covers filter and export regressions"
|
|
},
|
|
"input_fingerprints": {
|
|
"query_model": "c0ffee0000000000000000000000000000000000000000000000000000000000",
|
|
"checklist_catalog": "deadbeef00000000000000000000000000000000000000000000000000000000",
|
|
"baseline_version": "ba5eba1100000000000000000000000000000000000000000000000000000000",
|
|
"parameters": "feedf00d00000000000000000000000000000000000000000000000000000000"
|
|
},
|
|
"parameters": [
|
|
{
|
|
"name": "test_date",
|
|
"label": "Test date",
|
|
"type": "date",
|
|
"required": true,
|
|
"source": "user",
|
|
"validation": null,
|
|
"affected_logical_step_ids": [
|
|
"79512b6c-4f55-95aa-c266-42029595a544"
|
|
]
|
|
},
|
|
{
|
|
"name": "counterparty",
|
|
"label": "Counterparty",
|
|
"type": "string",
|
|
"required": true,
|
|
"source": "user",
|
|
"validation": null,
|
|
"affected_logical_step_ids": [
|
|
"79512b6c-4f55-95aa-c266-42029595a544"
|
|
]
|
|
}
|
|
],
|
|
"verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []},
|
|
"phases": [
|
|
"setup",
|
|
"interact",
|
|
"observe",
|
|
"assert",
|
|
"evidence",
|
|
"report"
|
|
],
|
|
"steps": [
|
|
{
|
|
"phase": "setup",
|
|
"title": "Open dashboard",
|
|
"tool": "browser",
|
|
"action": "open_dashboard",
|
|
"inputs": [
|
|
{
|
|
"name": "context.dashboard.route",
|
|
"kind": "context",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-1-B01-open_dashboard.dashboard_loaded",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "dashboard_loaded == true"
|
|
},
|
|
"depends_on": [],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "5effc389-922f-2b67-077f-fd6d14b1d6d4",
|
|
"step_key": "phase-1-B01-open_dashboard",
|
|
"position": 0,
|
|
"step_content_hash": "fe100f77e32efcc2e7e78d4a804eb1d77cdf7f7ec1dfce3cc006f8e17c780500"
|
|
},
|
|
{
|
|
"phase": "interact",
|
|
"title": "Apply dashboard filters",
|
|
"tool": "browser",
|
|
"action": "apply_filters",
|
|
"inputs": [
|
|
{
|
|
"name": "param.test_date",
|
|
"kind": "parameter",
|
|
"value_type": "date"
|
|
},
|
|
{
|
|
"name": "param.counterparty",
|
|
"kind": "parameter",
|
|
"value_type": "string"
|
|
},
|
|
{
|
|
"name": "step.phase-1-B01-open_dashboard.dashboard_loaded",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-2-B01-apply_filters.filter_state",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "filter_state == normalized"
|
|
},
|
|
"depends_on": [
|
|
"5effc389-922f-2b67-077f-fd6d14b1d6d4"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "UI_INTERACTION",
|
|
"logical_step_id": "79512b6c-4f55-95aa-c266-42029595a544",
|
|
"step_key": "phase-2-B01-apply_filters",
|
|
"position": 1,
|
|
"step_content_hash": "6d57aadd54c9260f9320f3f9b75d7314addb38fd2a4895e714b4c117b7c744b9"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Execute chart query via Superset API",
|
|
"tool": "superset_api",
|
|
"action": "execute_metric",
|
|
"inputs": [
|
|
{
|
|
"name": "context.dashboard.dashboard_id",
|
|
"kind": "context",
|
|
"value_type": "integer"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-3-B01-execute_metric.metric_value",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "baseline_ref",
|
|
"ref": "baseline.metric_revenue_q2",
|
|
"predicate": "approx_eq",
|
|
"description": "compare to approved baseline"
|
|
},
|
|
"depends_on": [
|
|
"79512b6c-4f55-95aa-c266-42029595a544"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "d91845fd-ba40-016d-4007-111dc1ee5aee",
|
|
"step_key": "phase-3-B01-execute_metric",
|
|
"position": 2,
|
|
"step_content_hash": "ce28f275509a6cb1fbbcd0faab1bee816597ac32761752b10ec1ccd855177ded"
|
|
},
|
|
{
|
|
"phase": "interact",
|
|
"title": "Download XLSX export",
|
|
"tool": "browser",
|
|
"action": "download_xlsx",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-2-B01-apply_filters.filter_state",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-4-C04-download_xlsx.xlsx_file",
|
|
"kind": "step_output",
|
|
"value_type": "artifact"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "xlsx file downloaded"
|
|
},
|
|
"depends_on": [
|
|
"79512b6c-4f55-95aa-c266-42029595a544"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C04"
|
|
],
|
|
"risk": "UI_INTERACTION",
|
|
"logical_step_id": "c656a892-7ae9-0ca6-4a87-f6acd3d79516",
|
|
"step_key": "phase-4-C04-download_xlsx",
|
|
"position": 3,
|
|
"step_content_hash": "b5e50d8cb20b45d5b5a1b51f4561e23bce25364d849cec0a1f588fa10af614a6"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Parse XLSX metric values",
|
|
"tool": "xlsx",
|
|
"action": "parse_xlsx",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-4-C04-download_xlsx.xlsx_file",
|
|
"kind": "step_output",
|
|
"value_type": "artifact"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-5-C05-parse_xlsx_metric.xlsx_metric",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "workbook readable, metric present"
|
|
},
|
|
"depends_on": [
|
|
"c656a892-7ae9-0ca6-4a87-f6acd3d79516"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C05"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "6e685a71-3eb8-786c-9e35-0a8835ced1fc",
|
|
"step_key": "phase-5-C05-parse_xlsx_metric",
|
|
"position": 4,
|
|
"step_content_hash": "1b21621b14da0bb4f5c37497e0e293efc0248eaed1b30c57cb52eba9ebf8b9e9"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Verify persisted comment fields via dataset result",
|
|
"tool": "superset_api",
|
|
"action": "dataset_field_assert",
|
|
"inputs": [
|
|
{
|
|
"name": "context.query_model.dataset_fields",
|
|
"kind": "context",
|
|
"value_type": "string_list"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-6-T01-verify_fields.fields_ok",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "fields present and typed"
|
|
},
|
|
"depends_on": [
|
|
"79512b6c-4f55-95aa-c266-42029595a544"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"T01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "acb0a443-0777-cfed-f6b0-aa56655fd07a",
|
|
"step_key": "phase-6-T01-verify_fields",
|
|
"position": 5,
|
|
"step_content_hash": "2c2b7ae922868a99b9c13e40d8cfb1849d16300c7bf840a0dc994696617ad2f3"
|
|
},
|
|
{
|
|
"phase": "assert",
|
|
"title": "Compare XLSX metric to baseline",
|
|
"tool": "assertion",
|
|
"action": "compare_to_baseline",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-5-C05-parse_xlsx_metric.xlsx_metric",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
},
|
|
{
|
|
"name": "step.phase-3-B01-execute_metric.metric_value",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-7-C05-compare_to_baseline.comparison",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "baseline_ref",
|
|
"ref": "baseline.metric_revenue_q2",
|
|
"predicate": "approx_eq",
|
|
"description": "xlsx and api values within tolerance of baseline"
|
|
},
|
|
"depends_on": [
|
|
"6e685a71-3eb8-786c-9e35-0a8835ced1fc",
|
|
"d91845fd-ba40-016d-4007-111dc1ee5aee"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C05"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "2efc15a9-77d0-2ced-a44e-02811416ba5a",
|
|
"step_key": "phase-7-C05-compare_to_baseline",
|
|
"position": 6,
|
|
"step_content_hash": "eb9f9c8745911605f753aaeb6674718e92d0cb9539872c3b642b4f5b70c2ef5d"
|
|
},
|
|
{
|
|
"phase": "report",
|
|
"title": "Generate report",
|
|
"tool": "report",
|
|
"action": "generate_report",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-7-C05-compare_to_baseline.comparison",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
},
|
|
{
|
|
"name": "step.phase-6-T01-verify_fields.fields_ok",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "artifact.report",
|
|
"kind": "artifact",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "human_evidence",
|
|
"description": "report rendered for review"
|
|
},
|
|
"depends_on": [
|
|
"2efc15a9-77d0-2ced-a44e-02811416ba5a",
|
|
"acb0a443-0777-cfed-f6b0-aa56655fd07a"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C04"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "27d97074-280d-675c-2a00-57d8e8b499fd",
|
|
"step_key": "phase-8-C04-generate_report",
|
|
"position": 7,
|
|
"step_content_hash": "74a642d067e12f5b8f65774a957c01ef404e5976ba4c4bd5cb06c1e2e0cf7b9b"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "SQL step",
|
|
"tool": "superset_api",
|
|
"action": "raw_sql",
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"expected": {
|
|
"kind": "structural"
|
|
},
|
|
"depends_on": [],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"T02"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"description": "SELECT * FROM dashboard_80 WHERE comment IS NOT NULL",
|
|
"logical_step_id": "ec961dc8-505d-6ce1-1536-4103ad336071",
|
|
"step_key": "phase-9-sql-step",
|
|
"position": 8,
|
|
"step_content_hash": "d90abbc39cceef50ab67e7e30e0e6167b229d545ae98750a3d543fced4b77690"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "artifact.report",
|
|
"kind": "artifact",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"artifact_plan": [
|
|
{
|
|
"artifact_key": "scenario",
|
|
"kind": "scenario",
|
|
"relative_path_template": "scenario.yaml",
|
|
"template_id": "scenario_v1",
|
|
"input_refs": [],
|
|
"required": true
|
|
},
|
|
{
|
|
"artifact_key": "report",
|
|
"kind": "report_template",
|
|
"relative_path_template": "reports/report_template.md",
|
|
"template_id": "report_template_v1",
|
|
"input_refs": [
|
|
"artifact.report"
|
|
],
|
|
"required": true
|
|
}
|
|
],
|
|
"checklist_coverage": [
|
|
{
|
|
"case_id": "B01",
|
|
"classification": "automated",
|
|
"rationale": "browser + superset_api + assertion",
|
|
"logical_step_ids": [
|
|
"5effc389-922f-2b67-077f-fd6d14b1d6d4",
|
|
"79512b6c-4f55-95aa-c266-42029595a544",
|
|
"d91845fd-ba40-016d-4007-111dc1ee5aee"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "C04",
|
|
"classification": "automated",
|
|
"rationale": "browser download + xlsx parse",
|
|
"logical_step_ids": [
|
|
"c656a892-7ae9-0ca6-4a87-f6acd3d79516",
|
|
"27d97074-280d-675c-2a00-57d8e8b499fd"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "C05",
|
|
"classification": "automated",
|
|
"rationale": "browser + superset_api + xlsx + assertion",
|
|
"logical_step_ids": [
|
|
"6e685a71-3eb8-786c-9e35-0a8835ced1fc",
|
|
"2efc15a9-77d0-2ced-a44e-02811416ba5a"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "T01",
|
|
"classification": "automated",
|
|
"rationale": "dataset field read via Superset API",
|
|
"logical_step_ids": [
|
|
"acb0a443-0777-cfed-f6b0-aa56655fd07a"
|
|
]
|
|
}
|
|
],
|
|
"warnings": [],
|
|
"blockers": [],
|
|
"risk_summary": {
|
|
"max_risk": "READ_ONLY",
|
|
"mutation_steps": 0,
|
|
"needs_test_data": false
|
|
},
|
|
"scenario_key": "fi-0080_sql_injection",
|
|
"content_hash": "b03813705ae494c58256a03c9a0d7f38fadb7e49df14c40ff63917b7d8720eb9"
|
|
}
|