452 lines
12 KiB
JSON
452 lines
12 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"compiler_version": "038.1.0",
|
|
"template_version": "v1",
|
|
"action_registry_version": "v1",
|
|
"dashboard_context": {
|
|
"environment_id": "env-prod-01",
|
|
"dashboard_id": 80,
|
|
"dashboard_name": "FI-0080 Operational Reporting",
|
|
"route": "/superset/dashboard/80"
|
|
},
|
|
"objective": {
|
|
"goal": "Verify filters, metric, XLSX export, and baseline",
|
|
"selected_case_ids": [
|
|
"B01",
|
|
"C04",
|
|
"C05",
|
|
"T01"
|
|
],
|
|
"rationale": "Release candidate covers filter and export regressions"
|
|
},
|
|
"input_fingerprints": {
|
|
"query_model": "c0ffee0000000000000000000000000000000000000000000000000000000000",
|
|
"checklist_catalog": "deadbeef00000000000000000000000000000000000000000000000000000000",
|
|
"baseline_version": "ba5eba1100000000000000000000000000000000000000000000000000000000",
|
|
"parameters": "feedf00d00000000000000000000000000000000000000000000000000000000"
|
|
},
|
|
"parameters": [
|
|
{
|
|
"name": "test_date",
|
|
"label": "Test date",
|
|
"type": "date",
|
|
"required": true,
|
|
"source": "user",
|
|
"validation": null,
|
|
"affected_logical_step_ids": [
|
|
"d57c118b-2cec-78fd-521b-40f417b9cb9f"
|
|
]
|
|
},
|
|
{
|
|
"name": "counterparty",
|
|
"label": "Counterparty",
|
|
"type": "string",
|
|
"required": true,
|
|
"source": "user",
|
|
"validation": null,
|
|
"affected_logical_step_ids": [
|
|
"d57c118b-2cec-78fd-521b-40f417b9cb9f"
|
|
]
|
|
}
|
|
],
|
|
"verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []},
|
|
"phases": [
|
|
"setup",
|
|
"interact",
|
|
"observe",
|
|
"assert",
|
|
"evidence",
|
|
"report"
|
|
],
|
|
"steps": [
|
|
{
|
|
"phase": "setup",
|
|
"title": "Open dashboard",
|
|
"tool": "browser",
|
|
"action": "open_dashboard",
|
|
"inputs": [
|
|
{
|
|
"name": "context.dashboard.route",
|
|
"kind": "context",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-1-B01-open_dashboard.dashboard_loaded",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "dashboard_loaded == true"
|
|
},
|
|
"depends_on": [],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "350cc442-9dc4-8a87-8c05-10f9061bc333",
|
|
"step_key": "phase-1-B01-open_dashboard",
|
|
"position": 0,
|
|
"step_content_hash": "fe100f77e32efcc2e7e78d4a804eb1d77cdf7f7ec1dfce3cc006f8e17c780500"
|
|
},
|
|
{
|
|
"phase": "interact",
|
|
"title": "Apply dashboard filters",
|
|
"tool": "browser",
|
|
"action": "apply_filters",
|
|
"inputs": [
|
|
{
|
|
"name": "param.test_date",
|
|
"kind": "parameter",
|
|
"value_type": "date"
|
|
},
|
|
{
|
|
"name": "param.counterparty",
|
|
"kind": "parameter",
|
|
"value_type": "string"
|
|
},
|
|
{
|
|
"name": "step.phase-1-B01-open_dashboard.dashboard_loaded",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-2-B01-apply_filters.filter_state",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "filter_state == normalized"
|
|
},
|
|
"depends_on": [
|
|
"350cc442-9dc4-8a87-8c05-10f9061bc333"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "UI_INTERACTION",
|
|
"logical_step_id": "d57c118b-2cec-78fd-521b-40f417b9cb9f",
|
|
"step_key": "phase-2-B01-apply_filters",
|
|
"position": 1,
|
|
"step_content_hash": "6d57aadd54c9260f9320f3f9b75d7314addb38fd2a4895e714b4c117b7c744b9"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Execute chart query via Superset API",
|
|
"tool": "superset_api",
|
|
"action": "execute_metric",
|
|
"inputs": [
|
|
{
|
|
"name": "context.dashboard.dashboard_id",
|
|
"kind": "context",
|
|
"value_type": "integer"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-3-B01-execute_metric.metric_value",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "baseline_ref",
|
|
"ref": "baseline.metric_revenue_q2",
|
|
"predicate": "approx_eq",
|
|
"description": "compare to approved baseline"
|
|
},
|
|
"depends_on": [
|
|
"d57c118b-2cec-78fd-521b-40f417b9cb9f"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "e6b53b50-88ea-9c24-361a-217c9fa70f8a",
|
|
"step_key": "phase-3-B01-execute_metric",
|
|
"position": 2,
|
|
"step_content_hash": "ce28f275509a6cb1fbbcd0faab1bee816597ac32761752b10ec1ccd855177ded"
|
|
},
|
|
{
|
|
"phase": "interact",
|
|
"title": "Download XLSX export",
|
|
"tool": "browser",
|
|
"action": "download_xlsx",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-2-B01-apply_filters.filter_state",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-4-C04-download_xlsx.xlsx_file",
|
|
"kind": "step_output",
|
|
"value_type": "artifact"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "xlsx file downloaded"
|
|
},
|
|
"depends_on": [
|
|
"d57c118b-2cec-78fd-521b-40f417b9cb9f"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C04"
|
|
],
|
|
"risk": "UI_INTERACTION",
|
|
"logical_step_id": "059ec187-5e46-d174-cdfd-525b7e766437",
|
|
"step_key": "phase-4-C04-download_xlsx",
|
|
"position": 3,
|
|
"step_content_hash": "b5e50d8cb20b45d5b5a1b51f4561e23bce25364d849cec0a1f588fa10af614a6"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Parse XLSX metric values",
|
|
"tool": "xlsx",
|
|
"action": "parse_xlsx",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-4-C04-download_xlsx.xlsx_file",
|
|
"kind": "step_output",
|
|
"value_type": "artifact"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-5-C05-parse_xlsx_metric.xlsx_metric",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "workbook readable, metric present"
|
|
},
|
|
"depends_on": [
|
|
"059ec187-5e46-d174-cdfd-525b7e766437"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C05"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "d090622c-26f4-0e97-e31c-b38e375b933e",
|
|
"step_key": "phase-5-C05-parse_xlsx_metric",
|
|
"position": 4,
|
|
"step_content_hash": "1b21621b14da0bb4f5c37497e0e293efc0248eaed1b30c57cb52eba9ebf8b9e9"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Verify persisted comment fields via dataset result",
|
|
"tool": "superset_api",
|
|
"action": "dataset_field_assert",
|
|
"inputs": [
|
|
{
|
|
"name": "context.query_model.dataset_fields",
|
|
"kind": "context",
|
|
"value_type": "string_list"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-6-T01-verify_fields.fields_ok",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "baseline_ref",
|
|
"ref": null,
|
|
"predicate": "eq",
|
|
"description": "metric must equal 1234.56"
|
|
},
|
|
"depends_on": [
|
|
"d57c118b-2cec-78fd-521b-40f417b9cb9f"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"T01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "4eada846-3288-774a-ed0e-4909571a04d5",
|
|
"step_key": "phase-6-T01-verify_fields",
|
|
"position": 5,
|
|
"step_content_hash": "e04daa6a66c9e9bb66a4116fcf1bd967bd2e11d84f277098ebb053d9e257b874"
|
|
},
|
|
{
|
|
"phase": "assert",
|
|
"title": "Compare XLSX metric to baseline",
|
|
"tool": "assertion",
|
|
"action": "compare_to_baseline",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-5-C05-parse_xlsx_metric.xlsx_metric",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
},
|
|
{
|
|
"name": "step.phase-3-B01-execute_metric.metric_value",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-7-C05-compare_to_baseline.comparison",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "baseline_ref",
|
|
"ref": "baseline.metric_revenue_q2",
|
|
"predicate": "approx_eq",
|
|
"description": "xlsx and api values within tolerance of baseline"
|
|
},
|
|
"depends_on": [
|
|
"d090622c-26f4-0e97-e31c-b38e375b933e",
|
|
"e6b53b50-88ea-9c24-361a-217c9fa70f8a"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C05"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "faf07f7f-5a39-a2d8-56a4-258acc13ef22",
|
|
"step_key": "phase-7-C05-compare_to_baseline",
|
|
"position": 6,
|
|
"step_content_hash": "eb9f9c8745911605f753aaeb6674718e92d0cb9539872c3b642b4f5b70c2ef5d"
|
|
},
|
|
{
|
|
"phase": "report",
|
|
"title": "Generate report",
|
|
"tool": "report",
|
|
"action": "generate_report",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-7-C05-compare_to_baseline.comparison",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
},
|
|
{
|
|
"name": "step.phase-6-T01-verify_fields.fields_ok",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "artifact.report",
|
|
"kind": "artifact",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "human_evidence",
|
|
"description": "report rendered for review"
|
|
},
|
|
"depends_on": [
|
|
"faf07f7f-5a39-a2d8-56a4-258acc13ef22",
|
|
"4eada846-3288-774a-ed0e-4909571a04d5"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C04"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "97df0515-e508-ed05-4944-9c10302baa1b",
|
|
"step_key": "phase-8-C04-generate_report",
|
|
"position": 7,
|
|
"step_content_hash": "74a642d067e12f5b8f65774a957c01ef404e5976ba4c4bd5cb06c1e2e0cf7b9b"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "artifact.report",
|
|
"kind": "artifact",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"artifact_plan": [
|
|
{
|
|
"artifact_key": "scenario",
|
|
"kind": "scenario",
|
|
"relative_path_template": "scenario.yaml",
|
|
"template_id": "scenario_v1",
|
|
"input_refs": [],
|
|
"required": true
|
|
},
|
|
{
|
|
"artifact_key": "report",
|
|
"kind": "report_template",
|
|
"relative_path_template": "reports/report_template.md",
|
|
"template_id": "report_template_v1",
|
|
"input_refs": [
|
|
"artifact.report"
|
|
],
|
|
"required": true
|
|
}
|
|
],
|
|
"checklist_coverage": [
|
|
{
|
|
"case_id": "B01",
|
|
"classification": "automated",
|
|
"rationale": "browser + superset_api + assertion",
|
|
"logical_step_ids": [
|
|
"350cc442-9dc4-8a87-8c05-10f9061bc333",
|
|
"d57c118b-2cec-78fd-521b-40f417b9cb9f",
|
|
"e6b53b50-88ea-9c24-361a-217c9fa70f8a"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "C04",
|
|
"classification": "automated",
|
|
"rationale": "browser download + xlsx parse",
|
|
"logical_step_ids": [
|
|
"059ec187-5e46-d174-cdfd-525b7e766437",
|
|
"97df0515-e508-ed05-4944-9c10302baa1b"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "C05",
|
|
"classification": "automated",
|
|
"rationale": "browser + superset_api + xlsx + assertion",
|
|
"logical_step_ids": [
|
|
"d090622c-26f4-0e97-e31c-b38e375b933e",
|
|
"faf07f7f-5a39-a2d8-56a4-258acc13ef22"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "T01",
|
|
"classification": "automated",
|
|
"rationale": "dataset field read via Superset API",
|
|
"logical_step_ids": [
|
|
"4eada846-3288-774a-ed0e-4909571a04d5"
|
|
]
|
|
}
|
|
],
|
|
"warnings": [],
|
|
"blockers": [],
|
|
"risk_summary": {
|
|
"max_risk": "READ_ONLY",
|
|
"mutation_steps": 0,
|
|
"needs_test_data": false
|
|
},
|
|
"scenario_key": "fi-0080_raw_baseline",
|
|
"content_hash": "c5c4184173556d836aa86963f8093584d7771e664cb83eb1f6762bacc1cd5ee5"
|
|
}
|