477 lines
13 KiB
JSON
477 lines
13 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"compiler_version": "038.1.0",
|
|
"template_version": "v1",
|
|
"action_registry_version": "v1",
|
|
"dashboard_context": {
|
|
"environment_id": "env-prod-01",
|
|
"dashboard_id": 80,
|
|
"dashboard_name": "FI-0080 Operational Reporting",
|
|
"route": "/superset/dashboard/80"
|
|
},
|
|
"objective": {
|
|
"goal": "Verify filters, metric, XLSX export, and baseline",
|
|
"selected_case_ids": [
|
|
"B01",
|
|
"C04",
|
|
"C05",
|
|
"T01"
|
|
],
|
|
"rationale": "Release candidate covers filter and export regressions"
|
|
},
|
|
"input_fingerprints": {
|
|
"query_model": "c0ffee0000000000000000000000000000000000000000000000000000000000",
|
|
"checklist_catalog": "deadbeef00000000000000000000000000000000000000000000000000000000",
|
|
"baseline_version": "ba5eba1100000000000000000000000000000000000000000000000000000000",
|
|
"parameters": "feedf00d00000000000000000000000000000000000000000000000000000000"
|
|
},
|
|
"parameters": [
|
|
{
|
|
"name": "test_date",
|
|
"label": "Test date",
|
|
"type": "date",
|
|
"required": true,
|
|
"source": "user",
|
|
"validation": null,
|
|
"affected_logical_step_ids": [
|
|
"b4219430-b9b7-a57d-ead9-20c05061e61d"
|
|
]
|
|
},
|
|
{
|
|
"name": "counterparty",
|
|
"label": "Counterparty",
|
|
"type": "string",
|
|
"required": true,
|
|
"source": "user",
|
|
"validation": null,
|
|
"affected_logical_step_ids": [
|
|
"b4219430-b9b7-a57d-ead9-20c05061e61d"
|
|
]
|
|
}
|
|
],
|
|
"verification_program": {"navigation_program": [], "evidence_program": [], "transformation_program": [], "assertion_program": [], "semantic_evaluation_program": []},
|
|
"phases": [
|
|
"setup",
|
|
"interact",
|
|
"observe",
|
|
"assert",
|
|
"evidence",
|
|
"report"
|
|
],
|
|
"steps": [
|
|
{
|
|
"phase": "setup",
|
|
"title": "Open dashboard",
|
|
"tool": "browser",
|
|
"action": "open_dashboard",
|
|
"inputs": [
|
|
{
|
|
"name": "context.dashboard.route",
|
|
"kind": "context",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-1-B01-open_dashboard.dashboard_loaded",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "dashboard_loaded == true"
|
|
},
|
|
"depends_on": [],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "42f40cf4-2cac-9377-a7d6-b782a3444aed",
|
|
"step_key": "phase-1-B01-open_dashboard",
|
|
"position": 0,
|
|
"step_content_hash": "fe100f77e32efcc2e7e78d4a804eb1d77cdf7f7ec1dfce3cc006f8e17c780500"
|
|
},
|
|
{
|
|
"phase": "interact",
|
|
"title": "Apply dashboard filters",
|
|
"tool": "browser",
|
|
"action": "apply_filters",
|
|
"inputs": [
|
|
{
|
|
"name": "param.test_date",
|
|
"kind": "parameter",
|
|
"value_type": "date"
|
|
},
|
|
{
|
|
"name": "param.counterparty",
|
|
"kind": "parameter",
|
|
"value_type": "string"
|
|
},
|
|
{
|
|
"name": "step.phase-1-B01-open_dashboard.dashboard_loaded",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-2-B01-apply_filters.filter_state",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "filter_state == normalized"
|
|
},
|
|
"depends_on": [
|
|
"42f40cf4-2cac-9377-a7d6-b782a3444aed"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "UI_INTERACTION",
|
|
"logical_step_id": "b4219430-b9b7-a57d-ead9-20c05061e61d",
|
|
"step_key": "phase-2-B01-apply_filters",
|
|
"position": 1,
|
|
"step_content_hash": "6d57aadd54c9260f9320f3f9b75d7314addb38fd2a4895e714b4c117b7c744b9"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Execute chart query via Superset API",
|
|
"tool": "superset_api",
|
|
"action": "execute_metric",
|
|
"inputs": [
|
|
{
|
|
"name": "context.dashboard.dashboard_id",
|
|
"kind": "context",
|
|
"value_type": "integer"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-3-B01-execute_metric.metric_value",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "baseline_ref",
|
|
"ref": "baseline.metric_revenue_q2",
|
|
"predicate": "approx_eq",
|
|
"description": "compare to approved baseline"
|
|
},
|
|
"depends_on": [
|
|
"b4219430-b9b7-a57d-ead9-20c05061e61d"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "31e09885-d86c-1f26-035a-b20d6612a8aa",
|
|
"step_key": "phase-3-B01-execute_metric",
|
|
"position": 2,
|
|
"step_content_hash": "ce28f275509a6cb1fbbcd0faab1bee816597ac32761752b10ec1ccd855177ded"
|
|
},
|
|
{
|
|
"phase": "interact",
|
|
"title": "Download XLSX export",
|
|
"tool": "browser",
|
|
"action": "download_xlsx",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-2-B01-apply_filters.filter_state",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-4-C04-download_xlsx.xlsx_file",
|
|
"kind": "step_output",
|
|
"value_type": "artifact"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "xlsx file downloaded"
|
|
},
|
|
"depends_on": [
|
|
"b4219430-b9b7-a57d-ead9-20c05061e61d"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C04"
|
|
],
|
|
"risk": "UI_INTERACTION",
|
|
"logical_step_id": "9d94e3bf-72b9-43d9-404f-c9f68a76ae88",
|
|
"step_key": "phase-4-C04-download_xlsx",
|
|
"position": 3,
|
|
"step_content_hash": "b5e50d8cb20b45d5b5a1b51f4561e23bce25364d849cec0a1f588fa10af614a6"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Parse XLSX metric values",
|
|
"tool": "xlsx",
|
|
"action": "parse_xlsx",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-4-C04-download_xlsx.xlsx_file",
|
|
"kind": "step_output",
|
|
"value_type": "artifact"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-5-C05-parse_xlsx_metric.xlsx_metric",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "workbook readable, metric present"
|
|
},
|
|
"depends_on": [
|
|
"9d94e3bf-72b9-43d9-404f-c9f68a76ae88"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C05"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "cc423c40-39bb-11cf-02b3-dbfe17ad08d8",
|
|
"step_key": "phase-5-C05-parse_xlsx_metric",
|
|
"position": 4,
|
|
"step_content_hash": "1b21621b14da0bb4f5c37497e0e293efc0248eaed1b30c57cb52eba9ebf8b9e9"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Verify persisted comment fields via dataset result",
|
|
"tool": "superset_api",
|
|
"action": "dataset_field_assert",
|
|
"inputs": [
|
|
{
|
|
"name": "context.query_model.dataset_fields",
|
|
"kind": "context",
|
|
"value_type": "string_list"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-6-T01-verify_fields.fields_ok",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural",
|
|
"description": "fields present and typed"
|
|
},
|
|
"depends_on": [
|
|
"b4219430-b9b7-a57d-ead9-20c05061e61d"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"T01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "8c7a79d2-561d-4b91-434e-31584c5db2e5",
|
|
"step_key": "phase-6-T01-verify_fields",
|
|
"position": 5,
|
|
"step_content_hash": "2c2b7ae922868a99b9c13e40d8cfb1849d16300c7bf840a0dc994696617ad2f3"
|
|
},
|
|
{
|
|
"phase": "assert",
|
|
"title": "Compare XLSX metric to baseline",
|
|
"tool": "assertion",
|
|
"action": "compare_to_baseline",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-5-C05-parse_xlsx_metric.xlsx_metric",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
},
|
|
{
|
|
"name": "step.phase-3-B01-execute_metric.metric_value",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-7-C05-compare_to_baseline.comparison",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "baseline_ref",
|
|
"ref": "baseline.metric_revenue_q2",
|
|
"predicate": "approx_eq",
|
|
"description": "xlsx and api values within tolerance of baseline"
|
|
},
|
|
"depends_on": [
|
|
"cc423c40-39bb-11cf-02b3-dbfe17ad08d8",
|
|
"31e09885-d86c-1f26-035a-b20d6612a8aa"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C05"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "12f1b598-d06e-e2ae-12b2-2f1e17d61cc0",
|
|
"step_key": "phase-7-C05-compare_to_baseline",
|
|
"position": 6,
|
|
"step_content_hash": "eb9f9c8745911605f753aaeb6674718e92d0cb9539872c3b642b4f5b70c2ef5d"
|
|
},
|
|
{
|
|
"phase": "report",
|
|
"title": "Generate report",
|
|
"tool": "report",
|
|
"action": "generate_report",
|
|
"inputs": [
|
|
{
|
|
"name": "step.phase-7-C05-compare_to_baseline.comparison",
|
|
"kind": "step_output",
|
|
"value_type": "string"
|
|
},
|
|
{
|
|
"name": "step.phase-6-T01-verify_fields.fields_ok",
|
|
"kind": "step_output",
|
|
"value_type": "boolean"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "artifact.report",
|
|
"kind": "artifact",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "human_evidence",
|
|
"description": "report rendered for review"
|
|
},
|
|
"depends_on": [
|
|
"12f1b598-d06e-e2ae-12b2-2f1e17d61cc0",
|
|
"8c7a79d2-561d-4b91-434e-31584c5db2e5"
|
|
],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"C04"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "66b6393a-64e7-a2b7-c5ad-69e6ce78ad84",
|
|
"step_key": "phase-8-C04-generate_report",
|
|
"position": 7,
|
|
"step_content_hash": "74a642d067e12f5b8f65774a957c01ef404e5976ba4c4bd5cb06c1e2e0cf7b9b"
|
|
},
|
|
{
|
|
"phase": "observe",
|
|
"title": "Duplicate producer",
|
|
"tool": "superset_api",
|
|
"action": "execute_metric",
|
|
"inputs": [],
|
|
"outputs": [
|
|
{
|
|
"name": "step.phase-3-B01-execute_metric.metric_value",
|
|
"kind": "step_output",
|
|
"value_type": "number"
|
|
}
|
|
],
|
|
"expected": {
|
|
"kind": "structural"
|
|
},
|
|
"depends_on": [],
|
|
"automation_status": "ready",
|
|
"checklist_case_ids": [
|
|
"B01"
|
|
],
|
|
"risk": "READ_ONLY",
|
|
"logical_step_id": "d91edbd6-43a8-3295-8782-584b8122fc3d",
|
|
"step_key": "phase-9-dup-output",
|
|
"position": 8,
|
|
"step_content_hash": "1fe1de1e061fba27c1bbd20de879f562ffd0a06675a87902c67e2fee2b21f168"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"name": "artifact.report",
|
|
"kind": "artifact",
|
|
"value_type": "string"
|
|
}
|
|
],
|
|
"artifact_plan": [
|
|
{
|
|
"artifact_key": "scenario",
|
|
"kind": "scenario",
|
|
"relative_path_template": "scenario.yaml",
|
|
"template_id": "scenario_v1",
|
|
"input_refs": [],
|
|
"required": true
|
|
},
|
|
{
|
|
"artifact_key": "report",
|
|
"kind": "report_template",
|
|
"relative_path_template": "reports/report_template.md",
|
|
"template_id": "report_template_v1",
|
|
"input_refs": [
|
|
"artifact.report"
|
|
],
|
|
"required": true
|
|
}
|
|
],
|
|
"checklist_coverage": [
|
|
{
|
|
"case_id": "B01",
|
|
"classification": "automated",
|
|
"rationale": "browser + superset_api + assertion",
|
|
"logical_step_ids": [
|
|
"42f40cf4-2cac-9377-a7d6-b782a3444aed",
|
|
"b4219430-b9b7-a57d-ead9-20c05061e61d",
|
|
"31e09885-d86c-1f26-035a-b20d6612a8aa"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "C04",
|
|
"classification": "automated",
|
|
"rationale": "browser download + xlsx parse",
|
|
"logical_step_ids": [
|
|
"9d94e3bf-72b9-43d9-404f-c9f68a76ae88",
|
|
"66b6393a-64e7-a2b7-c5ad-69e6ce78ad84"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "C05",
|
|
"classification": "automated",
|
|
"rationale": "browser + superset_api + xlsx + assertion",
|
|
"logical_step_ids": [
|
|
"cc423c40-39bb-11cf-02b3-dbfe17ad08d8",
|
|
"12f1b598-d06e-e2ae-12b2-2f1e17d61cc0"
|
|
]
|
|
},
|
|
{
|
|
"case_id": "T01",
|
|
"classification": "automated",
|
|
"rationale": "dataset field read via Superset API",
|
|
"logical_step_ids": [
|
|
"8c7a79d2-561d-4b91-434e-31584c5db2e5"
|
|
]
|
|
}
|
|
],
|
|
"warnings": [],
|
|
"blockers": [],
|
|
"risk_summary": {
|
|
"max_risk": "READ_ONLY",
|
|
"mutation_steps": 0,
|
|
"needs_test_data": false
|
|
},
|
|
"scenario_key": "fi-0080_duplicate_output",
|
|
"content_hash": "a0871c087584b0263a8ffa002543db4cc872dfb48426d9fd370c55f63fa35b0a"
|
|
}
|