From 93359c87510ce3bde58360797530f4fecde05fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20H=C3=A9zs=C5=91?= Date: Sun, 25 Oct 2026 09:19:51 +0100 Subject: [PATCH] add a sample report matrix rendered through the real pipeline --- .gitignore | 4 + tests/fixtures/samples/aws_no_egress.json | 97 ++++ tests/fixtures/samples/aws_typical.json | 157 ++++++ tests/fixtures/samples/azure_typical.json | 133 +++++ tests/fixtures/samples/azure_uploaded.json | 138 +++++ tests/fixtures/samples/egress_free_tier.json | 66 +++ tests/fixtures/samples/egress_multi_tib.json | 80 +++ tests/fixtures/samples/empty.json | 26 + tests/fixtures/samples/no_risks.json | 47 ++ tests/fixtures/samples/stress.json | 211 ++++++++ .../samples/tfstate/sample_aws.tfstate | 104 ++++ .../samples/tfstate/sample_azure.tfstate | 76 +++ tests/fixtures/samples/tfstate_aws.json | 15 + tests/fixtures/samples/tfstate_azure.json | 15 + tests/fixtures/samples/uploaded_no_link.json | 46 ++ tests/report/test_basic_report.py | 27 +- tests/report/test_egress_report.py | 2 +- tests/samples/__init__.py | 0 tests/samples/__main__.py | 178 +++++++ tests/samples/catalogue.py | 222 ++++++++ tests/samples/render.py | 293 +++++++++++ tests/samples/scenarios.py | 290 +++++++++++ tests/samples/test_samples.py | 484 ++++++++++++++++++ tests/samples/test_scenarios.py | 245 +++++++++ tests/support/html.py | 58 +++ 25 files changed, 2987 insertions(+), 27 deletions(-) create mode 100644 tests/fixtures/samples/aws_no_egress.json create mode 100644 tests/fixtures/samples/aws_typical.json create mode 100644 tests/fixtures/samples/azure_typical.json create mode 100644 tests/fixtures/samples/azure_uploaded.json create mode 100644 tests/fixtures/samples/egress_free_tier.json create mode 100644 tests/fixtures/samples/egress_multi_tib.json create mode 100644 tests/fixtures/samples/empty.json create mode 100644 tests/fixtures/samples/no_risks.json create mode 100644 tests/fixtures/samples/stress.json create mode 100644 tests/fixtures/samples/tfstate/sample_aws.tfstate create mode 100644 tests/fixtures/samples/tfstate/sample_azure.tfstate create mode 100644 tests/fixtures/samples/tfstate_aws.json create mode 100644 tests/fixtures/samples/tfstate_azure.json create mode 100644 tests/fixtures/samples/uploaded_no_link.json create mode 100644 tests/samples/__init__.py create mode 100644 tests/samples/__main__.py create mode 100644 tests/samples/catalogue.py create mode 100644 tests/samples/render.py create mode 100644 tests/samples/scenarios.py create mode 100644 tests/samples/test_samples.py create mode 100644 tests/samples/test_scenarios.py create mode 100644 tests/support/html.py diff --git a/.gitignore b/.gitignore index b1aa32b..bca627c 100644 --- a/.gitignore +++ b/.gitignore @@ -100,6 +100,9 @@ web_modules/ # Generated reports reports/ +# Rendered sample reports (python -m tests.samples) +/samples/ + # Local configs (only the example templates are tracked) config/* !config/*_example.json @@ -107,6 +110,7 @@ config/* # Terraform state and downloaded datasets *.tfstate !tests/fixtures/smoke/aws.tfstate +!tests/fixtures/samples/tfstate/*.tfstate datasets/ # TypeScript cache diff --git a/tests/fixtures/samples/aws_no_egress.json b/tests/fixtures/samples/aws_no_egress.json new file mode 100644 index 0000000..1d056cb --- /dev/null +++ b/tests/fixtures/samples/aws_no_egress.json @@ -0,0 +1,97 @@ +{ + "name": "Sample – Internal Billing Platform", + "purpose": "The same estate without egress: no egress document, no egress sentence.", + "provider": "aws", + "exit_strategy": 3, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-central-1", + "count": 14 + }, + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-west-1", + "count": 6 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-central-1", + "count": 9 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-west-1", + "count": 3 + }, + { + "type": "AWS.ec2.describe_volumes.Volumes", + "region": "eu-central-1", + "count": 22 + }, + { + "type": "AWS.ec2.describe_volumes.Volumes", + "region": "eu-west-1", + "count": 8 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-central-1", + "count": 3 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-west-1", + "count": 1 + }, + { + "type": "AWS.dynamodb.list_tables.TableNames", + "region": "eu-central-1", + "count": 5 + }, + { + "type": "AWS.lambda.list_functions.Functions", + "region": "eu-central-1", + "count": 18 + }, + { + "type": "AWS.lambda.list_functions.Functions", + "region": "eu-west-1", + "count": 6 + }, + { + "type": "AWS.eks.list_clusters.clusters", + "region": "eu-central-1", + "count": 2 + }, + { + "type": "AWS.sqs.list_queues.QueueUrls", + "region": "eu-central-1", + "count": 7 + }, + { + "type": "AWS.sns.list_topics.Topics", + "region": "eu-central-1", + "count": 4 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-06": 8420.35, + "2026-07": 8791.1, + "2026-08": 9054.8, + "2026-09": 8632.45, + "2026-10": 9310.2, + "2026-11": 4120.75 + } + }, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0002" +} diff --git a/tests/fixtures/samples/aws_typical.json b/tests/fixtures/samples/aws_typical.json new file mode 100644 index 0000000..74ac827 --- /dev/null +++ b/tests/fixtures/samples/aws_typical.json @@ -0,0 +1,157 @@ +{ + "name": "Sample – Customer Self-Service Portal", + "purpose": "Several risk severities, USD costs with a provisional month, egress with archive tiers.", + "provider": "aws", + "exit_strategy": 3, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-central-1", + "count": 14 + }, + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-west-1", + "count": 6 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-central-1", + "count": 9 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-west-1", + "count": 3 + }, + { + "type": "AWS.ec2.describe_volumes.Volumes", + "region": "eu-central-1", + "count": 22 + }, + { + "type": "AWS.ec2.describe_volumes.Volumes", + "region": "eu-west-1", + "count": 8 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-central-1", + "count": 3 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-west-1", + "count": 1 + }, + { + "type": "AWS.dynamodb.list_tables.TableNames", + "region": "eu-central-1", + "count": 5 + }, + { + "type": "AWS.lambda.list_functions.Functions", + "region": "eu-central-1", + "count": 18 + }, + { + "type": "AWS.lambda.list_functions.Functions", + "region": "eu-west-1", + "count": 6 + }, + { + "type": "AWS.eks.list_clusters.clusters", + "region": "eu-central-1", + "count": 2 + }, + { + "type": "AWS.sqs.list_queues.QueueUrls", + "region": "eu-central-1", + "count": 7 + }, + { + "type": "AWS.sns.list_topics.Topics", + "region": "eu-central-1", + "count": 4 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-06": 8420.35, + "2026-07": 8791.1, + "2026-08": 9054.8, + "2026-09": 8632.45, + "2026-10": 9310.2, + "2026-11": 4120.75 + } + }, + "egress": { + "archive_tiers": [ + "Archive", + "Glacier", + "Deep Archive" + ], + "resources": [ + { + "type": "AWS.s3.list_buckets.Buckets", + "name": "sample-portal-assets", + "size_gib": 1800, + "tiers": { + "Standard": 1200, + "Standard-IA": 300, + "Glacier": 300 + } + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "name": "sample-portal-archive", + "size_gib": 5200, + "tiers": { + "Standard": 400, + "Glacier": 3800, + "Deep Archive": 1000 + } + }, + { + "type": "AWS.ec2.describe_volumes.Volumes", + "name": "vol-0a1b2c3d4e5f60789", + "size_gib": 500, + "flags": [ + "allocated (upper bound)" + ] + }, + { + "type": "AWS.ec2.describe_volumes.Volumes", + "name": "vol-0123456789abcdef0", + "size_gib": 250, + "flags": [ + "allocated (upper bound)" + ] + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "name": "sample-portal-db", + "size_gib": 320, + "flags": [ + "allocated (upper bound)" + ], + "notes": [ + "allocated: 320.0 GiB" + ] + }, + { + "type": "AWS.dynamodb.list_tables.TableNames", + "name": "sample-sessions", + "size_gib": 12 + } + ] + }, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0001" +} diff --git a/tests/fixtures/samples/azure_typical.json b/tests/fixtures/samples/azure_typical.json new file mode 100644 index 0000000..45d861f --- /dev/null +++ b/tests/fixtures/samples/azure_typical.json @@ -0,0 +1,133 @@ +{ + "name": "Sample – Document Archive", + "purpose": "EUR costs, a type without alternatives, egress with unsized types so the totals carry a +.", + "provider": "azure", + "exit_strategy": 1, + "scope": { + "tenantId": "00000000-0000-4000-8000-0000000000a1", + "subscriptionId": "00000000-0000-4000-8000-0000000000a2", + "resourceGroupName": "rg-sample-archive" + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines", + "region": "westeurope", + "count": 12 + }, + { + "type": "Microsoft.Compute/virtualMachines", + "region": "northeurope", + "count": 4 + }, + { + "type": "Microsoft.ContainerService/managedClusters", + "region": "westeurope", + "count": 2 + }, + { + "type": "Microsoft.Storage/storageAccounts", + "region": "westeurope", + "count": 6 + }, + { + "type": "Microsoft.Storage/storageAccounts", + "region": "northeurope", + "count": 2 + }, + { + "type": "Microsoft.Sql/servers/databases", + "region": "westeurope", + "count": 5 + }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "region": "westeurope", + "count": 1 + }, + { + "type": "Microsoft.DBforPostgreSQL/flexibleServers", + "region": "northeurope", + "count": 2 + }, + { + "type": "Microsoft.RecoveryServices/vaults", + "region": "westeurope", + "count": 1 + }, + { + "type": "Microsoft.ContainerRegistry/registries", + "region": "westeurope", + "count": 1 + } + ], + "costs": { + "currency": "EUR", + "months": { + "2026-06": 5120.4, + "2026-07": 5388.15, + "2026-08": 5290.0, + "2026-09": 5602.75, + "2026-10": 5711.3, + "2026-11": 2204.9 + } + }, + "egress": { + "archive_tiers": [ + "Archive" + ], + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "name": "sampledocsprod", + "size_gib": 2048, + "tiers": { + "Hot": 1500, + "Cool": 300, + "Archive": 248 + } + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "sampledocsbackup", + "size_gib": 900, + "tiers": { + "Cool": 400, + "Archive": 500 + } + }, + { + "type": "Microsoft.Compute/disks", + "name": "sample-docs-vm-osdisk", + "size_gib": 256, + "flags": [ + "allocated (upper bound)" + ] + }, + { + "type": "Microsoft.Sql/servers/databases", + "name": "sample-docs-metadata", + "size_gib": 140 + }, + { + "type": "Microsoft.Sql/servers/databases", + "name": "sample-docs-reporting", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "name": "sample-docs-search", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.RecoveryServices/vaults", + "name": "sample-docs-vault", + "size_gib": null + } + ] + }, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0003" +} diff --git a/tests/fixtures/samples/azure_uploaded.json b/tests/fixtures/samples/azure_uploaded.json new file mode 100644 index 0000000..6d65415 --- /dev/null +++ b/tests/fixtures/samples/azure_uploaded.json @@ -0,0 +1,138 @@ +{ + "name": "Sample – Document Archive", + "purpose": "The same run, uploaded with a link: the uploaded closing page and the link on page 2 of both documents.", + "provider": "azure", + "exit_strategy": 1, + "scope": { + "tenantId": "00000000-0000-4000-8000-0000000000a1", + "subscriptionId": "00000000-0000-4000-8000-0000000000a2", + "resourceGroupName": "rg-sample-archive" + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines", + "region": "westeurope", + "count": 12 + }, + { + "type": "Microsoft.Compute/virtualMachines", + "region": "northeurope", + "count": 4 + }, + { + "type": "Microsoft.ContainerService/managedClusters", + "region": "westeurope", + "count": 2 + }, + { + "type": "Microsoft.Storage/storageAccounts", + "region": "westeurope", + "count": 6 + }, + { + "type": "Microsoft.Storage/storageAccounts", + "region": "northeurope", + "count": 2 + }, + { + "type": "Microsoft.Sql/servers/databases", + "region": "westeurope", + "count": 5 + }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "region": "westeurope", + "count": 1 + }, + { + "type": "Microsoft.DBforPostgreSQL/flexibleServers", + "region": "northeurope", + "count": 2 + }, + { + "type": "Microsoft.RecoveryServices/vaults", + "region": "westeurope", + "count": 1 + }, + { + "type": "Microsoft.ContainerRegistry/registries", + "region": "westeurope", + "count": 1 + } + ], + "costs": { + "currency": "EUR", + "months": { + "2026-06": 5120.4, + "2026-07": 5388.15, + "2026-08": 5290.0, + "2026-09": 5602.75, + "2026-10": 5711.3, + "2026-11": 2204.9 + } + }, + "egress": { + "archive_tiers": [ + "Archive" + ], + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "name": "sampledocsprod", + "size_gib": 2048, + "tiers": { + "Hot": 1500, + "Cool": 300, + "Archive": 248 + } + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "sampledocsbackup", + "size_gib": 900, + "tiers": { + "Cool": 400, + "Archive": 500 + } + }, + { + "type": "Microsoft.Compute/disks", + "name": "sample-docs-vm-osdisk", + "size_gib": 256, + "flags": [ + "allocated (upper bound)" + ] + }, + { + "type": "Microsoft.Sql/servers/databases", + "name": "sample-docs-metadata", + "size_gib": 140 + }, + { + "type": "Microsoft.Sql/servers/databases", + "name": "sample-docs-reporting", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.DocumentDB/databaseAccounts", + "name": "sample-docs-search", + "size_gib": null, + "unknown": true + }, + { + "type": "Microsoft.RecoveryServices/vaults", + "name": "sample-docs-vault", + "size_gib": null + } + ] + }, + "upload": { + "status": "uploaded", + "url": "https://platform.example.test/assessments/00000000-0000-4000-8000-0000000000b1", + "assessment_id": "00000000-0000-4000-8000-0000000000b1", + "sync_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0004" + }, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0004" +} diff --git a/tests/fixtures/samples/egress_free_tier.json b/tests/fixtures/samples/egress_free_tier.json new file mode 100644 index 0000000..ede6e4a --- /dev/null +++ b/tests/fixtures/samples/egress_free_tier.json @@ -0,0 +1,66 @@ +{ + "name": "Sample – Small Data Platform", + "purpose": "Egress below the free tier: a fee of $0.00 and the headroom sentence.", + "provider": "aws", + "exit_strategy": 3, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-central-1", + "count": 3 + }, + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-central-1", + "count": 2 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-central-1", + "count": 1 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-06": 220.45, + "2026-07": 231.1, + "2026-08": 228.6, + "2026-09": 240.05, + "2026-10": 236.9, + "2026-11": 98.4 + } + }, + "egress": { + "archive_tiers": [ + "Archive", + "Glacier", + "Deep Archive" + ], + "resources": [ + { + "type": "AWS.s3.list_buckets.Buckets", + "name": "sample-data-landing", + "size_gib": 14, + "tiers": { + "Standard": 14 + } + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "name": "sample-data-db", + "size_gib": 6, + "flags": [ + "allocated (upper bound)" + ] + } + ] + }, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0010" +} diff --git a/tests/fixtures/samples/egress_multi_tib.json b/tests/fixtures/samples/egress_multi_tib.json new file mode 100644 index 0000000..a539a66 --- /dev/null +++ b/tests/fixtures/samples/egress_multi_tib.json @@ -0,0 +1,80 @@ +{ + "name": "Sample – Media Library", + "purpose": "Egress of tens of TiB: several price tiers and a large fee.", + "provider": "azure", + "exit_strategy": 3, + "scope": { + "tenantId": "00000000-0000-4000-8000-0000000000a1", + "subscriptionId": "00000000-0000-4000-8000-0000000000a2", + "resourceGroupName": "rg-sample-media" + }, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "region": "westeurope", + "count": 14 + }, + { + "type": "Microsoft.Compute/virtualMachines", + "region": "westeurope", + "count": 6 + }, + { + "type": "Microsoft.Sql/servers/databases", + "region": "westeurope", + "count": 2 + } + ], + "costs": { + "currency": "EUR", + "months": { + "2026-06": 18200.0, + "2026-07": 18950.5, + "2026-08": 19120.75, + "2026-09": 19810.2, + "2026-10": 20344.0, + "2026-11": 8120.6 + } + }, + "egress": { + "archive_tiers": [ + "Archive" + ], + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "name": "samplemediaoriginals", + "size_gib": 36864, + "tiers": { + "Hot": 12000, + "Cool": 14864, + "Archive": 10000 + } + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "samplemediaproxies", + "size_gib": 12288, + "tiers": { + "Hot": 12288 + } + }, + { + "type": "Microsoft.Compute/disks", + "name": "sample-media-render-disk", + "size_gib": 8192, + "flags": [ + "allocated (upper bound)" + ] + }, + { + "type": "Microsoft.Sql/servers/databases", + "name": "sample-media-metadata", + "size_gib": 1024 + } + ] + }, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0011" +} diff --git a/tests/fixtures/samples/empty.json b/tests/fixtures/samples/empty.json new file mode 100644 index 0000000..761edce --- /dev/null +++ b/tests/fixtures/samples/empty.json @@ -0,0 +1,26 @@ +{ + "name": "Sample – Empty Scope", + "purpose": "No resources and no costs.", + "provider": "aws", + "exit_strategy": 1, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [], + "costs": { + "currency": "USD", + "months": { + "2026-06": 0, + "2026-07": 0, + "2026-08": 0, + "2026-09": 0, + "2026-10": 0, + "2026-11": 0 + } + }, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0009" +} diff --git a/tests/fixtures/samples/no_risks.json b/tests/fixtures/samples/no_risks.json new file mode 100644 index 0000000..5a7feae --- /dev/null +++ b/tests/fixtures/samples/no_risks.json @@ -0,0 +1,47 @@ +{ + "name": "Sample – Static Website", + "purpose": "Every type has enough supported alternatives, so the Risk Inventory is empty.", + "provider": "aws", + "exit_strategy": 2, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.apigateway.get_rest_apis.items", + "region": "eu-central-1", + "count": 2 + }, + { + "type": "AWS.acm.list_certificates.CertificateSummaryList", + "region": "eu-central-1", + "count": 3 + }, + { + "type": "AWS.cloudwatch.describe_alarms.MetricAlarms", + "region": "eu-central-1", + "count": 6 + }, + { + "type": "AWS.backup.list_backup_plans.BackupPlansList", + "region": "eu-central-1", + "count": 1 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-06": 42.1, + "2026-07": 44.35, + "2026-08": 41.9, + "2026-09": 45.2, + "2026-10": 43.75, + "2026-11": 19.1 + } + }, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0008" +} diff --git a/tests/fixtures/samples/stress.json b/tests/fixtures/samples/stress.json new file mode 100644 index 0000000..17c7217 --- /dev/null +++ b/tests/fixtures/samples/stress.json @@ -0,0 +1,211 @@ +{ + "name": "Sample – A Deliberately Long Project Name That Keeps Going To Check How Titles, Headers And The Cover Wrap Over Several Lines", + "purpose": "Many types, long names, big counts, cent-level amounts and mixed currencies.", + "provider": "aws", + "exit_strategy": 1, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.acm-pca", + "region": "eu-central-1", + "count": 1 + }, + { + "type": "AWS.acm.list_certificates.CertificateSummaryList", + "region": "eu-west-1", + "count": 8 + }, + { + "type": "AWS.amplify.list_apps.apps", + "region": "us-east-1", + "count": 15 + }, + { + "type": "AWS.apigateway.get_rest_apis.items", + "region": "eu-central-1", + "count": 22 + }, + { + "type": "AWS.appconfig.list_applications.Items", + "region": "eu-west-1", + "count": 29 + }, + { + "type": "AWS.appflow.list_flows.flows", + "region": "us-east-1", + "count": 36 + }, + { + "type": "AWS.application-autoscaling", + "region": "eu-central-1", + "count": 2 + }, + { + "type": "AWS.appmesh.list_meshes.meshes", + "region": "eu-west-1", + "count": 9 + }, + { + "type": "AWS.apprunner.list_services.ServiceSummaryList", + "region": "us-east-1", + "count": 16 + }, + { + "type": "AWS.appstream.describe_images.Images", + "region": "eu-central-1", + "count": 23 + }, + { + "type": "AWS.appsync.list_graphql_apis.graphqlApis", + "region": "eu-west-1", + "count": 30 + }, + { + "type": "AWS.athena.list_databases.DatabaseList", + "region": "us-east-1", + "count": 37 + }, + { + "type": "AWS.auditmanager.list_assessments.assessmentMetadata", + "region": "eu-central-1", + "count": 3 + }, + { + "type": "AWS.backup.list_backup_plans.BackupPlansList", + "region": "eu-west-1", + "count": 10 + }, + { + "type": "AWS.batch.describe_job_queues.jobQueues", + "region": "us-east-1", + "count": 17 + }, + { + "type": "AWS.cloud9.list_environments.environmentIds", + "region": "eu-central-1", + "count": 24 + }, + { + "type": "AWS.cloudcontrol.list_resources.ResourceDescriptions", + "region": "eu-west-1", + "count": 31 + }, + { + "type": "AWS.cloudformation.list_stacks.StackSummaries", + "region": "us-east-1", + "count": 38 + }, + { + "type": "AWS.cloudfront.list_distributions.DistributionList.Items", + "region": "eu-central-1", + "count": 4 + }, + { + "type": "AWS.cloudtrail.describe_trails.trailList", + "region": "eu-west-1", + "count": 11 + }, + { + "type": "AWS.cloudwatch.describe_alarms.MetricAlarms", + "region": "us-east-1", + "count": 18 + }, + { + "type": "AWS.codeartifact.list_repositories.repositories", + "region": "eu-central-1", + "count": 25 + }, + { + "type": "AWS.codebuild.list_projects.projects", + "region": "eu-west-1", + "count": 32 + }, + { + "type": "AWS.codecommit.list_repositories.repositories", + "region": "us-east-1", + "count": 39 + }, + { + "type": "AWS.dynamodb.list_tables.TableNames", + "region": "eu-central-1", + "count": 5 + }, + { + "type": "AWS.ec2.describe_volumes.Volumes", + "region": "eu-west-1", + "count": 12 + }, + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "us-east-1", + "count": 19 + }, + { + "type": "AWS.eks.list_clusters.clusters", + "region": "eu-central-1", + "count": 26 + }, + { + "type": "AWS.elasticache.describe_cache_clusters.CacheClusters", + "region": "eu-west-1", + "count": 33 + }, + { + "type": "AWS.kinesis.list_streams.StreamNames", + "region": "us-east-1", + "count": 40 + }, + { + "type": "AWS.lambda.list_functions.Functions", + "region": "eu-central-1", + "count": 6 + }, + { + "type": "AWS.rds.describe_db_instances.DBInstances", + "region": "eu-west-1", + "count": 13 + }, + { + "type": "AWS.route53.list_hosted_zones.HostedZones", + "region": "us-east-1", + "count": 20 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-central-1", + "count": 27 + }, + { + "type": "AWS.sns.list_topics.Topics", + "region": "eu-west-1", + "count": 34 + }, + { + "type": "AWS.sqs.list_queues.QueueUrls", + "region": "us-east-1", + "count": 41 + }, + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "ap-southeast-2", + "count": 123 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-10": 0.07, + "2026-11": 0.03 + }, + "currencies": { + "2026-11": "EUR" + } + }, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0012" +} diff --git a/tests/fixtures/samples/tfstate/sample_aws.tfstate b/tests/fixtures/samples/tfstate/sample_aws.tfstate new file mode 100644 index 0000000..23ec7c1 --- /dev/null +++ b/tests/fixtures/samples/tfstate/sample_aws.tfstate @@ -0,0 +1,104 @@ +{ + "version": 4, + "terraform_version": "1.14.6", + "serial": 12, + "lineage": "00000000-0000-4000-8000-0000000000c1", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "aws_instance", + "name": "portal", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "availability_zone": "eu-central-1a", + "arn": "arn:aws:ec2:eu-central-1:123456789012:instance/i-0123456789abcdef0" + } + }, + { + "schema_version": 0, + "attributes": { + "availability_zone": "eu-central-1b", + "arn": "arn:aws:ec2:eu-central-1:123456789012:instance/i-0123456789abcdef1" + } + }, + { + "schema_version": 0, + "attributes": { + "availability_zone": "eu-west-1a", + "arn": "arn:aws:ec2:eu-west-1:123456789012:instance/i-0123456789abcdef2" + } + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "assets", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "region": "eu-central-1", + "bucket": "sample-portal-assets", + "arn": "arn:aws:s3:::sample-portal-assets" + } + }, + { + "schema_version": 0, + "attributes": { + "bucket": "sample-portal-exports", + "arn": "arn:aws:s3:::sample-portal-exports" + } + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "worker", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:lambda:eu-central-1:123456789012:function:sample-worker" + } + } + ] + }, + { + "mode": "managed", + "type": "aws_sqs_queue", + "name": "jobs", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "region": "eu-central-1", + "arn": "arn:aws:sqs:eu-central-1:123456789012:sample-jobs" + } + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_role", + "name": "worker", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::123456789012:role/sample-worker" + } + } + ] + } + ] +} diff --git a/tests/fixtures/samples/tfstate/sample_azure.tfstate b/tests/fixtures/samples/tfstate/sample_azure.tfstate new file mode 100644 index 0000000..b5d5b52 --- /dev/null +++ b/tests/fixtures/samples/tfstate/sample_azure.tfstate @@ -0,0 +1,76 @@ +{ + "version": 4, + "terraform_version": "1.14.6", + "serial": 12, + "lineage": "00000000-0000-4000-8000-0000000000c2", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "azurerm_virtual_machine", + "name": "app", + "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "/subscriptions/00000000-0000-4000-8000-0000000000a2/resourceGroups/rg-sample-archive/providers/Microsoft.Compute/virtualMachines/sample-app-1", + "location": "westeurope" + } + }, + { + "schema_version": 0, + "attributes": { + "id": "/subscriptions/00000000-0000-4000-8000-0000000000a2/resourceGroups/rg-sample-archive/providers/Microsoft.Compute/virtualMachines/sample-app-2", + "location": "northeurope" + } + } + ] + }, + { + "mode": "managed", + "type": "azurerm_storage_account", + "name": "docs", + "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "/subscriptions/00000000-0000-4000-8000-0000000000a2/resourceGroups/rg-sample-archive/providers/Microsoft.Storage/storageAccounts/sampledocsprod", + "location": "westeurope" + } + } + ] + }, + { + "mode": "managed", + "type": "azurerm_mssql_database", + "name": "metadata", + "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "/subscriptions/00000000-0000-4000-8000-0000000000a2/resourceGroups/rg-sample-archive/providers/Microsoft.Sql/servers/sample-sql/databases/metadata", + "location": "westeurope" + } + } + ] + }, + { + "mode": "managed", + "type": "azurerm_resource_group", + "name": "main", + "provider": "provider[\"registry.terraform.io/hashicorp/azurerm\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "/subscriptions/00000000-0000-4000-8000-0000000000a2/resourceGroups/rg-sample-archive", + "location": "westeurope" + } + } + ] + } + ] +} diff --git a/tests/fixtures/samples/tfstate_aws.json b/tests/fixtures/samples/tfstate_aws.json new file mode 100644 index 0000000..97561ba --- /dev/null +++ b/tests/fixtures/samples/tfstate_aws.json @@ -0,0 +1,15 @@ +{ + "name": "Sample – Customer Self-Service Portal (Terraform)", + "purpose": "A Terraform state: no costs, a resource without a region, never egress.", + "provider": "aws", + "exit_strategy": 1, + "scope": { + "tfstate": "sample_aws.tfstate" + }, + "resources": [], + "costs": null, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0006" +} diff --git a/tests/fixtures/samples/tfstate_azure.json b/tests/fixtures/samples/tfstate_azure.json new file mode 100644 index 0000000..3956583 --- /dev/null +++ b/tests/fixtures/samples/tfstate_azure.json @@ -0,0 +1,15 @@ +{ + "name": "Sample – Document Archive (Terraform)", + "purpose": "A Terraform state of an Azure resource group: the state scope rows.", + "provider": "azure", + "exit_strategy": 1, + "scope": { + "tfstate": "sample_azure.tfstate" + }, + "resources": [], + "costs": null, + "egress": null, + "upload": null, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0007" +} diff --git a/tests/fixtures/samples/uploaded_no_link.json b/tests/fixtures/samples/uploaded_no_link.json new file mode 100644 index 0000000..742bf06 --- /dev/null +++ b/tests/fixtures/samples/uploaded_no_link.json @@ -0,0 +1,46 @@ +{ + "name": "Sample – Field Service Scheduler", + "purpose": "Uploaded with an assessment ID and no link.", + "provider": "aws", + "exit_strategy": 2, + "scope": { + "accountId": "123456789012", + "region": "eu-central-1" + }, + "resources": [ + { + "type": "AWS.ec2.describe_instances.Reservations", + "region": "eu-central-1", + "count": 3 + }, + { + "type": "AWS.s3.list_buckets.Buckets", + "region": "eu-central-1", + "count": 2 + }, + { + "type": "AWS.lambda.list_functions.Functions", + "region": "eu-central-1", + "count": 4 + } + ], + "costs": { + "currency": "USD", + "months": { + "2026-06": 310.2, + "2026-07": 295.85, + "2026-08": 322.4, + "2026-09": 301.1, + "2026-10": 318.65, + "2026-11": 140.3 + } + }, + "egress": null, + "upload": { + "status": "uploaded", + "assessment_id": "sample-assessment-0001", + "sync_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0005" + }, + "clock": "2026-11-12T09:30:00Z", + "run_id": "01939b8e-7c2a-7d41-9a51-3c8f0a6e0005" +} diff --git a/tests/report/test_basic_report.py b/tests/report/test_basic_report.py index 5da2f82..a4b7dae 100644 --- a/tests/report/test_basic_report.py +++ b/tests/report/test_basic_report.py @@ -5,7 +5,6 @@ import tempfile import unittest from datetime import datetime, timezone -from html.parser import HTMLParser from pathlib import Path from unittest.mock import patch @@ -35,6 +34,7 @@ from tests.report import test_egress as egress_helpers from tests.report.test_no_credentials import pricing_for_both_clouds from tests.support.flow import build_flow_run +from tests.support.html import html_text from tests.support.report import ( build_empty_report_fixture, build_fixture_context, @@ -118,31 +118,6 @@ def uploaded(**changes): return provenance(uploaded=True, platform_url=URL, assessment_id="a-1", **changes) -class TextOfHtml(HTMLParser): - def __init__(self): - super().__init__() - self.parts = [] - self.skip = 0 - - def handle_starttag(self, tag, attrs): - if tag in ("script", "style"): - self.skip += 1 - - def handle_endtag(self, tag): - if tag in ("script", "style") and self.skip: - self.skip -= 1 - - def handle_data(self, data): - if not self.skip: - self.parts.append(data) - - -def html_text(html): - parser = TextOfHtml() - parser.feed(html) - return " ".join(" ".join(parser.parts).split()) - - def page_texts(reader): return [" ".join(page.extract_text().split()) for page in reader.pages] diff --git a/tests/report/test_egress_report.py b/tests/report/test_egress_report.py index efd0352..d1bfc4d 100644 --- a/tests/report/test_egress_report.py +++ b/tests/report/test_egress_report.py @@ -23,7 +23,6 @@ from tests.report.test_basic_report import ( COLLECTED, URL, - html_text, metadata_texts, outline_entries, page_texts, @@ -32,6 +31,7 @@ ) from tests.report.test_no_credentials import pricing_for_both_clouds from tests.support.flow import build_flow_run +from tests.support.html import html_text from tests.support.report import ( build_fixture_context, build_report_fixture, diff --git a/tests/samples/__init__.py b/tests/samples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/samples/__main__.py b/tests/samples/__main__.py new file mode 100644 index 0000000..1a7856c --- /dev/null +++ b/tests/samples/__main__.py @@ -0,0 +1,178 @@ +import argparse +import sys +from html import escape +from pathlib import Path + +from cloudexit.catalogue.store import active_catalogue_path +from tests.samples.render import render_cases +from tests.samples.scenarios import ScenarioError, load_scenarios + +DOCUMENTS = ( + ("report.pdf", "Executive Summary"), + ("egress.pdf", "Egress & Data Landscape"), +) +PAGE = (595, 842) + +STYLE = """ +body { font: 14px/1.45 system-ui, sans-serif; margin: 2rem; color: #1a1f24; } +table { border-collapse: collapse; width: 100%; } +th, td { text-align: left; vertical-align: top; padding: .45rem .7rem; + border-bottom: 1px solid #e3e7ea; } +th { background: #f5f7f8; } +a { color: #005470; } +.failed { color: #b8322a; font-weight: 600; } +.muted { color: #6b7680; } +""" + + +def parse(argv): + parser = argparse.ArgumentParser( + prog="python -m tests.samples", + description="Render the sample assessments through the real report pipeline.", + ) + parser.add_argument("--case", nargs="+", metavar="NAME", help="cases to render") + parser.add_argument("--out", default="samples", help="output folder") + parser.add_argument( + "--catalogue", help="catalogue to render with (default: the installed one)" + ) + parser.add_argument( + "--sheets", action="store_true", help="contact sheets, needs PyMuPDF" + ) + return parser.parse_args(argv) + + +def load_pymupdf(): + try: + import pymupdf + except ImportError: + try: + import fitz as pymupdf + except ImportError: + return None + return pymupdf + + +def write_sheets(result, pymupdf): + # Four pages to a sheet, two by two + sheets, number = [], 0 + for name, _title in DOCUMENTS: + if name not in result.pages: + continue + source = pymupdf.open(str(result.directory / name)) + for first in range(0, source.page_count, 4): + number += 1 + last = min(first + 4, source.page_count) + sheet = pymupdf.open() + page = sheet.new_page(width=2 * PAGE[0], height=2 * PAGE[1]) + for slot, index in enumerate(range(first, last)): + x, y = (slot % 2) * PAGE[0], (slot // 2) * PAGE[1] + page.show_pdf_page( + pymupdf.Rect(x, y, x + PAGE[0], y + PAGE[1]), source, index + ) + file = f"sheet-{number}.png" + page.get_pixmap(dpi=110).save(str(result.directory / file)) + sheets.append((f"{name} {first + 1}–{last}", file)) + source.close() + return sheets + + +def link(case, file, label): + return f'{escape(label)}' + + +def overview(scenarios, results, failures, sheets): + by_case = {result.case: result for result in results} + rows = [] + for case, scenario in scenarios.items(): + result = by_case.get(case) + if result is None: + rows.append( + f"{escape(case)}{escape(scenario.name)}" + f'{escape(scenario.provider)}' + f"{escape(failures.get(case, 'not rendered'))}" + ) + continue + pages = ", ".join( + f"{title}: {result.pages[name]}" + for name, title in DOCUMENTS + if name in result.pages + ) + links = [ + link(case, name, title) + for name, title in ( + ("report.pdf", "Executive Summary PDF"), + ("index.html", "Executive Summary HTML"), + ("egress.pdf", "Egress PDF"), + ("egress.html", "Egress HTML"), + ) + if name in result.files + ] + links += [link(case, file, label) for label, file in sheets.get(case, [])] + rows.append( + f"{escape(case)}{escape(scenario.name)}" + f"{escape(scenario.provider)}{escape(scenario.purpose)}" + f"{escape(pages)}{'
'.join(links)}" + ) + return ( + '' + '' + f"Sample Reports" + '

Sample reports

Fictional data, rendered through ' + "the real report pipeline.

" + "" + f"{''.join(rows)}
CaseNameProviderPurposePagesOpen
\n" + ) + + +def main(argv=None): + args = parse(argv) + try: + scenarios = load_scenarios() + except ScenarioError as exc: + print(f"Scenario error: {exc}", file=sys.stderr) + return 2 + + if args.case: + unknown = [case for case in args.case if case not in scenarios] + if unknown: + print( + f"No such case: {', '.join(unknown)}. Cases: {', '.join(scenarios)}", + file=sys.stderr, + ) + return 2 + scenarios = {case: scenarios[case] for case in scenarios if case in args.case} + + catalogue = Path(args.catalogue) if args.catalogue else active_catalogue_path() + if not catalogue.is_file(): + print( + f"No catalogue at {catalogue}. Run cloudexit once so that the " + "catalogue is downloaded, or pass --catalogue PATH.", + file=sys.stderr, + ) + return 2 + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + results, failures = render_cases(scenarios.values(), out, catalogue_path=catalogue) + + sheets = {} + if args.sheets: + pymupdf = load_pymupdf() + if pymupdf is None: + print("PyMuPDF is not installed: no contact sheets (pip install pymupdf).") + else: + sheets = {result.case: write_sheets(result, pymupdf) for result in results} + + (out / "index.html").write_text( + overview(scenarios, results, failures, sheets), encoding="utf-8" + ) + for result in results: + print(f"rendered {result.case}") + for message in failures.values(): + print(f"FAILED {message}", file=sys.stderr) + print(f"Overview: {out / 'index.html'}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/samples/catalogue.py b/tests/samples/catalogue.py new file mode 100644 index 0000000..668a2a0 --- /dev/null +++ b/tests/samples/catalogue.py @@ -0,0 +1,222 @@ +import sqlite3 + +from tests.make_smoke_dataset import TABLES +from tests.support.catalogue import build_catalogue +from tests.support.flow import RISKS + +EXTRA_TABLES = """ +CREATE TABLE resourcetype_data ( + id INTEGER PRIMARY KEY, resource_type INTEGER NOT NULL UNIQUE, + data_category TEXT NOT NULL, strategy TEXT NOT NULL, params TEXT, + status TEXT NOT NULL +); +CREATE TABLE egresspricing ( + id INTEGER PRIMARY KEY, csp INTEGER NOT NULL, component TEXT NOT NULL, + zone TEXT NOT NULL DEFAULT 'zone1', tier_from REAL NOT NULL, tier_to REAL, + price_per_unit REAL NOT NULL, currency TEXT NOT NULL DEFAULT 'USD', + unit TEXT NOT NULL DEFAULT 'GiB', valid_from TEXT NOT NULL, valid_to TEXT +); +""" + +# Terraform type to resource type code, for the state file fixtures +TF_CODES = { + "aws_instance": "AWS.ec2.describe_instances.Reservations", + "aws_s3_bucket": "AWS.s3.list_buckets.Buckets", + "aws_lambda_function": "AWS.lambda.list_functions.Functions", + "aws_sqs_queue": "AWS.sqs.list_queues.QueueUrls", + "azurerm_virtual_machine": "Microsoft.Compute/virtualMachines", + "azurerm_storage_account": "Microsoft.Storage/storageAccounts", + "azurerm_mssql_database": "Microsoft.Sql/servers/databases", +} + +NAMES = { + "AWS.ec2.describe_instances.Reservations": "EC2 Instance", + "AWS.s3.list_buckets.Buckets": "S3 Bucket", + "AWS.ec2.describe_volumes.Volumes": "EBS Volume", + "AWS.rds.describe_db_instances.DBInstances": "RDS Instance", + "AWS.dynamodb.list_tables.TableNames": "DynamoDB Table", + "AWS.lambda.list_functions.Functions": "Lambda Function", + "AWS.eks.list_clusters.clusters": "EKS Cluster", + "AWS.sqs.list_queues.QueueUrls": "SQS Queue", + "AWS.sns.list_topics.Topics": "SNS Topic", + "AWS.apigateway.get_rest_apis.items": "API Gateway", + "AWS.acm.list_certificates.CertificateSummaryList": "Certificate Manager", + "AWS.cloudwatch.describe_alarms.MetricAlarms": "CloudWatch Alarm", + "AWS.backup.list_backup_plans.BackupPlansList": "Backup Plan", + "Microsoft.Compute/virtualMachines": "Virtual Machine", + "Microsoft.ContainerService/managedClusters": "Kubernetes Cluster", + "Microsoft.Storage/storageAccounts": "Storage Account", + "Microsoft.Sql/servers/databases": "SQL Database", + "Microsoft.DocumentDB/databaseAccounts": "Cosmos DB Account", + "Microsoft.DBforPostgreSQL/flexibleServers": "PostgreSQL Flexible Server", + "Microsoft.RecoveryServices/vaults": "Recovery Services Vault", + "Microsoft.ContainerRegistry/registries": "Container Registry", + "Microsoft.Compute/disks": "Managed Disk", + # Deliberately long, for the stress case + "AWS.cloudformation.list_stacks.StackSummaries": ( + "Sample Infrastructure Stack Definition With An Unreasonably Long Name" + ), + "AWS.codeartifact.list_repositories.repositories": ( + "Sample Artifact Repository For Internally Built And Mirrored Packages" + ), + "AWS.cloudfront.list_distributions.DistributionList.Items": ( + "Sample Content Delivery Distribution For Static And Dynamic Content" + ), +} + +ICONS = { + "AWS.ec2.describe_instances.Reservations": "/icons/aws/Compute/EC2.png", + "AWS.s3.list_buckets.Buckets": "/icons/aws/Storage/Simple-Storage-Service.png", + "AWS.ec2.describe_volumes.Volumes": "/icons/aws/Storage/Elastic-Block-Store.png", + "AWS.rds.describe_db_instances.DBInstances": "/icons/aws/Database/RDS.png", + "AWS.dynamodb.list_tables.TableNames": "/icons/aws/Database/DynamoDB.png", + "AWS.lambda.list_functions.Functions": "/icons/aws/Compute/Lambda.png", + "Microsoft.Compute/virtualMachines": ( + "/icons/azure/compute/10021-icon-service-Virtual-Machine.png" + ), + "Microsoft.Storage/storageAccounts": ( + "/icons/azure/storage/10086-icon-service-Storage-Accounts.png" + ), + "Microsoft.Sql/servers/databases": ( + "/icons/azure/databases/10130-icon-service-SQL-Database.png" + ), + "Microsoft.Compute/disks": "/icons/azure/compute/10032-icon-service-Disks.png", +} + +# The data a type carries, for the egress estimate +CATEGORIES = { + "AWS.s3.list_buckets.Buckets": "object", + "AWS.ec2.describe_volumes.Volumes": "block", + "AWS.rds.describe_db_instances.DBInstances": "database", + "AWS.dynamodb.list_tables.TableNames": "database", + "Microsoft.Storage/storageAccounts": "object", + "Microsoft.Compute/disks": "block", + "Microsoft.Sql/servers/databases": "database", + "Microsoft.DocumentDB/databaseAccounts": "database", + "Microsoft.RecoveryServices/vaults": "backup", +} + +# Types that are not in the inventory catalogue but still carry data +INACTIVE = {"Microsoft.Compute/disks"} + +# (alternatives, of them with a support plan) per type; the rest has enough +ALTERNATIVES = { + "AWS.ec2.describe_volumes.Volumes": (0, 0), + "AWS.dynamodb.list_tables.TableNames": (0, 0), + "AWS.s3.list_buckets.Buckets": (2, 1), + "AWS.rds.describe_db_instances.DBInstances": (1, 0), + "AWS.eks.list_clusters.clusters": (4, 2), + "Microsoft.ContainerRegistry/registries": (0, 0), + "Microsoft.DocumentDB/databaseAccounts": (2, 2), + "Microsoft.Compute/virtualMachines": (5, 4), +} +DEFAULT_ALTERNATIVES = (3, 3) + +LADDERS = { + 1: (0.087, 0.083, 0.07, 0.05), + 2: (0.09, 0.085, 0.07, 0.05), +} + + +def placeholder_name(code): + if code in NAMES: + return NAMES[code] + service = code.split(".")[1] if code.startswith("AWS.") else code.split("/")[0] + return f"Sample {service} Resource" + + +def provider_of(code): + return 2 if code.startswith("AWS.") else 1 + + +def _pricing_rows(): + rows = [] + for csp, (first, second, third, fourth) in LADDERS.items(): + tiers = ( + (0, 100, 0.0), + (100, 10240, first), + (10240, 51200, second), + (51200, 153600, third), + (153600, None, fourth), + ) + for lower, upper, price in tiers: + rows.append((csp, "internet_egress", lower, upper, price)) + rows.append((csp, "archive_retrieval", 0, None, 0.02)) + return rows + + +def build_sample_catalogue(path, scenarios): + # The codes the scenarios use, and the ones the state files map from + codes = set(TF_CODES.values()) + for scenario in scenarios: + codes |= scenario.type_codes() + codes = sorted(codes) + tf_codes = {code: tf for tf, code in TF_CODES.items()} + + build_catalogue(path, version="2026-11-02") + conn = sqlite3.connect(path) + conn.executescript(TABLES) + conn.executescript(EXTRA_TABLES) + conn.execute("DELETE FROM resourcetype") + conn.executemany("INSERT INTO risk VALUES (?, ?, ?, ?)", RISKS) + + ids = {} + for index, code in enumerate(codes, start=100): + ids[code] = index + conn.execute( + "INSERT INTO resourcetype (id, csp, code, name, icon, status, tf_code) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + index, + provider_of(code), + code, + placeholder_name(code), + ICONS.get(code, ""), + "f" if code in INACTIVE else "t", + tf_codes.get(code), + ), + ) + if code in CATEGORIES: + conn.execute( + "INSERT INTO resourcetype_data (resource_type, data_category, " + "strategy, params, status) VALUES (?, ?, 'sample', '{}', 't')", + (index, CATEGORIES[code]), + ) + + conn.execute( + "INSERT INTO alternativetechnologyorganization (id, name, url, " + "stability_tier, country_code, years_in_business) " + "VALUES (1, 'Sample Vendor', NULL, 3, 'DE', 12)" + ) + technology = 0 + for code, resource_type in ids.items(): + total, supported = ALTERNATIVES.get(code, DEFAULT_ALTERNATIVES) + for position in range(total): + technology += 1 + conn.execute( + "INSERT INTO alternativetechnology (id, product_name, " + "product_description, product_url, open_source, support_plan, " + "status, organization_id) VALUES (?, ?, '', NULL, ?, ?, 't', 1)", + ( + technology, + f"Sample Technology {technology}", + "t" if position % 2 else "f", + "t" if position < supported else "f", + ), + ) + for strategy in (1, 2, 3): + conn.execute( + "INSERT INTO alternative (resource_type, " + "alternative_technology, strategy_type) VALUES (?, ?, ?)", + (resource_type, technology, strategy), + ) + + conn.executemany( + "INSERT INTO egresspricing (csp, component, zone, tier_from, tier_to, " + "price_per_unit, currency, unit, valid_from, valid_to) " + "VALUES (?, ?, 'zone1', ?, ?, ?, 'USD', 'GiB', '2026-07-01', NULL)", + _pricing_rows(), + ) + conn.commit() + conn.close() + return ids diff --git a/tests/samples/render.py b/tests/samples/render.py new file mode 100644 index 0000000..afe9947 --- /dev/null +++ b/tests/samples/render.py @@ -0,0 +1,293 @@ +import logging +import shutil +import sqlite3 +from contextlib import ExitStack +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from unittest.mock import patch + +from pypdf import PdfReader + +from cloudexit.catalogue import store +from cloudexit.catalogue.db import connect +from cloudexit.egress.estimate import estimate_egress +from cloudexit.egress.model import GIB, load_egress_registry, new_row +from cloudexit.engine import ( + create_resource_inventory, + generate_report, + perform_risk_assessment, +) +from cloudexit.report.assets import copy_assets +from cloudexit.report.egress import generate_egress_html_report +from tests.support.html import html_blocks + +# What a rendered case holds. Both HTML reports have a text dump, so a change +# to either can be reviewed with a diff. +EXPECTED_FILES = { + "always": ("report.pdf", "index.html", "texts/report.txt", "texts/index-html.txt"), + "egress": ( + "egress.pdf", + "egress.html", + "texts/egress.txt", + "texts/egress-html.txt", + ), +} +ASSESSMENT_TYPE = 1 + + +class RenderError(Exception): + pass + + +@dataclass +class RenderResult: + case: str + directory: Path + files: list = field(default_factory=list) + pages: dict = field(default_factory=dict) + + +def expected_files(scenario): + files = list(EXPECTED_FILES["always"]) + if scenario.egress is not None: + files += EXPECTED_FILES["egress"] + return files + + +def frozen_datetime(moment): + class FrozenDatetime(datetime): + @classmethod + def now(cls, tz=None): + return moment.astimezone(tz) if tz else moment.replace(tzinfo=None) + + return FrozenDatetime + + +def pdf_dump(path): + reader = PdfReader(str(path)) + parts = [] + for number, page in enumerate(reader.pages, start=1): + lines = [line.rstrip() for line in (page.extract_text() or "").splitlines()] + parts.append(f"--- page {number} ---\n" + "\n".join(lines)) + return "\n".join(parts) + "\n", len(reader.pages) + + +def _catalogue_types(catalogue_path, csp): + uri = f"{Path(catalogue_path).resolve().as_uri()}?mode=ro" + conn = sqlite3.connect(uri, uri=True) + try: + rows = conn.execute( + "SELECT code, id, status FROM resourcetype WHERE csp = ?", (csp,) + ).fetchall() + finally: + conn.close() + return {code: (type_id, status == "t") for code, type_id, status in rows} + + +def _check_types(scenario, catalogue_path): + types = _catalogue_types(catalogue_path, scenario.csp) + for resource in scenario.resources: + code = resource["type"] + if code not in types: + raise RenderError(f"resource type {code} is not in the catalogue") + if not types[code][1]: + raise RenderError(f"resource type {code} is not active in the catalogue") + return {code: type_id for code, (type_id, _active) in types.items()} + + +def _insert_inventory(scenario, report_path, type_ids): + db_path = Path(report_path) / "data" / "assessment.db" + with connect(db_path=db_path) as conn: + conn.executemany( + "INSERT INTO resource_inventory (resource_type, location, count) " + "VALUES (?, ?, ?) ON CONFLICT(resource_type, location) " + "DO UPDATE SET count = excluded.count", + [ + (type_ids[item["type"]], item["region"], item["count"]) + for item in scenario.resources + ], + ) + costs = scenario.costs + if costs: + currencies = costs.get("currencies") or {} + conn.executemany( + "INSERT INTO cost_inventory (month, cost, currency) VALUES (?, ?, ?) " + "ON CONFLICT(month) DO UPDATE SET cost = excluded.cost, " + "currency = excluded.currency", + [ + (f"{month}-01", amount, currencies.get(month, costs["currency"])) + for month, amount in sorted(costs["months"].items()) + ], + ) + conn.commit() + + +def _egress_rows(scenario): + # What the collectors would have found, in the shape they return + registry = load_egress_registry(scenario.csp) + rows = [] + for index, item in enumerate(scenario.egress["resources"], start=1): + key = item["type"].strip().lower() if scenario.csp == 1 else item["type"] + if key not in registry: + raise RenderError(f"egress type {item['type']} is not in the registry") + entry = registry[key] + row = new_row( + f"/sample/{index}", + item["name"], + item["type"], + entry["label"], + entry["category"], + ) + if item["size_gib"] is not None: + row["size_bytes"] = round(item["size_gib"] * GIB) + row["size_unknown"] = bool(item.get("unknown")) + if item.get("tiers"): + row["tier_bytes"] = { + tier: round(gib * GIB) for tier, gib in item["tiers"].items() + } + row["flags"] = list(item.get("flags", [])) + row["notes"] = list(item.get("notes", [])) + rows.append(row) + return rows, set(scenario.egress["archive_tiers"]) + + +def _estimate_egress(scenario, report_path, raw_data_path): + rows, archive_tiers = _egress_rows(scenario) + with ( + patch( + "cloudexit.providers.aws.egress.collect_aws_egress", + return_value=(rows, archive_tiers), + ), + patch( + "cloudexit.providers.azure.egress.collect_azure_egress", + return_value=(rows, archive_tiers), + ), + ): + result = estimate_egress( + scenario.csp, + scenario.provider_details(), + raw_data_path, + report_path=report_path, + name=scenario.name, + exit_strategy=scenario.exit_strategy, + assessment_type=ASSESSMENT_TYPE, + ) + if not result["success"]: + raise RenderError(f"egress estimate failed: {result['logs']}") + return result["json_path"] + + +def _write_dumps(scenario, directory, result): + texts = directory / "texts" + texts.mkdir() + for pdf, dump in (("report.pdf", "report.txt"), ("egress.pdf", "egress.txt")): + if (directory / pdf).is_file(): + text, pages = pdf_dump(directory / pdf) + (texts / dump).write_text(text, encoding="utf-8") + result.pages[pdf] = pages + for html, dump in ( + ("index.html", "index-html.txt"), + ("egress.html", "egress-html.txt"), + ): + if (directory / html).is_file(): + source = (directory / html).read_text(encoding="utf-8") + (texts / dump).write_text(html_blocks(source), encoding="utf-8") + + +def _render(scenario, directory, catalogue_path): + type_ids = _check_types(scenario, catalogue_path) + report_path = str(directory) + raw_data_path = str(directory / "raw_data") + (directory / "raw_data").mkdir(parents=True) + details = scenario.provider_details() + + if scenario.tfstate: + # The real inventory stage, which also copies the assets and the catalogue + result = create_resource_inventory( + scenario.csp, details, report_path, raw_data_path + ) + if not result["success"]: + raise RenderError(f"resource inventory failed: {result['logs']}") + else: + copy_assets(report_path, scenario.csp) + _insert_inventory(scenario, report_path, type_ids) + + egress_json = None + if scenario.egress is not None: + egress_json = _estimate_egress(scenario, report_path, raw_data_path) + + risks = perform_risk_assessment(scenario.exit_strategy, report_path) + if not risks["success"]: + raise RenderError(f"risk assessment failed: {risks['logs']}") + + if egress_json: + html = generate_egress_html_report( + report_path, egress_json, details, scenario.run_id + ) + if not html["success"]: + raise RenderError(f"egress.html failed: {html['logs']}") + + report = generate_report( + scenario.csp, + details, + scenario.exit_strategy, + ASSESSMENT_TYPE, + scenario.name, + report_path, + raw_data_path, + upload=scenario.upload, + sync_id=scenario.run_id, + egress_json_path=egress_json, + ) + if not report["success"]: + raise RenderError(f"report failed: {report['logs']}") + if report.get("egress_pdf_logs"): + raise RenderError(f"egress.pdf failed: {report['egress_pdf_logs']}") + + +def render_case(scenario, out_dir, *, catalogue_path): + directory = Path(out_dir) / scenario.case + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir(parents=True) + + previous = store._active_override + clock = frozen_datetime(scenario.clock) + try: + store.set_active_catalogue(catalogue_path) + with ExitStack() as stack: + # The warnings of the pipeline are about the sample data, not news + stack.callback(logging.disable, logging.root.manager.disable) + logging.disable(logging.WARNING) + for module in ("report.context", "engine", "egress.estimate"): + stack.enter_context(patch(f"cloudexit.{module}.datetime", clock)) + _render(scenario, directory, catalogue_path) + except RenderError as exc: + raise RenderError(f"{scenario.case}: {exc}") from exc + finally: + store.set_active_catalogue(previous) + + result = RenderResult(scenario.case, directory) + _write_dumps(scenario, directory, result) + result.files = [ + name for name in expected_files(scenario) if (directory / name).is_file() + ] + return result + + +def render_cases(scenarios, out_dir, *, catalogue_path): + # A case that fails is reported, and the others still render + results, failures = [], {} + for scenario in scenarios: + try: + results.append( + render_case(scenario, out_dir, catalogue_path=catalogue_path) + ) + except Exception as exc: + failures[scenario.case] = ( + str(exc) + if isinstance(exc, RenderError) + else f"{scenario.case}: {exc!r}" + ) + return results, failures diff --git a/tests/samples/scenarios.py b/tests/samples/scenarios.py new file mode 100644 index 0000000..94a2f46 --- /dev/null +++ b/tests/samples/scenarios.py @@ -0,0 +1,290 @@ +import json +import re +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from tests.support.paths import ROOT + +SAMPLES_DIR = ROOT / "tests" / "fixtures" / "samples" +TFSTATE_DIR = SAMPLES_DIR / "tfstate" + +PROVIDER_IDS = {"azure": 1, "aws": 2} +FIELDS = ( + "name", + "purpose", + "provider", + "exit_strategy", + "scope", + "resources", + "costs", + "egress", + "upload", + "clock", + "run_id", +) +SCOPE_FIELDS = { + "aws": ("accountId", "region"), + "azure": ("tenantId", "subscriptionId", "resourceGroupName"), +} +UPLOAD_FIELDS = ("status", "url", "assessment_id", "sync_id", "reason") +MONTH = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$") +CURRENCY = re.compile(r"^[A-Z]{3}$") +UUID7 = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" +) + + +class ScenarioError(ValueError): + pass + + +@dataclass(frozen=True) +class Scenario: + case: str + path: Path + name: str + purpose: str + provider: str + exit_strategy: int + scope: dict + resources: tuple + costs: dict | None + egress: dict | None + upload: dict | None + clock: datetime + run_id: str + + @property + def csp(self): + return PROVIDER_IDS[self.provider] + + @property + def tfstate(self): + name = self.scope.get("tfstate") + return str(TFSTATE_DIR / name) if name else None + + def provider_details(self): + # What the collectors would have been given, without any credential + if self.tfstate: + return {"tfstatePath": self.tfstate} + return dict(self.scope) + + def type_codes(self): + codes = {resource["type"] for resource in self.resources} + if self.egress: + codes |= {resource["type"] for resource in self.egress["resources"]} + return codes + + +def _fail(path, field, message): + raise ScenarioError(f"{Path(path).name}: {field}: {message}") + + +def _text(path, field, value): + if not isinstance(value, str) or not value.strip(): + _fail(path, field, "expected a non-empty string") + return value + + +def _number(path, field, value, *, allow_none=False): + if value is None and allow_none: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0: + _fail(path, field, "expected a number that is not negative") + return value + + +def _count(path, field, value): + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + _fail(path, field, "expected a whole number from 1") + return value + + +def _only(path, field, data, allowed): + for key in data: + if key not in allowed: + _fail(path, f"{field}.{key}", "unknown field") + + +def _scope(path, provider, scope): + if not isinstance(scope, dict): + _fail(path, "scope", "expected an object") + if "tfstate" in scope: + _only(path, "scope", scope, ("tfstate",)) + name = _text(path, "scope.tfstate", scope["tfstate"]) + if Path(name).name != name or not (TFSTATE_DIR / name).is_file(): + _fail(path, "scope.tfstate", f"no such file in {TFSTATE_DIR.name}/: {name}") + return scope + fields = SCOPE_FIELDS[provider] + _only(path, "scope", scope, fields) + for field in fields: + if field not in scope: + _fail(path, f"scope.{field}", "missing") + _text(path, f"scope.{field}", scope[field]) + return scope + + +def _resources(path, resources): + if not isinstance(resources, list): + _fail(path, "resources", "expected a list") + seen = set() + for index, resource in enumerate(resources): + field = f"resources[{index}]" + if not isinstance(resource, dict): + _fail(path, field, "expected an object") + _only(path, field, resource, ("type", "region", "count")) + for key in ("type", "region", "count"): + if key not in resource: + _fail(path, f"{field}.{key}", "missing") + _text(path, f"{field}.type", resource["type"]) + _text(path, f"{field}.region", resource["region"]) + _count(path, f"{field}.count", resource["count"]) + key = (resource["type"], resource["region"]) + if key in seen: + _fail(path, field, f"{key[0]} in {key[1]} is listed twice") + seen.add(key) + return tuple(resources) + + +def _costs(path, costs): + if costs is None: + return None + if not isinstance(costs, dict): + _fail(path, "costs", "expected an object or null") + _only(path, "costs", costs, ("currency", "months", "currencies")) + for key in ("currency", "months"): + if key not in costs: + _fail(path, f"costs.{key}", "missing") + if not CURRENCY.match(str(costs["currency"])): + _fail(path, "costs.currency", "expected a three-letter code such as USD") + if not isinstance(costs["months"], dict): + _fail(path, "costs.months", "expected an object of YYYY-MM: amount") + for month, amount in costs["months"].items(): + if not MONTH.match(month): + _fail(path, f"costs.months.{month}", "expected a month as YYYY-MM") + _number(path, f"costs.months.{month}", amount) + for month, currency in (costs.get("currencies") or {}).items(): + if month not in costs["months"]: + _fail(path, f"costs.currencies.{month}", "not a month of costs.months") + if not CURRENCY.match(str(currency)): + _fail(path, f"costs.currencies.{month}", "expected a three-letter code") + return costs + + +def _egress(path, egress): + if egress is None: + return None + if not isinstance(egress, dict): + _fail(path, "egress", "expected an object or null") + _only(path, "egress", egress, ("resources", "archive_tiers")) + tiers = egress.get("archive_tiers", []) + if not isinstance(tiers, list) or not all(isinstance(t, str) for t in tiers): + _fail(path, "egress.archive_tiers", "expected a list of tier names") + resources = egress.get("resources") + if not isinstance(resources, list): + _fail(path, "egress.resources", "expected a list") + for index, resource in enumerate(resources): + field = f"egress.resources[{index}]" + if not isinstance(resource, dict): + _fail(path, field, "expected an object") + _only( + path, + field, + resource, + ("type", "name", "size_gib", "unknown", "tiers", "flags", "notes"), + ) + for key in ("type", "name", "size_gib"): + if key not in resource: + _fail(path, f"{field}.{key}", "missing") + _text(path, f"{field}.type", resource["type"]) + _text(path, f"{field}.name", resource["name"]) + _number(path, f"{field}.size_gib", resource["size_gib"], allow_none=True) + for key in ("tiers",): + for tier, gib in (resource.get(key) or {}).items(): + _number(path, f"{field}.{key}.{tier}", gib) + for key in ("flags", "notes"): + values = resource.get(key, []) + if not isinstance(values, list) or not all( + isinstance(v, str) for v in values + ): + _fail(path, f"{field}.{key}", "expected a list of strings") + return egress + + +def _upload(path, upload): + if upload is None: + return None + if not isinstance(upload, dict): + _fail(path, "upload", "expected an object or null") + _only(path, "upload", upload, UPLOAD_FIELDS) + _text(path, "upload.status", upload.get("status")) + return upload + + +def _clock(path, value): + text = _text(path, "clock", value) + if not text.endswith("Z"): + _fail(path, "clock", "expected an ISO time in UTC ending in Z") + try: + return datetime.fromisoformat(text).astimezone(timezone.utc) + except ValueError: + _fail(path, "clock", f"not an ISO time: {text}") + + +def load_scenario(path): + path = Path(path) + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ScenarioError(f"{path.name}: cannot read the file: {exc}") from exc + if not isinstance(data, dict): + _fail(path, "(file)", "expected an object") + for field in FIELDS: + if field not in data: + _fail(path, field, "missing") + _only(path, "(file)", data, FIELDS) + + provider = data["provider"] + if provider not in PROVIDER_IDS: + _fail( + path, + "provider", + f"expected one of {sorted(PROVIDER_IDS)}, got {provider!r}", + ) + strategy = data["exit_strategy"] + if strategy not in (1, 2, 3) or isinstance(strategy, bool): + _fail(path, "exit_strategy", "expected 1, 2 or 3") + run_id = _text(path, "run_id", data["run_id"]) + if not UUID7.match(run_id): + _fail(path, "run_id", "expected a UUIDv7-shaped string") + + scope = _scope(path, provider, data["scope"]) + resources = _resources(path, data["resources"]) + costs = _costs(path, data["costs"]) + egress = _egress(path, data["egress"]) + if "tfstate" in scope and (resources or costs or egress): + _fail(path, "scope.tfstate", "a state file has no resources, costs or egress") + + return Scenario( + case=path.stem, + path=path, + name=_text(path, "name", data["name"]), + purpose=_text(path, "purpose", data["purpose"]), + provider=provider, + exit_strategy=strategy, + scope=scope, + resources=resources, + costs=costs, + egress=egress, + upload=_upload(path, data["upload"]), + clock=_clock(path, data["clock"]), + run_id=run_id, + ) + + +def load_scenarios(directory=SAMPLES_DIR): + return { + path.stem: load_scenario(path) + for path in sorted(Path(directory).glob("*.json")) + } diff --git a/tests/samples/test_samples.py b/tests/samples/test_samples.py new file mode 100644 index 0000000..fea1e96 --- /dev/null +++ b/tests/samples/test_samples.py @@ -0,0 +1,484 @@ +import io +import re +import sqlite3 +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +from pypdf import PdfReader + +from cloudexit.report.sections import AWS_ICONS_URL, AZURE_ICONS_URL +from cloudexit.report.wording import BLOCKS +from tests.report.test_basic_report import metadata_texts, page_texts +from tests.report.test_no_credentials import find_leaks, folder_text +from tests.samples import __main__ as samples_main +from tests.samples.catalogue import build_sample_catalogue +from tests.samples.render import ( + expected_files, + pdf_dump, + render_case, + render_cases, +) +from tests.samples.scenarios import load_scenarios +from tests.support.credentials import AWS_DETAILS, AZURE_DETAILS +from tests.support.html import html_blocks, html_text +from tests.support.report_terms import american_hits, find_terms, retired_term_hits +from tests.support.sync import SECRETS + +IDENTIFYING = ("accountId", "tenantId", "subscriptionId", "resourceGroupName") +ICON_MARKERS = {"azure": "Microsoft group", "aws": "Amazon.com"} +EGRESS_FILES = ( + "egress.pdf", + "egress.html", + "texts/egress.txt", + "texts/egress-html.txt", +) + +STATE = {} + + +def squashed(text): + return text.replace(" ", "") + + +def setUpModule(): + tmp = tempfile.TemporaryDirectory() + STATE["tmp"] = tmp + root = Path(tmp.name) + STATE["scenarios"] = scenarios = load_scenarios() + STATE["catalogue"] = catalogue = root / "catalogue.db" + build_sample_catalogue(catalogue, scenarios.values()) + STATE["out"] = root / "out" + results, failures = render_cases( + scenarios.values(), STATE["out"], catalogue_path=catalogue + ) + STATE["results"] = {result.case: result for result in results} + STATE["failures"] = failures + + +def tearDownModule(): + STATE["tmp"].cleanup() + STATE.clear() + + +class RenderedTestCase(unittest.TestCase): + def cases(self): + yield from STATE["scenarios"].items() + + def folder(self, case): + return STATE["results"][case].directory + + def text(self, case, name): + return (self.folder(case) / name).read_text(encoding="utf-8") + + def pdf_pages(self, case, name): + return page_texts(PdfReader(str(self.folder(case) / name))) + + +class RenderTests(RenderedTestCase): + def test_every_case_renders(self): + self.assertEqual(STATE["failures"], {}) + self.assertEqual(set(STATE["results"]), set(STATE["scenarios"])) + + def test_the_expected_files_exist_and_no_others_of_the_list(self): + for case, scenario in self.cases(): + with self.subTest(case): + for name in expected_files(scenario): + self.assertTrue((self.folder(case) / name).is_file(), name) + if scenario.egress is None: + for name in EGRESS_FILES: + self.assertFalse((self.folder(case) / name).exists(), name) + self.assertEqual(STATE["results"][case].files, expected_files(scenario)) + + def test_a_state_file_never_produces_egress(self): + for case, scenario in self.cases(): + if scenario.tfstate: + with self.subTest(case): + self.assertEqual(scenario.egress, None) + self.assertFalse((self.folder(case) / "egress.pdf").exists()) + + def test_the_dumps_match_the_files(self): + for case, scenario in self.cases(): + with self.subTest(case): + folder = self.folder(case) + for pdf, dump in ( + ("report.pdf", "report.txt"), + ("egress.pdf", "egress.txt"), + ): + if (folder / pdf).is_file(): + text, pages = pdf_dump(folder / pdf) + self.assertEqual(self.text(case, f"texts/{dump}"), text) + self.assertEqual(text.count("--- page "), pages) + for html, dump in ( + ("index.html", "index-html.txt"), + ("egress.html", "egress-html.txt"), + ): + if (folder / html).is_file(): + self.assertEqual( + self.text(case, f"texts/{dump}"), + html_blocks(self.text(case, html)), + ) + + def test_rendering_a_case_twice_gives_the_same_texts(self): + scenario = STATE["scenarios"]["aws_typical"] + with tempfile.TemporaryDirectory() as tmp: + again = render_case(scenario, tmp, catalogue_path=STATE["catalogue"]) + for name in expected_files(scenario): + if name.startswith("texts/"): + self.assertEqual( + (again.directory / name).read_text(encoding="utf-8"), + self.text("aws_typical", name), + name, + ) + + +class GuardTests(RenderedTestCase): + def test_no_credential_and_no_value_of_another_case(self): + for case, scenario in self.cases(): + with self.subTest(case): + folder = self.folder(case) + state = "" + if scenario.tfstate: + state = Path(scenario.tfstate).read_text(encoding="utf-8") + own = {v for k, v in scenario.scope.items() if k in IDENTIFYING} + others = { + v + for other, candidate in self.cases() + if other != case + for k, v in candidate.scope.items() + if k in IDENTIFYING + } + forbidden = {v for v in others - own if v not in state} + texts = folder_text(folder) + for name, text in texts.items(): + for value in forbidden: + self.assertNotIn(value, text, f"{name}: {value}") + for secret in SECRETS: + self.assertNotIn(secret, text, name) + self.assertIsNone(re.search(r"\bAKIA[0-9A-Z]{16}\b", text), name) + for details in (AWS_DETAILS, AZURE_DETAILS): + self.assertEqual(find_leaks(folder, details), []) + + def test_the_term_wording_and_retired_term_guards(self): + for case, scenario in self.cases(): + with self.subTest(case): + folder = self.folder(case) + texts = [] + for pdf in ("report.pdf", "egress.pdf"): + if (folder / pdf).is_file(): + reader = PdfReader(str(folder / pdf)) + for text in metadata_texts(reader): + self.assertEqual(find_terms(text), []) + texts += page_texts(reader) + for html in ("index.html", "egress.html"): + if (folder / html).is_file(): + source = self.text(case, html) + texts.append(html_text(source)) + self.assertEqual(find_terms(source.split("