Check that the CSV rows reach DynamoDB in CI

.github/workflows/main.yml

@@ -42,6 +42,9 @@ jobs:
         run: |
           pip install -r requirements.txt
 
+      - name: Configure the AWS profile
+        run: lstk setup aws --force --non-interactive
+
       - name: Start LocalStack
         env:
           LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
@@ -58,9 +61,7 @@ jobs:
           lstk cdk deploy --require-approval never
 
       - name: Run tests
-        run: |
-          export BUCKET_NAME=$(lstk aws s3 ls | grep sqsblogstack-inventoryupdatesbucketfe | awk '{print $3}')
-          lstk aws s3 cp sqs_blog/sample_file.csv s3://$BUCKET_NAME
+        run: make test
 
       - name: Send a Slack notification
         if: failure() || github.event_name != 'pull_request'

Makefile

@@ -24,4 +24,7 @@ deploy:		## Deploy infrastructure using CDK
 	lstk cdk bootstrap
 	lstk cdk deploy --require-approval never
 
-.PHONY: usage install start stop logs deploy
+test:		## Upload the sample CSV and verify the items in DynamoDB
+	./test.sh
+
+.PHONY: usage install start stop logs deploy test

test.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Upload the sample CSV and check that every row ends up in DynamoDB
+# (S3 -> CSV processing Lambda -> SQS -> DynamoDB writer Lambda).
+set -euo pipefail
+
+CSV=sqs_blog/sample_file.csv
+MAX_ATTEMPTS=${MAX_ATTEMPTS:-30}
+
+BUCKET_NAME=$(lstk aws s3api list-buckets --query "Buckets[?contains(Name, 'inventoryupdatesbucket')].Name | [0]" --output text)
+TABLE_NAME=$(lstk aws dynamodb list-tables --query "TableNames[?contains(@, 'InventoryUpdates')] | [0]" --output text)
+echo "Bucket: $BUCKET_NAME"
+echo "Table: $TABLE_NAME"
+
+count_items() {
+  lstk aws dynamodb scan --table-name "$TABLE_NAME" --select COUNT --query Count --output text
+}
+
+expected=$(tail -n +2 "$CSV" | grep -c .)
+before=$(count_items)
+
+lstk aws s3 cp "$CSV" "s3://$BUCKET_NAME/" > /dev/null
+echo "Uploaded $CSV ($expected rows)"
+
+for attempt in $(seq 1 "$MAX_ATTEMPTS"); do
+  added=$(( $(count_items) - before ))
+  if [ "$added" -ge "$expected" ]; then
+    break
+  fi
+  echo "Attempt $attempt/$MAX_ATTEMPTS: $added/$expected items in DynamoDB"
+  sleep 2
+done
+
+if [ "$added" -lt "$expected" ]; then
+  echo "Test failed: expected $expected new items, found $added"
+  exit 1
+fi
+
+item=$(lstk aws dynamodb scan --table-name "$TABLE_NAME" \
+  --filter-expression "product_id = :p" \
+  --expression-attribute-values '{":p": {"S": "P002"}}' \
+  --query "Items[0]")
+echo "Sample item: $item"
+if ! echo "$item" | jq -e '.location.S == "Warehouse B" and .quantity.N == "25"' > /dev/null; then
+  echo "Test failed: item for P002 does not match the CSV row"
+  exit 1
+fi
+
+echo "Test passed: all $expected rows were written to DynamoDB"