REL12
REL12-BP03 - Test functional requirements
REL12-BP03: Test functional requirements
Implement comprehensive functional testing to validate that all system components work correctly individually and together. Include unit testing, integration testing, regression testing, and end-to-end validation to ensure system reliability.
Implementation Steps
1. Develop Comprehensive Test Suites
Create unit, integration, and end-to-end tests covering all functional requirements.
2. Implement Automated Testing
Build automated test pipelines that run continuously and on deployment.
3. Establish Test Data Management
Create and maintain realistic test data sets for comprehensive validation.
4. Perform Cross-Service Testing
Validate interactions between different services and components.
5. Monitor Test Coverage and Quality
Track test coverage metrics and continuously improve test quality.
Detailed Implementation
{% raw %}
View code
import boto3
import json
import time
import logging
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any, Tuple
from dataclasses import dataclass, asdict
from enum import Enum
import threading
import requests
import subprocess
import uuid
class TestType(Enum):
UNIT = "unit"
INTEGRATION = "integration"
END_TO_END = "end_to_end"
REGRESSION = "regression"
SMOKE = "smoke"
CONTRACT = "contract"
class TestStatus(Enum):
PENDING = "pending"
RUNNING = "running"
PASSED = "passed"
FAILED = "failed"
SKIPPED = "skipped"
class TestEnvironment(Enum):
DEVELOPMENT = "development"
STAGING = "staging"
PRODUCTION = "production"
TEST = "test"
@dataclass
class TestCase:
test_id: str
name: str
description: str
test_type: TestType
service: str
environment: TestEnvironment
prerequisites: List[str]
test_steps: List[str]
expected_result: str
timeout_seconds: int
retry_count: int
@dataclass
class TestExecution:
execution_id: str
test_id: str
status: TestStatus
start_time: datetime
end_time: Optional[datetime]
duration_seconds: float
result_details: str
error_message: Optional[str]
artifacts: List[str]
class FunctionalTestingSystem:
def __init__(self, region: str = 'us-east-1'):
self.region = region
# AWS clients
self.lambda_client = boto3.client('lambda', region_name=region)
self.codebuild = boto3.client('codebuild', region_name=region)
self.s3 = boto3.client('s3', region_name=region)
self.cloudwatch = boto3.client('cloudwatch', region_name=region)
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
self.logger = logging.getLogger(__name__)
# Test management
self.test_cases: Dict[str, TestCase] = {}
self.test_executions: List[TestExecution] = []
self.test_suites: Dict[str, List[str]] = {}
# Thread safety
self.test_lock = threading.Lock()
def register_test_case(self, test_case: TestCase) -> bool:
"""Register a new test case"""
try:
self.test_cases[test_case.test_id] = test_case
self.logger.info(f"Registered test case: {test_case.name}")
return True
except Exception as e:
self.logger.error(f"Failed to register test case: {str(e)}")
return False
def create_standard_test_cases(self) -> List[TestCase]:
"""Create standard functional test cases"""
test_cases = []
try:
# API Endpoint Tests
api_test = TestCase(
test_id="api-health-check",
name="API Health Check",
description="Verify API endpoints respond correctly",
test_type=TestType.SMOKE,
service="api-gateway",
environment=TestEnvironment.STAGING,
prerequisites=["API deployed", "Database available"],
test_steps=[
"Send GET request to /health endpoint",
"Verify response status is 200",
"Verify response contains expected health data"
],
expected_result="HTTP 200 with valid health response",
timeout_seconds=30,
retry_count=3
)
test_cases.append(api_test)
self.register_test_case(api_test)
# Database Integration Test
db_test = TestCase(
test_id="database-crud-operations",
name="Database CRUD Operations",
description="Test create, read, update, delete operations",
test_type=TestType.INTEGRATION,
service="user-service",
environment=TestEnvironment.TEST,
prerequisites=["Database schema deployed", "Test data loaded"],
test_steps=[
"Create new user record",
"Read user record by ID",
"Update user record",
"Delete user record",
"Verify record is deleted"
],
expected_result="All CRUD operations complete successfully",
timeout_seconds=60,
retry_count=2
)
test_cases.append(db_test)
self.register_test_case(db_test)
# End-to-End User Journey
e2e_test = TestCase(
test_id="user-registration-journey",
name="Complete User Registration Journey",
description="Test full user registration and login flow",
test_type=TestType.END_TO_END,
service="web-application",
environment=TestEnvironment.STAGING,
prerequisites=["All services running", "Email service configured"],
test_steps=[
"Navigate to registration page",
"Fill registration form",
"Submit registration",
"Verify email sent",
"Click verification link",
"Login with new credentials",
"Verify user dashboard loads"
],
expected_result="User successfully registered and logged in",
timeout_seconds=120,
retry_count=1
)
test_cases.append(e2e_test)
self.register_test_case(e2e_test)
self.logger.info(f"Created {len(test_cases)} standard test cases")
return test_cases
except Exception as e:
self.logger.error(f"Failed to create standard test cases: {str(e)}")
return test_cases
def execute_test_case(self, test_id: str) -> str:
"""Execute a single test case"""
try:
test_case = self.test_cases.get(test_id)
if not test_case:
raise ValueError(f"Test case {test_id} not found")
execution_id = f"exec-{uuid.uuid4().hex[:8]}"
execution = TestExecution(
execution_id=execution_id,
test_id=test_id,
status=TestStatus.RUNNING,
start_time=datetime.utcnow(),
end_time=None,
duration_seconds=0.0,
result_details="",
error_message=None,
artifacts=[]
)
with self.test_lock:
self.test_executions.append(execution)
# Execute test based on type
start_time = time.time()
try:
if test_case.test_type == TestType.SMOKE:
result = self._execute_smoke_test(test_case)
elif test_case.test_type == TestType.INTEGRATION:
result = self._execute_integration_test(test_case)
elif test_case.test_type == TestType.END_TO_END:
result = self._execute_e2e_test(test_case)
elif test_case.test_type == TestType.UNIT:
result = self._execute_unit_test(test_case)
else:
result = self._execute_generic_test(test_case)
execution.status = TestStatus.PASSED if result['success'] else TestStatus.FAILED
execution.result_details = result['details']
execution.artifacts = result.get('artifacts', [])
except Exception as e:
execution.status = TestStatus.FAILED
execution.error_message = str(e)
execution.result_details = f"Test execution failed: {str(e)}"
execution.duration_seconds = time.time() - start_time
execution.end_time = datetime.utcnow()
# Record metrics
self._record_test_metrics(execution, test_case)
self.logger.info(f"Test execution completed: {execution_id} - {execution.status.value}")
return execution_id
except Exception as e:
self.logger.error(f"Test execution failed: {str(e)}")
return ""
def _execute_smoke_test(self, test_case: TestCase) -> Dict[str, Any]:
"""Execute smoke test"""
try:
if test_case.test_id == "api-health-check":
# Simulate API health check
response = requests.get("https://api.example.com/health", timeout=10)
success = response.status_code == 200
details = f"Status: {response.status_code}, Response: {response.text[:100]}"
return {
'success': success,
'details': details,
'artifacts': [f"response-{int(time.time())}.json"]
}
return {'success': True, 'details': 'Smoke test completed'}
except Exception as e:
return {'success': False, 'details': f'Smoke test failed: {str(e)}'}
def _execute_integration_test(self, test_case: TestCase) -> Dict[str, Any]:
"""Execute integration test"""
try:
if test_case.test_id == "database-crud-operations":
# Simulate database operations
operations = ['CREATE', 'READ', 'UPDATE', 'DELETE']
results = []
for operation in operations:
# Simulate operation
time.sleep(0.1) # Simulate processing time
results.append(f"{operation}: SUCCESS")
return {
'success': True,
'details': '; '.join(results),
'artifacts': [f"db-test-{int(time.time())}.log"]
}
return {'success': True, 'details': 'Integration test completed'}
except Exception as e:
return {'success': False, 'details': f'Integration test failed: {str(e)}'}
def _execute_e2e_test(self, test_case: TestCase) -> Dict[str, Any]:
"""Execute end-to-end test"""
try:
if test_case.test_id == "user-registration-journey":
# Simulate user journey steps
steps = [
"Navigate to registration page",
"Fill registration form",
"Submit registration",
"Verify email sent",
"Click verification link",
"Login with credentials",
"Verify dashboard loads"
]
completed_steps = []
for step in steps:
time.sleep(0.2) # Simulate step execution
completed_steps.append(f"✓ {step}")
return {
'success': True,
'details': '\n'.join(completed_steps),
'artifacts': [
f"screenshot-{int(time.time())}.png",
f"browser-log-{int(time.time())}.txt"
]
}
return {'success': True, 'details': 'E2E test completed'}
except Exception as e:
return {'success': False, 'details': f'E2E test failed: {str(e)}'}
def _execute_unit_test(self, test_case: TestCase) -> Dict[str, Any]:
"""Execute unit test"""
try:
# Simulate unit test execution
return {
'success': True,
'details': 'All unit tests passed',
'artifacts': [f"unit-test-report-{int(time.time())}.xml"]
}
except Exception as e:
return {'success': False, 'details': f'Unit test failed: {str(e)}'}
def _execute_generic_test(self, test_case: TestCase) -> Dict[str, Any]:
"""Execute generic test"""
try:
return {
'success': True,
'details': f'Generic test {test_case.name} completed',
'artifacts': []
}
except Exception as e:
return {'success': False, 'details': f'Generic test failed: {str(e)}'}
def execute_test_suite(self, suite_name: str) -> Dict[str, Any]:
"""Execute a complete test suite"""
try:
test_ids = self.test_suites.get(suite_name, [])
if not test_ids:
raise ValueError(f"Test suite {suite_name} not found or empty")
suite_results = {
'suite_name': suite_name,
'total_tests': len(test_ids),
'passed': 0,
'failed': 0,
'skipped': 0,
'start_time': datetime.utcnow().isoformat(),
'executions': []
}
for test_id in test_ids:
execution_id = self.execute_test_case(test_id)
if execution_id:
execution = next((e for e in self.test_executions if e.execution_id == execution_id), None)
if execution:
suite_results['executions'].append({
'test_id': test_id,
'execution_id': execution_id,
'status': execution.status.value,
'duration': execution.duration_seconds
})
if execution.status == TestStatus.PASSED:
suite_results['passed'] += 1
elif execution.status == TestStatus.FAILED:
suite_results['failed'] += 1
else:
suite_results['skipped'] += 1
suite_results['end_time'] = datetime.utcnow().isoformat()
suite_results['success_rate'] = (suite_results['passed'] / suite_results['total_tests']) * 100
self.logger.info(f"Test suite {suite_name} completed: {suite_results['passed']}/{suite_results['total_tests']} passed")
return suite_results
except Exception as e:
self.logger.error(f"Test suite execution failed: {str(e)}")
return {}
def create_test_suite(self, suite_name: str, test_ids: List[str]) -> bool:
"""Create a new test suite"""
try:
# Validate test IDs exist
for test_id in test_ids:
if test_id not in self.test_cases:
raise ValueError(f"Test case {test_id} not found")
self.test_suites[suite_name] = test_ids
self.logger.info(f"Created test suite {suite_name} with {len(test_ids)} tests")
return True
except Exception as e:
self.logger.error(f"Failed to create test suite: {str(e)}")
return False
def _record_test_metrics(self, execution: TestExecution, test_case: TestCase) -> None:
"""Record test metrics to CloudWatch"""
try:
# Record test duration
self.cloudwatch.put_metric_data(
Namespace='Testing/Functional',
MetricData=[
{
'MetricName': 'TestDuration',
'Dimensions': [
{'Name': 'TestType', 'Value': test_case.test_type.value},
{'Name': 'Service', 'Value': test_case.service},
{'Name': 'Environment', 'Value': test_case.environment.value}
],
'Value': execution.duration_seconds,
'Unit': 'Seconds'
}
]
)
# Record test result
self.cloudwatch.put_metric_data(
Namespace='Testing/Functional',
MetricData=[
{
'MetricName': 'TestResult',
'Dimensions': [
{'Name': 'TestType', 'Value': test_case.test_type.value},
{'Name': 'Service', 'Value': test_case.service},
{'Name': 'Status', 'Value': execution.status.value}
],
'Value': 1,
'Unit': 'Count'
}
]
)
except Exception as e:
self.logger.warning(f"Failed to record test metrics: {str(e)}")
def get_test_coverage_report(self) -> Dict[str, Any]:
"""Generate test coverage report"""
try:
# Analyze test coverage by service and type
coverage_by_service = {}
coverage_by_type = {}
for test_case in self.test_cases.values():
service = test_case.service
test_type = test_case.test_type.value
if service not in coverage_by_service:
coverage_by_service[service] = {'total': 0, 'types': {}}
coverage_by_service[service]['total'] += 1
if test_type not in coverage_by_service[service]['types']:
coverage_by_service[service]['types'][test_type] = 0
coverage_by_service[service]['types'][test_type] += 1
if test_type not in coverage_by_type:
coverage_by_type[test_type] = 0
coverage_by_type[test_type] += 1
# Calculate recent execution statistics
recent_executions = [
e for e in self.test_executions
if e.start_time > datetime.utcnow() - timedelta(days=7)
]
passed_count = len([e for e in recent_executions if e.status == TestStatus.PASSED])
total_count = len(recent_executions)
success_rate = (passed_count / total_count * 100) if total_count > 0 else 0
report = {
'total_test_cases': len(self.test_cases),
'coverage_by_service': coverage_by_service,
'coverage_by_type': coverage_by_type,
'recent_executions': {
'total': total_count,
'passed': passed_count,
'failed': total_count - passed_count,
'success_rate': success_rate
},
'test_suites': len(self.test_suites),
'recommendations': self._generate_coverage_recommendations(coverage_by_service, coverage_by_type)
}
return report
except Exception as e:
self.logger.error(f"Coverage report generation failed: {str(e)}")
return {}
def _generate_coverage_recommendations(self, service_coverage: Dict, type_coverage: Dict) -> List[str]:
"""Generate test coverage recommendations"""
recommendations = []
try:
# Check for services with low test coverage
for service, coverage in service_coverage.items():
if coverage['total'] < 3:
recommendations.append(f"Increase test coverage for {service} service")
# Check for missing test types
if 'unit' not in coverage['types']:
recommendations.append(f"Add unit tests for {service} service")
if 'integration' not in coverage['types']:
recommendations.append(f"Add integration tests for {service} service")
# Check overall test type balance
total_tests = sum(type_coverage.values())
if total_tests > 0:
unit_percentage = type_coverage.get('unit', 0) / total_tests * 100
if unit_percentage < 60:
recommendations.append("Increase unit test coverage (should be 60%+ of total tests)")
e2e_percentage = type_coverage.get('end_to_end', 0) / total_tests * 100
if e2e_percentage > 20:
recommendations.append("Consider reducing E2E test percentage (should be <20% of total tests)")
return recommendations
except Exception as e:
self.logger.error(f"Recommendation generation failed: {str(e)}")
return []
# Example usage
def main():
# Initialize functional testing system
testing_system = FunctionalTestingSystem(region='us-east-1')
# Create standard test cases
print("Creating standard functional test cases...")
test_cases = testing_system.create_standard_test_cases()
print(f"Created {len(test_cases)} test cases:")
for test_case in test_cases:
print(f"- {test_case.name} ({test_case.test_type.value})")
# Create test suites
testing_system.create_test_suite("smoke_tests", ["api-health-check"])
testing_system.create_test_suite("integration_tests", ["database-crud-operations"])
testing_system.create_test_suite("full_suite", [
"api-health-check",
"database-crud-operations",
"user-registration-journey"
])
# Execute individual test
print("\nExecuting API health check test...")
execution_id = testing_system.execute_test_case("api-health-check")
print(f"Test execution ID: {execution_id}")
# Execute test suite
print("\nExecuting smoke test suite...")
suite_results = testing_system.execute_test_suite("smoke_tests")
print(f"Suite results: {json.dumps(suite_results, indent=2, default=str)}")
# Generate coverage report
coverage_report = testing_system.get_test_coverage_report()
print(f"\nTest coverage report: {json.dumps(coverage_report, indent=2)}")
if __name__ == "__main__":
main(){% endraw %}
AWS Services
Primary Services
- AWS CodeBuild: Automated test execution and CI/CD integration
- AWS Lambda: Serverless test execution and validation
- Amazon S3: Test artifact storage and test data management
- Amazon CloudWatch: Test metrics and monitoring
Supporting Services
- AWS CodePipeline: Automated testing in deployment pipelines
- Amazon EC2: Test environment provisioning
- AWS Step Functions: Complex test workflow orchestration
- Amazon EventBridge: Event-driven test triggering
Benefits
- Comprehensive Validation: Ensure all functional requirements are met
- Early Issue Detection: Catch defects before production deployment
- Regression Prevention: Automated tests prevent reintroduction of bugs
- Quality Assurance: Maintain high code quality through systematic testing
- Confidence in Deployments: Thorough testing reduces deployment risks