"""
A/B testing module for App Store Optimization.
Plans and tracks A/B tests for metadata and visual assets.
"""
from typing import Dict, List, Any, Optional
import math
class ABTestPlanner:
"""Plans and tracks A/B tests for ASO elements."""
# Minimum detectable effect sizes (conservative estimates)
MIN_EFFECT_SIZES = {
'icon': 0.10, # 10% conversion improvement
'screenshot': 0.08, # 8% conversion improvement
'title': 0.05, # 5% conversion improvement
'description': 0.03 # 3% conversion improvement
}
# Statistical confidence levels
CONFIDENCE_LEVELS = {
'high': 0.95, # 95% confidence
'standard': 0.90, # 90% confidence
'exploratory': 0.80 # 80% confidence
}
def __init__(self):
"""Initialize A/B test planner."""
self.active_tests = []
def design_test(
self,
test_type: str,
variant_a: Dict[str, Any],
variant_b: Dict[str, Any],
hypothesis: str,
success_metric: str = 'conversion_rate'
) -> Dict[str, Any]:
"""
Design an A/B test with hypothesis and variables.
Args:
test_type: Type of test ('icon', 'screenshot', 'title', 'description')
variant_a: Control variant details
variant_b: Test variant details
hypothesis: Expected outcome hypothesis
success_metric: Metric to optimize
Returns:
Test design with configuration
"""
test_design = {
'test_id': self._generate_test_id(test_type),
'test_type': test_type,
'hypothesis': hypothesis,
'variants': {
'a': {
'name': 'Control',
'details': variant_a,
'traffic_split': 0.5
},
'b': {
'name': 'Variation',
'details': variant_b,
'traffic_split': 0.5
}
},
'success_metric': success_metric,
'secondary_metrics': self._get_secondary_metrics(test_type),
'minimum_effect_size': self.MIN_EFFECT_SIZES.get(test_type, 0.05),
'recommended_confidence': 'standard',
'best_practices': self._get_test_best_practices(test_type)
}
self.active_tests.append(test_design)
return test_design
def calculate_sample_size(
self,
baseline_conversion: float,
minimum_detectable_effect: float,
confidence_level: str = 'standard',
power: float = 0.80
) -> Dict[str, Any]:
"""
Calculate required sample size for statistical significance.
Args:
baseline_conversion: Current conversion rate (0-1)
minimum_detectable_effect: Minimum effect size to detect (0-1)
confidence_level: 'high', 'standard', or 'exploratory'
power: Statistical power (typically 0.80 or 0.90)
Returns:
Sample size calculation with duration estimates
"""
alpha = 1 - self.CONFIDENCE_LEVELS[confidence_level]
beta = 1 - power
# Expected conversion for variant B
expected_conversion_b = baseline_conversion * (1 + minimum_detectable_effect)
# Z-scores for alpha and beta
z_alpha = self._get_z_score(1 - alpha / 2) # Two-tailed test
z_beta = self._get_z_score(power)
# Pooled standard deviation
p_pooled = (baseline_conversion + expected_conversion_b) / 2
sd_pooled = math.sqrt(2 * p_pooled * (1 - p_pooled))
# Sample size per variant
n_per_variant = math.ceil(
((z_alpha + z_beta) ** 2 * sd_pooled ** 2) /
((expected_conversion_b - baseline_conversion) ** 2)
)
total_sample_size = n_per_variant * 2
# Estimate duration based on typical traffic
duration_estimates = self._estimate_test_duration(
total_sample_size,
baseline_conversion
)
return {
'sample_size_per_variant': n_per_variant,
'total_sample_size': total_sample_size,
'baseline_conversion': baseline_conversion,
'expected_conversion_improvement': minimum_detectable_effect,
'expected_conversion_b': expected_conversion_b,
'confidence_level': confidence_level,
'statistical_power': power,
'duration_estimates': duration_estimates,
'recommendations': self._generate_sample_size_recommendations(
n_per_variant,
duration_estimates
)
}
def calculate_significance(
self,
variant_a_conversions: int,
variant_a_visitors: int,
variant_b_conversions: int,
variant_b_visitors: int
) -> Dict[str, Any]:
"""
Calculate statistical significance of test results.
Args:
variant_a_conversions: Conversions for control
variant_a_visitors: Visitors for control
variant_b_conversions: Conversions for variation
variant_b_visitors: Visitors for variation
Returns:
Significance analysis with decision recommendation
"""
# Calculate conversion rates
rate_a = variant_a_conversions / variant_a_visitors if variant_a_visitors > 0 else 0
rate_b = variant_b_conversions / variant_b_visitors if variant_b_visitors > 0 else 0
# Calculate improvement
if rate_a > 0:
relative_improvement = (rate_b - rate_a) / rate_a
else:
relative_improvement = 0
absolute_improvement = rate_b - rate_a
# Calculate standard error
se_a = math.sqrt(rate_a * (1 - rate_a) / variant_a_visitors) if variant_a_visitors > 0 else 0
se_b = math.sqrt(rate_b * (1 - rate_b) / variant_b_visitors) if variant_b_visitors > 0 else 0
se_diff = math.sqrt(se_a**2 + se_b**2)
# Calculate z-score
z_score = absolute_improvement / se_diff if se_diff > 0 else 0
# Calculate p-value (two-tailed)
p_value = 2 * (1 - self._standard_normal_cdf(abs(z_score)))
# Determine significance
is_significant_95 = p_value < 0.05
is_significant_90 = p_value < 0.10
# Generate decision
decision = self._generate_test_decision(
relative_improvement,
is_significant_95,
is_significant_90,
variant_a_visitors + variant_b_visitors
)
return {
'variant_a': {
'conversions': variant_a_conversions,
'visitors': variant_a_visitors,
'conversion_rate': round(rate_a, 4)
},
'variant_b': {
'conversions': variant_b_conversions,
'visitors': variant_b_visitors,
'conversion_rate': round(rate_b, 4)
},
'improvement': {
'absolute': round(absolute_improvement, 4),
'relative_percentage': round(relative_improvement * 100, 2)
},
'statistical_analysis': {
'z_score': round(z_score, 3),
'p_value': round(p_value, 4),
'is_significant_95': is_significant_95,
'is_significant_90': is_significant_90,
'confidence_level': '95%' if is_significant_95 else ('90%' if is_significant_90 else 'Not significant')
},
'decision': decision
}
def track_test_results(
self,
test_id: str,
results_data: Dict[str, Any]
) -> Dict[str, Any]:
"""
Track ongoing test results and provide recommendations.
Args:
test_id: Test identifier
results_data: Current test results
Returns:
Test tracking report with next steps
"""
# Find test
test = next((t for t in self.active_tests if t['test_id'] == test_id), None)
if not test:
return {'error': f'Test {test_id} not found'}
# Calculate significance
significance = self.calculate_significance(
results_data['variant_a_conversions'],
results_data['variant_a_visitors'],
results_data['variant_b_conversions'],
results_data['variant_b_visitors']
)
# Calculate test progress
total_visitors = results_data['variant_a_visitors'] + results_data['variant_b_visitors']
required_sample = results_data.get('required_sample_size', 10000)
progress_percentage = min((total_visitors / required_sample) * 100, 100)
# Generate recommendations
recommendations = self._generate_tracking_recommendations(
significance,
progress_percentage,
test['test_type']
)
return {
'test_id': test_id,
'test_type': test['test_type'],
'progress': {
'total_visitors': total_visitors,
'required_sample_size': required_sample,
'progress_percentage': round(progress_percentage, 1),
'is_complete': progress_percentage >= 100
},
'current_results': significance,
'recommendations': recommendations,
'next_steps': self._determine_next_steps(
significance,
progress_percentage
)
}
def generate_test_report(
self,
test_id: str,
final_results: Dict[str, Any]
) -> Dict[str, Any]:
"""
Generate final test report with insights and recommendations.
Args:
test_id: Test identifier
final_results: Final test results
Returns:
Comprehensive test report
"""
test = next((t for t in self.active_tests if t['test_id'] == test_id), None)
if not test:
return {'error': f'Test {test_id} not found'}
significance = self.calculate_significance(
final_results['variant_a_conversions'],
final_results['variant_a_visitors'],
final_results['variant_b_conversions'],
final_results['variant_b_visitors']
)
# Generate insights
insights = self._generate_test_insights(
test,
significance,
final_results
)
# Implementation plan
implementation_plan = self._create_implementation_plan(
test,
significance
)
return {
'test_summary': {
'test_id': test_id,
'test_type': test['test_type'],
'hypothesis': test['hypothesis'],
'duration_days': final_results.get('duration_days', 'N/A')
},
'results': significance,
'insights': insights,
'implementation_plan': implementation_plan,
'learnings': self._extract_learnings(test, significance)
}
def _generate_test_id(self, test_type: str) -> str:
"""Generate unique test ID."""
import time
timestamp = int(time.time())
return f"{test_type}_{timestamp}"
def _get_secondary_metrics(self, test_type: str) -> List[str]:
"""Get secondary metrics to track for test type."""
metrics_map = {
'icon': ['tap_through_rate', 'impression_count', 'brand_recall'],
'screenshot': ['tap_through_rate', 'time_on_page', 'scroll_depth'],
'title': ['impression_count', 'tap_through_rate', 'search_visibility'],
'description': ['time_on_page', 'scroll_depth', 'tap_through_rate']
}
return metrics_map.get(test_type, ['tap_through_rate'])
def _get_test_best_practices(self, test_type: str) -> List[str]:
"""Get best practices for specific test type."""
practices_map = {
'icon': [
'Test only one element at a time (color vs. style vs. symbolism)',
'Ensure icon is recognizable at small sizes (60x60px)',
'Consider cultural context for global audience',
'Test against top competitor icons'
],
'screenshot': [
'Test order of screenshots (users see first 2-3)',
'Use captions to tell story',
'Show key features and benefits',
'Test with and without device frames'
],
'title': [
'Test keyword variations, not major rebrand',
'Keep brand name consistent',
'Ensure title fits within character limits',
'Test on both search and browse contexts'
],
'description': [
'Test structure (bullet points vs. paragraphs)',
'Test call-to-action placement',
'Test feature vs. benefit focus',
'Maintain keyword density'
]
}
return practices_map.get(test_type, ['Test one variable at a time'])
def _estimate_test_duration(
self,
required_sample_size: int,
baseline_conversion: float
) -> Dict[str, Any]:
"""Estimate test duration based on typical traffic levels."""
# Assume different daily traffic scenarios
traffic_scenarios = {
'low': 100, # 100 page views/day
'medium': 1000, # 1000 page views/day
'high': 10000 # 10000 page views/day
}
estimates = {}
for scenario, daily_views in traffic_scenarios.items():
days = math.ceil(required_sample_size / daily_views)
estimates[scenario] = {
'daily_page_views': daily_views,
'estimated_days': days,
'estimated_weeks': round(days / 7, 1)
}
return estimates
def _generate_sample_size_recommendations(
self,
sample_size: int,
duration_estimates: Dict[str, Any]
) -> List[str]:
"""Generate recommendations based on sample size."""
recommendations = []
if sample_size > 50000:
recommendations.append(
"Large sample size required - consider testing smaller effect size or increasing traffic"
)
if duration_estimates['medium']['estimated_days'] > 30:
recommendations.append(
"Long test duration - consider higher minimum detectable effect or focus on high-impact changes"
)
if duration_estimates['low']['estimated_days'] > 60:
recommendations.append(
"Insufficient traffic for reliable testing - consider user acquisition or broader targeting"
)
if not recommendations:
recommendations.append("Sample size and duration are reasonable for this test")
return recommendations
def _get_z_score(self, percentile: float) -> float:
"""Get z-score for given percentile (approximation)."""
# Common z-scores
z_scores = {
0.80: 0.84,
0.85: 1.04,
0.90: 1.28,
0.95: 1.645,
0.975: 1.96,
0.99: 2.33
}
return z_scores.get(percentile, 1.96)
def _standard_normal_cdf(self, z: float) -> float:
"""Approximate standard normal cumulative distribution function."""
# Using error function approximation
t = 1.0 / (1.0 + 0.2316419 * abs(z))
d = 0.3989423 * math.exp(-z * z / 2.0)
p = d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274))))
if z > 0:
return 1.0 - p
else:
return p
def _generate_test_decision(
self,
improvement: float,
is_significant_95: bool,
is_significant_90: bool,
total_visitors: int
) -> Dict[str, Any]:
"""Generate test decision and recommendation."""
if total_visitors < 1000:
return {
'decision': 'continue',
'rationale': 'Insufficient data - continue test to reach minimum sample size',
'action': 'Keep test running'
}
if is_significant_95:
if improvement > 0:
return {
'decision': 'implement_b',
'rationale': f'Variant B shows {improvement*100:.1f}% improvement with 95% confidence',
'action': 'Implement Variant B'
}
else:
return {
'decision': 'keep_a',
'rationale': 'Variant A performs better with 95% confidence',
'action': 'Keep current version (A)'
}
elif is_significant_90:
if improvement > 0:
return {
'decision': 'implement_b_cautiously',
'rationale': f'Variant B shows {improvement*100:.1f}% improvement with 90% confidence',
'action': 'Consider implementing B, monitor closely'
}
else:
return {
'decision': 'keep_a',
'rationale': 'Variant A performs better with 90% confidence',
'action': 'Keep current version (A)'
}
else:
return {
'decision': 'inconclusive',
'rationale': 'No statistically significant difference detected',
'action': 'Either keep A or test different hypothesis'
}
def _generate_tracking_recommendations(
self,
significance: Dict[str, Any],
progress: float,
test_type: str
) -> List[str]:
"""Generate recommendations for ongoing test."""
recommendations = []
if progress < 50:
recommendations.append(
f"Test is {progress:.0f}% complete - continue collecting data"
)
if progress >= 100:
if significance['statistical_analysis']['is_significant_95']:
recommendations.append(
"Sufficient data collected with significant results - ready to conclude test"
)
else:
recommendations.append(
"Sample size reached but no significant difference - consider extending test or concluding"
)
return recommendations
def _determine_next_steps(
self,
significance: Dict[str, Any],
progress: float
) -> str:
"""Determine next steps for test."""
if progress < 100:
return f"Continue test until reaching 100% sample size (currently {progress:.0f}%)"
decision = significance.get('decision', {}).get('decision', 'inconclusive')
if decision == 'implement_b':
return "Implement Variant B and monitor metrics for 2 weeks"
elif decision == 'keep_a':
return "Keep Variant A and design new test with different hypothesis"
else:
return "Test inconclusive - either keep A or design new test"
def _generate_test_insights(
self,
test: Dict[str, Any],
significance: Dict[str, Any],
results: Dict[str, Any]
) -> List[str]:
"""Generate insights from test results."""
insights = []
improvement = significance['improvement']['relative_percentage']
if significance['statistical_analysis']['is_significant_95']:
insights.append(
f"Strong evidence: Variant B {'improved' if improvement > 0 else 'decreased'} "
f"conversion by {abs(improvement):.1f}% with 95% confidence"
)
insights.append(
f"Tested {test['test_type']} changes: {test['hypothesis']}"
)
# Add context-specific insights
if test['test_type'] == 'icon' and improvement > 5:
insights.append(
"Icon change had substantial impact - visual first impression is critical"
)
return insights
def _create_implementation_plan(
self,
test: Dict[str, Any],
significance: Dict[str, Any]
) -> List[Dict[str, str]]:
"""Create implementation plan for winning variant."""
plan = []
if significance.get('decision', {}).get('decision') == 'implement_b':
plan.append({
'step': '1. Update store listing',
'details': f"Replace {test['test_type']} with Variant B across all platforms"
})
plan.append({
'step': '2. Monitor metrics',
'details': 'Track conversion rate for 2 weeks to confirm sustained improvement'
})
plan.append({
'step': '3. Document learnings',
'details': 'Record insights for future optimization'
})
return plan
def _extract_learnings(
self,
test: Dict[str, Any],
significance: Dict[str, Any]
) -> List[str]:
"""Extract key learnings from test."""
learnings = []
improvement = significance['improvement']['relative_percentage']
learnings.append(
f"Testing {test['test_type']} can yield {abs(improvement):.1f}% conversion change"
)
if test['test_type'] == 'title':
learnings.append(
"Title changes affect search visibility and user perception"
)
elif test['test_type'] == 'screenshot':
learnings.append(
"First 2-3 screenshots are critical for conversion"
)
return learnings
def plan_ab_test(
test_type: str,
variant_a: Dict[str, Any],
variant_b: Dict[str, Any],
hypothesis: str,
baseline_conversion: float
) -> Dict[str, Any]:
"""
Convenience function to plan an A/B test.
Args:
test_type: Type of test
variant_a: Control variant
variant_b: Test variant
hypothesis: Test hypothesis
baseline_conversion: Current conversion rate
Returns:
Complete test plan
"""
planner = ABTestPlanner()
test_design = planner.design_test(
test_type,
variant_a,
variant_b,
hypothesis
)
sample_size = planner.calculate_sample_size(
baseline_conversion,
planner.MIN_EFFECT_SIZES.get(test_type, 0.05)
)
return {
'test_design': test_design,
'sample_size_requirements': sample_size
}
