Cost-Aware AI Architecture

# GPU monthly cost calculator

def estimate_monthly_cost(
    gpu_type: str,
    num_gpus: int,
    pricing: str = "on_demand",  # on_demand, spot, reserved_1y, reserved_3y
    utilization_hours_per_day: float = 24,  # 24 = always on
    days_per_month: int = 30
):
    prices = {
        "T4":       {"on_demand": 0.53, "spot": 0.16, "reserved_1y": 0.33, "reserved_3y": 0.22},
        "A10G":     {"on_demand": 1.01, "spot": 0.30, "reserved_1y": 0.63, "reserved_3y": 0.42},
        "L4":       {"on_demand": 0.81, "spot": 0.24, "reserved_1y": 0.51, "reserved_3y": 0.34},
        "A100_40":  {"on_demand": 3.67, "spot": 1.10, "reserved_1y": 2.29, "reserved_3y": 1.53},
        "A100_80":  {"on_demand": 4.90, "spot": 1.47, "reserved_1y": 3.06, "reserved_3y": 2.04},
        "H100":     {"on_demand": 8.80, "spot": 2.64, "reserved_1y": 5.50, "reserved_3y": 3.67},
    }

    hourly = prices[gpu_type][pricing]
    monthly = hourly * utilization_hours_per_day * days_per_month * num_gpus
    return monthly

# Example: 8 A10G GPUs, on-demand, 24/7
print(estimate_monthly_cost("A10G", 8, "on_demand"))    # $5,818/month

# Same setup with spot instances (for training)
print(estimate_monthly_cost("A10G", 8, "spot"))          # $1,728/month (70% savings)

# Same setup with 1-year reserved (for serving)
print(estimate_monthly_cost("A10G", 8, "reserved_1y"))   # $3,629/month (38% savings)

# Multi-tier serving architecture
# Saves 40-60% on inference costs by routing intelligently

class MultiTierRouter:
    def __init__(self):
        self.tiers = {
            "tier1_cache": {"cost_per_req": 0.00001, "latency_ms": 2},
            "tier2_small": {"cost_per_req": 0.0005,  "latency_ms": 10,  "model": "distilbert"},
            "tier3_medium": {"cost_per_req": 0.003,  "latency_ms": 50,  "model": "gpt-4o-mini"},
            "tier4_large": {"cost_per_req": 0.03,    "latency_ms": 500, "model": "gpt-4o"},
        }

    async def route_and_predict(self, query: str) -> dict:
        # Tier 1: Exact cache match (handles ~30% of traffic)
        cached = await self.cache.get(query_hash(query))
        if cached:
            return {"result": cached, "tier": 1, "cost": 0.00001}

        # Tier 2: Simple classifier for easy queries (handles ~40% of traffic)
        # Examples: "What's your return policy?" → FAQ lookup
        complexity = self.complexity_classifier.predict(query)
        if complexity == "simple":
            result = await self.small_model.predict(query)
            if result.confidence > 0.95:
                return {"result": result, "tier": 2, "cost": 0.0005}

        # Tier 3: Medium model for moderate queries (handles ~20% of traffic)
        result = await self.medium_model.predict(query)
        if result.confidence > 0.85:
            return {"result": result, "tier": 3, "cost": 0.003}

        # Tier 4: Large model for complex queries (handles ~10% of traffic)
        result = await self.large_model.predict(query)
        return {"result": result, "tier": 4, "cost": 0.03}

# Cost comparison at 1M requests/day:
# Without multi-tier: 1M × $0.03 = $30,000/day
# With multi-tier:
#   300K × $0.00001 (cache)  =     $3/day
#   400K × $0.0005 (small)   =   $200/day
#   200K × $0.003 (medium)   =   $600/day
#   100K × $0.03 (large)     = $3,000/day
#   Total:                    = $3,803/day (87% savings)

# Cost monitoring dashboard metrics (export to Grafana/Datadog)

cost_metrics = {
    # Per-request metrics
    "inference_cost_per_request": {
        "description": "Cost of each inference call",
        "calculation": "gpu_hour_cost / requests_served_per_hour",
        "alert": "Avg cost per request > $0.05 for 1 hour"
    },

    # GPU utilization
    "gpu_utilization_percent": {
        "description": "Percentage of GPU compute being used",
        "target": "65-80%",
        "alert_low": "< 40% for 30 min → consider scaling down",
        "alert_high": "> 90% for 10 min → consider scaling up"
    },

    # Daily/monthly spend
    "daily_gpu_spend": {
        "description": "Total GPU cost for the day",
        "calculation": "sum(gpu_hours × hourly_rate) across all instances",
        "alert": "Daily spend > 120% of budget"
    },

    # Cost by model tier
    "cost_by_tier": {
        "description": "Spending breakdown by model tier",
        "alert": "Tier 4 (expensive) usage > 15% of total requests"
    },

    # Cost per business outcome
    "cost_per_conversion": {
        "description": "AI infrastructure cost per user conversion",
        "calculation": "total_ai_cost / conversions_attributed_to_ai",
        "target": "< 5% of revenue per conversion"
    },

    # Waste detection
    "idle_gpu_hours": {
        "description": "GPU hours with < 10% utilization",
        "alert": "Any GPU idle > 10% for 2+ hours"
    }
}

# Budget guardrails
budget_config = {
    "daily_hard_limit": 5000,       # Kill non-essential jobs above this
    "daily_soft_limit": 4000,       # Alert and reduce batch jobs
    "monthly_budget": 120000,       # Total monthly budget
    "emergency_shutdown_percent": 150  # Auto-shutdown at 150% of budget
}