ladill-servers/app/Services/Hosting/HostingResourcePolicyService.php

<?php

namespace App\Services\Hosting;

use App\Models\CustomerHostingOrder;
use App\Models\HostingAccount;
use App\Models\HostingAccountAlert;
use App\Models\HostingProduct;
use App\Notifications\HostingResourceWarningNotification;
use App\Services\Hosting\Providers\SharedNodeProvider;

class HostingResourcePolicyService
{
    public const DISK_WARNING_THRESHOLD = 90;
    public const INODE_WARNING_THRESHOLD = 90;
    private const AUTOMATIC_SUSPENSION_GRACE_HOURS = 24;
    private const AUTOMATIC_SUSPENSION_MIN_BREACH_STREAK = 12;

    public function __construct(
        private SharedNodeProvider $sharedNodeProvider
    ) {
    }

    public function defaultLimitsForProduct(?HostingProduct $product): array
    {
        $defaults = match ($product?->type) {
            HostingProduct::TYPE_WORDPRESS => [
                'cpu_limit_percent' => 113,
                'memory_limit_mb' => 1152,
                'process_limit' => 23,
                'io_limit_mb' => 12,
                'inode_limit' => 450000,
            ],
            HostingProduct::TYPE_MULTI_DOMAIN => [
                'cpu_limit_percent' => 150,
                'memory_limit_mb' => 1536,
                'process_limit' => 30,
                'io_limit_mb' => 15,
                'inode_limit' => 525000,
            ],
            default => [
                'cpu_limit_percent' => 75,
                'memory_limit_mb' => 768,
                'process_limit' => 15,
                'io_limit_mb' => 8,
                'inode_limit' => 375000,
            ],
        };

        $defaults = array_merge($defaults, $this->productSpecificLimits($product));

        $overrides = (array) data_get($product?->features, 'resource_limits', []);

        return array_merge($defaults, array_filter($overrides, fn ($value) => $value !== null));
    }

    private function productSpecificLimits(?HostingProduct $product): array
    {
        return match ($product?->slug) {
            'starter-plan' => [
                'inode_limit' => 225000,
            ],
            'basic-plan' => [
                'inode_limit' => 450000,
            ],
            'plus-plan' => [
                'inode_limit' => 450000,
            ],
            'growth-plan' => [
                'inode_limit' => 750000,
            ],
            'pro-plan' => [
                'inode_limit' => 1200000,
            ],
            default => [],
        };
    }

    public function applyDefaultLimits(HostingAccount $account, bool $persist = true): array
    {
        $defaults = $this->defaultLimitsForProduct($account->product);
        $resourceLimits = (array) ($account->resource_limits ?? []);

        $updates = [];
        foreach ($defaults as $column => $value) {
            if ($account->{$column} === null) {
                $updates[$column] = $value;
            }

            $resourceLimits[$column] = $account->{$column} ?? $value;
        }

        if (! isset($resourceLimits['disk_gb'])) {
            $resourceLimits['disk_gb'] = $account->requestedDiskGb();
        }

        if ($persist) {
            $updates['resource_limits'] = $resourceLimits;
            $updates['resource_status'] = $account->resource_status ?: HostingAccount::RESOURCE_STATUS_ACTIVE;

            if ($updates !== []) {
                $account->update($updates);
                $account->refresh();
            }
        } else {
            $account->forceFill(array_merge($updates, [
                'resource_limits' => $resourceLimits,
                'resource_status' => $account->resource_status ?: HostingAccount::RESOURCE_STATUS_ACTIVE,
            ]));
        }

        return $resourceLimits;
    }

    public function summarize(HostingAccount $account): array
    {
        $this->applyDefaultLimits($account);

        return [
            'resource_status' => $account->resource_status ?: HostingAccount::RESOURCE_STATUS_ACTIVE,
            'uploads_restricted' => $account->uploadsAreRestricted(),
            'disk_usage_percent' => $account->diskUsagePercent(),
            'inode_usage_percent' => $account->inodeUsagePercent(),
            'cpu_usage_percent' => (float) ($account->cpu_usage_percent ?? 0),
            'memory_used_mb' => (int) ($account->memory_used_mb ?? 0),
            'process_count' => (int) ($account->process_count ?? 0),
            'io_usage_mb' => $account->io_usage_mb !== null ? (float) $account->io_usage_mb : null,
            'cpu_limit_percent' => (int) ($account->cpu_limit_percent ?? 0),
            'memory_limit_mb' => (int) ($account->memory_limit_mb ?? 0),
            'process_limit' => (int) ($account->process_limit ?? 0),
            'io_limit_mb' => (int) ($account->io_limit_mb ?? 0),
            'inode_limit' => (int) ($account->inode_limit ?? 0),
            'warning_count' => (int) ($account->warning_count ?? 0),
            'is_flagged' => (bool) ($account->is_flagged ?? false),
            'last_usage_sync_at' => $account->last_usage_sync_at,
            'last_warning_at' => $account->last_warning_at,
            'throttled_at' => $account->throttled_at,
        ];
    }

    public function process(HostingAccount $account): array
    {
        $account->loadMissing(['product', 'user', 'latestOrder']);
        $this->applyDefaultLimits($account);

        $evaluation = $this->evaluate($account);
        $breaches = $evaluation['breaches'];
        $warnings = $evaluation['warnings'];
        $warningMessages = $evaluation['warning_messages'];
        $primaryReason = $evaluation['primary_reason'];

        $this->updateBreachCounters($account, $breaches);

        $account->forceFill([
            'uploads_restricted' => array_key_exists('inode', $breaches) || $account->inode_count >= (int) $account->inode_limit,
            'is_flagged' => $warnings !== [] || $breaches !== [],
            'metadata' => $account->metadata, // Save CPU history from evaluation
        ]);

        if ($warnings !== [] || $breaches !== []) {
            $account->forceFill([
                'warning_count' => (int) $account->warning_count + 1,
                'last_warning_at' => now(),
                'last_resource_breach_at' => now(),
            ]);
        }

        $account->save();
        $account->refresh();

        foreach ($warningMessages as $type => $message) {
            $severity = array_key_exists($type, $breaches) ? 'critical' : 'warning';
            $alertType = array_key_exists($type, $breaches)
                ? HostingAccountAlert::TYPE_LIMIT_EXCEEDED
                : HostingAccountAlert::TYPE_WARNING;

            $this->recordAlert($account, $alertType, $severity, $message, $evaluation['usage']);
        }

        if ($breaches !== []) {
            if ($account->isThrottled()) {
                if ($this->shouldAutomaticallySuspendForSustainedBreach($account, $breaches)) {
                    $this->suspendAccount($account, $primaryReason, false);
                }
            } else {
                $this->throttleAccount($account, $primaryReason, false);
            }
        } elseif ($warnings !== []) {
            $this->notifyAccount($account, 'Hosting usage warning', implode(' ', array_values($warningMessages)), true);
        } else {
            $this->clearWarningState($account);
        }

        if ($breaches === []) {
            if ($this->shouldAutomaticallyUnsuspend($account)) {
                $this->unsuspendAccount($account, 'Usage returned within allowed thresholds.');
            } elseif ($account->isThrottled() && $account->status === HostingAccount::STATUS_ACTIVE) {
                $this->restoreAccount($account, 'Usage returned within allowed thresholds.', false);
            }
        }

        return $evaluation;
    }

    public function throttleAccount(HostingAccount $account, string $reason, bool $manual = true): void
    {
        $account->loadMissing(['user', 'latestOrder']);

        if ($account->status === HostingAccount::STATUS_SUSPENDED) {
            return;
        }

        $this->sharedNodeProvider->applyAccountResourceProfile($account, true);

        $metadata = array_merge((array) ($account->metadata ?? []), [
            'throttle_reason' => $reason,
            'throttle_origin' => $manual ? 'manual' : 'automatic',
            'throttled_at' => now()->toIso8601String(),
        ]);

        $account->update([
            'resource_status' => HostingAccount::RESOURCE_STATUS_THROTTLED,
            'throttled_at' => now(),
            'metadata' => $metadata,
            'uploads_restricted' => $account->uploadsAreRestricted(),
        ]);

        $this->syncLatestOrderState($account, CustomerHostingOrder::STATUS_ACTIVE, $reason);
        $this->recordAlert($account, HostingAccountAlert::TYPE_THROTTLED, 'critical', $reason, $this->usageSnapshot($account));
        $this->notifyAccount($account, 'Hosting account throttled', $reason, false);
    }

    public function restoreAccount(HostingAccount $account, ?string $reason = null, bool $manual = true): void
    {
        $account->loadMissing(['user', 'latestOrder']);

        if ($account->status === HostingAccount::STATUS_SUSPENDED) {
            return;
        }

        $this->sharedNodeProvider->applyAccountResourceProfile($account, false);

        $metadata = (array) ($account->metadata ?? []);
        $metadata['restored_at'] = now()->toIso8601String();
        $metadata['restore_origin'] = $manual ? 'manual' : 'automatic';
        // Clear CPU history on restore
        unset($metadata['cpu_history'], $metadata['cpu_averages']);

        $account->update([
            'resource_status' => HostingAccount::RESOURCE_STATUS_ACTIVE,
            'uploads_restricted' => false,
            'is_flagged' => false,
            'cpu_breach_streak' => 0,
            'memory_breach_streak' => 0,
            'process_breach_streak' => 0,
            'io_breach_streak' => 0,
            'inode_breach_streak' => 0,
            'throttled_at' => null,
            'metadata' => $metadata,
        ]);

        $this->resolveAlerts($account);

        if ($reason) {
            $this->notifyAccount($account, 'Hosting account restored', $reason, false);
        }
    }

    public function suspendAccount(HostingAccount $account, string $reason, bool $manual = true): void
    {
        $account->loadMissing(['sites', 'user', 'latestOrder']);

        if ($account->status === HostingAccount::STATUS_SUSPENDED) {
            return;
        }

        $this->sharedNodeProvider->suspendAccount($account, $reason);

        $metadata = array_merge((array) ($account->metadata ?? []), [
            'suspension_reason' => $reason,
            'suspension_origin' => $manual ? 'manual' : 'automatic',
            'suspended_at' => now()->toIso8601String(),
        ]);

        $account->update([
            'status' => HostingAccount::STATUS_SUSPENDED,
            'resource_status' => HostingAccount::RESOURCE_STATUS_SUSPENDED,
            'suspension_reason' => $reason,
            'suspended_at' => now(),
            'is_flagged' => true,
            'metadata' => $metadata,
        ]);

        $this->syncLatestOrderState($account, CustomerHostingOrder::STATUS_SUSPENDED, $reason);
        $this->recordAlert($account, HostingAccountAlert::TYPE_SUSPENDED, 'critical', $reason, $this->usageSnapshot($account));
    }

    public function unsuspendAccount(HostingAccount $account, ?string $reason = null): void
    {
        $account->loadMissing(['sites', 'user', 'latestOrder']);

        if ($account->status !== HostingAccount::STATUS_SUSPENDED) {
            return;
        }

        $this->sharedNodeProvider->unsuspendAccount($account);
        $this->sharedNodeProvider->applyAccountResourceProfile($account, false);

        $metadata = (array) ($account->metadata ?? []);
        $metadata['unsuspended_at'] = now()->toIso8601String();

        $account->update([
            'status' => HostingAccount::STATUS_ACTIVE,
            'resource_status' => HostingAccount::RESOURCE_STATUS_ACTIVE,
            'suspension_reason' => null,
            'suspended_at' => null,
            'uploads_restricted' => false,
            'metadata' => $metadata,
        ]);

        $this->syncLatestOrderState($account, CustomerHostingOrder::STATUS_ACTIVE, $reason);
        $this->resolveAlerts($account);
    }

    private function evaluate(HostingAccount $account): array
    {
        $usage = $this->usageSnapshot($account);
        $warnings = [];
        $breaches = [];
        $messages = [];

        if ($usage['disk_usage_percent'] >= self::DISK_WARNING_THRESHOLD) {
            $warnings['disk'] = true;
            $messages['disk'] = sprintf(
                'Disk usage is at %.1f%% of the allocated quota.',
                $usage['disk_usage_percent']
            );
        }

        if ($usage['inode_usage_percent'] >= self::INODE_WARNING_THRESHOLD) {
            $warnings['inode'] = true;
            $messages['inode'] = sprintf(
                'Inode usage is at %.1f%% of the allowed limit.',
                $usage['inode_usage_percent']
            );
        }

        // CPU: Use time-based sustained usage, not instant spikes
        if ($account->cpu_limit_percent) {
            $cpuEvaluation = $this->evaluateCpuUsage($account, $usage['cpu_usage_percent'], $account->cpu_limit_percent);
            if ($cpuEvaluation['is_breach']) {
                $breaches['cpu'] = true;
                $messages['cpu'] = $cpuEvaluation['message'];
            } elseif ($cpuEvaluation['is_warning']) {
                $warnings['cpu'] = true;
                $messages['cpu'] = $cpuEvaluation['message'];
            }
        }

        if ($account->memory_limit_mb && $usage['memory_used_mb'] > $account->memory_limit_mb) {
            $breaches['memory'] = true;
            $messages['memory'] = sprintf(
                'Memory usage reached %d MB against a %d MB limit.',
                $usage['memory_used_mb'],
                $account->memory_limit_mb
            );
        }

        if ($account->process_limit && $usage['process_count'] > $account->process_limit) {
            $breaches['process'] = true;
            $messages['process'] = sprintf(
                'Process count reached %d against a %d process limit.',
                $usage['process_count'],
                $account->process_limit
            );
        }

        if ($account->io_limit_mb && $usage['io_usage_mb'] !== null && $usage['io_usage_mb'] > $account->io_limit_mb) {
            $breaches['io'] = true;
            $messages['io'] = sprintf(
                'I/O usage reached %.1f MB against a %d MB limit.',
                $usage['io_usage_mb'],
                $account->io_limit_mb
            );
        }

        if ($account->inode_limit && $usage['inode_count'] > $account->inode_limit) {
            $breaches['inode'] = true;
            $messages['inode_limit'] = sprintf(
                'Inode count reached %d against a %d inode limit.',
                $usage['inode_count'],
                $account->inode_limit
            );
        }

        $primaryReason = $messages[array_key_first($breaches)] ?? implode(' ', array_values($messages)) ?: 'Resource policy triggered.';

        return [
            'usage' => $usage,
            'warnings' => $warnings,
            'breaches' => $breaches,
            'warning_messages' => $messages,
            'primary_reason' => $primaryReason,
        ];
    }

    private function updateBreachCounters(HostingAccount $account, array $breaches): void
    {
        foreach ([
            'cpu',
            'memory',
            'process',
            'io',
            'inode',
        ] as $metric) {
            $column = "{$metric}_breach_streak";
            $account->{$column} = array_key_exists($metric, $breaches)
                ? ((int) $account->{$column}) + 1
                : 0;
        }
    }

    private function evaluateCpuUsage(HostingAccount $account, float $currentCpu, int $limit): array
    {
        $metadata = (array) ($account->metadata ?? []);
        $cpuHistory = (array) ($metadata['cpu_history'] ?? []);
        
        // Add current reading with timestamp
        $now = now()->timestamp;
        $cpuHistory[] = ['ts' => $now, 'cpu' => $currentCpu];
        
        // Keep only last 30 minutes of data (sync runs every 5 min = ~6 readings)
        $cutoff = $now - 1800; // 30 minutes
        $cpuHistory = array_filter($cpuHistory, fn($r) => $r['ts'] > $cutoff);
        $cpuHistory = array_values($cpuHistory);
        
        // Calculate averages over time windows
        $windows = [
            '1m' => 60,   // 1 minute
            '5m' => 300,  // 5 minutes
            '15m' => 900, // 15 minutes
        ];
        
        $averages = [];
        foreach ($windows as $name => $seconds) {
            $windowCutoff = $now - $seconds;
            $windowReadings = array_filter($cpuHistory, fn($r) => $r['ts'] > $windowCutoff);
            $count = count($windowReadings);
            
            if ($count > 0) {
                $sum = array_sum(array_column($windowReadings, 'cpu'));
                $averages[$name] = $sum / $count;
            } else {
                $averages[$name] = $currentCpu;
            }
        }
        
        // Save updated history
        $metadata['cpu_history'] = $cpuHistory;
        $metadata['cpu_averages'] = $averages;
        $account->metadata = $metadata;
        
        // Evaluation rules:
        // - Warning threshold: 80% of limit, sustained for 5 minutes
        // - Breach threshold: 100% of limit, sustained for 10 minutes
        // - Extreme breach: 120% of limit, sustained for 5 minutes
        
        $warningThreshold = $limit * 0.8;
        $breachThreshold = $limit;
        $extremeThreshold = $limit * 1.2;
        
        $isWarning = false;
        $isBreach = false;
        $message = '';
        
        // Check for extreme breach (immediate action)
        if ($averages['5m'] >= $extremeThreshold) {
            $isBreach = true;
            $message = sprintf(
                'CPU usage critically high: %.1f%% average over 5 minutes (limit: %d%%).',
                $averages['5m'],
                $limit
            );
        }
        // Check for sustained breach (10+ minutes at or above limit)
        elseif ($averages['15m'] >= $breachThreshold) {
            $isBreach = true;
            $message = sprintf(
                'CPU usage sustained above limit: %.1f%% average over 15 minutes (limit: %d%%).',
                $averages['15m'],
                $limit
            );
        }
        // Check for sustained warning (5+ minutes at 80%+ of limit)
        elseif ($averages['5m'] >= $warningThreshold) {
            $isWarning = true;
            $message = sprintf(
                'CPU usage elevated: %.1f%% average over 5 minutes (limit: %d%%). Monitor closely.',
                $averages['5m'],
                $limit
            );
        }
        
        return [
            'is_breach' => $isBreach,
            'is_warning' => $isWarning,
            'message' => $message,
            'averages' => $averages,
        ];
    }

    private function usageSnapshot(HostingAccount $account): array
    {
        return [
            'disk_used_bytes' => (int) $account->disk_used_bytes,
            'disk_usage_percent' => $account->diskUsagePercent(),
            'inode_count' => (int) $account->inode_count,
            'inode_usage_percent' => $account->inodeUsagePercent(),
            'cpu_usage_percent' => (float) ($account->cpu_usage_percent ?? 0),
            'memory_used_mb' => (int) ($account->memory_used_mb ?? 0),
            'process_count' => (int) ($account->process_count ?? 0),
            'io_usage_mb' => $account->io_usage_mb !== null ? (float) $account->io_usage_mb : null,
        ];
    }

    private function clearWarningState(HostingAccount $account): void
    {
        $account->update([
            'is_flagged' => false,
            'uploads_restricted' => false,
            'cpu_breach_streak' => 0,
            'memory_breach_streak' => 0,
            'process_breach_streak' => 0,
            'io_breach_streak' => 0,
            'inode_breach_streak' => 0,
        ]);

        $this->resolveAlerts($account);
    }

    private function shouldAutomaticallyUnsuspend(HostingAccount $account): bool
    {
        return $account->status === HostingAccount::STATUS_SUSPENDED
            && $account->resource_status === HostingAccount::RESOURCE_STATUS_SUSPENDED
            && data_get($account->metadata, 'suspension_origin') === 'automatic';
    }

    private function shouldAutomaticallySuspendForSustainedBreach(HostingAccount $account, array $breaches): bool
    {
        if (! $account->throttled_at || $account->throttled_at->gt(now()->subHours(self::AUTOMATIC_SUSPENSION_GRACE_HOURS))) {
            return false;
        }

        foreach (array_keys($breaches) as $metric) {
            $column = "{$metric}_breach_streak";

            if ((int) ($account->{$column} ?? 0) >= self::AUTOMATIC_SUSPENSION_MIN_BREACH_STREAK) {
                return true;
            }
        }

        return false;
    }

    private function recordAlert(HostingAccount $account, string $alertType, string $severity, string $message, array $usage): void
    {
        $existing = $account->alerts()
            ->where('alert_type', $alertType)
            ->where('is_resolved', false)
            ->latest()
            ->first();

        if ($existing) {
            $existing->update([
                'severity' => $severity,
                'message' => $message,
                'resource_usage' => $usage,
            ]);

            return;
        }

        $account->alerts()->create([
            'alert_type' => $alertType,
            'severity' => $severity,
            'message' => $message,
            'resource_usage' => $usage,
        ]);
    }

    private function resolveAlerts(HostingAccount $account): void
    {
        $account->alerts()
            ->unresolved()
            ->update([
                'is_resolved' => true,
                'resolved_at' => now(),
            ]);
    }

    private function notifyAccount(HostingAccount $account, string $subject, string $message, bool $respectCooldown): void
    {
        if (! $account->user) {
            return;
        }

        if (
            $respectCooldown
            && (int) $account->warning_count > 1
            && $account->last_warning_at
            && $account->last_warning_at->gt(now()->subHours(6))
        ) {
            return;
        }

        $account->user->notify(new HostingResourceWarningNotification($account, $subject, $message));
    }

    private function syncLatestOrderState(HostingAccount $account, string $status, ?string $reason = null): void
    {
        $order = $account->latestOrder;

        if (! $order) {
            return;
        }

        if ($status === CustomerHostingOrder::STATUS_SUSPENDED) {
            $order->suspend($reason);
            return;
        }

        $meta = array_merge((array) ($order->meta ?? []), [
            'resource_policy_note' => $reason,
        ]);

        $order->update([
            'status' => $status,
            'suspended_at' => $status === CustomerHostingOrder::STATUS_SUSPENDED ? now() : null,
            'meta' => $meta,
        ]);
    }
}