ladill-hosting/app/Services/Hosting/NodeMonitoringService.php

<?php

namespace App\Services\Hosting;

use App\Models\HostingNode;
use App\Models\NodeCapacityAlert;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Process;
use phpseclib3\Crypt\PublicKeyLoader;
use phpseclib3\Net\SSH2;

class NodeMonitoringService
{
    private ?SSH2 $ssh = null;
    private bool $isLocalNode = false;

    public const HEALTH_OK = 'healthy';
    public const HEALTH_WARNING = 'warning';
    public const HEALTH_CRITICAL = 'critical';
    public const HEALTH_UNREACHABLE = 'unreachable';

    public const CPU_WARNING_THRESHOLD = 75;
    public const CPU_CRITICAL_THRESHOLD = 90;
    public const RAM_WARNING_THRESHOLD = 80;
    public const RAM_CRITICAL_THRESHOLD = 95;
    public const DISK_WARNING_THRESHOLD = 80;
    public const DISK_CRITICAL_THRESHOLD = 90;
    public const LOAD_WARNING_MULTIPLIER = 1.5;
    public const LOAD_CRITICAL_MULTIPLIER = 2.0;

    public function checkAllNodes(): array
    {
        $results = [];

        $nodes = HostingNode::whereIn('status', [
            HostingNode::STATUS_ACTIVE,
            HostingNode::STATUS_FULL,
            HostingNode::STATUS_MAINTENANCE,
        ])->get();

        foreach ($nodes as $node) {
            $results[$node->id] = $this->checkNode($node);
        }

        return $results;
    }

    public function checkNode(HostingNode $node): array
    {
        $startTime = microtime(true);

        try {
            $ssh = $this->connect($node);

            $metrics = $this->collectMetrics($ssh, $node);
            $metrics['response_time_ms'] = round((microtime(true) - $startTime) * 1000);
            $metrics['checked_at'] = now()->toIso8601String();

            $healthStatus = $this->evaluateHealth($metrics, $node);
            $metrics['health'] = $healthStatus;

            $node->update([
                'health_status' => $metrics,
                'last_health_check_at' => now(),
            ]);

            $this->processAlerts($node, $metrics, $healthStatus);

            Log::info("Node health check completed", [
                'node_id' => $node->id,
                'node_name' => $node->name,
                'health' => $healthStatus,
            ]);

            return $metrics;

        } catch (\Exception $e) {
            $metrics = [
                'health' => self::HEALTH_UNREACHABLE,
                'error' => $e->getMessage(),
                'checked_at' => now()->toIso8601String(),
                'response_time_ms' => round((microtime(true) - $startTime) * 1000),
            ];

            $node->update([
                'health_status' => $metrics,
                'last_health_check_at' => now(),
            ]);

            $this->createUnreachableAlert($node, $e->getMessage());

            Log::error("Node health check failed", [
                'node_id' => $node->id,
                'node_name' => $node->name,
                'error' => $e->getMessage(),
            ]);

            return $metrics;
        } finally {
            $this->disconnect();
        }
    }

    private function isLocal(HostingNode $node): bool
    {
        return ($node->provider === 'local' || $node->ip_address === '127.0.0.1' || $node->ip_address === 'localhost')
            && blank($node->ssh_private_key);
    }

    private function connect(HostingNode $node): ?SSH2
    {
        if ($this->isLocal($node)) {
            $this->isLocalNode = true;
            return null;
        }

        $this->isLocalNode = false;

        // For local nodes with SSH keys, connect to 127.0.0.1
        $host = $this->isLocalIp($node) ? '127.0.0.1' : $node->ip_address;
        $this->ssh = new SSH2($host, (int) ($node->ssh_port ?: 22));
        $this->ssh->setTimeout(30);

        // Load the private key properly for phpseclib3
        $credential = $node->ssh_private_key;
        if (str_contains($credential, '-----BEGIN') || str_contains($credential, 'PRIVATE KEY')) {
            try {
                $credential = PublicKeyLoader::load($credential);
            } catch (\Throwable $e) {
                throw new \RuntimeException("Invalid SSH key for node {$node->hostname}: " . $e->getMessage());
            }
        }

        if (!$this->ssh->login('root', $credential)) {
            throw new \RuntimeException("SSH authentication failed for node: {$node->hostname}");
        }

        return $this->ssh;
    }

    private function isLocalIp(HostingNode $node): bool
    {
        return $node->provider === 'local' || $node->ip_address === '127.0.0.1' || $node->ip_address === 'localhost';
    }

    private function disconnect(): void
    {
        if ($this->ssh) {
            $this->ssh->disconnect();
            $this->ssh = null;
        }

        $this->isLocalNode = false;
    }

    private function exec(?SSH2 $ssh, string $command): string
    {
        if ($this->isLocalNode) {
            $result = Process::run($command);
            return trim($result->output() . $result->errorOutput());
        }

        return trim($ssh->exec($command));
    }

    private function collectMetrics(?SSH2 $ssh, HostingNode $node): array
    {
        return [
            'cpu' => $this->getCpuMetrics($ssh),
            'memory' => $this->getMemoryMetrics($ssh),
            'disk' => $this->getDiskMetrics($ssh),
            'load' => $this->getLoadMetrics($ssh, $node),
            'uptime' => $this->getUptime($ssh),
            'processes' => $this->getProcessCount($ssh),
            'services' => $this->checkServices($ssh),
            'network' => $this->getNetworkMetrics($ssh),
        ];
    }

    private function getCpuMetrics(?SSH2 $ssh): array
    {
        $output = $this->exec($ssh, "top -bn1 | grep 'Cpu(s)' | awk '{print \$2 + \$4}'");
        $usagePercent = is_numeric($output) ? (float) $output : 0;

        $coresOutput = $this->exec($ssh, "nproc");
        $cores = is_numeric($coresOutput) ? (int) $coresOutput : 1;

        return [
            'usage_percent' => round($usagePercent, 1),
            'cores' => $cores,
        ];
    }

    private function getMemoryMetrics(?SSH2 $ssh): array
    {
        $output = $this->exec($ssh, "free -b | grep Mem");
        $parts = preg_split('/\s+/', trim($output));

        $total = isset($parts[1]) && is_numeric($parts[1]) ? (int) $parts[1] : 0;
        $used = isset($parts[2]) && is_numeric($parts[2]) ? (int) $parts[2] : 0;
        $free = isset($parts[3]) && is_numeric($parts[3]) ? (int) $parts[3] : 0;
        $available = isset($parts[6]) && is_numeric($parts[6]) ? (int) $parts[6] : $free;

        $usagePercent = $total > 0 ? (($total - $available) / $total) * 100 : 0;

        return [
            'total_bytes' => $total,
            'used_bytes' => $used,
            'available_bytes' => $available,
            'usage_percent' => round($usagePercent, 1),
        ];
    }

    private function getDiskMetrics(?SSH2 $ssh): array
    {
        $output = $this->exec($ssh, "df -B1 / | tail -1");
        $parts = preg_split('/\s+/', trim($output));

        $total = isset($parts[1]) && is_numeric($parts[1]) ? (int) $parts[1] : 0;
        $used = isset($parts[2]) && is_numeric($parts[2]) ? (int) $parts[2] : 0;
        $available = isset($parts[3]) && is_numeric($parts[3]) ? (int) $parts[3] : 0;
        $usagePercent = isset($parts[4]) ? (float) rtrim($parts[4], '%') : 0;

        $homeOutput = $this->exec($ssh, "df -B1 /home 2>/dev/null | tail -1");
        $homeParts = preg_split('/\s+/', trim($homeOutput));
        $homeUsagePercent = isset($homeParts[4]) ? (float) rtrim($homeParts[4], '%') : $usagePercent;

        return [
            'root' => [
                'total_bytes' => $total,
                'used_bytes' => $used,
                'available_bytes' => $available,
                'usage_percent' => $usagePercent,
            ],
            'home_usage_percent' => $homeUsagePercent,
        ];
    }

    private function getLoadMetrics(?SSH2 $ssh, HostingNode $node): array
    {
        $output = $this->exec($ssh, "cat /proc/loadavg");
        $parts = explode(' ', $output);

        $load1 = isset($parts[0]) && is_numeric($parts[0]) ? (float) $parts[0] : 0;
        $load5 = isset($parts[1]) && is_numeric($parts[1]) ? (float) $parts[1] : 0;
        $load15 = isset($parts[2]) && is_numeric($parts[2]) ? (float) $parts[2] : 0;

        $cores = $node->cpu_cores ?: 1;
        $loadPerCore = $load1 / $cores;

        return [
            'load_1' => $load1,
            'load_5' => $load5,
            'load_15' => $load15,
            'load_per_core' => round($loadPerCore, 2),
            'cores' => $cores,
        ];
    }

    private function getUptime(?SSH2 $ssh): array
    {
        $output = $this->exec($ssh, "cat /proc/uptime | awk '{print \$1}'");
        $uptimeSeconds = is_numeric($output) ? (int) $output : 0;

        $days = floor($uptimeSeconds / 86400);
        $hours = floor(($uptimeSeconds % 86400) / 3600);
        $minutes = floor(($uptimeSeconds % 3600) / 60);

        return [
            'seconds' => $uptimeSeconds,
            'formatted' => "{$days}d {$hours}h {$minutes}m",
        ];
    }

    private function getProcessCount(?SSH2 $ssh): array
    {
        $total = (int) $this->exec($ssh, "ps aux | wc -l") - 1;
        $running = (int) $this->exec($ssh, "ps aux | grep -c ' R'");

        return [
            'total' => $total,
            'running' => $running,
        ];
    }

    private function checkServices(?SSH2 $ssh): array
    {
        // Check for PHP-FPM with dynamic version detection
        $phpVersion = $this->exec($ssh, "php -v | head -n1 | cut -d' ' -f2 | cut -d'.' -f1,2");
        $phpVersion = preg_match('/^\d+\.\d+$/', $phpVersion) ? $phpVersion : '8.2';
        $phpFpmService = "php{$phpVersion}-fpm";
        
        $services = ['nginx', $phpFpmService, 'mysql', 'ssh'];
        $status = [];

        foreach ($services as $service) {
            $output = $this->exec($ssh, "systemctl is-active {$service} 2>/dev/null || echo 'inactive'");
            $status[$service] = $output === 'active';
        }

        return $status;
    }

    private function getNetworkMetrics(?SSH2 $ssh): array
    {
        $output = $this->exec($ssh, "cat /proc/net/dev | grep -E 'eth0|ens|enp' | head -1");
        $parts = preg_split('/\s+/', trim($output));

        $rxBytes = isset($parts[1]) && is_numeric($parts[1]) ? (int) $parts[1] : 0;
        $txBytes = isset($parts[9]) && is_numeric($parts[9]) ? (int) $parts[9] : 0;

        $connections = (int) $this->exec($ssh, "ss -tun | wc -l") - 1;

        return [
            'rx_bytes_total' => $rxBytes,
            'tx_bytes_total' => $txBytes,
            'active_connections' => max(0, $connections),
        ];
    }

    private function evaluateHealth(array $metrics, HostingNode $node): string
    {
        $cpuUsage = $metrics['cpu']['usage_percent'] ?? 0;
        $ramUsage = $metrics['memory']['usage_percent'] ?? 0;
        $diskUsage = $metrics['disk']['root']['usage_percent'] ?? 0;
        $loadPerCore = $metrics['load']['load_per_core'] ?? 0;

        $services = $metrics['services'] ?? [];
        $criticalServices = ['nginx', 'php8.2-fpm'];
        foreach ($criticalServices as $service) {
            if (isset($services[$service]) && !$services[$service]) {
                return self::HEALTH_CRITICAL;
            }
        }

        if (
            $cpuUsage >= self::CPU_CRITICAL_THRESHOLD ||
            $ramUsage >= self::RAM_CRITICAL_THRESHOLD ||
            $diskUsage >= self::DISK_CRITICAL_THRESHOLD ||
            $loadPerCore >= self::LOAD_CRITICAL_MULTIPLIER
        ) {
            return self::HEALTH_CRITICAL;
        }

        if (
            $cpuUsage >= self::CPU_WARNING_THRESHOLD ||
            $ramUsage >= self::RAM_WARNING_THRESHOLD ||
            $diskUsage >= self::DISK_WARNING_THRESHOLD ||
            $loadPerCore >= self::LOAD_WARNING_MULTIPLIER
        ) {
            return self::HEALTH_WARNING;
        }

        return self::HEALTH_OK;
    }

    private function processAlerts(HostingNode $node, array $metrics, string $healthStatus): void
    {
        $existingAlert = NodeCapacityAlert::where('hosting_node_id', $node->id)
            ->where('is_resolved', false)
            ->whereIn('alert_type', ['resource_high', 'service_down'])
            ->first();

        if ($healthStatus === self::HEALTH_OK) {
            if ($existingAlert) {
                $existingAlert->update([
                    'is_resolved' => true,
                    'resolved_at' => now(),
                    'resolution_notes' => 'Auto-resolved: Node health returned to normal.',
                ]);
            }
            return;
        }

        $issues = $this->identifyIssues($metrics);

        if (empty($issues)) {
            return;
        }

        $message = "Node {$node->name} health issues: " . implode(', ', $issues);

        if ($existingAlert) {
            $existingAlert->update([
                'alert_type' => $healthStatus === self::HEALTH_CRITICAL ? 'resource_high' : 'capacity_warning',
                'resource_usage' => $metrics,
                'message' => $message,
            ]);
        } else {
            NodeCapacityAlert::create([
                'hosting_node_id' => $node->id,
                'alert_type' => $healthStatus === self::HEALTH_CRITICAL ? 'resource_high' : 'capacity_warning',
                'current_accounts' => $node->current_accounts,
                'max_accounts' => $node->max_accounts ?? 0,
                'capacity_percent' => $node->max_accounts > 0
                    ? ($node->current_accounts / $node->max_accounts) * 100
                    : 0,
                'resource_usage' => $metrics,
                'message' => $message,
            ]);
        }
    }

    private function identifyIssues(array $metrics): array
    {
        $issues = [];

        $cpuUsage = $metrics['cpu']['usage_percent'] ?? 0;
        if ($cpuUsage >= self::CPU_WARNING_THRESHOLD) {
            $issues[] = "CPU at {$cpuUsage}%";
        }

        $ramUsage = $metrics['memory']['usage_percent'] ?? 0;
        if ($ramUsage >= self::RAM_WARNING_THRESHOLD) {
            $issues[] = "RAM at {$ramUsage}%";
        }

        $diskUsage = $metrics['disk']['root']['usage_percent'] ?? 0;
        if ($diskUsage >= self::DISK_WARNING_THRESHOLD) {
            $issues[] = "Disk at {$diskUsage}%";
        }

        $loadPerCore = $metrics['load']['load_per_core'] ?? 0;
        if ($loadPerCore >= self::LOAD_WARNING_MULTIPLIER) {
            $issues[] = "Load per core: {$loadPerCore}";
        }

        $services = $metrics['services'] ?? [];
        foreach ($services as $service => $running) {
            if (!$running) {
                $issues[] = "{$service} is down";
            }
        }

        return $issues;
    }

    private function createUnreachableAlert(HostingNode $node, string $error): void
    {
        $existingAlert = NodeCapacityAlert::where('hosting_node_id', $node->id)
            ->where('is_resolved', false)
            ->where('alert_type', 'resource_high')
            ->first();

        $message = "Node {$node->name} is unreachable: {$error}";

        if ($existingAlert) {
            $existingAlert->update([
                'message' => $message,
                'resource_usage' => ['error' => $error],
            ]);
        } else {
            NodeCapacityAlert::create([
                'hosting_node_id' => $node->id,
                'alert_type' => 'resource_high',
                'current_accounts' => $node->current_accounts,
                'max_accounts' => $node->max_accounts ?? 0,
                'capacity_percent' => $node->max_accounts > 0
                    ? ($node->current_accounts / $node->max_accounts) * 100
                    : 0,
                'resource_usage' => ['error' => $error],
                'message' => $message,
            ]);
        }
    }

    public function getNodeSummary(HostingNode $node): array
    {
        $healthStatus = $node->health_status ?? [];

        return [
            'id' => $node->id,
            'name' => $node->name,
            'hostname' => $node->hostname,
            'ip_address' => $node->ip_address,
            'status' => $node->status,
            'health' => $healthStatus['health'] ?? 'unknown',
            'last_check' => $node->last_health_check_at?->diffForHumans() ?? 'Never',
            'cpu_usage' => $healthStatus['cpu']['usage_percent'] ?? null,
            'ram_usage' => $healthStatus['memory']['usage_percent'] ?? null,
            'disk_usage' => $healthStatus['disk']['root']['usage_percent'] ?? null,
            'load' => $healthStatus['load']['load_1'] ?? null,
            'uptime' => $healthStatus['uptime']['formatted'] ?? null,
            'logical_capacity_percent' => $node->logicalCapacityPercent(),
            'current_load_percent' => (float) $node->current_load_percent,
            'allocated_disk_gb' => $node->allocated_disk_gb,
            'logical_disk_gb' => $node->logicalCapacityGb(),
            'accounts' => [
                'current' => $node->current_accounts,
                'max' => $node->max_accounts,
                'percent' => $node->max_accounts > 0
                    ? round(($node->current_accounts / $node->max_accounts) * 100, 1)
                    : 0,
            ],
        ];
    }
}