get(); foreach ($nodes as $node) { $results[$node->id] = $this->checkNode($node); } return $results; } public function checkNode(HostingNode $node): array { $startTime = microtime(true); try { $ssh = $this->connect($node); $metrics = $this->collectMetrics($ssh, $node); $metrics['response_time_ms'] = round((microtime(true) - $startTime) * 1000); $metrics['checked_at'] = now()->toIso8601String(); $healthStatus = $this->evaluateHealth($metrics, $node); $metrics['health'] = $healthStatus; $node->update([ 'health_status' => $metrics, 'last_health_check_at' => now(), ]); $this->processAlerts($node, $metrics, $healthStatus); Log::info("Node health check completed", [ 'node_id' => $node->id, 'node_name' => $node->name, 'health' => $healthStatus, ]); return $metrics; } catch (\Exception $e) { $metrics = [ 'health' => self::HEALTH_UNREACHABLE, 'error' => $e->getMessage(), 'checked_at' => now()->toIso8601String(), 'response_time_ms' => round((microtime(true) - $startTime) * 1000), ]; $node->update([ 'health_status' => $metrics, 'last_health_check_at' => now(), ]); $this->createUnreachableAlert($node, $e->getMessage()); Log::error("Node health check failed", [ 'node_id' => $node->id, 'node_name' => $node->name, 'error' => $e->getMessage(), ]); return $metrics; } finally { $this->disconnect(); } } private function isLocal(HostingNode $node): bool { return ($node->provider === 'local' || $node->ip_address === '127.0.0.1' || $node->ip_address === 'localhost') && blank($node->ssh_private_key); } private function connect(HostingNode $node): ?SSH2 { if ($this->isLocal($node)) { $this->isLocalNode = true; return null; } $this->isLocalNode = false; // For local nodes with SSH keys, connect to 127.0.0.1 $host = $this->isLocalIp($node) ? '127.0.0.1' : $node->ip_address; $this->ssh = new SSH2($host, (int) ($node->ssh_port ?: 22)); $this->ssh->setTimeout(30); // Load the private key properly for phpseclib3 $credential = $node->ssh_private_key; if (str_contains($credential, '-----BEGIN') || str_contains($credential, 'PRIVATE KEY')) { try { $credential = PublicKeyLoader::load($credential); } catch (\Throwable $e) { throw new \RuntimeException("Invalid SSH key for node {$node->hostname}: " . $e->getMessage()); } } if (!$this->ssh->login('root', $credential)) { throw new \RuntimeException("SSH authentication failed for node: {$node->hostname}"); } return $this->ssh; } private function isLocalIp(HostingNode $node): bool { return $node->provider === 'local' || $node->ip_address === '127.0.0.1' || $node->ip_address === 'localhost'; } private function disconnect(): void { if ($this->ssh) { $this->ssh->disconnect(); $this->ssh = null; } $this->isLocalNode = false; } private function exec(?SSH2 $ssh, string $command): string { if ($this->isLocalNode) { $result = Process::run($command); return trim($result->output() . $result->errorOutput()); } return trim($ssh->exec($command)); } private function collectMetrics(?SSH2 $ssh, HostingNode $node): array { return [ 'cpu' => $this->getCpuMetrics($ssh), 'memory' => $this->getMemoryMetrics($ssh), 'disk' => $this->getDiskMetrics($ssh), 'load' => $this->getLoadMetrics($ssh, $node), 'uptime' => $this->getUptime($ssh), 'processes' => $this->getProcessCount($ssh), 'services' => $this->checkServices($ssh), 'network' => $this->getNetworkMetrics($ssh), ]; } private function getCpuMetrics(?SSH2 $ssh): array { $output = $this->exec($ssh, "top -bn1 | grep 'Cpu(s)' | awk '{print \$2 + \$4}'"); $usagePercent = is_numeric($output) ? (float) $output : 0; $coresOutput = $this->exec($ssh, "nproc"); $cores = is_numeric($coresOutput) ? (int) $coresOutput : 1; return [ 'usage_percent' => round($usagePercent, 1), 'cores' => $cores, ]; } private function getMemoryMetrics(?SSH2 $ssh): array { $output = $this->exec($ssh, "free -b | grep Mem"); $parts = preg_split('/\s+/', trim($output)); $total = isset($parts[1]) && is_numeric($parts[1]) ? (int) $parts[1] : 0; $used = isset($parts[2]) && is_numeric($parts[2]) ? (int) $parts[2] : 0; $free = isset($parts[3]) && is_numeric($parts[3]) ? (int) $parts[3] : 0; $available = isset($parts[6]) && is_numeric($parts[6]) ? (int) $parts[6] : $free; $usagePercent = $total > 0 ? (($total - $available) / $total) * 100 : 0; return [ 'total_bytes' => $total, 'used_bytes' => $used, 'available_bytes' => $available, 'usage_percent' => round($usagePercent, 1), ]; } private function getDiskMetrics(?SSH2 $ssh): array { $output = $this->exec($ssh, "df -B1 / | tail -1"); $parts = preg_split('/\s+/', trim($output)); $total = isset($parts[1]) && is_numeric($parts[1]) ? (int) $parts[1] : 0; $used = isset($parts[2]) && is_numeric($parts[2]) ? (int) $parts[2] : 0; $available = isset($parts[3]) && is_numeric($parts[3]) ? (int) $parts[3] : 0; $usagePercent = isset($parts[4]) ? (float) rtrim($parts[4], '%') : 0; $homeOutput = $this->exec($ssh, "df -B1 /home 2>/dev/null | tail -1"); $homeParts = preg_split('/\s+/', trim($homeOutput)); $homeUsagePercent = isset($homeParts[4]) ? (float) rtrim($homeParts[4], '%') : $usagePercent; return [ 'root' => [ 'total_bytes' => $total, 'used_bytes' => $used, 'available_bytes' => $available, 'usage_percent' => $usagePercent, ], 'home_usage_percent' => $homeUsagePercent, ]; } private function getLoadMetrics(?SSH2 $ssh, HostingNode $node): array { $output = $this->exec($ssh, "cat /proc/loadavg"); $parts = explode(' ', $output); $load1 = isset($parts[0]) && is_numeric($parts[0]) ? (float) $parts[0] : 0; $load5 = isset($parts[1]) && is_numeric($parts[1]) ? (float) $parts[1] : 0; $load15 = isset($parts[2]) && is_numeric($parts[2]) ? (float) $parts[2] : 0; $cores = $node->cpu_cores ?: 1; $loadPerCore = $load1 / $cores; return [ 'load_1' => $load1, 'load_5' => $load5, 'load_15' => $load15, 'load_per_core' => round($loadPerCore, 2), 'cores' => $cores, ]; } private function getUptime(?SSH2 $ssh): array { $output = $this->exec($ssh, "cat /proc/uptime | awk '{print \$1}'"); $uptimeSeconds = is_numeric($output) ? (int) $output : 0; $days = floor($uptimeSeconds / 86400); $hours = floor(($uptimeSeconds % 86400) / 3600); $minutes = floor(($uptimeSeconds % 3600) / 60); return [ 'seconds' => $uptimeSeconds, 'formatted' => "{$days}d {$hours}h {$minutes}m", ]; } private function getProcessCount(?SSH2 $ssh): array { $total = (int) $this->exec($ssh, "ps aux | wc -l") - 1; $running = (int) $this->exec($ssh, "ps aux | grep -c ' R'"); return [ 'total' => $total, 'running' => $running, ]; } private function checkServices(?SSH2 $ssh): array { // Check for PHP-FPM with dynamic version detection $phpVersion = $this->exec($ssh, "php -v | head -n1 | cut -d' ' -f2 | cut -d'.' -f1,2"); $phpVersion = preg_match('/^\d+\.\d+$/', $phpVersion) ? $phpVersion : '8.2'; $phpFpmService = "php{$phpVersion}-fpm"; $services = ['nginx', $phpFpmService, 'mysql', 'ssh']; $status = []; foreach ($services as $service) { $output = $this->exec($ssh, "systemctl is-active {$service} 2>/dev/null || echo 'inactive'"); $status[$service] = $output === 'active'; } return $status; } private function getNetworkMetrics(?SSH2 $ssh): array { $output = $this->exec($ssh, "cat /proc/net/dev | grep -E 'eth0|ens|enp' | head -1"); $parts = preg_split('/\s+/', trim($output)); $rxBytes = isset($parts[1]) && is_numeric($parts[1]) ? (int) $parts[1] : 0; $txBytes = isset($parts[9]) && is_numeric($parts[9]) ? (int) $parts[9] : 0; $connections = (int) $this->exec($ssh, "ss -tun | wc -l") - 1; return [ 'rx_bytes_total' => $rxBytes, 'tx_bytes_total' => $txBytes, 'active_connections' => max(0, $connections), ]; } private function evaluateHealth(array $metrics, HostingNode $node): string { $cpuUsage = $metrics['cpu']['usage_percent'] ?? 0; $ramUsage = $metrics['memory']['usage_percent'] ?? 0; $diskUsage = $metrics['disk']['root']['usage_percent'] ?? 0; $loadPerCore = $metrics['load']['load_per_core'] ?? 0; $services = $metrics['services'] ?? []; $criticalServices = ['nginx', 'php8.2-fpm']; foreach ($criticalServices as $service) { if (isset($services[$service]) && !$services[$service]) { return self::HEALTH_CRITICAL; } } if ( $cpuUsage >= self::CPU_CRITICAL_THRESHOLD || $ramUsage >= self::RAM_CRITICAL_THRESHOLD || $diskUsage >= self::DISK_CRITICAL_THRESHOLD || $loadPerCore >= self::LOAD_CRITICAL_MULTIPLIER ) { return self::HEALTH_CRITICAL; } if ( $cpuUsage >= self::CPU_WARNING_THRESHOLD || $ramUsage >= self::RAM_WARNING_THRESHOLD || $diskUsage >= self::DISK_WARNING_THRESHOLD || $loadPerCore >= self::LOAD_WARNING_MULTIPLIER ) { return self::HEALTH_WARNING; } return self::HEALTH_OK; } private function processAlerts(HostingNode $node, array $metrics, string $healthStatus): void { $existingAlert = NodeCapacityAlert::where('hosting_node_id', $node->id) ->where('is_resolved', false) ->whereIn('alert_type', ['resource_high', 'service_down']) ->first(); if ($healthStatus === self::HEALTH_OK) { if ($existingAlert) { $existingAlert->update([ 'is_resolved' => true, 'resolved_at' => now(), 'resolution_notes' => 'Auto-resolved: Node health returned to normal.', ]); } return; } $issues = $this->identifyIssues($metrics); if (empty($issues)) { return; } $message = "Node {$node->name} health issues: " . implode(', ', $issues); if ($existingAlert) { $existingAlert->update([ 'alert_type' => $healthStatus === self::HEALTH_CRITICAL ? 'resource_high' : 'capacity_warning', 'resource_usage' => $metrics, 'message' => $message, ]); } else { NodeCapacityAlert::create([ 'hosting_node_id' => $node->id, 'alert_type' => $healthStatus === self::HEALTH_CRITICAL ? 'resource_high' : 'capacity_warning', 'current_accounts' => $node->current_accounts, 'max_accounts' => $node->max_accounts ?? 0, 'capacity_percent' => $node->max_accounts > 0 ? ($node->current_accounts / $node->max_accounts) * 100 : 0, 'resource_usage' => $metrics, 'message' => $message, ]); } } private function identifyIssues(array $metrics): array { $issues = []; $cpuUsage = $metrics['cpu']['usage_percent'] ?? 0; if ($cpuUsage >= self::CPU_WARNING_THRESHOLD) { $issues[] = "CPU at {$cpuUsage}%"; } $ramUsage = $metrics['memory']['usage_percent'] ?? 0; if ($ramUsage >= self::RAM_WARNING_THRESHOLD) { $issues[] = "RAM at {$ramUsage}%"; } $diskUsage = $metrics['disk']['root']['usage_percent'] ?? 0; if ($diskUsage >= self::DISK_WARNING_THRESHOLD) { $issues[] = "Disk at {$diskUsage}%"; } $loadPerCore = $metrics['load']['load_per_core'] ?? 0; if ($loadPerCore >= self::LOAD_WARNING_MULTIPLIER) { $issues[] = "Load per core: {$loadPerCore}"; } $services = $metrics['services'] ?? []; foreach ($services as $service => $running) { if (!$running) { $issues[] = "{$service} is down"; } } return $issues; } private function createUnreachableAlert(HostingNode $node, string $error): void { $existingAlert = NodeCapacityAlert::where('hosting_node_id', $node->id) ->where('is_resolved', false) ->where('alert_type', 'resource_high') ->first(); $message = "Node {$node->name} is unreachable: {$error}"; if ($existingAlert) { $existingAlert->update([ 'message' => $message, 'resource_usage' => ['error' => $error], ]); } else { NodeCapacityAlert::create([ 'hosting_node_id' => $node->id, 'alert_type' => 'resource_high', 'current_accounts' => $node->current_accounts, 'max_accounts' => $node->max_accounts ?? 0, 'capacity_percent' => $node->max_accounts > 0 ? ($node->current_accounts / $node->max_accounts) * 100 : 0, 'resource_usage' => ['error' => $error], 'message' => $message, ]); } } public function getNodeSummary(HostingNode $node): array { $healthStatus = $node->health_status ?? []; return [ 'id' => $node->id, 'name' => $node->name, 'hostname' => $node->hostname, 'ip_address' => $node->ip_address, 'status' => $node->status, 'health' => $healthStatus['health'] ?? 'unknown', 'last_check' => $node->last_health_check_at?->diffForHumans() ?? 'Never', 'cpu_usage' => $healthStatus['cpu']['usage_percent'] ?? null, 'ram_usage' => $healthStatus['memory']['usage_percent'] ?? null, 'disk_usage' => $healthStatus['disk']['root']['usage_percent'] ?? null, 'load' => $healthStatus['load']['load_1'] ?? null, 'uptime' => $healthStatus['uptime']['formatted'] ?? null, 'logical_capacity_percent' => $node->logicalCapacityPercent(), 'current_load_percent' => (float) $node->current_load_percent, 'allocated_disk_gb' => $node->allocated_disk_gb, 'logical_disk_gb' => $node->logicalCapacityGb(), 'accounts' => [ 'current' => $node->current_accounts, 'max' => $node->max_accounts, 'percent' => $node->max_accounts > 0 ? round(($node->current_accounts / $node->max_accounts) * 100, 1) : 0, ], ]; } }