Deploy Ladill Hosting / deploy (push) Failing after 17s
Shared web hosting extracted from the platform monolith, with CI deploy to /var/www/ladill-hosting matching Bird/Domains/Email. Co-authored-by: Cursor <cursoragent@cursor.com>
377 lines
13 KiB
PHP
377 lines
13 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Hosting;
|
|
|
|
use App\Models\HostingAccount;
|
|
use App\Models\HostingNode;
|
|
use App\Models\HostingProduct;
|
|
use App\Models\NodeCapacityAlert;
|
|
|
|
class NodeCapacityService
|
|
{
|
|
public function __construct(
|
|
private HostingNodePoolService $nodePool,
|
|
) {}
|
|
|
|
public const WARNING_THRESHOLD = 80;
|
|
public const CRITICAL_THRESHOLD = 95;
|
|
public const LOCAL_NODE_LIMIT = 50;
|
|
private const BYTES_PER_GB = 1073741824;
|
|
|
|
public function checkAllNodes(): array
|
|
{
|
|
$alerts = [];
|
|
|
|
$nodes = HostingNode::query()
|
|
->whereIn('status', [HostingNode::STATUS_ACTIVE, HostingNode::STATUS_FULL])
|
|
->where('type', HostingNode::TYPE_SHARED)
|
|
->get();
|
|
|
|
foreach ($nodes as $node) {
|
|
$alert = $this->checkNode($node);
|
|
|
|
if ($alert) {
|
|
$alerts[] = $alert;
|
|
}
|
|
}
|
|
|
|
return $alerts;
|
|
}
|
|
|
|
public function checkNode(HostingNode $node): ?NodeCapacityAlert
|
|
{
|
|
$node = $this->refreshNodeResourceTotals($node);
|
|
$capacityPercent = $this->nodePressurePercent($node);
|
|
|
|
$existingAlert = NodeCapacityAlert::where('hosting_node_id', $node->id)
|
|
->where('is_resolved', false)
|
|
->whereIn('alert_type', [
|
|
NodeCapacityAlert::TYPE_CAPACITY_WARNING,
|
|
NodeCapacityAlert::TYPE_CAPACITY_CRITICAL,
|
|
NodeCapacityAlert::TYPE_NEEDS_EXPANSION,
|
|
])
|
|
->first();
|
|
|
|
if ($capacityPercent >= self::CRITICAL_THRESHOLD) {
|
|
return $this->createOrUpdateAlert(
|
|
$node,
|
|
NodeCapacityAlert::TYPE_CAPACITY_CRITICAL,
|
|
$capacityPercent,
|
|
sprintf(
|
|
'Node %s is at %.1f%% pressure (%s). Immediate action required.',
|
|
$node->name,
|
|
$capacityPercent,
|
|
$this->dominantPressureLabel($node)
|
|
),
|
|
$existingAlert
|
|
);
|
|
}
|
|
|
|
if ($capacityPercent >= self::WARNING_THRESHOLD) {
|
|
return $this->createOrUpdateAlert(
|
|
$node,
|
|
NodeCapacityAlert::TYPE_CAPACITY_WARNING,
|
|
$capacityPercent,
|
|
sprintf(
|
|
'Node %s is at %.1f%% pressure (%s). Capacity is nearing exhaustion.',
|
|
$node->name,
|
|
$capacityPercent,
|
|
$this->dominantPressureLabel($node)
|
|
),
|
|
$existingAlert
|
|
);
|
|
}
|
|
|
|
if ($existingAlert) {
|
|
$existingAlert->update([
|
|
'is_resolved' => true,
|
|
'resolved_at' => now(),
|
|
'resolution_notes' => 'Auto-resolved: Node pressure dropped below warning threshold.',
|
|
]);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public function canProvision(HostingNode|int $node, int $requestedDiskGb): bool
|
|
{
|
|
$node = $node instanceof HostingNode ? $node : HostingNode::query()->findOrFail($node);
|
|
$node = $this->refreshNodeResourceTotals($node);
|
|
|
|
return $node->canProvision($requestedDiskGb);
|
|
}
|
|
|
|
public function checkCapacityForProduct(HostingProduct $product): array
|
|
{
|
|
$requestedDiskGb = (int) ($product->disk_gb ?? 0);
|
|
$segment = $product->preferredNodeSegment();
|
|
$nodes = $this->preparedCandidateNodes($segment)
|
|
->map(fn (HostingNode $node): array => $this->nodeSnapshot($node, $requestedDiskGb, $segment))
|
|
->values();
|
|
|
|
$selected = $this->findAvailableNodeForProduct($product);
|
|
|
|
return [
|
|
'available' => $selected !== null,
|
|
'requested_disk_gb' => $requestedDiskGb,
|
|
'segment' => $segment,
|
|
'selected_node_id' => $selected?->id,
|
|
'selected_node_name' => $selected?->name,
|
|
'nodes' => $nodes,
|
|
];
|
|
}
|
|
|
|
public function findAvailableNode(int $requestedDiskGb = 0, ?string $segment = null): ?HostingNode
|
|
{
|
|
$primaries = $this->preparedCandidateNodes($segment)
|
|
->filter(fn (HostingNode $node) => $node->isPrimary());
|
|
|
|
$bestMember = null;
|
|
$bestLoad = PHP_FLOAT_MAX;
|
|
|
|
foreach ($primaries as $primary) {
|
|
$member = $this->nodePool->findProvisioningMember($primary, $requestedDiskGb, $segment);
|
|
if (! $member) {
|
|
continue;
|
|
}
|
|
|
|
if ($this->isLocalNode($member) && $member->accountCapacityPercent() >= self::LOCAL_NODE_LIMIT) {
|
|
continue;
|
|
}
|
|
|
|
$load = (float) $member->current_load_percent;
|
|
if ($load < $bestLoad) {
|
|
$bestLoad = $load;
|
|
$bestMember = $member;
|
|
}
|
|
}
|
|
|
|
if ($bestMember) {
|
|
return $bestMember;
|
|
}
|
|
|
|
foreach ($this->preparedCandidateNodes($segment) as $node) {
|
|
if (! $node->canProvision($requestedDiskGb, $segment)) {
|
|
continue;
|
|
}
|
|
|
|
if ($this->isLocalNode($node) && $node->accountCapacityPercent() >= self::LOCAL_NODE_LIMIT) {
|
|
continue;
|
|
}
|
|
|
|
return $node;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public function findAvailableNodeForProduct(HostingProduct $product): ?HostingNode
|
|
{
|
|
return $this->findAvailableNode(
|
|
(int) ($product->disk_gb ?? 0),
|
|
$product->preferredNodeSegment()
|
|
);
|
|
}
|
|
|
|
public function needsNewNode(): bool
|
|
{
|
|
return $this->findAvailableNode() === null;
|
|
}
|
|
|
|
public function getCapacitySummary(): array
|
|
{
|
|
$nodes = HostingNode::query()
|
|
->whereIn('status', [HostingNode::STATUS_ACTIVE, HostingNode::STATUS_FULL])
|
|
->where('type', HostingNode::TYPE_SHARED)
|
|
->get()
|
|
->map(fn (HostingNode $node): HostingNode => $this->refreshNodeResourceTotals($node));
|
|
|
|
$totalDisk = $nodes->sum('disk_gb');
|
|
$totalLogicalCapacity = $nodes->sum(fn (HostingNode $node): float => $node->logicalCapacityGb());
|
|
$totalAllocated = $nodes->sum('allocated_disk_gb');
|
|
$totalUsed = $nodes->sum('used_disk_gb');
|
|
$totalAccountCapacity = $nodes->sum('max_accounts');
|
|
$totalAccounts = $nodes->sum('current_accounts');
|
|
|
|
return [
|
|
'total_nodes' => $nodes->count(),
|
|
'total_capacity' => $totalAccountCapacity,
|
|
'total_used' => $totalAccounts,
|
|
'available_slots' => max($totalAccountCapacity - $totalAccounts, 0),
|
|
'overall_percent' => $totalAccountCapacity > 0 ? round(($totalAccounts / $totalAccountCapacity) * 100, 1) : 0,
|
|
'physical_disk_gb' => $totalDisk,
|
|
'logical_disk_gb' => round($totalLogicalCapacity, 2),
|
|
'allocated_disk_gb' => $totalAllocated,
|
|
'used_disk_gb' => $totalUsed,
|
|
'logical_disk_percent' => $totalDisk > 0 ? round(($totalUsed / $totalDisk) * 100, 1) : 0,
|
|
'physical_disk_percent' => $totalDisk > 0 ? round(($totalUsed / $totalDisk) * 100, 1) : 0,
|
|
'needs_expansion' => $this->needsNewNode(),
|
|
'unresolved_alerts' => NodeCapacityAlert::unresolved()->count(),
|
|
];
|
|
}
|
|
|
|
public function refreshNodeResourceTotals(HostingNode $node): HostingNode
|
|
{
|
|
$aggregate = HostingAccount::query()
|
|
->where('hosting_node_id', $node->id)
|
|
->where('type', HostingAccount::TYPE_SHARED)
|
|
->selectRaw('COUNT(*) as account_count')
|
|
->selectRaw('COALESCE(SUM(allocated_disk_gb), 0) as allocated_disk_gb')
|
|
->selectRaw('COALESCE(SUM(disk_used_bytes), 0) as used_disk_bytes')
|
|
->first();
|
|
|
|
$currentAccounts = (int) ($aggregate->account_count ?? 0);
|
|
$allocatedDiskGb = (int) ($aggregate->allocated_disk_gb ?? 0);
|
|
$usedDiskGb = (int) ceil(((int) ($aggregate->used_disk_bytes ?? 0)) / self::BYTES_PER_GB);
|
|
$logicalCapacity = $node->disk_gb && $node->disk_gb > 0
|
|
? $node->disk_gb * max((float) ($node->oversell_ratio ?: 1), 1)
|
|
: 0;
|
|
$physicalPercent = $node->disk_gb && $node->disk_gb > 0 ? ($usedDiskGb / $node->disk_gb) * 100 : 0;
|
|
$accountPercent = $node->max_accounts && $node->max_accounts > 0 ? ($currentAccounts / $node->max_accounts) * 100 : 0;
|
|
$currentLoadPercent = round(max($physicalPercent, $accountPercent), 2);
|
|
|
|
$updates = [
|
|
'current_accounts' => $currentAccounts,
|
|
'allocated_disk_gb' => $allocatedDiskGb,
|
|
'used_disk_gb' => $usedDiskGb,
|
|
'current_load_percent' => $currentLoadPercent,
|
|
];
|
|
|
|
if (in_array($node->status, [HostingNode::STATUS_ACTIVE, HostingNode::STATUS_FULL], true)) {
|
|
$updates['status'] = $this->shouldMarkFull($node, $currentAccounts, $usedDiskGb)
|
|
? HostingNode::STATUS_FULL
|
|
: HostingNode::STATUS_ACTIVE;
|
|
}
|
|
|
|
$node->forceFill($updates);
|
|
|
|
if ($node->exists && $node->isDirty()) {
|
|
$node->save();
|
|
}
|
|
|
|
return $node->refresh();
|
|
}
|
|
|
|
private function candidateNodes(?string $segment)
|
|
{
|
|
return HostingNode::query()
|
|
->whereIn('status', [HostingNode::STATUS_ACTIVE, HostingNode::STATUS_FULL])
|
|
->where('type', HostingNode::TYPE_SHARED)
|
|
->when($segment, function ($query) use ($segment) {
|
|
$query->whereIn('segment', [$segment, HostingNode::SEGMENT_GENERAL]);
|
|
})
|
|
->get();
|
|
}
|
|
|
|
private function preparedCandidateNodes(?string $segment)
|
|
{
|
|
return $this->candidateNodes($segment)
|
|
->map(fn (HostingNode $node): HostingNode => $this->refreshNodeResourceTotals($node))
|
|
->sortBy(fn (HostingNode $node) => sprintf(
|
|
'%012.2f-%012d-%012d',
|
|
(float) $node->current_load_percent,
|
|
$node->allocated_disk_gb,
|
|
$node->current_accounts
|
|
))
|
|
->values();
|
|
}
|
|
|
|
private function shouldMarkFull(HostingNode $node, int $currentAccounts, int $usedDiskGb): bool
|
|
{
|
|
if ($node->max_accounts && $currentAccounts >= $node->max_accounts) {
|
|
return true;
|
|
}
|
|
|
|
if ($node->disk_gb && $node->disk_gb > 0) {
|
|
return $usedDiskGb >= $node->disk_gb;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private function nodePressurePercent(HostingNode $node): float
|
|
{
|
|
return round(max(
|
|
$node->physicalDiskPercent(),
|
|
$node->accountCapacityPercent()
|
|
), 2);
|
|
}
|
|
|
|
private function dominantPressureLabel(HostingNode $node): string
|
|
{
|
|
$pressures = [
|
|
'disk usage' => $node->physicalDiskPercent(),
|
|
'account slots' => $node->accountCapacityPercent(),
|
|
];
|
|
|
|
arsort($pressures);
|
|
|
|
$label = array_key_first($pressures);
|
|
|
|
return sprintf('%s %.1f%%', $label, $pressures[$label] ?? 0);
|
|
}
|
|
|
|
private function nodeSnapshot(HostingNode $node, int $requestedDiskGb, ?string $segment = null): array
|
|
{
|
|
return [
|
|
'id' => $node->id,
|
|
'name' => $node->name,
|
|
'segment' => $node->segment,
|
|
'can_provision' => $node->canProvision($requestedDiskGb, $segment),
|
|
'requested_disk_gb' => $requestedDiskGb,
|
|
'physical_disk_gb' => $node->disk_gb,
|
|
'logical_disk_gb' => $node->logicalCapacityGb(),
|
|
'allocated_disk_gb' => $node->allocated_disk_gb,
|
|
'used_disk_gb' => $node->used_disk_gb,
|
|
'available_logical_disk_gb' => $node->availableLogicalDiskGb(),
|
|
'oversell_ratio' => (float) $node->oversell_ratio,
|
|
'current_accounts' => $node->current_accounts,
|
|
'max_accounts' => $node->max_accounts,
|
|
'logical_capacity_percent' => $node->logicalCapacityPercent(),
|
|
'physical_disk_percent' => $node->physicalDiskPercent(),
|
|
'account_capacity_percent' => $node->accountCapacityPercent(),
|
|
'current_load_percent' => (float) $node->current_load_percent,
|
|
];
|
|
}
|
|
|
|
private function isLocalNode(HostingNode $node): bool
|
|
{
|
|
return $node->provider === HostingNode::PROVIDER_LOCAL
|
|
|| $node->ip_address === '127.0.0.1'
|
|
|| $node->ip_address === 'localhost';
|
|
}
|
|
|
|
private function createOrUpdateAlert(
|
|
HostingNode $node,
|
|
string $type,
|
|
float $capacityPercent,
|
|
string $message,
|
|
?NodeCapacityAlert $existingAlert
|
|
): NodeCapacityAlert {
|
|
$data = [
|
|
'hosting_node_id' => $node->id,
|
|
'alert_type' => $type,
|
|
'current_accounts' => $node->current_accounts,
|
|
'max_accounts' => $node->max_accounts ?? 0,
|
|
'capacity_percent' => $capacityPercent,
|
|
'resource_usage' => [
|
|
'logical_capacity_percent' => $node->logicalCapacityPercent(),
|
|
'physical_disk_percent' => $node->physicalDiskPercent(),
|
|
'account_capacity_percent' => $node->accountCapacityPercent(),
|
|
'allocated_disk_gb' => $node->allocated_disk_gb,
|
|
'used_disk_gb' => $node->used_disk_gb,
|
|
'logical_disk_gb' => $node->logicalCapacityGb(),
|
|
],
|
|
'message' => $message,
|
|
];
|
|
|
|
if ($existingAlert) {
|
|
$existingAlert->update($data);
|
|
|
|
return $existingAlert;
|
|
}
|
|
|
|
return NodeCapacityAlert::create($data);
|
|
}
|
|
}
|