Initial Ladill Hosting app with Gitea deploy pipeline.

Shared web hosting extracted from the platform monolith, with CI deploy to /var/www/ladill-hosting matching Bird/Domains/Email. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-06 16:24:20 +00:00
commit e251a4cf60
367 changed files with 66268 additions and 0 deletions
@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\HostingAccount;
+use App\Models\Mailbox;
+use App\Services\Hosting\HostingMailboxService;
+use App\Services\Mail\MailServerProvisioner;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class DeactivateMailboxJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+
+    public int $backoff = 60;
+
+    public function __construct(public int $mailboxId)
+    {
+    }
+
+    public function handle(MailServerProvisioner $provisioner, HostingMailboxService $hostingMailboxes): void
+    {
+        $mailbox = Mailbox::query()->find($this->mailboxId);
+        if (! $mailbox) {
+            return;
+        }
+
+        $hostingAccountId = $mailbox->hosting_account_id;
+
+        try {
+            if ($provisioner->isConfigured()) {
+                $provisioner->deactivateMailbox($mailbox);
+            }
+        } catch (\Throwable $exception) {
+            $mailbox->update(['status' => 'failed']);
+            Log::error('DeactivateMailboxJob: Remote deactivation failed', [
+                'mailbox_id' => $mailbox->id,
+                'address' => $mailbox->address,
+                'error' => $exception->getMessage(),
+            ]);
+
+            throw $exception;
+        }
+
+        $mailbox->delete();
+
+        if ($hostingAccountId) {
+            $account = HostingAccount::query()->with(['product', 'mailboxes'])->find($hostingAccountId);
+            if ($account) {
+                $hostingMailboxes->syncAddonInvoice($account);
+            }
+        }
+    }
+}
@@ -0,0 +1,120 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Domain;
+use App\Models\DomainNsCheck;
+use App\Services\Domain\DomainDkimService;
+use App\Services\Mail\MailServerProvisioner;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class MailProvisioningJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+    public int $backoff = 60;
+
+    public function __construct(public int $domainId)
+    {
+    }
+
+    public function handle(MailServerProvisioner $provisioner, DomainDkimService $dkim): void
+    {
+        $domain = Domain::query()->with('mailboxes')->find($this->domainId);
+        if (! $domain) {
+            return;
+        }
+
+        if (! $domain->isActiveForMail()) {
+            return;
+        }
+
+        if (! $provisioner->isConfigured()) {
+            Log::info('MailProvisioningJob: Skipped — mail server DB not configured', ['domain_id' => $domain->id]);
+            return;
+        }
+
+        $key = $dkim->ensureActive($domain);
+
+        $beforeStatus = (string) $domain->status;
+        $beforeState = (string) ($domain->onboarding_state ?? '');
+
+        try {
+            // 1. Provision domain in Postfix/Dovecot DB
+            $provisioner->provisionDomain($domain);
+
+            // 2. Provision each mailbox in Postfix/Dovecot DB
+            foreach ($domain->mailboxes as $mailbox) {
+                try {
+                    $provisioner->provisionMailbox($mailbox);
+                } catch (\Throwable $e) {
+                    Log::warning('MailProvisioningJob: Mailbox provision skipped (may need password)', [
+                        'email' => $mailbox->address,
+                        'error' => $e->getMessage(),
+                    ]);
+                }
+            }
+
+            // 3. Deploy DKIM key to mail server
+            $provisioner->deployDkim($domain, $key);
+
+            $this->markReady($domain);
+            $afterStatus = (string) $domain->status;
+            $afterState = (string) ($domain->onboarding_state ?? '');
+            $this->logCheck($domain, 'success', 'Mail stack provisioned via direct DB', [], $beforeStatus, $beforeState, $afterStatus, $afterState);
+        } catch (\Throwable $exception) {
+            $this->markFailed($domain);
+            $afterStatus = (string) $domain->status;
+            $afterState = (string) ($domain->onboarding_state ?? '');
+            $this->logCheck($domain, 'failed', $exception->getMessage(), [], $beforeStatus, $beforeState, $afterStatus, $afterState);
+            Log::error('Mail provisioning failed', ['domain_id' => $domain->id, 'error' => $exception->getMessage()]);
+            throw $exception;
+        }
+    }
+
+    private function markReady(Domain $domain): void
+    {
+        $domain->update([
+            'mail_ready_at' => $domain->mail_ready_at ?? now(),
+            'active_at' => $domain->active_at ?? now(),
+            'status' => 'verified',
+            'onboarding_state' => Domain::STATE_ACTIVE,
+        ]);
+    }
+
+    private function markFailed(Domain $domain): void
+    {
+        $domain->update([
+            'status' => 'verifying',
+            'onboarding_state' => Domain::STATE_MAIL_READY,
+        ]);
+    }
+
+    private function logCheck(Domain $domain, string $result, string $notes, array $payload = []): void
+    {
+        $stateBefore = $stateAfter = (string) ($domain->onboarding_state ?? '');
+        $statusBefore = $statusAfter = (string) $domain->status;
+
+        if (! empty($payload)) {
+            $notes .= ' '.json_encode($payload);
+        }
+
+        DomainNsCheck::create([
+            'domain_id' => $domain->id,
+            'check_type' => 'mail_provision',
+            'result' => $result,
+            'notes' => $notes.(! empty($payload) ? ' '.json_encode($payload) : ''),
+            'status_before' => $statusBefore,
+            'status_after' => $statusAfter,
+            'state_before' => $stateBefore,
+            'state_after' => $stateAfter,
+            'checked_at' => now(),
+        ]);
+    }
+}
@@ -0,0 +1,109 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\HostingAccount;
+use App\Models\HostingNode;
+use App\Services\Hosting\Providers\SharedNodeProvider;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\DB;
+
+class MigrateHostingAccountJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $timeout = 3600; // 1 hour timeout for large migrations
+    public int $tries = 1; // Don't retry migrations automatically
+
+    public function __construct(
+        public int $accountId,
+        public int $destinationNodeId,
+        public ?int $initiatedBy = null
+    ) {}
+
+    public function handle(SharedNodeProvider $provider): void
+    {
+        $account = HostingAccount::with(['node', 'sites', 'databases'])->find($this->accountId);
+        $destinationNode = HostingNode::find($this->destinationNodeId);
+
+        if (!$account) {
+            Log::error("Migration job failed: Account {$this->accountId} not found");
+            return;
+        }
+
+        if (!$destinationNode) {
+            Log::error("Migration job failed: Destination node {$this->destinationNodeId} not found");
+            return;
+        }
+
+        $sourceNode = $account->node;
+        $sourceNodeId = $sourceNode?->id;
+
+        Log::info("Starting migration job for account {$account->username}", [
+            'account_id' => $account->id,
+            'source_node' => $sourceNode?->name,
+            'destination_node' => $destinationNode->name,
+            'initiated_by' => $this->initiatedBy,
+        ]);
+
+        try {
+            // Update account status to migrating
+            $account->update(['status' => 'migrating']);
+
+            // Perform the migration
+            $result = $provider->migrateAccount($account, $destinationNode, function ($message) use ($account) {
+                Log::info("Migration progress [{$account->username}]: {$message}");
+            });
+
+            // Update account with new node
+            DB::transaction(function () use ($account, $destinationNode, $sourceNodeId) {
+                $account->update([
+                    'hosting_node_id' => $destinationNode->id,
+                    'status' => 'active',
+                ]);
+
+                // Update node account counts
+                $destinationNode->incrementAccountCount();
+
+                if ($sourceNodeId) {
+                    HostingNode::where('id', $sourceNodeId)->decrement('current_accounts');
+                }
+            });
+
+            Log::info("Migration completed successfully for account {$account->username}", [
+                'account_id' => $account->id,
+                'new_node' => $destinationNode->name,
+            ]);
+
+        } catch (\Throwable $e) {
+            Log::error("Migration failed for account {$account->username}: " . $e->getMessage(), [
+                'account_id' => $account->id,
+                'exception' => $e,
+            ]);
+
+            // Revert status
+            $account->update(['status' => 'active']);
+
+            throw $e;
+        }
+    }
+
+    public function failed(\Throwable $exception): void
+    {
+        Log::error("Migration job failed permanently for account {$this->accountId}", [
+            'destination_node' => $this->destinationNodeId,
+            'error' => $exception->getMessage(),
+        ]);
+
+        // Try to revert account status
+        $account = HostingAccount::find($this->accountId);
+        if ($account && $account->status === 'migrating') {
+            $account->update(['status' => 'active']);
+        }
+    }
+}
@@ -0,0 +1,58 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\ContaboInfrastructureProvision;
+use App\Services\Infrastructure\ContaboInfrastructureProvisioner;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Facades\Log;
+
+class ProvisionContaboInfrastructureJob implements ShouldQueue
+{
+    use Queueable;
+
+    public int $tries = 30;
+
+    public int $backoff = 60;
+
+    public function __construct(
+        public ContaboInfrastructureProvision $provision,
+    ) {}
+
+    public function handle(ContaboInfrastructureProvisioner $provisioner): void
+    {
+        $provision = $this->provision->fresh();
+
+        if (! $provision) {
+            return;
+        }
+
+        try {
+            if ($provision->status === ContaboInfrastructureProvision::STATUS_PENDING) {
+                $provisioner->createContaboInstance($provision);
+
+                return;
+            }
+
+            if ($provision->status === ContaboInfrastructureProvision::STATUS_WAITING_IP) {
+                if (! $provisioner->pollInstanceIp($provision)) {
+                    self::dispatch($provision)->delay(now()->addSeconds(30));
+                }
+
+                return;
+            }
+
+            if ($provision->status === ContaboInfrastructureProvision::STATUS_BOOTSTRAPPING) {
+                self::dispatch($provision)->delay(now()->addMinutes(2));
+            }
+        } catch (\Throwable $e) {
+            Log::error('Contabo infrastructure provision failed', [
+                'provision_id' => $provision->id,
+                'error' => $e->getMessage(),
+            ]);
+            $provision->markFailed($e->getMessage());
+            throw $e;
+        }
+    }
+}
@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Domain;
+use App\Services\Dns\PowerDnsClient;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+class ProvisionDomainSlaveZoneJob implements ShouldQueue
+{
+    use Dispatchable;
+    use InteractsWithQueue;
+    use Queueable;
+    use SerializesModels;
+
+    public int $tries = 3;
+
+    public int $backoff = 90;
+
+    public function __construct(public int $domainId)
+    {
+    }
+
+    public function handle(PowerDnsClient $client): void
+    {
+        $domain = Domain::query()->find($this->domainId);
+        if (!$domain || !$domain->isActiveForMail()) {
+            return;
+        }
+
+        $masters = config('pdns.slave_masters', []);
+        $client->ensureSlaveZone($domain, $masters);
+    }
+}
@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Domain;
+use App\Models\DomainDkimKey;
+use App\Models\DomainNsCheck;
+use App\Jobs\ProvisionDomainSlaveZoneJob;
+use App\Services\Dns\PowerDnsClient;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Http\Client\RequestException;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\File;
+use Illuminate\Support\Str;
+
+class ProvisionDomainZoneJob implements ShouldQueue
+{
+    use Dispatchable;
+    use InteractsWithQueue;
+    use Queueable;
+    use SerializesModels;
+
+    public int $tries = 5;
+
+    public int $backoff = 60;
+
+    public function __construct(public int $domainId)
+    {
+    }
+
+    public function handle(PowerDnsClient $client): void
+    {
+        $domain = Domain::query()->find($this->domainId);
+        if (! $domain) {
+            return;
+        }
+
+        // Ensure DKIM keys exist before provisioning zone
+        $this->ensureDkimKeys($domain);
+
+        try {
+            $client->ensureZone($domain);
+            $this->logCheck($domain, 'success', 'PDNS zone provisioned successfully.');
+            ProvisionDomainSlaveZoneJob::dispatch($domain->id);
+        } catch (RequestException $exception) {
+            $this->logCheck($domain, 'failed', $exception->getMessage());
+            throw $exception;
+        }
+    }
+
+    private function ensureDkimKeys(Domain $domain): void
+    {
+        $existing = DomainDkimKey::query()
+            ->where('domain_id', $domain->id)
+            ->where('status', 'active')
+            ->latest('id')
+            ->first();
+
+        if ($existing) {
+            return;
+        }
+
+        $prefix = (string) config('mailinfra.dkim_selector_prefix', 'ladill');
+        $selector = Str::lower(trim($prefix)) ?: 'ladill';
+        $selector .= '1';
+
+        $config = [
+            'digest_alg' => 'sha512',
+            'private_key_bits' => 2048,
+            'private_key_type' => OPENSSL_KEYTYPE_RSA,
+        ];
+
+        $res = openssl_pkey_new($config);
+        openssl_pkey_export($res, $privateKey);
+        $details = openssl_pkey_get_details($res);
+        $publicKey = $details['key'] ?? '';
+
+        $publicKey = preg_replace('/-----BEGIN PUBLIC KEY-----|-----END PUBLIC KEY-----|\s/', '', $publicKey);
+
+        $privatePath = null;
+        if (is_string($privateKey) && trim($privateKey) !== '') {
+            $safeHost = Str::slug($domain->host, '_');
+            $directory = storage_path('app/mail/dkim');
+            File::ensureDirectoryExists($directory);
+            $privatePath = $directory.'/'.$safeHost.'_'.$selector.'.key';
+            File::put($privatePath, $privateKey);
+        }
+
+        DomainDkimKey::query()->create([
+            'domain_id' => $domain->id,
+            'selector' => $selector,
+            'public_key_txt' => $publicKey,
+            'private_key_path' => $privatePath,
+            'status' => 'active',
+        ]);
+    }
+
+    private function logCheck(Domain $domain, string $result, string $notes): void
+    {
+        DomainNsCheck::create([
+            'domain_id' => $domain->id,
+            'check_type' => 'pdns_provision',
+            'result' => $result,
+            'status_before' => (string) $domain->status,
+            'status_after' => (string) $domain->status,
+            'state_before' => (string) ($domain->onboarding_state ?? ''),
+            'state_after' => (string) ($domain->onboarding_state ?? ''),
+            'notes' => $notes,
+            'checked_at' => now(),
+        ]);
+    }
+}
@@ -0,0 +1,154 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\HostingAccount;
+use App\Models\HostingNode;
+use App\Services\Hosting\NodeCapacityService;
+use App\Services\Hosting\Providers\SharedNodeProvider;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ProvisionHostingAccountJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+    public int $timeout = 300; // 5 minutes for SSH operations
+    public array $backoff = [30, 60, 120]; // Retry after 30s, 60s, 120s
+
+    public function __construct(
+        private int $accountId,
+    ) {
+    }
+
+    public function handle(SharedNodeProvider $provider, NodeCapacityService $capacityService): void
+    {
+        $account = HostingAccount::with(['node', 'product'])->find($this->accountId);
+
+        if (! $account) {
+            Log::warning('ProvisionHostingAccountJob: Account not found', ['account_id' => $this->accountId]);
+            return;
+        }
+
+        // Skip if already provisioned or not in pending status
+        if ($account->status !== 'pending' && $account->provisioned_at !== null) {
+            Log::info('ProvisionHostingAccountJob: Already provisioned', ['account_id' => $this->accountId, 'status' => $account->status]);
+            return;
+        }
+
+        $node = $account->node;
+        if (! $node) {
+            Log::error('ProvisionHostingAccountJob: No node assigned', ['account_id' => $this->accountId]);
+            $account->update(['status' => 'failed', 'notes' => 'No hosting node assigned']);
+            return;
+        }
+
+        $product = $account->product;
+        if (! $product) {
+            Log::error('ProvisionHostingAccountJob: No product found', ['account_id' => $this->accountId]);
+            $account->update(['status' => 'failed', 'notes' => 'No hosting product found']);
+            return;
+        }
+
+        // Mark as provisioning
+        $account->update(['status' => 'provisioning']);
+
+        try {
+            $password = \Illuminate\Support\Str::password(16, true, true, false, false);
+            $domain = $account->primary_domain ?? $account->username . '.ladill.com';
+
+            $result = $provider->createAccountOnNode($node, [
+                'username' => $account->username,
+                'password' => $password,
+                'domain' => $domain,
+                'php_version' => '8.4',
+                'disk_quota_mb' => max((int) ($product->disk_gb ?? 0), 1) * 1024,
+            ]);
+
+            $docRoot = $result['document_root'] ?? "/home/{$account->username}/public_html";
+            
+            $account->update([
+                'status' => 'active',
+                'home_directory' => $result['home_directory'] ?? "/home/{$account->username}",
+                'provisioned_at' => now(),
+                'metadata' => array_merge((array) $account->metadata, [
+                    'provisioned_password' => $password, // Store temporarily for admin reference
+                    'provisioned_at' => now()->toISOString(),
+                ]),
+            ]);
+
+            // Apply resource limits
+            $provider->applyAccountResourceProfile($account->fresh(), false);
+
+            // Install WordPress if it's a WordPress hosting product
+            if ($product->type === 'wordpress') {
+                $wpResult = $provider->installWordPress($node, $account->username, $domain, [
+                    'document_root' => $docRoot,
+                    'admin_email' => $account->user?->email ?? 'admin@' . $domain,
+                    'site_title' => $domain,
+                ]);
+
+                // Store WordPress credentials in account metadata
+                $account->update([
+                    'metadata' => array_merge((array) $account->metadata, [
+                        'wp_admin_user' => $wpResult['admin_user'] ?? 'admin',
+                        'wp_admin_password' => $wpResult['admin_password'] ?? null,
+                        'wp_db_name' => $wpResult['db_name'] ?? null,
+                        'wp_db_user' => $wpResult['db_user'] ?? null,
+                        'wp_db_password' => $wpResult['db_password'] ?? null,
+                    ]),
+                ]);
+
+                Log::info('ProvisionHostingAccountJob: WordPress installed', [
+                    'account_id' => $this->accountId,
+                    'domain' => $domain,
+                ]);
+            }
+
+            // Refresh node capacity
+            $capacityService->refreshNodeResourceTotals($node);
+
+            Log::info('ProvisionHostingAccountJob: Successfully provisioned', [
+                'account_id' => $this->accountId,
+                'username' => $account->username,
+                'node_id' => $node->id,
+            ]);
+        } catch (\Throwable $exception) {
+            Log::error('ProvisionHostingAccountJob: Failed to provision', [
+                'account_id' => $this->accountId,
+                'username' => $account->username,
+                'error' => $exception->getMessage(),
+                'trace' => $exception->getTraceAsString(),
+            ]);
+
+            $account->update([
+                'status' => 'failed',
+                'notes' => 'Provisioning failed: ' . $exception->getMessage(),
+            ]);
+
+            // Re-throw to trigger retry
+            throw $exception;
+        }
+    }
+
+    public function failed(\Throwable $exception): void
+    {
+        Log::error('ProvisionHostingAccountJob: Job failed after all retries', [
+            'account_id' => $this->accountId,
+            'error' => $exception->getMessage(),
+        ]);
+
+        $account = HostingAccount::find($this->accountId);
+        if ($account) {
+            $account->update([
+                'status' => 'failed',
+                'notes' => 'Provisioning failed after retries: ' . $exception->getMessage(),
+            ]);
+        }
+    }
+}
@@ -0,0 +1,403 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Exceptions\ContaboApiException;
+use App\Models\CustomerHostingOrder;
+use App\Models\HostedSite;
+use App\Models\HostingAccount;
+use App\Models\HostingNode;
+use App\Models\ProvisioningQueueItem;
+use App\Services\Hosting\HostingResourcePolicyService;
+use App\Services\Hosting\NodeCapacityService;
+use App\Services\Hosting\Providers\ContaboProvider;
+use App\Services\Hosting\ServerOrderService;
+use App\Services\Hosting\ServerAgentBootstrapService;
+use App\Services\Hosting\ServerAgentInstallerService;
+use App\Services\Hosting\Providers\SharedNodeProvider;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Str;
+
+class ProvisionHostingOrderJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+    public int $backoff = 300;
+
+    public function __construct(
+        public CustomerHostingOrder $order
+    ) {}
+
+    public function handle(
+        NodeCapacityService $capacityService,
+        HostingResourcePolicyService $resourcePolicyService,
+        SharedNodeProvider $sharedProvider,
+        ContaboProvider $contaboProvider,
+        ServerOrderService $serverOrders,
+        ServerAgentBootstrapService $serverAgentBootstrap,
+        ServerAgentInstallerService $serverAgentInstaller,
+    ): void {
+        if ($this->order->status !== CustomerHostingOrder::STATUS_APPROVED) {
+            return;
+        }
+
+        $this->order->markAsProvisioning();
+
+        $product = $this->order->product;
+        $queueItem = $this->getOrCreateQueueItem();
+
+        try {
+            $queueItem->markAsProcessing();
+
+            if ($product->isSharedHosting()) {
+                $this->provisionSharedHosting($capacityService, $resourcePolicyService, $sharedProvider, $queueItem);
+            } elseif ($product->isVps()) {
+                $this->provisionVps($contaboProvider, $serverOrders, $serverAgentBootstrap, $serverAgentInstaller, $queueItem);
+            } elseif ($product->isDedicated()) {
+                $this->provisionDedicated($contaboProvider, $serverOrders, $serverAgentBootstrap, $serverAgentInstaller, $queueItem);
+            } else {
+                throw new \Exception("Unknown product category: {$product->category}");
+            }
+
+            $queueItem->markAsCompleted();
+
+        } catch (\Throwable $e) {
+            if ($this->shouldHoldForAdmin($e)) {
+                $message = $e->getMessage();
+                $queueItem->markAsFailed($message);
+                $this->order->fresh()->holdForAdminProvisioning(
+                    $message,
+                    $e instanceof ContaboApiException ? $e->httpStatus : null,
+                );
+
+                return;
+            }
+
+            $queueItem->markAsFailed($e->getMessage());
+
+            if ($queueItem->canRetry()) {
+                $this->release($this->backoff);
+            } else {
+                $this->order->markAsFailed($e->getMessage());
+            }
+
+            throw $e;
+        }
+    }
+
+    private function shouldHoldForAdmin(\Throwable $e): bool
+    {
+        if ($e instanceof ContaboApiException) {
+            return $e->isPaymentRequired();
+        }
+
+        $product = $this->order->product;
+        if ($product?->requiresContabo() !== true) {
+            return false;
+        }
+
+        return ContaboApiException::messageIndicatesPaymentRequired($e->getMessage());
+    }
+
+    private function provisionSharedHosting(
+        NodeCapacityService $capacityService,
+        HostingResourcePolicyService $resourcePolicyService,
+        SharedNodeProvider $sharedProvider,
+        ProvisioningQueueItem $queueItem
+    ): void {
+        $queueItem->addLog('Finding available shared hosting node...');
+
+        $product = $this->order->product;
+        $requestedDiskGb = (int) ($product->disk_gb ?? 0);
+        $segment = $product->preferredNodeSegment();
+        $node = $capacityService->findAvailableNodeForProduct($product);
+
+        if (!$node) {
+            throw new \Exception('No available shared hosting nodes. Please provision a new node.');
+        }
+
+        $queueItem->addLog("Selected node: {$node->name} ({$node->ip_address}) [segment={$segment}, requested_disk={$requestedDiskGb}GB]");
+
+        // Generate username
+        $username = $this->generateUsername($this->order->domain_name);
+        $password = Str::password(16, true, true, false, false);
+
+        $queueItem->addLog("Creating account: {$username}");
+        $resourceLimits = $resourcePolicyService->defaultLimitsForProduct($product);
+
+        // Create account on node via SSH
+        $result = $sharedProvider->createAccountOnNode($node, [
+            'username' => $username,
+            'password' => $password,
+            'domain' => $this->order->domain_name,
+            'email' => $this->order->user->email ?: $this->order->guest_email,
+            'disk_quota_mb' => max($requestedDiskGb, 1) * 1024,
+            'bandwidth_mb' => (($product->bandwidth_gb ?? 100) > 0 ? $product->bandwidth_gb : 100) * 1024,
+        ]);
+
+        // Create hosting account record
+        $account = HostingAccount::create([
+            'user_id' => $this->order->user_id,
+            'hosting_node_id' => $node->id,
+            'hosting_product_id' => $product->id,
+            'username' => $username,
+            'primary_domain' => $this->order->domain_name,
+            'type' => HostingAccount::TYPE_SHARED,
+            'status' => HostingAccount::STATUS_ACTIVE,
+            'home_directory' => $result['home_directory'] ?? "/home/{$username}",
+            'allocated_disk_gb' => $requestedDiskGb,
+            'disk_used_bytes' => 0,
+            'bandwidth_used_bytes' => 0,
+            'cpu_limit_percent' => $resourceLimits['cpu_limit_percent'],
+            'memory_limit_mb' => $resourceLimits['memory_limit_mb'],
+            'process_limit' => $resourceLimits['process_limit'],
+            'io_limit_mb' => $resourceLimits['io_limit_mb'],
+            'inode_limit' => $resourceLimits['inode_limit'],
+            'resource_status' => HostingAccount::RESOURCE_STATUS_ACTIVE,
+            'uploads_restricted' => false,
+            'is_flagged' => false,
+            'warning_count' => 0,
+            'suspended_at' => null,
+            'provisioned_at' => now(),
+            'metadata' => [
+                'assigned_duration_months' => $this->order->getCycleLengthInMonths(),
+                'initial_password' => $password,
+                'provisioned_from_order_id' => $this->order->id,
+                'provisioned_on_node' => $node->name,
+            ],
+            'resource_limits' => array_merge([
+                'disk_gb' => $requestedDiskGb,
+                'bandwidth_gb' => $product->bandwidth_gb,
+                'max_domains' => $product->max_domains,
+                'max_databases' => $product->max_databases,
+            ], $resourceLimits),
+        ]);
+
+        HostedSite::create([
+            'hosting_account_id' => $account->id,
+            'domain_id' => $this->order->domain_id,
+            'domain' => $this->order->domain_name,
+            'document_root' => ($result['document_root'] ?? "/home/{$username}/public_html"),
+            'type' => 'primary',
+            'status' => 'active',
+            'php_version' => '8.4',
+            'ssl_enabled' => false,
+        ]);
+
+        $sharedProvider->applyAccountResourceProfile($account, false);
+
+        $node = $capacityService->refreshNodeResourceTotals($node);
+        $capacityService->checkNode($node);
+
+        // Install WordPress if it's a WordPress hosting plan
+        $wpCredentials = [];
+        if ($product->type === 'wordpress') {
+            $queueItem->addLog('Installing WordPress...');
+            $docRoot = $result['document_root'] ?? "/home/{$username}/public_html";
+            $wpCredentials = $sharedProvider->installWordPress($node, $username, $this->order->domain_name, [
+                'document_root' => $docRoot,
+                'admin_email' => $this->order->user->email ?: $this->order->guest_email,
+                'site_title' => $this->order->domain_name,
+            ]);
+        }
+
+        $queueItem->addLog('Provisioning completed successfully.');
+
+        // Update order with provisioning details
+        $orderMeta = array_merge($this->order->meta ?? [], [
+            'provisioned_on_node' => $node->name,
+            'node_segment' => $node->segment,
+            'requested_disk_gb' => $requestedDiskGb,
+            'initial_password' => $password,
+        ]);
+
+        // Add WordPress credentials if installed
+        if (!empty($wpCredentials)) {
+            $orderMeta['wp_admin_user'] = $wpCredentials['admin_user'] ?? 'admin';
+            $orderMeta['wp_admin_password'] = $wpCredentials['admin_password'] ?? null;
+            $orderMeta['wp_admin_email'] = $wpCredentials['admin_email'] ?? null;
+            $orderMeta['wp_db_name'] = $wpCredentials['db_name'] ?? null;
+            $orderMeta['wp_db_user'] = $wpCredentials['db_user'] ?? null;
+            $orderMeta['wp_db_password'] = $wpCredentials['db_password'] ?? null;
+        }
+
+        $this->order->markAsActive([
+            'hosting_node_id' => $node->id,
+            'hosting_account_id' => $account->id,
+            'server_username' => $username,
+            'server_ip' => $node->ip_address,
+            'control_panel_url' => "https://{$this->order->domain_name}/cpanel",
+            'expires_at' => $this->order->calculateExpiryDate(),
+            'meta' => $orderMeta,
+        ]);
+    }
+
+    private function provisionVps(ContaboProvider $contaboProvider, ServerOrderService $serverOrders, ServerAgentBootstrapService $serverAgentBootstrap, ServerAgentInstallerService $serverAgentInstaller, ProvisioningQueueItem $queueItem): void
+    {
+        $queueItem->addLog('Provisioning VPS via Contabo API...');
+
+        $product = $this->order->product;
+
+        if (!$product->contabo_product_id) {
+            throw new \Exception('Product does not have a Contabo product ID configured.');
+        }
+
+        $provisioning = $this->serverProvisioningConfig($serverOrders, 'VPS');
+        if ((bool) ($provisioning['panel_snapshot']['managed_stack_candidate'] ?? false)) {
+            $bootstrap = $serverAgentBootstrap->ensureBootstrap($this->order->fresh());
+            $provisioning = $serverAgentInstaller->augmentProvisioningConfig($this->order->fresh(), $provisioning, $bootstrap);
+            $queueItem->addLog('Managed stack bootstrap and Ladill server agent install prepared.');
+        }
+        $result = $contaboProvider->createInstance($provisioning);
+
+        $queueItem->addLog("VPS created with instance ID: {$result['instance_id']}");
+        foreach ($provisioning['manual_follow_up'] ?? [] as $item) {
+            $queueItem->addLog("Manual follow-up required for {$item}.");
+        }
+
+        $this->order->markAsActive([
+            'server_ip' => $result['ip_address'] ?? null,
+            'server_username' => (string) ($provisioning['default_user'] ?? 'root'),
+            'control_panel_url' => $this->serverControlPanelUrl($provisioning),
+            'expires_at' => $this->order->calculateExpiryDate(),
+            'meta' => array_merge($this->order->meta ?? [], [
+                'contabo_instance_id' => $result['instance_id'],
+                'server_panel_runtime' => $provisioning['panel_snapshot'] ?? data_get($this->order->meta, 'server_panel_runtime', []),
+                'server_panel' => [
+                    'panel' => $provisioning['panel'] ?? null,
+                    'instance_status' => $result['status'] ?? null,
+                    'power_status' => 'on',
+                    'region' => $result['region'] ?? null,
+                    'datacenter' => $result['datacenter'] ?? null,
+                    'image' => $result['image'] ?? null,
+                    'last_synced_at' => now()->toIso8601String(),
+                ],
+            ]),
+        ]);
+
+        if ((bool) ($provisioning['panel_snapshot']['managed_stack_candidate'] ?? false)) {
+            $serverAgentBootstrap->ensureBootstrap($this->order->fresh());
+        }
+    }
+
+    private function provisionDedicated(ContaboProvider $contaboProvider, ServerOrderService $serverOrders, ServerAgentBootstrapService $serverAgentBootstrap, ServerAgentInstallerService $serverAgentInstaller, ProvisioningQueueItem $queueItem): void
+    {
+        $queueItem->addLog('Provisioning Dedicated Server via Contabo API...');
+
+        $product = $this->order->product;
+
+        if (!$product->contabo_product_id) {
+            throw new \Exception('Product does not have a Contabo product ID configured.');
+        }
+
+        $provisioning = $this->serverProvisioningConfig($serverOrders, 'Dedicated');
+        if ((bool) ($provisioning['panel_snapshot']['managed_stack_candidate'] ?? false)) {
+            $bootstrap = $serverAgentBootstrap->ensureBootstrap($this->order->fresh());
+            $provisioning = $serverAgentInstaller->augmentProvisioningConfig($this->order->fresh(), $provisioning, $bootstrap);
+            $queueItem->addLog('Managed stack bootstrap and Ladill server agent install prepared.');
+        }
+        $result = $contaboProvider->createInstance($provisioning);
+
+        $queueItem->addLog("Dedicated server created with instance ID: {$result['instance_id']}");
+        foreach ($provisioning['manual_follow_up'] ?? [] as $item) {
+            $queueItem->addLog("Manual follow-up required for {$item}.");
+        }
+
+        $this->order->markAsActive([
+            'server_ip' => $result['ip_address'] ?? null,
+            'server_username' => (string) ($provisioning['default_user'] ?? 'root'),
+            'control_panel_url' => $this->serverControlPanelUrl($provisioning),
+            'expires_at' => $this->order->calculateExpiryDate(),
+            'meta' => array_merge($this->order->meta ?? [], [
+                'contabo_instance_id' => $result['instance_id'],
+                'server_panel_runtime' => $provisioning['panel_snapshot'] ?? data_get($this->order->meta, 'server_panel_runtime', []),
+                'server_panel' => [
+                    'panel' => $provisioning['panel'] ?? null,
+                    'instance_status' => $result['status'] ?? null,
+                    'power_status' => 'on',
+                    'region' => $result['region'] ?? null,
+                    'datacenter' => $result['datacenter'] ?? null,
+                    'image' => $result['image'] ?? null,
+                    'last_synced_at' => now()->toIso8601String(),
+                ],
+            ]),
+        ]);
+
+        if ((bool) ($provisioning['panel_snapshot']['managed_stack_candidate'] ?? false)) {
+            $serverAgentBootstrap->ensureBootstrap($this->order->fresh());
+        }
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    private function serverProvisioningConfig(ServerOrderService $serverOrders, string $prefix): array
+    {
+        $meta = (array) ($this->order->meta ?? []);
+        $serverOrder = (array) ($meta['server_order'] ?? []);
+        $selection = (array) ($serverOrder['selection'] ?? []);
+        $passwordEncrypted = $serverOrder['server_password_encrypted'] ?? null;
+
+        $config = $serverOrders->provisioningConfig(
+            $this->order->product,
+            $selection,
+            "{$prefix}-{$this->order->id}-{$this->order->domain_name}",
+            $this->order->billing_cycle
+        );
+
+        if (is_string($passwordEncrypted) && $passwordEncrypted !== '') {
+            $config['root_password'] = decrypt($passwordEncrypted);
+        }
+
+        return $config;
+    }
+
+    /**
+     * @param  array<string, mixed>  $provisioning
+     */
+    private function serverControlPanelUrl(array $provisioning): ?string
+    {
+        if (($provisioning['panel'] ?? null) !== 'ladill') {
+            return null;
+        }
+
+        return route('hosting.server-panel.show', $this->order);
+    }
+
+    private function getOrCreateQueueItem(): ProvisioningQueueItem
+    {
+        $existing = ProvisioningQueueItem::where('customer_hosting_order_id', $this->order->id)->first();
+
+        if ($existing) {
+            return $existing;
+        }
+
+        $provider = match ($this->order->product->category) {
+            'shared' => ProvisioningQueueItem::PROVIDER_SHARED_NODE,
+            'vps' => ProvisioningQueueItem::PROVIDER_CONTABO_VPS,
+            'dedicated' => ProvisioningQueueItem::PROVIDER_CONTABO_DEDICATED,
+            default => ProvisioningQueueItem::PROVIDER_SHARED_NODE,
+        };
+
+        return ProvisioningQueueItem::create([
+            'customer_hosting_order_id' => $this->order->id,
+            'status' => ProvisioningQueueItem::STATUS_QUEUED,
+            'provider' => $provider,
+        ]);
+    }
+
+    private function generateUsername(string $domain): string
+    {
+        // Remove TLD and special characters
+        $base = preg_replace('/[^a-z0-9]/', '', strtolower(explode('.', $domain)[0]));
+        $base = substr($base, 0, 8);
+
+        // Add random suffix to ensure uniqueness
+        return $base . Str::random(4);
+    }
+}
@@ -0,0 +1,114 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\HostedSite;
+use App\Services\Hosting\Providers\SharedNodeProvider;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ProvisionHostingSslJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 8;
+    public int $timeout = 180; // 3 minutes for certbot
+    public array $backoff = [300, 600, 1800, 3600, 7200, 14400, 21600]; // Retry across typical DNS propagation windows
+
+    public function __construct(
+        private int $siteId,
+    ) {
+    }
+
+    public function handle(SharedNodeProvider $provider): void
+    {
+        $site = HostedSite::with(['account.node', 'account.user'])->find($this->siteId);
+
+        if (! $site) {
+            Log::warning('ProvisionHostingSslJob: Site not found', ['site_id' => $this->siteId]);
+            return;
+        }
+
+        // Skip if already has SSL
+        if ($site->ssl_enabled && $site->ssl_status === 'issued') {
+            Log::info('ProvisionHostingSslJob: SSL already active', ['site_id' => $this->siteId, 'domain' => $site->domain]);
+            return;
+        }
+
+        $account = $site->account;
+        if (! $account || ! $account->node) {
+            Log::error('ProvisionHostingSslJob: No account or node', ['site_id' => $this->siteId]);
+            return;
+        }
+
+        // Skip if account is not active
+        if ($account->status !== 'active') {
+            Log::info('ProvisionHostingSslJob: Account not active, skipping SSL', [
+                'site_id' => $this->siteId,
+                'account_status' => $account->status,
+            ]);
+            return;
+        }
+
+        $site->update(['ssl_status' => 'provisioning']);
+
+        try {
+            $provider->requestLetsEncryptCertificate($site);
+
+            $site->update([
+                'ssl_enabled' => true,
+                'ssl_status' => 'issued',
+                'ssl_provisioned_at' => now(),
+                'ssl_expires_at' => now()->addDays(90),
+            ]);
+
+            Log::info('ProvisionHostingSslJob: SSL provisioned successfully', [
+                'site_id' => $this->siteId,
+                'domain' => $site->domain,
+            ]);
+        } catch (\Throwable $exception) {
+            $message = $exception->getMessage();
+
+            // Check if it's a DNS propagation issue (should retry)
+            $isDnsIssue = str_contains($message, 'DNS problem')
+                || str_contains($message, 'NXDOMAIN')
+                || str_contains($message, 'no valid A records')
+                || str_contains($message, 'Timeout');
+
+            $site->update([
+                'ssl_status' => 'failed',
+                'ssl_error' => $message,
+            ]);
+
+            Log::error('ProvisionHostingSslJob: Failed to provision SSL', [
+                'site_id' => $this->siteId,
+                'domain' => $site->domain,
+                'error' => $message,
+                'is_dns_issue' => $isDnsIssue,
+            ]);
+
+            // Re-throw to trigger retry (especially for DNS issues)
+            throw $exception;
+        }
+    }
+
+    public function failed(\Throwable $exception): void
+    {
+        Log::error('ProvisionHostingSslJob: Job failed after all retries', [
+            'site_id' => $this->siteId,
+            'error' => $exception->getMessage(),
+        ]);
+
+        $site = HostedSite::find($this->siteId);
+        if ($site) {
+            $site->update([
+                'ssl_status' => 'failed',
+                'ssl_error' => 'SSL provisioning failed after multiple attempts: ' . $exception->getMessage(),
+            ]);
+        }
+    }
+}
@@ -0,0 +1,70 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Mailbox;
+use App\Services\Mail\MailServerProvisioner;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ProvisionMailboxJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+
+    public int $backoff = 60;
+
+    public function __construct(
+        public int $mailboxId,
+        public ?string $encryptedPassword = null,
+    ) {
+    }
+
+    public function handle(MailServerProvisioner $provisioner): void
+    {
+        $mailbox = Mailbox::query()
+            ->with(['domain', 'emailDomain'])
+            ->find($this->mailboxId);
+
+        if (! $mailbox || (string) $mailbox->status === 'deleting') {
+            return;
+        }
+
+        if (! $provisioner->isConfigured()) {
+            Log::info('ProvisionMailboxJob: Skipped — mail server DB not configured', [
+                'mailbox_id' => $mailbox->id,
+            ]);
+            $mailbox->update(['status' => 'active']);
+
+            return;
+        }
+
+        $password = $this->encryptedPassword !== null
+            ? decrypt($this->encryptedPassword)
+            : null;
+
+        try {
+            if ($mailbox->email_domain_id !== null) {
+                $provisioner->provisionStandaloneMailbox($mailbox, $password);
+            } else {
+                $provisioner->provisionMailbox($mailbox, $password);
+            }
+
+            $mailbox->update(['status' => 'active']);
+        } catch (\Throwable $exception) {
+            $mailbox->update(['status' => 'failed']);
+            Log::error('ProvisionMailboxJob: Failed', [
+                'mailbox_id' => $mailbox->id,
+                'address' => $mailbox->address,
+                'error' => $exception->getMessage(),
+            ]);
+
+            throw $exception;
+        }
+    }
+}
@@ -0,0 +1,106 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Domain;
+use App\Models\DomainNsCheck;
+use App\Notifications\SslProvisionedNotification;
+use App\Services\Ssl\SslProvisioner;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class SslProvisioningJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+
+    public int $backoff = 120;
+
+    public function __construct(public int $domainId)
+    {
+    }
+
+    public function handle(SslProvisioner $provisioner): void
+    {
+        $domain = Domain::query()->find($this->domainId);
+        if (! $domain) {
+            return;
+        }
+
+        if ((string) $domain->status !== 'verified' || (string) $domain->onboarding_state !== Domain::STATE_ACTIVE) {
+            return;
+        }
+
+        if ((string) $domain->ssl_status === 'active') {
+            return;
+        }
+
+        if (! $provisioner->isConfigured()) {
+            Log::info('SslProvisioningJob: Skipped — SSL not configured', ['domain_id' => $domain->id]);
+
+            return;
+        }
+
+        $beforeStatus = (string) $domain->status;
+        $beforeState = (string) ($domain->onboarding_state ?? '');
+
+        $domain->update(['ssl_status' => 'provisioning']);
+
+        try {
+            $certOk = $provisioner->provisionCertificate($domain);
+            if (! $certOk) {
+                throw new \RuntimeException('certbot failed for ' . $domain->host);
+            }
+
+            $nginxOk = $provisioner->generateNginxConfig($domain);
+            if (! $nginxOk) {
+                throw new \RuntimeException('Nginx config generation failed for ' . $domain->host);
+            }
+
+            $reloadOk = $provisioner->reloadNginx();
+            if (! $reloadOk) {
+                throw new \RuntimeException('Nginx reload failed after provisioning ' . $domain->host);
+            }
+
+            $domain->update([
+                'ssl_status' => 'active',
+                'ssl_provisioned_at' => now(),
+                'ssl_expires_at' => now()->addDays(90),
+            ]);
+
+            $this->logCheck($domain, 'success', 'SSL certificate provisioned and Nginx configured', $beforeStatus, $beforeState);
+
+            // Notify user that SSL is active
+            $user = $domain->website?->user;
+            if ($user) {
+                $user->notify(new SslProvisionedNotification($domain->fresh()));
+            }
+        } catch (\Throwable $exception) {
+            $domain->update(['ssl_status' => 'failed']);
+            $this->logCheck($domain, 'failed', $exception->getMessage(), $beforeStatus, $beforeState);
+            Log::error('SslProvisioningJob: Failed', ['domain_id' => $domain->id, 'error' => $exception->getMessage()]);
+
+            throw $exception;
+        }
+    }
+
+    private function logCheck(Domain $domain, string $result, string $notes, string $statusBefore, string $stateBefore): void
+    {
+        DomainNsCheck::create([
+            'domain_id' => $domain->id,
+            'check_type' => 'ssl_provision',
+            'result' => $result,
+            'notes' => $notes,
+            'status_before' => $statusBefore,
+            'status_after' => (string) $domain->status,
+            'state_before' => $stateBefore,
+            'state_after' => (string) ($domain->onboarding_state ?? ''),
+            'checked_at' => now(),
+        ]);
+    }
+}