Fix display voice prompts requiring browser unlock and failed acks.

Add tap-to-enable speech overlay, stop marking announcements played on TTS errors, pick voices by locale, and scope pending announcements to the display's queues.

Co-authored-by: Cursor <cursoragent@cursor.com>

app/Services/Qms/DisplayService.php

@@ -55,8 +55,8 @@ class DisplayService
             ->where('status', 'completed')
             ->whereDate('issued_at', today())
             ->whereNotNull('called_at')
-            ->selectRaw('AVG(TIMESTAMPDIFF(SECOND, issued_at, called_at)) as avg_wait')
-            ->value('avg_wait');
+            ->get(['issued_at', 'called_at'])
+            ->avg(fn (Ticket $t) => $t->called_at->diffInSeconds($t->issued_at));
 
         return [
             'screen' => [
@@ -71,7 +71,10 @@ class DisplayService
                 'prefix' => $q->prefix,
                 'is_paused' => $q->is_paused,
             ]),
-            'announcements' => app(VoiceAnnouncementService::class)->pendingForBranch((int) $screen->branch_id),
+            'announcements' => app(VoiceAnnouncementService::class)->pendingForBranch(
+                (int) $screen->branch_id,
+                array_map('intval', $screen->service_queue_ids ?? []),
+            ),
             'updated_at' => now()->toIso8601String(),
         ];
     }

app/Services/Qms/VoiceAnnouncementService.php

@@ -5,7 +5,6 @@ namespace App\Services\Qms;
 use App\Models\Counter;
 use App\Models\Ticket;
 use App\Models\VoiceAnnouncement;
-use Illuminate\Support\Str;
 
 class VoiceAnnouncementService
 {
@@ -41,13 +40,17 @@ class VoiceAnnouncementService
     }
 
     /**
+     * @param  list<int>|null  $queueIds
      * @return list<array<string, mixed>>
      */
-    public function pendingForBranch(int $branchId, int $limit = 10): array
+    public function pendingForBranch(int $branchId, ?array $queueIds = null, int $limit = 10): array
     {
         return VoiceAnnouncement::query()
             ->where('branch_id', $branchId)
             ->where('status', 'pending')
+            ->when($queueIds, function ($query, array $queueIds) {
+                $query->whereHas('ticket', fn ($ticket) => $ticket->whereIn('service_queue_id', $queueIds));
+            })
             ->orderBy('id')
             ->limit($limit)
             ->get()

resources/js/kiosk-flow.js

@@ -1,3 +1,42 @@
+const DISPLAY_AUDIO_KEY = 'qms_display_audio';
+
+function localeToSpeechLang(locale) {
+    if (locale === 'fr') return 'fr-FR';
+    if (locale === 'tw') return 'ak-GH';
+
+    return 'en-US';
+}
+
+function pickVoice(voices, locale, preference) {
+    const list = voices.length ? voices : window.speechSynthesis?.getVoices() ?? [];
+    if (! list.length) {
+        return null;
+    }
+
+    const lang = localeToSpeechLang(locale).toLowerCase();
+    const langMatches = list.filter((voice) => voice.lang?.toLowerCase().startsWith(lang.slice(0, 2)));
+    const pool = langMatches.length ? langMatches : list;
+    const wantFemale = preference !== 'male';
+
+    const scored = pool.map((voice) => {
+        const name = voice.name.toLowerCase();
+        const female = /female|woman|zira|samantha|karen|victoria|fiona|moira|tessa|veena|lekha/.test(name);
+        const male = /male|man|david|daniel|james|fred|thomas|lee|ralph/.test(name);
+
+        let score = 0;
+        if (wantFemale && female) score += 2;
+        if (! wantFemale && male) score += 2;
+        if (voice.default) score += 1;
+        if (voice.localService) score += 1;
+
+        return { voice, score };
+    });
+
+    scored.sort((a, b) => b.score - a.score);
+
+    return scored[0]?.voice ?? pool[0] ?? null;
+}
+
 export function registerKioskFlow(Alpine) {
     Alpine.data('kioskFlow', (config = {}) => ({
         step: 'select',
@@ -58,6 +97,8 @@ export function registerKioskFlow(Alpine) {
         payload: null,
         announcedIds: {},
         isAnnouncing: false,
+        speechReady: false,
+        voices: [],
         dataUrl: typeof config === 'string' ? config : config.dataUrl,
         playedUrlTemplate: config.playedUrl ?? null,
         pollMs: config.pollMs ?? 1200,
@@ -73,8 +114,48 @@ export function registerKioskFlow(Alpine) {
             }
         },
 
+        primeSpeech() {
+            if (! window.speechSynthesis) {
+                return;
+            }
+
+            const loadVoices = () => {
+                this.voices = window.speechSynthesis.getVoices() || [];
+            };
+
+            loadVoices();
+            window.speechSynthesis.addEventListener('voiceschanged', loadVoices);
+        },
+
+        unlockSpeech() {
+            if (! window.speechSynthesis) {
+                return;
+            }
+
+            window.speechSynthesis.resume?.();
+            window.speechSynthesis.cancel();
+
+            const utter = new SpeechSynthesisUtterance(' ');
+            utter.volume = 0;
+            utter.onend = () => this.markSpeechReady();
+            utter.onerror = () => this.markSpeechReady();
+            window.speechSynthesis.speak(utter);
+        },
+
+        markSpeechReady() {
+            this.speechReady = true;
+
+            try {
+                sessionStorage.setItem(DISPLAY_AUDIO_KEY, '1');
+            } catch (e) {
+                // ignore storage failures
+            }
+
+            this.maybeAnnounce(this.payload?.announcements ?? []);
+        },
+
         maybeAnnounce(announcements) {
-            if (! window.speechSynthesis || this.isAnnouncing || ! announcements.length) {
+            if (! window.speechSynthesis || ! this.speechReady || this.isAnnouncing || ! announcements.length) {
                 return;
             }
 
@@ -92,20 +173,59 @@ export function registerKioskFlow(Alpine) {
             const repeats = Math.max(1, Number(item.repeat_count) || 1);
             const volume = Math.min(1, Math.max(0, (Number(item.volume) || 80) / 100));
             let remaining = repeats;
+            let watchdog = null;
+
+            const release = () => {
+                if (watchdog) {
+                    clearTimeout(watchdog);
+                    watchdog = null;
+                }
+                this.isAnnouncing = false;
+            };
 
             const finish = () => {
-                this.isAnnouncing = false;
+                release();
                 this.ackPlayed(item.id);
                 this.maybeAnnounce(this.payload?.announcements ?? []);
             };
 
+            const fail = (reason) => {
+                console.error('Announcement speech failed', reason);
+                release();
+                delete this.announcedIds[item.id];
+            };
+
             const speakOnce = () => {
                 window.speechSynthesis.resume?.();
+                window.speechSynthesis.cancel();
 
                 const utter = new SpeechSynthesisUtterance(item.message);
                 utter.volume = volume;
-                utter.lang = item.locale === 'fr' ? 'fr-FR' : item.locale === 'tw' ? 'ak-GH' : 'en-US';
+                utter.lang = localeToSpeechLang(item.locale);
+                const voice = pickVoice(this.voices, item.locale, item.voice);
+                if (voice) {
+                    utter.voice = voice;
+                }
+
+                const estimatedMs = Math.max(4000, (item.message?.length || 24) * 140);
+                watchdog = setTimeout(() => {
+                    if (! this.isAnnouncing) {
+                        return;
+                    }
+                    window.speechSynthesis.cancel();
+                    remaining -= 1;
+                    if (remaining > 0) {
+                        speakOnce();
+                    } else {
+                        finish();
+                    }
+                }, estimatedMs);
+
                 utter.onend = () => {
+                    if (watchdog) {
+                        clearTimeout(watchdog);
+                        watchdog = null;
+                    }
                     remaining -= 1;
                     if (remaining > 0) {
                         speakOnce();
@@ -113,10 +233,7 @@ export function registerKioskFlow(Alpine) {
                         finish();
                     }
                 };
-                utter.onerror = () => {
-                    delete this.announcedIds[item.id];
-                    finish();
-                };
+                utter.onerror = (event) => fail(event);
 
                 window.speechSynthesis.speak(utter);
             };
@@ -132,13 +249,16 @@ export function registerKioskFlow(Alpine) {
             const url = this.playedUrlTemplate.replace('__ID__', String(id));
 
             try {
-                await fetch(url, {
+                const res = await fetch(url, {
                     method: 'POST',
                     headers: {
                         Accept: 'application/json',
                         'X-CSRF-TOKEN': this.csrf,
                     },
                 });
+                if (! res.ok) {
+                    throw new Error(`Ack failed (${res.status})`);
+                }
             } catch (e) {
                 console.error('Announcement ack failed', e);
                 delete this.announcedIds[id];
@@ -146,6 +266,16 @@ export function registerKioskFlow(Alpine) {
         },
 
         init() {
+            this.primeSpeech();
+
+            try {
+                if (sessionStorage.getItem(DISPLAY_AUDIO_KEY) === '1') {
+                    this.unlockSpeech();
+                }
+            } catch (e) {
+                // ignore storage failures
+            }
+
             this.poll();
             setInterval(() => this.poll(), this.pollMs);
         },

resources/views/qms/display/public.blade.php

@@ -8,6 +8,22 @@
     @vite(['resources/css/app.css', 'resources/js/app.js'])
 </head>
 <body class="min-h-screen p-8" x-data="displayBoard({ dataUrl: '{{ $dataUrl }}', playedUrl: '{{ $playedUrl }}', pollMs: {{ $pollMs }}, csrf: '{{ csrf_token() }}' })" x-init="init()">
+    <div
+        x-cloak
+        x-show="!speechReady"
+        @click="unlockSpeech()"
+        @keydown.enter.prevent="unlockSpeech()"
+        tabindex="0"
+        class="fixed inset-0 z-50 flex cursor-pointer items-center justify-center bg-slate-950/95 p-8 text-center outline-none"
+        role="button"
+        aria-label="Enable voice announcements"
+    >
+        <div>
+            <p class="text-3xl font-semibold text-white">Tap to enable voice announcements</p>
+            <p class="mt-3 text-slate-400">Browsers require one tap before the display can speak ticket calls.</p>
+        </div>
+    </div>
+
     <div class="mx-auto max-w-6xl">
         <div class="mb-8 flex items-center justify-between">
             <h1 class="text-4xl font-bold" x-text="payload?.screen?.name ?? '{{ $screen->name }}'"></h1>

tests/Feature/DisplayVoiceTest.php

@@ -0,0 +1,117 @@
+<?php
+
+namespace Tests\Feature;
+
+use App\Models\Branch;
+use App\Models\Counter;
+use App\Models\DisplayScreen;
+use App\Models\Organization;
+use App\Models\ServiceQueue;
+use App\Models\User;
+use App\Models\VoiceAnnouncement;
+use App\Services\Qms\OrganizationResolver;
+use App\Services\Qms\QueueEngine;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Tests\TestCase;
+
+class DisplayVoiceTest extends TestCase
+{
+    use RefreshDatabase;
+
+    /**
+     * @return array{0: User, 1: Organization, 2: Branch, 3: ServiceQueue}
+     */
+    protected function setUpOrg(): array
+    {
+        $user = User::create([
+            'public_id' => 'test-owner-uuid',
+            'name' => 'Test User',
+            'email' => 'test@example.com',
+            'password' => bcrypt('password'),
+        ]);
+        $resolver = app(OrganizationResolver::class);
+        $org = $resolver->completeOnboarding($user, [
+            'organization_name' => 'Test Org',
+            'industry' => 'retail',
+            'appointment_mode' => 'hybrid',
+            'branch_name' => 'Main',
+            'timezone' => 'UTC',
+        ]);
+        $branch = Branch::first();
+        $queue = ServiceQueue::create([
+            'owner_ref' => $user->public_id,
+            'organization_id' => $org->id,
+            'branch_id' => $branch->id,
+            'name' => 'Reception',
+            'prefix' => 'A',
+            'strategy' => 'fifo',
+            'is_active' => true,
+        ]);
+
+        return [$user, $org, $branch, $queue];
+    }
+
+    public function test_display_data_includes_pending_announcement_after_call(): void
+    {
+        [$user, , $branch, $queue] = $this->setUpOrg();
+
+        $counter = Counter::create([
+            'owner_ref' => $user->public_id,
+            'organization_id' => $queue->organization_id,
+            'branch_id' => $queue->branch_id,
+            'name' => 'Counter 1',
+            'status' => 'available',
+            'is_active' => true,
+        ]);
+
+        $screen = DisplayScreen::create([
+            'owner_ref' => $user->public_id,
+            'organization_id' => $queue->organization_id,
+            'branch_id' => $branch->id,
+            'name' => 'Lobby',
+            'layout' => 'standard',
+            'service_queue_ids' => [$queue->id],
+            'is_active' => true,
+        ]);
+
+        $engine = app(QueueEngine::class);
+        $ticket = $engine->issueTicket($queue, $user->public_id);
+        $engine->callTicket($ticket, $counter, $user->public_id);
+
+        $response = $this->getJson(route('qms.display.data', $screen->access_token));
+
+        $response->assertOk()
+            ->assertJsonPath('announcements.0.message', 'Ticket A001, please proceed to Counter 1.');
+    }
+
+    public function test_display_marks_announcement_played(): void
+    {
+        [$user, , $branch, $queue] = $this->setUpOrg();
+
+        $screen = DisplayScreen::create([
+            'owner_ref' => $user->public_id,
+            'organization_id' => $queue->organization_id,
+            'branch_id' => $branch->id,
+            'name' => 'Lobby',
+            'layout' => 'standard',
+            'service_queue_ids' => [$queue->id],
+            'is_active' => true,
+        ]);
+
+        $announcement = VoiceAnnouncement::create([
+            'owner_ref' => $user->public_id,
+            'organization_id' => $queue->organization_id,
+            'branch_id' => $branch->id,
+            'locale' => 'en',
+            'message' => 'Ticket A001, please proceed to Counter 1.',
+            'status' => 'pending',
+        ]);
+
+        $this->postJson(route('qms.display.announcement.played', [
+            'token' => $screen->access_token,
+            'announcement' => $announcement->id,
+        ]))->assertOk();
+
+        $this->assertSame('played', $announcement->fresh()->status);
+    }
+}