diff --git a/app/Http/Controllers/Meet/MeetingRoomController.php b/app/Http/Controllers/Meet/MeetingRoomController.php index e9fbeb3..b069b15 100644 --- a/app/Http/Controllers/Meet/MeetingRoomController.php +++ b/app/Http/Controllers/Meet/MeetingRoomController.php @@ -617,8 +617,7 @@ class MeetingRoomController extends Controller return response()->json(['ok' => false], 422); } - $transcript = $session->transcripts()->latest()->first() - ?? $this->transcripts->ensureForSession($session); + $transcript = $this->transcripts->getOrCreateForSession($session); $this->transcripts->appendSegment($transcript, $participant->display_name, $validated['text']); diff --git a/app/Jobs/GenerateAiSummary.php b/app/Jobs/GenerateAiSummary.php index a09ae88..db30e96 100644 --- a/app/Jobs/GenerateAiSummary.php +++ b/app/Jobs/GenerateAiSummary.php @@ -11,6 +11,8 @@ class GenerateAiSummary implements ShouldQueue { use Queueable; + public int $tries = 2; + public function __construct( public int $transcriptId, ) {} diff --git a/app/Services/Meet/MeetAiService.php b/app/Services/Meet/MeetAiService.php index 8250e5a..b0002eb 100644 --- a/app/Services/Meet/MeetAiService.php +++ b/app/Services/Meet/MeetAiService.php @@ -2,9 +2,9 @@ namespace App\Services\Meet; +use App\Models\ActionItem; use App\Models\AiSummary; use App\Models\Transcript; -use App\Models\ActionItem; use Illuminate\Support\Facades\Http; use Illuminate\Support\Str; use RuntimeException; @@ -19,15 +19,35 @@ class MeetAiService public function summarize(Transcript $transcript): AiSummary { - $summary = AiSummary::create([ - 'owner_ref' => $transcript->owner_ref, - 'session_id' => $transcript->session_id, - 'transcript_id' => $transcript->id, - 'status' => 'processing', - ]); + $transcript->loadMissing('session.room'); + + if ($transcript->session) { + app(TranscriptService::class)->refreshContent($transcript, $transcript->session); + $transcript->refresh(); + } + + $summary = AiSummary::query() + ->where('transcript_id', $transcript->id) + ->first(); + + if ($summary) { + $summary->actionItems()->delete(); + $summary->update([ + 'status' => 'processing', + 'summary' => '', + 'decisions' => [], + ]); + } else { + $summary = AiSummary::create([ + 'owner_ref' => $transcript->owner_ref, + 'session_id' => $transcript->session_id, + 'transcript_id' => $transcript->id, + 'status' => 'processing', + ]); + } try { - $result = $this->generateSummary((string) $transcript->content); + $result = $this->generateSummary($transcript); $summary->update([ 'status' => 'ready', 'summary' => $result['summary'] ?? '', @@ -57,21 +77,15 @@ class MeetAiService /** * @return array{summary: string, decisions: array, action_items: array} */ - protected function generateSummary(string $transcript): array + protected function generateSummary(Transcript $transcript): array { + $content = trim((string) $transcript->content); + if (! $this->enabled()) { - return $this->stubSummary($transcript); + return $this->stubSummary($content); } - $prompt = <<buildPrompt($transcript, $content); $reply = $this->hasLocalKey() ? $this->viaOpenAi($prompt) @@ -85,6 +99,55 @@ PROMPT; return $parsed; } + protected function buildPrompt(Transcript $transcript, string $content): string + { + $session = $transcript->session; + $room = $session?->room; + $title = $room?->title ?? 'Meeting'; + $duration = 'unknown'; + + if ($session?->started_at && $session?->ended_at) { + $minutes = max(1, $session->started_at->diffInMinutes($session->ended_at)); + $duration = $minutes.' minutes'; + } + + $participantCount = $session?->participants()->where('status', 'joined')->count() ?? 0; + $segments = $transcript->segments ?? []; + $chatLineCount = collect($segments)->where('source', 'chat')->count(); + $captionLineCount = collect($segments)->filter( + fn (array $segment) => ($segment['source'] ?? 'caption') === 'caption', + )->count(); + + $isSparse = $content === '' + || $content === 'No transcript content available.' + || strlen($content) < 120; + + $sourceNote = "The transcript combines in-meeting chat ({$chatLineCount} messages) " + ."and live speech captions ({$captionLineCount} lines). " + ."Caption lines are the primary record of spoken conversation — use them when chat is minimal.\n\n"; + + $sparseNote = $isSparse + ? "IMPORTANT: The transcript below is very limited and does not include a full audio transcription. " + ."If it does not contain enough detail to summarize the meeting accurately, say so clearly in the summary. " + ."Do not invent topics, decisions, or action items that are not supported by the transcript.\n\n" + : "Only summarize what is explicitly supported by the transcript below.\n\n"; + + return <<acceptJson()->timeout(60) + $res = Http::withToken((string) config('meet.ai.api_key'))->acceptJson()->timeout(120) ->post('https://api.openai.com/v1/chat/completions', [ 'model' => config('meet.ai.model', 'gpt-4o-mini'), 'temperature' => 0.2, 'response_format' => ['type' => 'json_object'], 'messages' => [ - ['role' => 'system', 'content' => 'You summarize Ladill Meet transcripts. Return valid JSON only.'], + [ + 'role' => 'system', + 'content' => 'You summarize Ladill Meet sessions from text transcripts. Return valid JSON only. Never fabricate content missing from the transcript.', + ], ['role' => 'user', 'content' => $prompt], ], ]); @@ -119,12 +185,12 @@ PROMPT; protected function viaPlatform(string $prompt): string { $base = rtrim((string) config('meet.ai.platform_api_url'), '/'); - $res = Http::withToken((string) config('meet.ai.platform_api_key'))->acceptJson()->timeout(60) + $res = Http::withToken((string) config('meet.ai.platform_api_key'))->acceptJson()->timeout(120) ->post($base.'/afia/chat', [ 'product' => 'meet', 'message' => $prompt, 'history' => [], - 'system_prompt' => 'You summarize Ladill Meet transcripts. Return valid JSON only.', + 'system_prompt' => 'You summarize Ladill Meet sessions from text transcripts. Return valid JSON only. Never fabricate content missing from the transcript.', ]); if ($res->failed()) { diff --git a/app/Services/Meet/RecordingService.php b/app/Services/Meet/RecordingService.php index ffcec84..2a4ff9b 100644 --- a/app/Services/Meet/RecordingService.php +++ b/app/Services/Meet/RecordingService.php @@ -175,7 +175,7 @@ class RecordingService app(MeetBillingService::class)->chargeRecordingStorageInitial($recording); if ($session->room->setting('auto_ai_summary', false) || config('meet.ai.auto_summarize', true)) { - app(TranscriptService::class)->ensureForSession($session, $recording); + app(TranscriptService::class)->finalizeAndSummarize($session, $recording); } return $recording->fresh(); @@ -231,7 +231,7 @@ class RecordingService app(MeetBillingService::class)->chargeRecordingStorageInitial($recording->fresh()); if ($session->room->setting('auto_ai_summary', false) || config('meet.ai.auto_summarize', true)) { - app(TranscriptService::class)->ensureForSession($session, $recording); + app(TranscriptService::class)->finalizeAndSummarize($session, $recording); } return $recording->fresh(); diff --git a/app/Services/Meet/TranscriptService.php b/app/Services/Meet/TranscriptService.php index 40b3488..36c8efc 100644 --- a/app/Services/Meet/TranscriptService.php +++ b/app/Services/Meet/TranscriptService.php @@ -2,29 +2,43 @@ namespace App\Services\Meet; +use App\Jobs\GenerateAiSummary; use App\Models\Recording; use App\Models\Session; use App\Models\Transcript; -use App\Jobs\GenerateAiSummary; class TranscriptService { - public function ensureForSession(Session $session, ?Recording $recording = null): Transcript + public function getOrCreateForSession(Session $session, ?Recording $recording = null): Transcript { $existing = $session->transcripts()->where('status', '!=', 'failed')->latest()->first(); if ($existing) { + if ($recording && ! $existing->recording_id) { + $existing->update(['recording_id' => $recording->id]); + } + return $existing; } - $transcript = Transcript::create([ + return Transcript::create([ 'owner_ref' => $session->owner_ref, 'session_id' => $session->id, 'recording_id' => $recording?->id, 'status' => 'processing', 'live_captions_enabled' => (bool) $session->room->setting('live_captions', false), ]); + } - $this->populateFromChat($transcript, $session); + /** @deprecated Use finalizeAndSummarize() when a recording is ready. */ + public function ensureForSession(Session $session, ?Recording $recording = null): Transcript + { + return $this->finalizeAndSummarize($session, $recording); + } + + public function finalizeAndSummarize(Session $session, ?Recording $recording = null): Transcript + { + $transcript = $this->getOrCreateForSession($session, $recording); + $this->refreshContent($transcript, $session); GenerateAiSummary::dispatch($transcript->id); @@ -38,9 +52,11 @@ class TranscriptService 'speaker' => $speaker, 'text' => trim($text), 'at' => now()->toIso8601String(), + 'source' => 'caption', ]; - $content = collect($segments)->map(fn ($s) => ($s['speaker'] ?? 'Speaker').': '.($s['text'] ?? ''))->implode("\n"); + $segments = $this->compactCaptionSegments($segments); + $content = $this->segmentsToContent($segments); $transcript->update([ 'segments' => $segments, @@ -51,21 +67,128 @@ class TranscriptService return $transcript; } - protected function populateFromChat(Transcript $transcript, Session $session): void + public function refreshContent(Transcript $transcript, Session $session): void { - $messages = $session->messages()->orderBy('created_at')->get(); - $segments = $messages->map(fn ($m) => [ - 'speaker' => $m->sender_name, - 'text' => $m->body, - 'at' => $m->created_at->toIso8601String(), - ])->all(); + $segments = $session->messages() + ->orderBy('created_at') + ->get() + ->map(fn ($message) => [ + 'speaker' => $message->sender_name, + 'text' => $message->body, + 'at' => $message->created_at->toIso8601String(), + 'source' => 'chat', + ]) + ->all(); - $content = $messages->map(fn ($m) => $m->sender_name.': '.$m->body)->implode("\n"); + foreach ($transcript->segments ?? [] as $segment) { + $source = $segment['source'] ?? null; + if ($source === 'chat') { + continue; + } + + $text = trim((string) ($segment['text'] ?? '')); + if ($text === '') { + continue; + } + + $segments[] = array_merge($segment, ['source' => 'caption']); + } + + $segments = $this->compactCaptionSegments($segments); + usort($segments, fn (array $a, array $b) => strcmp((string) ($a['at'] ?? ''), (string) ($b['at'] ?? ''))); + $content = $this->segmentsToContent($segments); $transcript->update([ 'segments' => $segments, - 'content' => $content ?: 'No transcript content available.', + 'content' => $content !== '' ? $content : 'No transcript content available.', 'status' => 'ready', ]); } + + /** + * @param array> $segments + * @return array> + */ + protected function compactCaptionSegments(array $segments): array + { + $compact = []; + + foreach ($segments as $segment) { + if (($segment['source'] ?? '') === 'chat') { + $compact[] = $segment; + + continue; + } + + $speaker = (string) ($segment['speaker'] ?? 'Speaker'); + $text = trim((string) ($segment['text'] ?? '')); + if ($text === '') { + continue; + } + + $lastCaptionIndex = $this->lastCaptionIndex($compact); + if ($lastCaptionIndex !== null && ($compact[$lastCaptionIndex]['speaker'] ?? '') === $speaker) { + $lastText = trim((string) ($compact[$lastCaptionIndex]['text'] ?? '')); + + if ($this->captionTextsEquivalent($lastText, $text)) { + continue; + } + + if ($lastText !== '' && $this->captionTextExtends($lastText, $text)) { + $compact[$lastCaptionIndex] = $segment; + + continue; + } + + if ($lastText !== '' && $this->captionTextExtends($text, $lastText)) { + continue; + } + } + + $compact[] = $segment; + } + + return $compact; + } + + /** + * @param array> $segments + */ + protected function lastCaptionIndex(array $segments): ?int + { + for ($index = count($segments) - 1; $index >= 0; $index -= 1) { + if (($segments[$index]['source'] ?? '') !== 'chat') { + return $index; + } + } + + return null; + } + + protected function captionTextsEquivalent(string $left, string $right): bool + { + return mb_strtolower(trim($left)) === mb_strtolower(trim($right)); + } + + protected function captionTextExtends(string $shorter, string $longer): bool + { + $shorter = mb_strtolower(trim($shorter)); + $longer = mb_strtolower(trim($longer)); + + if ($shorter === '' || $longer === '') { + return false; + } + + return str_starts_with($longer, $shorter); + } + + /** + * @param array> $segments + */ + protected function segmentsToContent(array $segments): string + { + return collect($segments) + ->map(fn ($segment) => ($segment['speaker'] ?? 'Speaker').': '.($segment['text'] ?? '')) + ->implode("\n"); + } } diff --git a/resources/js/meet-room.js b/resources/js/meet-room.js index 3ac4c34..59acdd7 100644 --- a/resources/js/meet-room.js +++ b/resources/js/meet-room.js @@ -370,8 +370,18 @@ function meetRoom() { }); room.on(RoomEvent.TrackUnsubscribed, (track, publication, participant) => { track.detach(); - if (track.kind === Track.Kind.Video && publication.source !== Track.Source.ScreenShare) { + + const isScreen = publication?.source === Track.Source.ScreenShare + || track.source === Track.Source.ScreenShare; + + if (isScreen) { + this.clearScreenShareOverlay(); + return; + } + + if (track.kind === Track.Kind.Video) { this.updateParticipantTile(participant); + this.reattachParticipantVideo(participant); } }); room.on(RoomEvent.TrackMuted, (publication, participant) => { @@ -408,7 +418,11 @@ function meetRoom() { }); room.on(RoomEvent.LocalTrackUnpublished, (publication, participant) => { if (publication.kind === Track.Kind.Video) { - this.updateParticipantTile(participant); + if (publication.source === Track.Source.ScreenShare) { + this.clearScreenShareOverlay(); + } else { + this.updateParticipantTile(participant); + } } this.syncLocalMediaState(); }); @@ -660,6 +674,15 @@ function meetRoom() { const local = room.localParticipant; this.isMuted = !local.isMicrophoneEnabled; this.isVideoOff = !local.isCameraEnabled; + this.isScreenSharing = Array.from(local.videoTrackPublications.values()) + .some((pub) => pub.source === Track.Source.ScreenShare && pub.track && !pub.isMuted); + + if (!this.isScreenSharing) { + const screen = document.getElementById('screen-share'); + if (screen && !screen.classList.contains('hidden')) { + this.clearScreenShareOverlay(); + } + } }, participantInitials(name) { @@ -1657,12 +1680,15 @@ function meetRoom() { if (hasVideo) { avatar?.classList.add('hidden'); - if (!stage?.querySelector('video')) { - const publication = Array.from(participant.videoTrackPublications.values()) - .find((pub) => pub.source !== Track.Source.ScreenShare && pub.track && !pub.isMuted); - if (publication?.track) { - this.attachVideoToTile(participant, publication.track); - } + const publication = Array.from(participant.videoTrackPublications.values()) + .find((pub) => pub.source !== Track.Source.ScreenShare && pub.track && !pub.isMuted); + const video = stage?.querySelector('video'); + const needsAttach = !video + || video.readyState === HTMLMediaElement.HAVE_NOTHING + || !video.srcObject; + + if (publication?.track && needsAttach) { + this.attachVideoToTile(participant, publication.track); } return; @@ -1707,27 +1733,76 @@ function meetRoom() { this.updateParticipantTile(participant); }, + publishCaption(text) { + const normalized = text.trim(); + if (!normalized || !this.config.captionUrl) { + return; + } + + if (normalized === this._lastCaptionSent) { + this.captionText = normalized; + return; + } + + this._lastCaptionSent = normalized; + this.captionText = normalized; + + fetch(this.config.captionUrl, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ text: normalized }), + }).catch(() => {}); + }, + setupCaptionPipeline() { if (!this.liveCaptions || !this.config.captionUrl) return; - if (room?.localParticipant?.isMicrophoneEnabled) return; const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; if (!SpeechRecognition) return; + if (this._captionRecognition) { + return; + } + const recognition = new SpeechRecognition(); recognition.continuous = true; recognition.interimResults = true; recognition.onresult = (event) => { - const text = event.results[event.results.length - 1][0].transcript.trim(); - if (!text) return; - this.captionText = text; - fetch(this.config.captionUrl, { - method: 'POST', - headers: this.headers(), - body: JSON.stringify({ text }), - }).catch(() => {}); + let interim = ''; + + for (let index = event.resultIndex; index < event.results.length; index += 1) { + const result = event.results[index]; + const text = result[0]?.transcript?.trim() ?? ''; + if (!text) { + continue; + } + + if (result.isFinal) { + this.publishCaption(text); + } else { + interim = text; + } + } + + if (interim) { + this.captionText = interim; + } }; recognition.onerror = () => {}; + recognition.onend = () => { + if (!this.liveCaptions || !room) { + return; + } + + try { + recognition.start(); + } catch (e) { + // browser may block without gesture + } + }; + + this._captionRecognition = recognition; + try { recognition.start(); } catch (e) { @@ -2297,6 +2372,34 @@ function meetRoom() { this.updateParticipantTile(participant); }, + clearScreenShareOverlay() { + const screen = document.getElementById('screen-share'); + if (!screen) { + return; + } + + screen.querySelectorAll('video').forEach((element) => { + element.srcObject = null; + }); + screen.classList.add('hidden'); + screen.replaceChildren(); + + if (room?.localParticipant) { + this.isScreenSharing = Array.from(room.localParticipant.videoTrackPublications.values()) + .some((pub) => pub.source === Track.Source.ScreenShare && pub.track && !pub.isMuted); + } else { + this.isScreenSharing = false; + } + + if (!room || this.audioOnly) { + return; + } + + const participants = [room.localParticipant, ...room.remoteParticipants.values()]; + participants.forEach((participant) => this.reattachParticipantVideo(participant)); + this.syncParticipants(); + }, + attachTrack(track, publication, participant) { if (track.kind === Track.Kind.Audio) { if (participant === room?.localParticipant) return; @@ -2334,6 +2437,15 @@ function meetRoom() { const el = track.attach(); el.className = 'h-full w-full rounded-xl object-contain'; container.appendChild(el); + + track.mediaStreamTrack?.addEventListener('ended', () => { + this.clearScreenShareOverlay(); + }, { once: true }); + + if (participant === room?.localParticipant) { + this.isScreenSharing = true; + } + return; } @@ -2484,11 +2596,7 @@ function meetRoom() { this.isScreenSharing = !this.isScreenSharing; await room.localParticipant.setScreenShareEnabled(this.isScreenSharing); if (!this.isScreenSharing) { - const screen = document.getElementById('screen-share'); - if (screen) { - screen.classList.add('hidden'); - screen.innerHTML = ''; - } + this.clearScreenShareOverlay(); } }, diff --git a/tests/Feature/MeetTranscriptServiceTest.php b/tests/Feature/MeetTranscriptServiceTest.php new file mode 100644 index 0000000..5b4aa74 --- /dev/null +++ b/tests/Feature/MeetTranscriptServiceTest.php @@ -0,0 +1,166 @@ +withoutMiddleware(EnsurePlatformSession::class); + + $this->user = User::create([ + 'public_id' => 'test-user-transcript', + 'name' => 'Test User', + 'email' => 'transcript@example.com', + ]); + + $this->organization = Organization::create([ + 'owner_ref' => $this->user->public_id, + 'name' => 'Test Org', + 'slug' => 'test-org-transcript', + 'timezone' => 'UTC', + 'settings' => ['onboarded' => true], + ]); + + Member::create([ + 'owner_ref' => $this->user->public_id, + 'organization_id' => $this->organization->id, + 'user_ref' => $this->user->public_id, + 'role' => 'owner', + ]); + } + + public function test_finalize_and_summarize_merges_chat_and_captions_before_dispatching_summary(): void + { + Queue::fake(); + + [$session] = $this->liveSession(); + + Message::create([ + 'owner_ref' => $this->user->public_id, + 'session_id' => $session->id, + 'sender_ref' => $this->user->public_id, + 'sender_name' => 'Host', + 'body' => 'Let us review the launch timeline.', + 'type' => 'public', + ]); + + $service = app(TranscriptService::class); + $transcript = $service->getOrCreateForSession($session); + $service->appendSegment($transcript, 'Host', 'We should ship the dashboard next week.'); + + $recording = Recording::create([ + 'owner_ref' => $this->user->public_id, + 'session_id' => $session->id, + 'status' => 'ready', + 'layout' => 'gallery', + 'started_by_ref' => $this->user->public_id, + 'started_at' => now()->subMinutes(30), + 'ended_at' => now(), + ]); + + $service->finalizeAndSummarize($session, $recording); + + $transcript->refresh(); + + $this->assertSame($recording->id, $transcript->recording_id); + $this->assertStringContainsString('Let us review the launch timeline.', $transcript->content); + $this->assertStringContainsString('We should ship the dashboard next week.', $transcript->content); + + Queue::assertPushed(GenerateAiSummary::class, function (GenerateAiSummary $job) use ($transcript) { + return $job->transcriptId === $transcript->id; + }); + } + + public function test_caption_append_does_not_queue_summary_job(): void + { + Queue::fake(); + + [$session] = $this->liveSession(); + $service = app(TranscriptService::class); + $transcript = $service->getOrCreateForSession($session); + $service->appendSegment($transcript, 'Host', 'Opening remarks.'); + + Queue::assertNothingPushed(); + } + + public function test_caption_segments_collapse_progressive_duplicates(): void + { + Queue::fake(); + + [$session] = $this->liveSession(); + $service = app(TranscriptService::class); + $transcript = $service->getOrCreateForSession($session); + + $service->appendSegment($transcript, 'Trypose', 'Yeah'); + $service->appendSegment($transcript, 'Trypose', 'Yeah I want to ask'); + $service->appendSegment($transcript, 'Trypose', 'Yeah I want to ask question please'); + $service->appendSegment($transcript, 'Trypose', 'Okay'); + $service->appendSegment($transcript, 'Trypose', 'Okay'); + + $transcript->refresh(); + + $this->assertSame([ + 'Trypose: Yeah I want to ask question please', + 'Trypose: Okay', + ], explode("\n", $transcript->content)); + } + + /** @return array{0: Session, 1: Participant} */ + protected function liveSession(): array + { + $room = Room::create([ + 'organization_id' => $this->organization->id, + 'owner_ref' => $this->user->public_id, + 'host_user_ref' => $this->user->public_id, + 'title' => 'Transcript meeting', + 'type' => 'instant', + 'status' => 'live', + 'timezone' => 'UTC', + 'settings' => ['live_captions' => true], + ]); + + $session = Session::create([ + 'room_id' => $room->id, + 'owner_ref' => $this->user->public_id, + 'media_room_name' => 'room-'.$room->uuid, + 'status' => 'ended', + 'started_at' => now()->subMinutes(30), + 'ended_at' => now(), + ]); + + $participant = Participant::create([ + 'owner_ref' => $this->user->public_id, + 'session_id' => $session->id, + 'user_ref' => $this->user->public_id, + 'role' => 'host', + 'display_name' => 'Host', + 'status' => 'joined', + 'joined_at' => now()->subMinutes(30), + ]); + + return [$session, $participant]; + } +}