Fix screen-share tile recovery and make AI summaries use full transcripts.
Deploy Ladill Meet / deploy (push) Successful in 1m17s
Deploy Ladill Meet / deploy (push) Successful in 1m17s
Defer summarization until recordings finish, merge chat with deduplicated live captions, and clear the screen-share overlay so participant cards return after sharing ends. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -617,8 +617,7 @@ class MeetingRoomController extends Controller
|
|||||||
return response()->json(['ok' => false], 422);
|
return response()->json(['ok' => false], 422);
|
||||||
}
|
}
|
||||||
|
|
||||||
$transcript = $session->transcripts()->latest()->first()
|
$transcript = $this->transcripts->getOrCreateForSession($session);
|
||||||
?? $this->transcripts->ensureForSession($session);
|
|
||||||
|
|
||||||
$this->transcripts->appendSegment($transcript, $participant->display_name, $validated['text']);
|
$this->transcripts->appendSegment($transcript, $participant->display_name, $validated['text']);
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ class GenerateAiSummary implements ShouldQueue
|
|||||||
{
|
{
|
||||||
use Queueable;
|
use Queueable;
|
||||||
|
|
||||||
|
public int $tries = 2;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
public int $transcriptId,
|
public int $transcriptId,
|
||||||
) {}
|
) {}
|
||||||
|
|||||||
@@ -2,9 +2,9 @@
|
|||||||
|
|
||||||
namespace App\Services\Meet;
|
namespace App\Services\Meet;
|
||||||
|
|
||||||
|
use App\Models\ActionItem;
|
||||||
use App\Models\AiSummary;
|
use App\Models\AiSummary;
|
||||||
use App\Models\Transcript;
|
use App\Models\Transcript;
|
||||||
use App\Models\ActionItem;
|
|
||||||
use Illuminate\Support\Facades\Http;
|
use Illuminate\Support\Facades\Http;
|
||||||
use Illuminate\Support\Str;
|
use Illuminate\Support\Str;
|
||||||
use RuntimeException;
|
use RuntimeException;
|
||||||
@@ -19,15 +19,35 @@ class MeetAiService
|
|||||||
|
|
||||||
public function summarize(Transcript $transcript): AiSummary
|
public function summarize(Transcript $transcript): AiSummary
|
||||||
{
|
{
|
||||||
$summary = AiSummary::create([
|
$transcript->loadMissing('session.room');
|
||||||
'owner_ref' => $transcript->owner_ref,
|
|
||||||
'session_id' => $transcript->session_id,
|
if ($transcript->session) {
|
||||||
'transcript_id' => $transcript->id,
|
app(TranscriptService::class)->refreshContent($transcript, $transcript->session);
|
||||||
'status' => 'processing',
|
$transcript->refresh();
|
||||||
]);
|
}
|
||||||
|
|
||||||
|
$summary = AiSummary::query()
|
||||||
|
->where('transcript_id', $transcript->id)
|
||||||
|
->first();
|
||||||
|
|
||||||
|
if ($summary) {
|
||||||
|
$summary->actionItems()->delete();
|
||||||
|
$summary->update([
|
||||||
|
'status' => 'processing',
|
||||||
|
'summary' => '',
|
||||||
|
'decisions' => [],
|
||||||
|
]);
|
||||||
|
} else {
|
||||||
|
$summary = AiSummary::create([
|
||||||
|
'owner_ref' => $transcript->owner_ref,
|
||||||
|
'session_id' => $transcript->session_id,
|
||||||
|
'transcript_id' => $transcript->id,
|
||||||
|
'status' => 'processing',
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$result = $this->generateSummary((string) $transcript->content);
|
$result = $this->generateSummary($transcript);
|
||||||
$summary->update([
|
$summary->update([
|
||||||
'status' => 'ready',
|
'status' => 'ready',
|
||||||
'summary' => $result['summary'] ?? '',
|
'summary' => $result['summary'] ?? '',
|
||||||
@@ -57,21 +77,15 @@ class MeetAiService
|
|||||||
/**
|
/**
|
||||||
* @return array{summary: string, decisions: array, action_items: array}
|
* @return array{summary: string, decisions: array, action_items: array}
|
||||||
*/
|
*/
|
||||||
protected function generateSummary(string $transcript): array
|
protected function generateSummary(Transcript $transcript): array
|
||||||
{
|
{
|
||||||
|
$content = trim((string) $transcript->content);
|
||||||
|
|
||||||
if (! $this->enabled()) {
|
if (! $this->enabled()) {
|
||||||
return $this->stubSummary($transcript);
|
return $this->stubSummary($content);
|
||||||
}
|
}
|
||||||
|
|
||||||
$prompt = <<<PROMPT
|
$prompt = $this->buildPrompt($transcript, $content);
|
||||||
Analyze this meeting transcript and respond in JSON with keys:
|
|
||||||
- summary (string, 2-4 paragraphs)
|
|
||||||
- decisions (array of strings)
|
|
||||||
- action_items (array of {assignee, description, due_date or null})
|
|
||||||
|
|
||||||
Transcript:
|
|
||||||
{$transcript}
|
|
||||||
PROMPT;
|
|
||||||
|
|
||||||
$reply = $this->hasLocalKey()
|
$reply = $this->hasLocalKey()
|
||||||
? $this->viaOpenAi($prompt)
|
? $this->viaOpenAi($prompt)
|
||||||
@@ -85,6 +99,55 @@ PROMPT;
|
|||||||
return $parsed;
|
return $parsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected function buildPrompt(Transcript $transcript, string $content): string
|
||||||
|
{
|
||||||
|
$session = $transcript->session;
|
||||||
|
$room = $session?->room;
|
||||||
|
$title = $room?->title ?? 'Meeting';
|
||||||
|
$duration = 'unknown';
|
||||||
|
|
||||||
|
if ($session?->started_at && $session?->ended_at) {
|
||||||
|
$minutes = max(1, $session->started_at->diffInMinutes($session->ended_at));
|
||||||
|
$duration = $minutes.' minutes';
|
||||||
|
}
|
||||||
|
|
||||||
|
$participantCount = $session?->participants()->where('status', 'joined')->count() ?? 0;
|
||||||
|
$segments = $transcript->segments ?? [];
|
||||||
|
$chatLineCount = collect($segments)->where('source', 'chat')->count();
|
||||||
|
$captionLineCount = collect($segments)->filter(
|
||||||
|
fn (array $segment) => ($segment['source'] ?? 'caption') === 'caption',
|
||||||
|
)->count();
|
||||||
|
|
||||||
|
$isSparse = $content === ''
|
||||||
|
|| $content === 'No transcript content available.'
|
||||||
|
|| strlen($content) < 120;
|
||||||
|
|
||||||
|
$sourceNote = "The transcript combines in-meeting chat ({$chatLineCount} messages) "
|
||||||
|
."and live speech captions ({$captionLineCount} lines). "
|
||||||
|
."Caption lines are the primary record of spoken conversation — use them when chat is minimal.\n\n";
|
||||||
|
|
||||||
|
$sparseNote = $isSparse
|
||||||
|
? "IMPORTANT: The transcript below is very limited and does not include a full audio transcription. "
|
||||||
|
."If it does not contain enough detail to summarize the meeting accurately, say so clearly in the summary. "
|
||||||
|
."Do not invent topics, decisions, or action items that are not supported by the transcript.\n\n"
|
||||||
|
: "Only summarize what is explicitly supported by the transcript below.\n\n";
|
||||||
|
|
||||||
|
return <<<PROMPT
|
||||||
|
{$sourceNote}{$sparseNote}Meeting context:
|
||||||
|
- Title: {$title}
|
||||||
|
- Duration: {$duration}
|
||||||
|
- Participants in call: {$participantCount}
|
||||||
|
|
||||||
|
Analyze this meeting transcript and respond in JSON with keys:
|
||||||
|
- summary (string, 2-4 paragraphs)
|
||||||
|
- decisions (array of strings)
|
||||||
|
- action_items (array of {assignee, description, due_date or null})
|
||||||
|
|
||||||
|
Transcript:
|
||||||
|
{$content}
|
||||||
|
PROMPT;
|
||||||
|
}
|
||||||
|
|
||||||
protected function stubSummary(string $transcript): array
|
protected function stubSummary(string $transcript): array
|
||||||
{
|
{
|
||||||
$preview = Str::limit($transcript, 500);
|
$preview = Str::limit($transcript, 500);
|
||||||
@@ -98,13 +161,16 @@ PROMPT;
|
|||||||
|
|
||||||
protected function viaOpenAi(string $prompt): string
|
protected function viaOpenAi(string $prompt): string
|
||||||
{
|
{
|
||||||
$res = Http::withToken((string) config('meet.ai.api_key'))->acceptJson()->timeout(60)
|
$res = Http::withToken((string) config('meet.ai.api_key'))->acceptJson()->timeout(120)
|
||||||
->post('https://api.openai.com/v1/chat/completions', [
|
->post('https://api.openai.com/v1/chat/completions', [
|
||||||
'model' => config('meet.ai.model', 'gpt-4o-mini'),
|
'model' => config('meet.ai.model', 'gpt-4o-mini'),
|
||||||
'temperature' => 0.2,
|
'temperature' => 0.2,
|
||||||
'response_format' => ['type' => 'json_object'],
|
'response_format' => ['type' => 'json_object'],
|
||||||
'messages' => [
|
'messages' => [
|
||||||
['role' => 'system', 'content' => 'You summarize Ladill Meet transcripts. Return valid JSON only.'],
|
[
|
||||||
|
'role' => 'system',
|
||||||
|
'content' => 'You summarize Ladill Meet sessions from text transcripts. Return valid JSON only. Never fabricate content missing from the transcript.',
|
||||||
|
],
|
||||||
['role' => 'user', 'content' => $prompt],
|
['role' => 'user', 'content' => $prompt],
|
||||||
],
|
],
|
||||||
]);
|
]);
|
||||||
@@ -119,12 +185,12 @@ PROMPT;
|
|||||||
protected function viaPlatform(string $prompt): string
|
protected function viaPlatform(string $prompt): string
|
||||||
{
|
{
|
||||||
$base = rtrim((string) config('meet.ai.platform_api_url'), '/');
|
$base = rtrim((string) config('meet.ai.platform_api_url'), '/');
|
||||||
$res = Http::withToken((string) config('meet.ai.platform_api_key'))->acceptJson()->timeout(60)
|
$res = Http::withToken((string) config('meet.ai.platform_api_key'))->acceptJson()->timeout(120)
|
||||||
->post($base.'/afia/chat', [
|
->post($base.'/afia/chat', [
|
||||||
'product' => 'meet',
|
'product' => 'meet',
|
||||||
'message' => $prompt,
|
'message' => $prompt,
|
||||||
'history' => [],
|
'history' => [],
|
||||||
'system_prompt' => 'You summarize Ladill Meet transcripts. Return valid JSON only.',
|
'system_prompt' => 'You summarize Ladill Meet sessions from text transcripts. Return valid JSON only. Never fabricate content missing from the transcript.',
|
||||||
]);
|
]);
|
||||||
|
|
||||||
if ($res->failed()) {
|
if ($res->failed()) {
|
||||||
|
|||||||
@@ -175,7 +175,7 @@ class RecordingService
|
|||||||
app(MeetBillingService::class)->chargeRecordingStorageInitial($recording);
|
app(MeetBillingService::class)->chargeRecordingStorageInitial($recording);
|
||||||
|
|
||||||
if ($session->room->setting('auto_ai_summary', false) || config('meet.ai.auto_summarize', true)) {
|
if ($session->room->setting('auto_ai_summary', false) || config('meet.ai.auto_summarize', true)) {
|
||||||
app(TranscriptService::class)->ensureForSession($session, $recording);
|
app(TranscriptService::class)->finalizeAndSummarize($session, $recording);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $recording->fresh();
|
return $recording->fresh();
|
||||||
@@ -231,7 +231,7 @@ class RecordingService
|
|||||||
app(MeetBillingService::class)->chargeRecordingStorageInitial($recording->fresh());
|
app(MeetBillingService::class)->chargeRecordingStorageInitial($recording->fresh());
|
||||||
|
|
||||||
if ($session->room->setting('auto_ai_summary', false) || config('meet.ai.auto_summarize', true)) {
|
if ($session->room->setting('auto_ai_summary', false) || config('meet.ai.auto_summarize', true)) {
|
||||||
app(TranscriptService::class)->ensureForSession($session, $recording);
|
app(TranscriptService::class)->finalizeAndSummarize($session, $recording);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $recording->fresh();
|
return $recording->fresh();
|
||||||
|
|||||||
@@ -2,29 +2,43 @@
|
|||||||
|
|
||||||
namespace App\Services\Meet;
|
namespace App\Services\Meet;
|
||||||
|
|
||||||
|
use App\Jobs\GenerateAiSummary;
|
||||||
use App\Models\Recording;
|
use App\Models\Recording;
|
||||||
use App\Models\Session;
|
use App\Models\Session;
|
||||||
use App\Models\Transcript;
|
use App\Models\Transcript;
|
||||||
use App\Jobs\GenerateAiSummary;
|
|
||||||
|
|
||||||
class TranscriptService
|
class TranscriptService
|
||||||
{
|
{
|
||||||
public function ensureForSession(Session $session, ?Recording $recording = null): Transcript
|
public function getOrCreateForSession(Session $session, ?Recording $recording = null): Transcript
|
||||||
{
|
{
|
||||||
$existing = $session->transcripts()->where('status', '!=', 'failed')->latest()->first();
|
$existing = $session->transcripts()->where('status', '!=', 'failed')->latest()->first();
|
||||||
if ($existing) {
|
if ($existing) {
|
||||||
|
if ($recording && ! $existing->recording_id) {
|
||||||
|
$existing->update(['recording_id' => $recording->id]);
|
||||||
|
}
|
||||||
|
|
||||||
return $existing;
|
return $existing;
|
||||||
}
|
}
|
||||||
|
|
||||||
$transcript = Transcript::create([
|
return Transcript::create([
|
||||||
'owner_ref' => $session->owner_ref,
|
'owner_ref' => $session->owner_ref,
|
||||||
'session_id' => $session->id,
|
'session_id' => $session->id,
|
||||||
'recording_id' => $recording?->id,
|
'recording_id' => $recording?->id,
|
||||||
'status' => 'processing',
|
'status' => 'processing',
|
||||||
'live_captions_enabled' => (bool) $session->room->setting('live_captions', false),
|
'live_captions_enabled' => (bool) $session->room->setting('live_captions', false),
|
||||||
]);
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
$this->populateFromChat($transcript, $session);
|
/** @deprecated Use finalizeAndSummarize() when a recording is ready. */
|
||||||
|
public function ensureForSession(Session $session, ?Recording $recording = null): Transcript
|
||||||
|
{
|
||||||
|
return $this->finalizeAndSummarize($session, $recording);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function finalizeAndSummarize(Session $session, ?Recording $recording = null): Transcript
|
||||||
|
{
|
||||||
|
$transcript = $this->getOrCreateForSession($session, $recording);
|
||||||
|
$this->refreshContent($transcript, $session);
|
||||||
|
|
||||||
GenerateAiSummary::dispatch($transcript->id);
|
GenerateAiSummary::dispatch($transcript->id);
|
||||||
|
|
||||||
@@ -38,9 +52,11 @@ class TranscriptService
|
|||||||
'speaker' => $speaker,
|
'speaker' => $speaker,
|
||||||
'text' => trim($text),
|
'text' => trim($text),
|
||||||
'at' => now()->toIso8601String(),
|
'at' => now()->toIso8601String(),
|
||||||
|
'source' => 'caption',
|
||||||
];
|
];
|
||||||
|
|
||||||
$content = collect($segments)->map(fn ($s) => ($s['speaker'] ?? 'Speaker').': '.($s['text'] ?? ''))->implode("\n");
|
$segments = $this->compactCaptionSegments($segments);
|
||||||
|
$content = $this->segmentsToContent($segments);
|
||||||
|
|
||||||
$transcript->update([
|
$transcript->update([
|
||||||
'segments' => $segments,
|
'segments' => $segments,
|
||||||
@@ -51,21 +67,128 @@ class TranscriptService
|
|||||||
return $transcript;
|
return $transcript;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function populateFromChat(Transcript $transcript, Session $session): void
|
public function refreshContent(Transcript $transcript, Session $session): void
|
||||||
{
|
{
|
||||||
$messages = $session->messages()->orderBy('created_at')->get();
|
$segments = $session->messages()
|
||||||
$segments = $messages->map(fn ($m) => [
|
->orderBy('created_at')
|
||||||
'speaker' => $m->sender_name,
|
->get()
|
||||||
'text' => $m->body,
|
->map(fn ($message) => [
|
||||||
'at' => $m->created_at->toIso8601String(),
|
'speaker' => $message->sender_name,
|
||||||
])->all();
|
'text' => $message->body,
|
||||||
|
'at' => $message->created_at->toIso8601String(),
|
||||||
|
'source' => 'chat',
|
||||||
|
])
|
||||||
|
->all();
|
||||||
|
|
||||||
$content = $messages->map(fn ($m) => $m->sender_name.': '.$m->body)->implode("\n");
|
foreach ($transcript->segments ?? [] as $segment) {
|
||||||
|
$source = $segment['source'] ?? null;
|
||||||
|
if ($source === 'chat') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$text = trim((string) ($segment['text'] ?? ''));
|
||||||
|
if ($text === '') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$segments[] = array_merge($segment, ['source' => 'caption']);
|
||||||
|
}
|
||||||
|
|
||||||
|
$segments = $this->compactCaptionSegments($segments);
|
||||||
|
usort($segments, fn (array $a, array $b) => strcmp((string) ($a['at'] ?? ''), (string) ($b['at'] ?? '')));
|
||||||
|
$content = $this->segmentsToContent($segments);
|
||||||
|
|
||||||
$transcript->update([
|
$transcript->update([
|
||||||
'segments' => $segments,
|
'segments' => $segments,
|
||||||
'content' => $content ?: 'No transcript content available.',
|
'content' => $content !== '' ? $content : 'No transcript content available.',
|
||||||
'status' => 'ready',
|
'status' => 'ready',
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param array<int, array<string, mixed>> $segments
|
||||||
|
* @return array<int, array<string, mixed>>
|
||||||
|
*/
|
||||||
|
protected function compactCaptionSegments(array $segments): array
|
||||||
|
{
|
||||||
|
$compact = [];
|
||||||
|
|
||||||
|
foreach ($segments as $segment) {
|
||||||
|
if (($segment['source'] ?? '') === 'chat') {
|
||||||
|
$compact[] = $segment;
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$speaker = (string) ($segment['speaker'] ?? 'Speaker');
|
||||||
|
$text = trim((string) ($segment['text'] ?? ''));
|
||||||
|
if ($text === '') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$lastCaptionIndex = $this->lastCaptionIndex($compact);
|
||||||
|
if ($lastCaptionIndex !== null && ($compact[$lastCaptionIndex]['speaker'] ?? '') === $speaker) {
|
||||||
|
$lastText = trim((string) ($compact[$lastCaptionIndex]['text'] ?? ''));
|
||||||
|
|
||||||
|
if ($this->captionTextsEquivalent($lastText, $text)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($lastText !== '' && $this->captionTextExtends($lastText, $text)) {
|
||||||
|
$compact[$lastCaptionIndex] = $segment;
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($lastText !== '' && $this->captionTextExtends($text, $lastText)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$compact[] = $segment;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $compact;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param array<int, array<string, mixed>> $segments
|
||||||
|
*/
|
||||||
|
protected function lastCaptionIndex(array $segments): ?int
|
||||||
|
{
|
||||||
|
for ($index = count($segments) - 1; $index >= 0; $index -= 1) {
|
||||||
|
if (($segments[$index]['source'] ?? '') !== 'chat') {
|
||||||
|
return $index;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function captionTextsEquivalent(string $left, string $right): bool
|
||||||
|
{
|
||||||
|
return mb_strtolower(trim($left)) === mb_strtolower(trim($right));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function captionTextExtends(string $shorter, string $longer): bool
|
||||||
|
{
|
||||||
|
$shorter = mb_strtolower(trim($shorter));
|
||||||
|
$longer = mb_strtolower(trim($longer));
|
||||||
|
|
||||||
|
if ($shorter === '' || $longer === '') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return str_starts_with($longer, $shorter);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param array<int, array<string, mixed>> $segments
|
||||||
|
*/
|
||||||
|
protected function segmentsToContent(array $segments): string
|
||||||
|
{
|
||||||
|
return collect($segments)
|
||||||
|
->map(fn ($segment) => ($segment['speaker'] ?? 'Speaker').': '.($segment['text'] ?? ''))
|
||||||
|
->implode("\n");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+130
-22
@@ -370,8 +370,18 @@ function meetRoom() {
|
|||||||
});
|
});
|
||||||
room.on(RoomEvent.TrackUnsubscribed, (track, publication, participant) => {
|
room.on(RoomEvent.TrackUnsubscribed, (track, publication, participant) => {
|
||||||
track.detach();
|
track.detach();
|
||||||
if (track.kind === Track.Kind.Video && publication.source !== Track.Source.ScreenShare) {
|
|
||||||
|
const isScreen = publication?.source === Track.Source.ScreenShare
|
||||||
|
|| track.source === Track.Source.ScreenShare;
|
||||||
|
|
||||||
|
if (isScreen) {
|
||||||
|
this.clearScreenShareOverlay();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (track.kind === Track.Kind.Video) {
|
||||||
this.updateParticipantTile(participant);
|
this.updateParticipantTile(participant);
|
||||||
|
this.reattachParticipantVideo(participant);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
room.on(RoomEvent.TrackMuted, (publication, participant) => {
|
room.on(RoomEvent.TrackMuted, (publication, participant) => {
|
||||||
@@ -408,7 +418,11 @@ function meetRoom() {
|
|||||||
});
|
});
|
||||||
room.on(RoomEvent.LocalTrackUnpublished, (publication, participant) => {
|
room.on(RoomEvent.LocalTrackUnpublished, (publication, participant) => {
|
||||||
if (publication.kind === Track.Kind.Video) {
|
if (publication.kind === Track.Kind.Video) {
|
||||||
this.updateParticipantTile(participant);
|
if (publication.source === Track.Source.ScreenShare) {
|
||||||
|
this.clearScreenShareOverlay();
|
||||||
|
} else {
|
||||||
|
this.updateParticipantTile(participant);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
this.syncLocalMediaState();
|
this.syncLocalMediaState();
|
||||||
});
|
});
|
||||||
@@ -660,6 +674,15 @@ function meetRoom() {
|
|||||||
const local = room.localParticipant;
|
const local = room.localParticipant;
|
||||||
this.isMuted = !local.isMicrophoneEnabled;
|
this.isMuted = !local.isMicrophoneEnabled;
|
||||||
this.isVideoOff = !local.isCameraEnabled;
|
this.isVideoOff = !local.isCameraEnabled;
|
||||||
|
this.isScreenSharing = Array.from(local.videoTrackPublications.values())
|
||||||
|
.some((pub) => pub.source === Track.Source.ScreenShare && pub.track && !pub.isMuted);
|
||||||
|
|
||||||
|
if (!this.isScreenSharing) {
|
||||||
|
const screen = document.getElementById('screen-share');
|
||||||
|
if (screen && !screen.classList.contains('hidden')) {
|
||||||
|
this.clearScreenShareOverlay();
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
participantInitials(name) {
|
participantInitials(name) {
|
||||||
@@ -1657,12 +1680,15 @@ function meetRoom() {
|
|||||||
|
|
||||||
if (hasVideo) {
|
if (hasVideo) {
|
||||||
avatar?.classList.add('hidden');
|
avatar?.classList.add('hidden');
|
||||||
if (!stage?.querySelector('video')) {
|
const publication = Array.from(participant.videoTrackPublications.values())
|
||||||
const publication = Array.from(participant.videoTrackPublications.values())
|
.find((pub) => pub.source !== Track.Source.ScreenShare && pub.track && !pub.isMuted);
|
||||||
.find((pub) => pub.source !== Track.Source.ScreenShare && pub.track && !pub.isMuted);
|
const video = stage?.querySelector('video');
|
||||||
if (publication?.track) {
|
const needsAttach = !video
|
||||||
this.attachVideoToTile(participant, publication.track);
|
|| video.readyState === HTMLMediaElement.HAVE_NOTHING
|
||||||
}
|
|| !video.srcObject;
|
||||||
|
|
||||||
|
if (publication?.track && needsAttach) {
|
||||||
|
this.attachVideoToTile(participant, publication.track);
|
||||||
}
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
@@ -1707,27 +1733,76 @@ function meetRoom() {
|
|||||||
this.updateParticipantTile(participant);
|
this.updateParticipantTile(participant);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
publishCaption(text) {
|
||||||
|
const normalized = text.trim();
|
||||||
|
if (!normalized || !this.config.captionUrl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalized === this._lastCaptionSent) {
|
||||||
|
this.captionText = normalized;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this._lastCaptionSent = normalized;
|
||||||
|
this.captionText = normalized;
|
||||||
|
|
||||||
|
fetch(this.config.captionUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: this.headers(),
|
||||||
|
body: JSON.stringify({ text: normalized }),
|
||||||
|
}).catch(() => {});
|
||||||
|
},
|
||||||
|
|
||||||
setupCaptionPipeline() {
|
setupCaptionPipeline() {
|
||||||
if (!this.liveCaptions || !this.config.captionUrl) return;
|
if (!this.liveCaptions || !this.config.captionUrl) return;
|
||||||
if (room?.localParticipant?.isMicrophoneEnabled) return;
|
|
||||||
|
|
||||||
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||||
if (!SpeechRecognition) return;
|
if (!SpeechRecognition) return;
|
||||||
|
|
||||||
|
if (this._captionRecognition) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const recognition = new SpeechRecognition();
|
const recognition = new SpeechRecognition();
|
||||||
recognition.continuous = true;
|
recognition.continuous = true;
|
||||||
recognition.interimResults = true;
|
recognition.interimResults = true;
|
||||||
recognition.onresult = (event) => {
|
recognition.onresult = (event) => {
|
||||||
const text = event.results[event.results.length - 1][0].transcript.trim();
|
let interim = '';
|
||||||
if (!text) return;
|
|
||||||
this.captionText = text;
|
for (let index = event.resultIndex; index < event.results.length; index += 1) {
|
||||||
fetch(this.config.captionUrl, {
|
const result = event.results[index];
|
||||||
method: 'POST',
|
const text = result[0]?.transcript?.trim() ?? '';
|
||||||
headers: this.headers(),
|
if (!text) {
|
||||||
body: JSON.stringify({ text }),
|
continue;
|
||||||
}).catch(() => {});
|
}
|
||||||
|
|
||||||
|
if (result.isFinal) {
|
||||||
|
this.publishCaption(text);
|
||||||
|
} else {
|
||||||
|
interim = text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (interim) {
|
||||||
|
this.captionText = interim;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
recognition.onerror = () => {};
|
recognition.onerror = () => {};
|
||||||
|
recognition.onend = () => {
|
||||||
|
if (!this.liveCaptions || !room) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
recognition.start();
|
||||||
|
} catch (e) {
|
||||||
|
// browser may block without gesture
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
this._captionRecognition = recognition;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
recognition.start();
|
recognition.start();
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -2297,6 +2372,34 @@ function meetRoom() {
|
|||||||
this.updateParticipantTile(participant);
|
this.updateParticipantTile(participant);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
clearScreenShareOverlay() {
|
||||||
|
const screen = document.getElementById('screen-share');
|
||||||
|
if (!screen) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
screen.querySelectorAll('video').forEach((element) => {
|
||||||
|
element.srcObject = null;
|
||||||
|
});
|
||||||
|
screen.classList.add('hidden');
|
||||||
|
screen.replaceChildren();
|
||||||
|
|
||||||
|
if (room?.localParticipant) {
|
||||||
|
this.isScreenSharing = Array.from(room.localParticipant.videoTrackPublications.values())
|
||||||
|
.some((pub) => pub.source === Track.Source.ScreenShare && pub.track && !pub.isMuted);
|
||||||
|
} else {
|
||||||
|
this.isScreenSharing = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!room || this.audioOnly) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const participants = [room.localParticipant, ...room.remoteParticipants.values()];
|
||||||
|
participants.forEach((participant) => this.reattachParticipantVideo(participant));
|
||||||
|
this.syncParticipants();
|
||||||
|
},
|
||||||
|
|
||||||
attachTrack(track, publication, participant) {
|
attachTrack(track, publication, participant) {
|
||||||
if (track.kind === Track.Kind.Audio) {
|
if (track.kind === Track.Kind.Audio) {
|
||||||
if (participant === room?.localParticipant) return;
|
if (participant === room?.localParticipant) return;
|
||||||
@@ -2334,6 +2437,15 @@ function meetRoom() {
|
|||||||
const el = track.attach();
|
const el = track.attach();
|
||||||
el.className = 'h-full w-full rounded-xl object-contain';
|
el.className = 'h-full w-full rounded-xl object-contain';
|
||||||
container.appendChild(el);
|
container.appendChild(el);
|
||||||
|
|
||||||
|
track.mediaStreamTrack?.addEventListener('ended', () => {
|
||||||
|
this.clearScreenShareOverlay();
|
||||||
|
}, { once: true });
|
||||||
|
|
||||||
|
if (participant === room?.localParticipant) {
|
||||||
|
this.isScreenSharing = true;
|
||||||
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2484,11 +2596,7 @@ function meetRoom() {
|
|||||||
this.isScreenSharing = !this.isScreenSharing;
|
this.isScreenSharing = !this.isScreenSharing;
|
||||||
await room.localParticipant.setScreenShareEnabled(this.isScreenSharing);
|
await room.localParticipant.setScreenShareEnabled(this.isScreenSharing);
|
||||||
if (!this.isScreenSharing) {
|
if (!this.isScreenSharing) {
|
||||||
const screen = document.getElementById('screen-share');
|
this.clearScreenShareOverlay();
|
||||||
if (screen) {
|
|
||||||
screen.classList.add('hidden');
|
|
||||||
screen.innerHTML = '';
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,166 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Tests\Feature;
|
||||||
|
|
||||||
|
use App\Http\Middleware\EnsurePlatformSession;
|
||||||
|
use App\Jobs\GenerateAiSummary;
|
||||||
|
use App\Models\Member;
|
||||||
|
use App\Models\Message;
|
||||||
|
use App\Models\Organization;
|
||||||
|
use App\Models\Participant;
|
||||||
|
use App\Models\Recording;
|
||||||
|
use App\Models\Room;
|
||||||
|
use App\Models\Session;
|
||||||
|
use App\Models\User;
|
||||||
|
use App\Services\Meet\TranscriptService;
|
||||||
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||||
|
use Illuminate\Support\Facades\Queue;
|
||||||
|
use Tests\TestCase;
|
||||||
|
|
||||||
|
class MeetTranscriptServiceTest extends TestCase
|
||||||
|
{
|
||||||
|
use RefreshDatabase;
|
||||||
|
|
||||||
|
protected User $user;
|
||||||
|
|
||||||
|
protected Organization $organization;
|
||||||
|
|
||||||
|
protected function setUp(): void
|
||||||
|
{
|
||||||
|
parent::setUp();
|
||||||
|
$this->withoutMiddleware(EnsurePlatformSession::class);
|
||||||
|
|
||||||
|
$this->user = User::create([
|
||||||
|
'public_id' => 'test-user-transcript',
|
||||||
|
'name' => 'Test User',
|
||||||
|
'email' => 'transcript@example.com',
|
||||||
|
]);
|
||||||
|
|
||||||
|
$this->organization = Organization::create([
|
||||||
|
'owner_ref' => $this->user->public_id,
|
||||||
|
'name' => 'Test Org',
|
||||||
|
'slug' => 'test-org-transcript',
|
||||||
|
'timezone' => 'UTC',
|
||||||
|
'settings' => ['onboarded' => true],
|
||||||
|
]);
|
||||||
|
|
||||||
|
Member::create([
|
||||||
|
'owner_ref' => $this->user->public_id,
|
||||||
|
'organization_id' => $this->organization->id,
|
||||||
|
'user_ref' => $this->user->public_id,
|
||||||
|
'role' => 'owner',
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function test_finalize_and_summarize_merges_chat_and_captions_before_dispatching_summary(): void
|
||||||
|
{
|
||||||
|
Queue::fake();
|
||||||
|
|
||||||
|
[$session] = $this->liveSession();
|
||||||
|
|
||||||
|
Message::create([
|
||||||
|
'owner_ref' => $this->user->public_id,
|
||||||
|
'session_id' => $session->id,
|
||||||
|
'sender_ref' => $this->user->public_id,
|
||||||
|
'sender_name' => 'Host',
|
||||||
|
'body' => 'Let us review the launch timeline.',
|
||||||
|
'type' => 'public',
|
||||||
|
]);
|
||||||
|
|
||||||
|
$service = app(TranscriptService::class);
|
||||||
|
$transcript = $service->getOrCreateForSession($session);
|
||||||
|
$service->appendSegment($transcript, 'Host', 'We should ship the dashboard next week.');
|
||||||
|
|
||||||
|
$recording = Recording::create([
|
||||||
|
'owner_ref' => $this->user->public_id,
|
||||||
|
'session_id' => $session->id,
|
||||||
|
'status' => 'ready',
|
||||||
|
'layout' => 'gallery',
|
||||||
|
'started_by_ref' => $this->user->public_id,
|
||||||
|
'started_at' => now()->subMinutes(30),
|
||||||
|
'ended_at' => now(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$service->finalizeAndSummarize($session, $recording);
|
||||||
|
|
||||||
|
$transcript->refresh();
|
||||||
|
|
||||||
|
$this->assertSame($recording->id, $transcript->recording_id);
|
||||||
|
$this->assertStringContainsString('Let us review the launch timeline.', $transcript->content);
|
||||||
|
$this->assertStringContainsString('We should ship the dashboard next week.', $transcript->content);
|
||||||
|
|
||||||
|
Queue::assertPushed(GenerateAiSummary::class, function (GenerateAiSummary $job) use ($transcript) {
|
||||||
|
return $job->transcriptId === $transcript->id;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public function test_caption_append_does_not_queue_summary_job(): void
|
||||||
|
{
|
||||||
|
Queue::fake();
|
||||||
|
|
||||||
|
[$session] = $this->liveSession();
|
||||||
|
$service = app(TranscriptService::class);
|
||||||
|
$transcript = $service->getOrCreateForSession($session);
|
||||||
|
$service->appendSegment($transcript, 'Host', 'Opening remarks.');
|
||||||
|
|
||||||
|
Queue::assertNothingPushed();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function test_caption_segments_collapse_progressive_duplicates(): void
|
||||||
|
{
|
||||||
|
Queue::fake();
|
||||||
|
|
||||||
|
[$session] = $this->liveSession();
|
||||||
|
$service = app(TranscriptService::class);
|
||||||
|
$transcript = $service->getOrCreateForSession($session);
|
||||||
|
|
||||||
|
$service->appendSegment($transcript, 'Trypose', 'Yeah');
|
||||||
|
$service->appendSegment($transcript, 'Trypose', 'Yeah I want to ask');
|
||||||
|
$service->appendSegment($transcript, 'Trypose', 'Yeah I want to ask question please');
|
||||||
|
$service->appendSegment($transcript, 'Trypose', 'Okay');
|
||||||
|
$service->appendSegment($transcript, 'Trypose', 'Okay');
|
||||||
|
|
||||||
|
$transcript->refresh();
|
||||||
|
|
||||||
|
$this->assertSame([
|
||||||
|
'Trypose: Yeah I want to ask question please',
|
||||||
|
'Trypose: Okay',
|
||||||
|
], explode("\n", $transcript->content));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array{0: Session, 1: Participant} */
|
||||||
|
protected function liveSession(): array
|
||||||
|
{
|
||||||
|
$room = Room::create([
|
||||||
|
'organization_id' => $this->organization->id,
|
||||||
|
'owner_ref' => $this->user->public_id,
|
||||||
|
'host_user_ref' => $this->user->public_id,
|
||||||
|
'title' => 'Transcript meeting',
|
||||||
|
'type' => 'instant',
|
||||||
|
'status' => 'live',
|
||||||
|
'timezone' => 'UTC',
|
||||||
|
'settings' => ['live_captions' => true],
|
||||||
|
]);
|
||||||
|
|
||||||
|
$session = Session::create([
|
||||||
|
'room_id' => $room->id,
|
||||||
|
'owner_ref' => $this->user->public_id,
|
||||||
|
'media_room_name' => 'room-'.$room->uuid,
|
||||||
|
'status' => 'ended',
|
||||||
|
'started_at' => now()->subMinutes(30),
|
||||||
|
'ended_at' => now(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$participant = Participant::create([
|
||||||
|
'owner_ref' => $this->user->public_id,
|
||||||
|
'session_id' => $session->id,
|
||||||
|
'user_ref' => $this->user->public_id,
|
||||||
|
'role' => 'host',
|
||||||
|
'display_name' => 'Host',
|
||||||
|
'status' => 'joined',
|
||||||
|
'joined_at' => now()->subMinutes(30),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return [$session, $participant];
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user