import json import logging import asyncio import uuid from AssistantFnc import AssistantFnc from typing import Annotated from dotenv import load_dotenv from livekit.agents import ( AutoSubscribe, JobContext, JobProcess, WorkerOptions, cli, llm, metrics, ) from PIL import Image from livekit.agents.pipeline import VoicePipelineAgent from livekit.plugins import cartesia, openai, deepgram, silero, turn_detector, anthropic import io import base64 from livekit.agents.llm import ChatMessage, ChatImage import os from jira import JIRA from livekit.agents.utils.images import encode, EncodeOptions, ResizeOptions # Load environment variables load_dotenv('.env.local') logger = logging.getLogger("voice-agent") def prewarm(proc: JobProcess): proc.userdata["vad"] = silero.VAD.load() import json # When receiving the data from the client async def entrypoint(ctx: JobContext): metadata = json.loads(ctx.job.metadata) token = metadata.get("token") logger.info(f"Room metadata is {metadata}") initial_ctx = llm.ChatContext().append( role="system", text=( """You are an AI assistant designed to support healthcare providers. Your role is to assist with clinical decision-making, patient management, documentation, scheduling, and communication, using all available tools and resources. Prioritize patient safety, evidence-based practice, and confidentiality. Always act as a supportive, efficient, and knowledgeable assistant, but defer final decisions to the licensed healthcare provider. When performing any action that requires multiple pieces of information: 1. Ask for details one by one in a conversational manner rather than requesting all information at once 2. Confirm each piece of information before moving to the next question 3. Summarize all collected information before executing the final action 4. For forms or complex data entry, guide the user through each field step by step 5. If the user provides multiple pieces of information at once, acknowledge them and confirm before proceeding For example, when adding a practitioner: - First ask for the practitioner's name - Then ask for their email address - Then ask for additional details like gender, date of birth, etc. - Confirm all details before submitting This approach makes the interaction more natural and ensures all necessary information is collected accurately.""" ), ) _active_tasks = [] fnc_ctx = AssistantFnc(ctx=ctx) chunks = [] logger.info(f"connecting to room {ctx.room.name}") await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) participant = await ctx.wait_for_participant() logger.info(f"starting voice assistant for participant {participant.identity}") agent = VoicePipelineAgent( vad=ctx.proc.userdata["vad"], stt=deepgram.STT(), llm=openai.LLM(), tts=deepgram.TTS(), turn_detector=turn_detector.EOUModel(), # minimum delay for endpointing, used when turn detector believes the user is done with their turn min_endpointing_delay=0.5, # maximum delay for endpointing, used when turn detector does not believe the user is done with their turn max_endpointing_delay=10.0, max_nested_fnc_calls= 3, chat_ctx=initial_ctx, fnc_ctx=fnc_ctx, ) usage_collector = metrics.UsageCollector() @agent.on("metrics_collected") def on_metrics_collected(agent_metrics: metrics.AgentMetrics): metrics.log_metrics(agent_metrics) usage_collector.collect(agent_metrics) agent.start(ctx.room, participant) # The agent should be polite and greet the user when it joins :) await agent.say("Hey, I am Adi, how can I help you today?", allow_interruptions=True) if __name__ == "__main__": cli.run_app( WorkerOptions( entrypoint_fnc=entrypoint, prewarm_fnc=prewarm, agent_name='Provider_DashBoard_Assistant', ), )