124 lines
4.0 KiB
Python
124 lines
4.0 KiB
Python
import json
|
|
import logging
|
|
import asyncio
|
|
import uuid
|
|
from AssistantFnc import AssistantFnc
|
|
from typing import Annotated
|
|
from dotenv import load_dotenv
|
|
from livekit.agents import (
|
|
AutoSubscribe,
|
|
JobContext,
|
|
JobProcess,
|
|
WorkerOptions,
|
|
cli,
|
|
llm,
|
|
metrics,
|
|
)
|
|
from PIL import Image
|
|
from livekit.agents.pipeline import VoicePipelineAgent
|
|
from livekit.plugins import cartesia, openai, deepgram, silero, turn_detector, anthropic
|
|
import io
|
|
import base64
|
|
from livekit.agents.llm import ChatMessage, ChatImage
|
|
import os
|
|
from jira import JIRA
|
|
from livekit.agents.utils.images import encode, EncodeOptions, ResizeOptions
|
|
|
|
# Load environment variables
|
|
load_dotenv('.env.local')
|
|
logger = logging.getLogger("voice-agent")
|
|
|
|
|
|
def prewarm(proc: JobProcess):
|
|
proc.userdata["vad"] = silero.VAD.load()
|
|
|
|
import json
|
|
|
|
# When receiving the data from the client
|
|
|
|
async def entrypoint(ctx: JobContext):
|
|
|
|
metadata = json.loads(ctx.job.metadata)
|
|
token = metadata.get("token")
|
|
logger.info(f"Room metadata is {metadata}")
|
|
|
|
initial_ctx = llm.ChatContext().append(
|
|
role="system",
|
|
text=(
|
|
"""You are an AI assistant designed to support healthcare providers. Your role is to assist with clinical decision-making, patient management, documentation, scheduling, and communication, using all available tools and resources. Prioritize patient safety, evidence-based practice, and confidentiality. Always act as a supportive, efficient, and knowledgeable assistant, but defer final decisions to the licensed healthcare provider.
|
|
|
|
When performing any action that requires multiple pieces of information:
|
|
1. Ask for details one by one in a conversational manner rather than requesting all information at once
|
|
2. Confirm each piece of information before moving to the next question
|
|
3. Summarize all collected information before executing the final action
|
|
4. For forms or complex data entry, guide the user through each field step by step
|
|
5. If the user provides multiple pieces of information at once, acknowledge them and confirm before proceeding
|
|
|
|
For example, when adding a practitioner:
|
|
- First ask for the practitioner's name
|
|
- Then ask for their email address
|
|
- Then ask for additional details like gender, date of birth, etc.
|
|
- Confirm all details before submitting
|
|
|
|
This approach makes the interaction more natural and ensures all necessary information is collected accurately."""
|
|
),
|
|
)
|
|
_active_tasks = []
|
|
|
|
|
|
|
|
fnc_ctx = AssistantFnc(ctx=ctx)
|
|
chunks = []
|
|
|
|
|
|
|
|
logger.info(f"connecting to room {ctx.room.name}")
|
|
await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
|
|
|
|
|
|
participant = await ctx.wait_for_participant()
|
|
|
|
|
|
logger.info(f"starting voice assistant for participant {participant.identity}")
|
|
|
|
agent = VoicePipelineAgent(
|
|
vad=ctx.proc.userdata["vad"],
|
|
stt=deepgram.STT(),
|
|
llm=openai.LLM(),
|
|
tts=deepgram.TTS(),
|
|
turn_detector=turn_detector.EOUModel(),
|
|
# minimum delay for endpointing, used when turn detector believes the user is done with their turn
|
|
min_endpointing_delay=0.5,
|
|
# maximum delay for endpointing, used when turn detector does not believe the user is done with their turn
|
|
max_endpointing_delay=10.0,
|
|
max_nested_fnc_calls= 3,
|
|
chat_ctx=initial_ctx,
|
|
fnc_ctx=fnc_ctx,
|
|
)
|
|
|
|
usage_collector = metrics.UsageCollector()
|
|
|
|
@agent.on("metrics_collected")
|
|
def on_metrics_collected(agent_metrics: metrics.AgentMetrics):
|
|
metrics.log_metrics(agent_metrics)
|
|
usage_collector.collect(agent_metrics)
|
|
|
|
agent.start(ctx.room, participant)
|
|
|
|
# The agent should be polite and greet the user when it joins :)
|
|
await agent.say("Hey, I am Adi, how can I help you today?", allow_interruptions=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli.run_app(
|
|
WorkerOptions(
|
|
entrypoint_fnc=entrypoint,
|
|
prewarm_fnc=prewarm,
|
|
agent_name='Provider_DashBoard_Assistant',
|
|
),
|
|
)
|