first commit

.gitignore

@@ -0,0 +1,3 @@
+.myenv
+.env.local
+__pycache__

agent.py

@@ -0,0 +1,123 @@
+import json
+import logging
+import asyncio
+import uuid
+from AssistantFnc import AssistantFnc
+from typing import Annotated
+from dotenv import load_dotenv
+from livekit.agents import (
+    AutoSubscribe,
+    JobContext,
+    JobProcess,
+    WorkerOptions,
+    cli,
+    llm,
+    metrics,
+)
+from PIL import Image
+from livekit.agents.pipeline import VoicePipelineAgent
+from livekit.plugins import cartesia, openai, deepgram, silero, turn_detector, anthropic
+import io
+import base64
+from livekit.agents.llm import ChatMessage, ChatImage
+import os
+from jira import JIRA
+from livekit.agents.utils.images import encode, EncodeOptions, ResizeOptions
+
+# Load environment variables
+load_dotenv('.env.local')
+logger = logging.getLogger("voice-agent")
+
+
+def prewarm(proc: JobProcess):
+    proc.userdata["vad"] = silero.VAD.load()
+
+import json
+
+# When receiving the data from the client
+
+async def entrypoint(ctx: JobContext):
+  
+    metadata = json.loads(ctx.job.metadata)
+    token = metadata.get("token")
+    logger.info(f"Room metadata is {metadata}")
+   
+    initial_ctx = llm.ChatContext().append(
+        role="system",
+        text=(
+           """You are an AI assistant designed to support healthcare providers. Your role is to assist with clinical decision-making, patient management, documentation, scheduling, and communication, using all available tools and resources. Prioritize patient safety, evidence-based practice, and confidentiality. Always act as a supportive, efficient, and knowledgeable assistant, but defer final decisions to the licensed healthcare provider.
+
+When performing any action that requires multiple pieces of information:
+1. Ask for details one by one in a conversational manner rather than requesting all information at once
+2. Confirm each piece of information before moving to the next question
+3. Summarize all collected information before executing the final action
+4. For forms or complex data entry, guide the user through each field step by step
+5. If the user provides multiple pieces of information at once, acknowledge them and confirm before proceeding
+
+For example, when adding a practitioner:
+- First ask for the practitioner's name
+- Then ask for their email address
+- Then ask for additional details like gender, date of birth, etc.
+- Confirm all details before submitting
+
+This approach makes the interaction more natural and ensures all necessary information is collected accurately."""
+        ),
+    )
+    _active_tasks = []
+
+    
+
+    fnc_ctx = AssistantFnc(ctx=ctx)
+    chunks = []
+   
+       
+    
+    logger.info(f"connecting to room {ctx.room.name}")
+    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
+
+    
+    participant = await ctx.wait_for_participant()
+    
+
+    logger.info(f"starting voice assistant for participant {participant.identity}")
+    
+    agent = VoicePipelineAgent(
+        vad=ctx.proc.userdata["vad"],
+        stt=deepgram.STT(),
+        llm=openai.LLM(),
+        tts=deepgram.TTS(),
+        turn_detector=turn_detector.EOUModel(),
+        # minimum delay for endpointing, used when turn detector believes the user is done with their turn
+        min_endpointing_delay=0.5,
+        # maximum delay for endpointing, used when turn detector does not believe the user is done with their turn
+        max_endpointing_delay=10.0,
+        max_nested_fnc_calls= 3,
+        chat_ctx=initial_ctx,
+        fnc_ctx=fnc_ctx,
+    )
+
+    usage_collector = metrics.UsageCollector()
+
+    @agent.on("metrics_collected")
+    def on_metrics_collected(agent_metrics: metrics.AgentMetrics):
+        metrics.log_metrics(agent_metrics)
+        usage_collector.collect(agent_metrics)
+
+    agent.start(ctx.room, participant)
+
+    # The agent should be polite and greet the user when it joins :)
+    await agent.say("Hey, I am Adi, how can I help you today?", allow_interruptions=True)
+
+    
+        
+
+    
+    
+if __name__ == "__main__":
+    cli.run_app(
+        WorkerOptions(
+            entrypoint_fnc=entrypoint,
+            prewarm_fnc=prewarm,
+            agent_name='Provider_DashBoard_Assistant',
+        ),
+    )

requirements.txt

@@ -0,0 +1,9 @@
+livekit-agents>=0.12.11,<1.0.0
+livekit-plugins-openai>=0.10.17,<1.0.0
+livekit-plugins-cartesia>=0.4.7,<1.0.0
+livekit-plugins-deepgram>=0.6.17,<1.0.0
+livekit-plugins-silero>=0.7.4,<1.0.0
+livekit-plugins-turn-detector>=0.4.0,<1.0.0
+livekit-plugins-anthropic>=0.2.13,<1.0.0
+jira~=3.8.0
+python-dotenv~=1.0

sample.json

@@ -0,0 +1,34 @@
+{
+  "getAppointment": {
+    "method": "POST",
+    "url": "/api/get-appointment-list-date",
+    "fields": [
+      {
+        "name": "date",
+        "description": "date of appointment"
+      },
+      {
+        "name": "patient_name",
+        "description": "patient's name"
+      }
+    ],
+    "payload": {
+      "start_date": "2025-04-01",
+      "end_date": "2025-04-30"
+    }
+  },
+  "start_call": {
+    "method": "POST",
+    "url": "/api/get-appointment-list-date",
+    "fields": [
+      {
+        "name": "patient_id",
+        "description": "Id of patient"
+      },
+      {
+        "name": "appoinment_id",
+        "description": "appoinment id of patient"
+      }
+    ]
+  }
+}