From 7f27c206b5ba09a9d71ae8bfa71b9566124896ba Mon Sep 17 00:00:00 2001 From: Deep Koluguri Date: Tue, 26 May 2026 22:56:14 -0400 Subject: [PATCH] feat: Add Vision Agents (RxVision and DocumentIntelligence) --- backend/agents/AgentManager.js | 4 ++ backend/agents/DocumentIntelligenceAgent.js | 47 +++++++++++++++++++++ backend/agents/RxVisionAgent.js | 44 +++++++++++++++++++ backend/db.js | 4 +- settings.html | 22 ++++++++++ 5 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 backend/agents/DocumentIntelligenceAgent.js create mode 100644 backend/agents/RxVisionAgent.js diff --git a/backend/agents/AgentManager.js b/backend/agents/AgentManager.js index 536829b..9b3cf2d 100644 --- a/backend/agents/AgentManager.js +++ b/backend/agents/AgentManager.js @@ -4,6 +4,8 @@ const TriageAgent = require('./TriageAgent'); const FollowUpAgent = require('./FollowUpAgent'); const InventoryAgent = require('./InventoryAgent'); const CDSSAgent = require('./CDSSAgent'); +const RxVisionAgent = require('./RxVisionAgent'); +const DocumentIntelligenceAgent = require('./DocumentIntelligenceAgent'); class AgentManager { constructor() { @@ -13,6 +15,8 @@ class AgentManager { this.registerAgent('FOLLOWUP', FollowUpAgent); this.registerAgent('INVENTORY', InventoryAgent); this.registerAgent('CDSS', CDSSAgent); + this.registerAgent('RX_VISION', RxVisionAgent); + this.registerAgent('DOC_INTELLIGENCE', DocumentIntelligenceAgent); } /** diff --git a/backend/agents/DocumentIntelligenceAgent.js b/backend/agents/DocumentIntelligenceAgent.js new file mode 100644 index 0000000..c137e36 --- /dev/null +++ b/backend/agents/DocumentIntelligenceAgent.js @@ -0,0 +1,47 @@ +const BaseAgent = require('./BaseAgent'); + +class DocumentIntelligenceAgent extends BaseAgent { + constructor() { + super('DocumentIntelligenceAgent'); + this.responseMocks = { + 'default': JSON.stringify({ + documentType: 'LAB_REPORT', + patientName: 'Jane Doe', + date: '2026-05-26', + metrics: [ + { name: 'Fasting Blood Sugar', value: '110 mg/dL', status: 'BORDERLINE' }, + { name: 'Total Cholesterol', value: '240 mg/dL', status: 'HIGH' } + ], + summary: 'Elevated cholesterol levels detected.' + }) + }; + } + + async execute(config, payload) { + const { documentUrl, documentTypeHint } = payload; + + console.log(`[DocumentIntelligenceAgent] Processing document image/pdf: ${documentUrl}`); + + const prompt = ` + Act as an expert medical document intelligence system. Analyze the provided image or PDF. + Hint: This is likely a ${documentTypeHint}. + Extract the document type, patient name, date, and key metrics/values. + Flag any metrics that are out of normal ranges. + Return a JSON object: { "documentType": "...", "patientName": "...", "date": "...", "metrics": [{ "name", "value", "status" }], "summary": "..." } + `; + + // Simulate sending image to Vision API + const llmResponse = await this.mockLLMCall(prompt, this.responseMocks); + + try { + const result = JSON.parse(llmResponse); + console.log(`[DocumentIntelligenceAgent] Extracted ${result.metrics?.length || 0} metrics from ${result.documentType}.`); + return result; + } catch (e) { + console.error('[DocumentIntelligenceAgent] Failed to parse Vision LLM response', e); + return { documentType: 'UNKNOWN', metrics: [] }; + } + } +} + +module.exports = new DocumentIntelligenceAgent(); diff --git a/backend/agents/RxVisionAgent.js b/backend/agents/RxVisionAgent.js new file mode 100644 index 0000000..d8295ee --- /dev/null +++ b/backend/agents/RxVisionAgent.js @@ -0,0 +1,44 @@ +const BaseAgent = require('./BaseAgent'); + +class RxVisionAgent extends BaseAgent { + constructor() { + super('RxVisionAgent'); + this.responseMocks = { + 'default': JSON.stringify({ + medications: [ + { name: 'Amoxicillin', dosage: '500mg', frequency: '1-0-1', duration: '5 days' }, + { name: 'Paracetamol', dosage: '650mg', frequency: '1-1-1', duration: '3 days' } + ], + doctorNotes: 'Drink plenty of fluids. Rest for 3 days.', + nextFollowUp: '2026-06-02' + }) + }; + } + + async execute(config, payload) { + const { imageUrl } = payload; + + console.log(`[RxVisionAgent] Processing prescription image: ${imageUrl}`); + + const prompt = ` + Act as an expert medical OCR system. Analyze the provided image of a handwritten prescription. + Extract the medications, dosages, frequencies, and duration. + Also extract any general doctor notes and the next follow-up date. + Return a JSON object: { "medications": [{ "name", "dosage", "frequency", "duration" }], "doctorNotes": "...", "nextFollowUp": "YYYY-MM-DD" } + `; + + // Simulate sending image to Vision API + const llmResponse = await this.mockLLMCall(prompt, this.responseMocks); + + try { + const result = JSON.parse(llmResponse); + console.log(`[RxVisionAgent] Successfully extracted ${result.medications.length} medications.`); + return result; + } catch (e) { + console.error('[RxVisionAgent] Failed to parse Vision LLM response', e); + return { medications: [], doctorNotes: '', nextFollowUp: null }; + } + } +} + +module.exports = new RxVisionAgent(); diff --git a/backend/db.js b/backend/db.js index e7d6cc4..739b70b 100644 --- a/backend/db.js +++ b/backend/db.js @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS transactions ( CREATE TABLE IF NOT EXISTS agent_configs ( id SERIAL PRIMARY KEY, tenant_id TEXT REFERENCES tenants(id) ON DELETE CASCADE, - agent_type TEXT NOT NULL CHECK (agent_type IN ('BILLING', 'TRIAGE', 'FOLLOWUP', 'INVENTORY', 'CDSS')), + agent_type TEXT NOT NULL CHECK (agent_type IN ('BILLING', 'TRIAGE', 'FOLLOWUP', 'INVENTORY', 'CDSS', 'RX_VISION', 'DOC_INTELLIGENCE')), is_active BOOLEAN DEFAULT false, config JSONB DEFAULT '{}', created_at TIMESTAMPTZ DEFAULT now(), @@ -98,6 +98,8 @@ ALTER TABLE users ADD COLUMN IF NOT EXISTS q_and_a JSONB DEFAULT '[]'; ALTER TABLE users ADD COLUMN IF NOT EXISTS blocked_dates JSONB DEFAULT '[]'; ALTER TABLE users DROP CONSTRAINT IF EXISTS users_role_check; ALTER TABLE users ADD CONSTRAINT users_role_check CHECK (role IN ('SUPER_ADMIN','OWNER','DOCTOR','RECEPTIONIST','PHARMACIST','LAB_TECH','PATIENT')); +ALTER TABLE agent_configs DROP CONSTRAINT IF EXISTS agent_configs_agent_type_check; +ALTER TABLE agent_configs ADD CONSTRAINT agent_configs_agent_type_check CHECK (agent_type IN ('BILLING', 'TRIAGE', 'FOLLOWUP', 'INVENTORY', 'CDSS', 'RX_VISION', 'DOC_INTELLIGENCE')); `; // ── Seed super admin ───────────────────────────────────────────────────────── diff --git a/settings.html b/settings.html index 332e6a3..635725d 100644 --- a/settings.html +++ b/settings.html @@ -167,6 +167,28 @@

Analyzes prescriptions against patient history to warn doctors of drug-drug interactions.

+ +
+
+ 📷 Prescription Vision Parser + +
+

Automatically converts handwritten prescription photos into digital, structured medication lists.

+
+ +
+
+ 📄 Document Intelligence + +
+

Extracts data from uploaded lab reports and medical documents into structured records.

+