#!/usr/bin/env node /** * Memvid Prompt Indexer * * Usage: node memvid-index-prompt.cjs * * This script chunks a large custom prompt and stores it in Memvid for semantic search. */ const { spawn } = require('child_process'); const path = require('path'); const fs = require('fs'); // Parse arguments const args = process.argv.slice(2); if (args.length < 2) { console.error('Usage: node memvid-index-prompt.cjs '); process.exit(1); } const memoryFilePath = args[0]; const promptText = args[1]; // MCP configuration const mcpCommand = 'uv'; const mcpArgs = [ 'run', '--directory', 'D:\\works\\projects-windows\\memvid-agent-mcp', 'memvid_mcp_server.py' ]; const mcpEnv = { ...process.env, PYTHONUNBUFFERED: '1', MEMVID_LOG_LEVEL: 'ERROR' }; // JSON-RPC communication let messageId = 1; function createRequest(method, params = {}) { return { jsonrpc: '2.0', id: messageId++, method, params }; } function sendRequest(proc, request) { const message = JSON.stringify(request) + '\n'; proc.stdin.write(message); } /** * Chunk prompt by semantic sections * Looks for markdown headers, paragraphs, or splits by double newlines */ function chunkPrompt(text) { const chunks = []; // Split by markdown headers (## or ###) const headerRegex = /^(#{2,3})\s+(.+)$/gm; const matches = [...text.matchAll(headerRegex)]; if (matches.length > 0) { // Has headers: split by sections let lastIndex = 0; for (let i = 0; i < matches.length; i++) { const match = matches[i]; const nextMatch = matches[i + 1]; const start = match.index; const end = nextMatch ? nextMatch.index : text.length; const section = text.substring(start, end).trim(); if (section.length > 0) { chunks.push({ title: match[2], content: section, category: match[1] === '##' ? 'section' : 'subsection' }); } } // Add content before first header if (matches[0].index > 0) { const preamble = text.substring(0, matches[0].index).trim(); if (preamble.length > 100) { chunks.unshift({ title: 'General Instructions', content: preamble, category: 'general' }); } } } else { // No headers: split by double newlines const paragraphs = text.split(/\n\n+/).filter(p => p.trim().length > 50); paragraphs.forEach((para, index) => { // Try to extract a title from the first line const lines = para.trim().split('\n'); const firstLine = lines[0]; const title = firstLine.length < 80 ? firstLine : `Instruction ${index + 1}`; chunks.push({ title: title, content: para.trim(), category: 'instruction' }); }); } return chunks; } // Main execution async function main() { return new Promise((resolve, reject) => { const proc = spawn(mcpCommand, mcpArgs, { env: mcpEnv, stdio: ['pipe', 'pipe', 'pipe'] }); let outputBuffer = ''; let errorBuffer = ''; let initialized = false; let timeoutId = null; const chunks = chunkPrompt(promptText); let currentChunkIndex = 0; let addedCount = 0; function cleanup() { if (timeoutId) clearTimeout(timeoutId); proc.kill(); } function processNextChunk() { if (currentChunkIndex >= chunks.length) { // All chunks added console.log(JSON.stringify({ success: true, added: addedCount, total: chunks.length })); cleanup(); setTimeout(() => process.exit(0), 100); resolve(); return; } const chunk = chunks[currentChunkIndex]; currentChunkIndex++; const addRequest = createRequest('tools/call', { name: 'memvid_add_text', arguments: { file_path: memoryFilePath, content: chunk.content, title: chunk.title, tags: { type: 'instruction', category: chunk.category, priority: chunk.category === 'general' ? 'high' : 'medium' } } }); sendRequest(proc, addRequest); } proc.stdout.on('data', (data) => { outputBuffer += data.toString(); const lines = outputBuffer.split('\n'); outputBuffer = lines.pop() || ''; for (const line of lines) { if (!line.trim()) continue; try { const response = JSON.parse(line); // Handle initialization if (!initialized && response.id === 1) { initialized = true; // Start processing chunks processNextChunk(); } // Handle add result else if (initialized && response.result) { addedCount++; // Process next chunk processNextChunk(); } // Handle errors else if (response.error) { console.error(JSON.stringify({ error: response.error.message || 'MCP error', chunk: currentChunkIndex - 1 })); cleanup(); setTimeout(() => process.exit(1), 100); reject(new Error(response.error.message)); } } catch (e) { // Ignore non-JSON lines } } }); proc.stderr.on('data', (data) => { errorBuffer += data.toString(); }); proc.on('error', (error) => { console.error(JSON.stringify({ error: error.message })); cleanup(); reject(error); }); proc.on('exit', (code) => { if (code !== 0 && code !== null) { console.error(JSON.stringify({ error: `MCP server exited with code ${code}`, stderr: errorBuffer })); reject(new Error(`Process exited with code ${code}`)); } }); // Initialize MCP connection const initRequest = createRequest('initialize', { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'claudian-memvid-indexer', version: '1.0.0' } }); sendRequest(proc, initRequest); // Timeout after 30 seconds timeoutId = setTimeout(() => { proc.kill(); console.error(JSON.stringify({ error: 'Indexing timeout' })); reject(new Error('Timeout')); }, 30000); }); } main().catch((error) => { process.exit(1); });