my-vault/.claude/scripts/memvid-index-prompt.cjs

#!/usr/bin/env node

/**
 * Memvid Prompt Indexer
 *
 * Usage: node memvid-index-prompt.cjs <memory_file_path> <prompt_text>
 *
 * This script chunks a large custom prompt and stores it in Memvid for semantic search.
 */

const { spawn } = require('child_process');
const path = require('path');
const fs = require('fs');

// Parse arguments
const args = process.argv.slice(2);
if (args.length < 2) {
  console.error('Usage: node memvid-index-prompt.cjs <memory_file_path> <prompt_text>');
  process.exit(1);
}

const memoryFilePath = args[0];
const promptText = args[1];

// MCP configuration
const mcpCommand = 'uv';
const mcpArgs = [
  'run',
  '--directory',
  'D:\\works\\projects-windows\\memvid-agent-mcp',
  'memvid_mcp_server.py'
];

const mcpEnv = {
  ...process.env,
  PYTHONUNBUFFERED: '1',
  MEMVID_LOG_LEVEL: 'ERROR'
};

// JSON-RPC communication
let messageId = 1;

function createRequest(method, params = {}) {
  return {
    jsonrpc: '2.0',
    id: messageId++,
    method,
    params
  };
}

function sendRequest(proc, request) {
  const message = JSON.stringify(request) + '\n';
  proc.stdin.write(message);
}

/**
 * Chunk prompt by semantic sections
 * Looks for markdown headers, paragraphs, or splits by double newlines
 */
function chunkPrompt(text) {
  const chunks = [];

  // Split by markdown headers (## or ###)
  const headerRegex = /^(#{2,3})\s+(.+)$/gm;
  const matches = [...text.matchAll(headerRegex)];

  if (matches.length > 0) {
    // Has headers: split by sections
    let lastIndex = 0;

    for (let i = 0; i < matches.length; i++) {
      const match = matches[i];
      const nextMatch = matches[i + 1];
      const start = match.index;
      const end = nextMatch ? nextMatch.index : text.length;

      const section = text.substring(start, end).trim();
      if (section.length > 0) {
        chunks.push({
          title: match[2],
          content: section,
          category: match[1] === '##' ? 'section' : 'subsection'
        });
      }
    }

    // Add content before first header
    if (matches[0].index > 0) {
      const preamble = text.substring(0, matches[0].index).trim();
      if (preamble.length > 100) {
        chunks.unshift({
          title: 'General Instructions',
          content: preamble,
          category: 'general'
        });
      }
    }
  } else {
    // No headers: split by double newlines
    const paragraphs = text.split(/\n\n+/).filter(p => p.trim().length > 50);

    paragraphs.forEach((para, index) => {
      // Try to extract a title from the first line
      const lines = para.trim().split('\n');
      const firstLine = lines[0];
      const title = firstLine.length < 80 ? firstLine : `Instruction ${index + 1}`;

      chunks.push({
        title: title,
        content: para.trim(),
        category: 'instruction'
      });
    });
  }

  return chunks;
}

// Main execution
async function main() {
  return new Promise((resolve, reject) => {
    const proc = spawn(mcpCommand, mcpArgs, {
      env: mcpEnv,
      stdio: ['pipe', 'pipe', 'pipe']
    });

    let outputBuffer = '';
    let errorBuffer = '';
    let initialized = false;
    let timeoutId = null;

    const chunks = chunkPrompt(promptText);
    let currentChunkIndex = 0;
    let addedCount = 0;

    function cleanup() {
      if (timeoutId) clearTimeout(timeoutId);
      proc.kill();
    }

    function processNextChunk() {
      if (currentChunkIndex >= chunks.length) {
        // All chunks added
        console.log(JSON.stringify({
          success: true,
          added: addedCount,
          total: chunks.length
        }));

        cleanup();
        setTimeout(() => process.exit(0), 100);
        resolve();
        return;
      }

      const chunk = chunks[currentChunkIndex];
      currentChunkIndex++;

      const addRequest = createRequest('tools/call', {
        name: 'memvid_add_text',
        arguments: {
          file_path: memoryFilePath,
          content: chunk.content,
          title: chunk.title,
          tags: {
            type: 'instruction',
            category: chunk.category,
            priority: chunk.category === 'general' ? 'high' : 'medium'
          }
        }
      });

      sendRequest(proc, addRequest);
    }

    proc.stdout.on('data', (data) => {
      outputBuffer += data.toString();

      const lines = outputBuffer.split('\n');
      outputBuffer = lines.pop() || '';

      for (const line of lines) {
        if (!line.trim()) continue;

        try {
          const response = JSON.parse(line);

          // Handle initialization
          if (!initialized && response.id === 1) {
            initialized = true;
            // Start processing chunks
            processNextChunk();
          }
          // Handle add result
          else if (initialized && response.result) {
            addedCount++;
            // Process next chunk
            processNextChunk();
          }
          // Handle errors
          else if (response.error) {
            console.error(JSON.stringify({
              error: response.error.message || 'MCP error',
              chunk: currentChunkIndex - 1
            }));

            cleanup();
            setTimeout(() => process.exit(1), 100);
            reject(new Error(response.error.message));
          }
        } catch (e) {
          // Ignore non-JSON lines
        }
      }
    });

    proc.stderr.on('data', (data) => {
      errorBuffer += data.toString();
    });

    proc.on('error', (error) => {
      console.error(JSON.stringify({ error: error.message }));
      cleanup();
      reject(error);
    });

    proc.on('exit', (code) => {
      if (code !== 0 && code !== null) {
        console.error(JSON.stringify({
          error: `MCP server exited with code ${code}`,
          stderr: errorBuffer
        }));
        reject(new Error(`Process exited with code ${code}`));
      }
    });

    // Initialize MCP connection
    const initRequest = createRequest('initialize', {
      protocolVersion: '2024-11-05',
      capabilities: {},
      clientInfo: {
        name: 'claudian-memvid-indexer',
        version: '1.0.0'
      }
    });

    sendRequest(proc, initRequest);

    // Timeout after 30 seconds
    timeoutId = setTimeout(() => {
      proc.kill();
      console.error(JSON.stringify({ error: 'Indexing timeout' }));
      reject(new Error('Timeout'));
    }, 30000);
  });
}

main().catch((error) => {
  process.exit(1);
});