vault backup: 2026-01-08 09:34:22

2026-01-08 09:34:22 +08:00
parent e967fa874a
commit 0e4d21f721
28 changed files with 3164 additions and 271 deletions
@@ -0,0 +1,261 @@
+#!/usr/bin/env node
+
+/**
+ * Memvid Prompt Indexer
+ *
+ * Usage: node memvid-index-prompt.cjs <memory_file_path> <prompt_text>
+ *
+ * This script chunks a large custom prompt and stores it in Memvid for semantic search.
+ */
+
+const { spawn } = require('child_process');
+const path = require('path');
+const fs = require('fs');
+
+// Parse arguments
+const args = process.argv.slice(2);
+if (args.length < 2) {
+  console.error('Usage: node memvid-index-prompt.cjs <memory_file_path> <prompt_text>');
+  process.exit(1);
+}
+
+const memoryFilePath = args[0];
+const promptText = args[1];
+
+// MCP configuration
+const mcpCommand = 'uv';
+const mcpArgs = [
+  'run',
+  '--directory',
+  'D:\\works\\projects-windows\\memvid-agent-mcp',
+  'memvid_mcp_server.py'
+];
+
+const mcpEnv = {
+  ...process.env,
+  PYTHONUNBUFFERED: '1',
+  MEMVID_LOG_LEVEL: 'ERROR'
+};
+
+// JSON-RPC communication
+let messageId = 1;
+
+function createRequest(method, params = {}) {
+  return {
+    jsonrpc: '2.0',
+    id: messageId++,
+    method,
+    params
+  };
+}
+
+function sendRequest(proc, request) {
+  const message = JSON.stringify(request) + '\n';
+  proc.stdin.write(message);
+}
+
+/**
+ * Chunk prompt by semantic sections
+ * Looks for markdown headers, paragraphs, or splits by double newlines
+ */
+function chunkPrompt(text) {
+  const chunks = [];
+
+  // Split by markdown headers (## or ###)
+  const headerRegex = /^(#{2,3})\s+(.+)$/gm;
+  const matches = [...text.matchAll(headerRegex)];
+
+  if (matches.length > 0) {
+    // Has headers: split by sections
+    let lastIndex = 0;
+
+    for (let i = 0; i < matches.length; i++) {
+      const match = matches[i];
+      const nextMatch = matches[i + 1];
+      const start = match.index;
+      const end = nextMatch ? nextMatch.index : text.length;
+
+      const section = text.substring(start, end).trim();
+      if (section.length > 0) {
+        chunks.push({
+          title: match[2],
+          content: section,
+          category: match[1] === '##' ? 'section' : 'subsection'
+        });
+      }
+    }
+
+    // Add content before first header
+    if (matches[0].index > 0) {
+      const preamble = text.substring(0, matches[0].index).trim();
+      if (preamble.length > 100) {
+        chunks.unshift({
+          title: 'General Instructions',
+          content: preamble,
+          category: 'general'
+        });
+      }
+    }
+  } else {
+    // No headers: split by double newlines
+    const paragraphs = text.split(/\n\n+/).filter(p => p.trim().length > 50);
+
+    paragraphs.forEach((para, index) => {
+      // Try to extract a title from the first line
+      const lines = para.trim().split('\n');
+      const firstLine = lines[0];
+      const title = firstLine.length < 80 ? firstLine : `Instruction ${index + 1}`;
+
+      chunks.push({
+        title: title,
+        content: para.trim(),
+        category: 'instruction'
+      });
+    });
+  }
+
+  return chunks;
+}
+
+// Main execution
+async function main() {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(mcpCommand, mcpArgs, {
+      env: mcpEnv,
+      stdio: ['pipe', 'pipe', 'pipe']
+    });
+
+    let outputBuffer = '';
+    let errorBuffer = '';
+    let initialized = false;
+    let timeoutId = null;
+
+    const chunks = chunkPrompt(promptText);
+    let currentChunkIndex = 0;
+    let addedCount = 0;
+
+    function cleanup() {
+      if (timeoutId) clearTimeout(timeoutId);
+      proc.kill();
+    }
+
+    function processNextChunk() {
+      if (currentChunkIndex >= chunks.length) {
+        // All chunks added
+        console.log(JSON.stringify({
+          success: true,
+          added: addedCount,
+          total: chunks.length
+        }));
+
+        cleanup();
+        setTimeout(() => process.exit(0), 100);
+        resolve();
+        return;
+      }
+
+      const chunk = chunks[currentChunkIndex];
+      currentChunkIndex++;
+
+      const addRequest = createRequest('tools/call', {
+        name: 'memvid_add_text',
+        arguments: {
+          file_path: memoryFilePath,
+          content: chunk.content,
+          title: chunk.title,
+          tags: {
+            type: 'instruction',
+            category: chunk.category,
+            priority: chunk.category === 'general' ? 'high' : 'medium'
+          }
+        }
+      });
+
+      sendRequest(proc, addRequest);
+    }
+
+    proc.stdout.on('data', (data) => {
+      outputBuffer += data.toString();
+
+      const lines = outputBuffer.split('\n');
+      outputBuffer = lines.pop() || '';
+
+      for (const line of lines) {
+        if (!line.trim()) continue;
+
+        try {
+          const response = JSON.parse(line);
+
+          // Handle initialization
+          if (!initialized && response.id === 1) {
+            initialized = true;
+            // Start processing chunks
+            processNextChunk();
+          }
+          // Handle add result
+          else if (initialized && response.result) {
+            addedCount++;
+            // Process next chunk
+            processNextChunk();
+          }
+          // Handle errors
+          else if (response.error) {
+            console.error(JSON.stringify({
+              error: response.error.message || 'MCP error',
+              chunk: currentChunkIndex - 1
+            }));
+
+            cleanup();
+            setTimeout(() => process.exit(1), 100);
+            reject(new Error(response.error.message));
+          }
+        } catch (e) {
+          // Ignore non-JSON lines
+        }
+      }
+    });
+
+    proc.stderr.on('data', (data) => {
+      errorBuffer += data.toString();
+    });
+
+    proc.on('error', (error) => {
+      console.error(JSON.stringify({ error: error.message }));
+      cleanup();
+      reject(error);
+    });
+
+    proc.on('exit', (code) => {
+      if (code !== 0 && code !== null) {
+        console.error(JSON.stringify({
+          error: `MCP server exited with code ${code}`,
+          stderr: errorBuffer
+        }));
+        reject(new Error(`Process exited with code ${code}`));
+      }
+    });
+
+    // Initialize MCP connection
+    const initRequest = createRequest('initialize', {
+      protocolVersion: '2024-11-05',
+      capabilities: {},
+      clientInfo: {
+        name: 'claudian-memvid-indexer',
+        version: '1.0.0'
+      }
+    });
+
+    sendRequest(proc, initRequest);
+
+    // Timeout after 30 seconds
+    timeoutId = setTimeout(() => {
+      proc.kill();
+      console.error(JSON.stringify({ error: 'Indexing timeout' }));
+      reject(new Error('Timeout'));
+    }, 30000);
+  });
+}
+
+main().catch((error) => {
+  process.exit(1);
+});
@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+
+/**
+ * Memvid Tag-Based Search Script
+ *
+ * Uses tag-based search with keyword mapping for reliable instruction retrieval
+ * Falls back to direct loading if search fails
+ */
+
+const { spawn } = require('child_process');
+const path = require('path');
+
+// Parse arguments
+const args = process.argv.slice(2);
+if (args.length < 2) {
+  console.error(JSON.stringify({
+    success: false,
+    error: 'Usage: memvid-search-tags.cjs <memory_file_path> <user_query> [top_k]'
+  }));
+  process.exit(1);
+}
+
+const memoryFilePath = args[0];
+const userQuery = args[1].toLowerCase();
+const topK = parseInt(args[2]) || 3;
+
+/**
+ * Map user query keywords to instruction categories
+ */
+function detectCategories(query) {
+  const categoryMap = {
+    'project-management': ['project', 'task', 'todo', 'gtd', 'deadline', 'status'],
+    'linking': ['link', 'connect', 'relation', 'backlink', 'reference', 'wiki'],
+    'writing': ['write', 'article', 'essay', 'content', 'edit', 'draft'],
+    'automation': ['automate', 'workflow', 'template', 'batch', 'process'],
+    'technical': ['code', 'api', 'programming', 'snippet', 'documentation'],
+    'organization': ['para', 'folder', 'structure', 'inbox', 'archive'],
+    'daily': ['daily', 'journal', 'note', 'log', 'capture'],
+    'resource': ['resource', 'reference', 'research', 'article', 'learning']
+  };
+
+  const matches = [];
+
+  for (const [category, keywords] of Object.entries(categoryMap)) {
+    if (keywords.some(kw => query.includes(kw))) {
+      matches.push(category);
+    }
+  }
+
+  // Default categories if no matches
+  if (matches.length === 0) {
+    matches.push('organization', 'project-management');
+  }
+
+  return matches;
+}
+
+/**
+ * Search Memvid using tag-based search
+ */
+async function searchByTags(categories) {
+  const results = [];
+
+  for (const category of categories) {
+    try {
+      const result = await callMemvidTool('memvid_search_by_tag', {
+        file_path: memoryFilePath,
+        tag_key: 'category',
+        tag_value: category
+      });
+
+      if (result && result.content) {
+        results.push({
+          category,
+          content: result.content
+        });
+      }
+    } catch (error) {
+      console.error(`[Tag Search] Failed for category ${category}:`, error.message);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Call Memvid MCP tool
+ */
+function callMemvidTool(toolName, args) {
+  return new Promise((resolve, reject) => {
+    // Get MCP server config from .claude/mcp.json
+    const fs = require('fs');
+    const memoryDir = path.dirname(memoryFilePath);
+    const claudeDir = path.dirname(memoryDir);
+    const mcpConfigPath = path.join(claudeDir, 'mcp.json');
+    const mcpConfig = JSON.parse(fs.readFileSync(mcpConfigPath, 'utf-8'));
+
+    const serverConfig = mcpConfig.mcpServers.memvid;
+    const proc = spawn(serverConfig.command, serverConfig.args, {
+      env: { ...process.env, ...serverConfig.env },
+      stdio: ['pipe', 'pipe', 'pipe']
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let timeoutId;
+
+    proc.stdout.on('data', (data) => {
+      stdout += data.toString();
+
+      // Look for JSON-RPC response
+      const lines = stdout.split('\n');
+      for (const line of lines) {
+        if (line.trim().startsWith('{')) {
+          try {
+            const response = JSON.parse(line);
+            if (response.result) {
+              cleanup();
+              resolve(response.result);
+              return;
+            }
+          } catch (e) {
+            // Not valid JSON, continue
+          }
+        }
+      }
+    });
+
+    proc.stderr.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    proc.on('error', (error) => {
+      cleanup();
+      reject(error);
+    });
+
+    proc.on('exit', (code) => {
+      cleanup();
+      if (code !== 0 && code !== null) {
+        reject(new Error(`Process exited with code ${code}: ${stderr}`));
+      }
+    });
+
+    // Send initialize request
+    const initRequest = {
+      jsonrpc: '2.0',
+      id: 1,
+      method: 'initialize',
+      params: {
+        protocolVersion: '2024-11-05',
+        capabilities: {},
+        clientInfo: { name: 'claudian-plugin', version: '1.0.0' }
+      }
+    };
+
+    proc.stdin.write(JSON.stringify(initRequest) + '\n');
+
+    // Wait a bit for initialization, then send tool call
+    setTimeout(() => {
+      const toolRequest = {
+        jsonrpc: '2.0',
+        id: 2,
+        method: 'tools/call',
+        params: {
+          name: toolName,
+          arguments: args
+        }
+      };
+
+      proc.stdin.write(JSON.stringify(toolRequest) + '\n');
+    }, 500);
+
+    // Set timeout
+    timeoutId = setTimeout(() => {
+      cleanup();
+      reject(new Error('Search timeout'));
+    }, 10000);
+
+    function cleanup() {
+      if (timeoutId) clearTimeout(timeoutId);
+      if (!proc.killed) {
+        proc.kill();
+      }
+    }
+  });
+}
+
+/**
+ * Format results
+ */
+function formatResults(results) {
+  if (results.length === 0) {
+    return null;
+  }
+
+  const formatted = results.map(r => {
+    const content = Array.isArray(r.content)
+      ? r.content.map(c => c.text || c).join('\n')
+      : (r.content.text || r.content);
+
+    return `### ${r.category}\n\n${content}`;
+  }).join('\n\n---\n\n');
+
+  return formatted;
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  try {
+    // Detect relevant categories from query
+    const categories = detectCategories(userQuery);
+
+    // Search by tags
+    const results = await searchByTags(categories.slice(0, topK));
+
+    // Format results
+    const formatted = formatResults(results);
+
+    if (formatted) {
+      console.log(JSON.stringify({
+        success: true,
+        result: {
+          content: [{ type: 'text', text: formatted }],
+          structuredContent: { result: formatted },
+          isError: false
+        }
+      }));
+    } else {
+      console.log(JSON.stringify({
+        success: false,
+        result: {
+          content: [{ type: 'text', text: `No results found for categories: ${categories.join(', ')}` }],
+          structuredContent: { result: `No results found` },
+          isError: false
+        }
+      }));
+    }
+
+    process.exit(0);
+  } catch (error) {
+    console.error(JSON.stringify({
+      success: false,
+      error: error.message
+    }));
+    process.exit(1);
+  }
+}
+
+main();
@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+
+/**
+ * Memvid MCP Search Helper
+ *
+ * Usage: node memvid-search.js <memory_file_path> <query> [top_k] [snippet_chars]
+ *
+ * This script calls the Memvid MCP server to perform semantic search.
+ */
+
+const { spawn } = require('child_process');
+const path = require('path');
+const fs = require('fs');
+
+// Parse arguments
+const args = process.argv.slice(2);
+if (args.length < 2) {
+  console.error('Usage: node memvid-search.js <memory_file_path> <query> [top_k] [snippet_chars]');
+  process.exit(1);
+}
+
+const memoryFilePath = args[0];
+const query = args[1];
+const topK = parseInt(args[2]) || 3;
+const snippetChars = parseInt(args[3]) || 500;
+
+// Validate memory file exists
+if (!fs.existsSync(memoryFilePath)) {
+  console.error(JSON.stringify({ error: `Memory file not found: ${memoryFilePath}` }));
+  process.exit(1);
+}
+
+// MCP configuration (from .claude/mcp.json)
+const mcpCommand = 'uv';
+const mcpArgs = [
+  'run',
+  '--directory',
+  'D:\\works\\projects-windows\\memvid-agent-mcp',
+  'memvid_mcp_server.py'
+];
+
+const mcpEnv = {
+  ...process.env,
+  PYTHONUNBUFFERED: '1',
+  MEMVID_LOG_LEVEL: 'ERROR' // Reduce noise
+};
+
+// JSON-RPC communication
+let messageId = 1;
+
+function createRequest(method, params = {}) {
+  return {
+    jsonrpc: '2.0',
+    id: messageId++,
+    method,
+    params
+  };
+}
+
+function sendRequest(proc, request) {
+  const message = JSON.stringify(request) + '\n';
+  proc.stdin.write(message);
+}
+
+// Main execution
+async function main() {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(mcpCommand, mcpArgs, {
+      env: mcpEnv,
+      stdio: ['pipe', 'pipe', 'pipe']
+    });
+
+    let outputBuffer = '';
+    let errorBuffer = '';
+    let initialized = false;
+    let searchRequestSent = false;
+    let timeoutId = null;
+
+    function cleanup() {
+      if (timeoutId) clearTimeout(timeoutId);
+      proc.kill();
+    }
+
+    proc.stdout.on('data', (data) => {
+      outputBuffer += data.toString();
+
+      // Process complete JSON-RPC messages
+      const lines = outputBuffer.split('\n');
+      outputBuffer = lines.pop() || ''; // Keep incomplete line
+
+      for (const line of lines) {
+        if (!line.trim()) continue;
+
+        try {
+          const response = JSON.parse(line);
+
+          // Handle initialization
+          if (!initialized && response.id === 1) {
+            initialized = true;
+
+            // Send search request
+            const searchRequest = createRequest('tools/call', {
+              name: 'memvid_search',
+              arguments: {
+                file_path: memoryFilePath,
+                query: query,
+                top_k: topK,
+                snippet_chars: snippetChars
+              }
+            });
+
+            sendRequest(proc, searchRequest);
+            searchRequestSent = true;
+          }
+          // Handle search result
+          else if (searchRequestSent && response.result) {
+            console.log(JSON.stringify({
+              success: true,
+              result: response.result
+            }));
+
+            cleanup();
+            setTimeout(() => process.exit(0), 100);
+            resolve();
+          }
+          // Handle errors
+          else if (response.error) {
+            console.error(JSON.stringify({
+              error: response.error.message || 'MCP error'
+            }));
+
+            cleanup();
+            setTimeout(() => process.exit(1), 100);
+            reject(new Error(response.error.message));
+          }
+        } catch (e) {
+          // Ignore non-JSON lines (logs, etc.)
+        }
+      }
+    });
+
+    proc.stderr.on('data', (data) => {
+      errorBuffer += data.toString();
+    });
+
+    proc.on('error', (error) => {
+      console.error(JSON.stringify({ error: error.message }));
+      cleanup();
+      reject(error);
+    });
+
+    proc.on('exit', (code) => {
+      if (code !== 0 && code !== null) {
+        console.error(JSON.stringify({
+          error: `MCP server exited with code ${code}`,
+          stderr: errorBuffer
+        }));
+        reject(new Error(`Process exited with code ${code}`));
+      }
+    });
+
+    // Initialize MCP connection
+    const initRequest = createRequest('initialize', {
+      protocolVersion: '2024-11-05',
+      capabilities: {},
+      clientInfo: {
+        name: 'claudian-memvid-helper',
+        version: '1.0.0'
+      }
+    });
+
+    sendRequest(proc, initRequest);
+
+    // Timeout after 10 seconds
+    timeoutId = setTimeout(() => {
+      proc.kill();
+      console.error(JSON.stringify({ error: 'Search timeout' }));
+      reject(new Error('Timeout'));
+    }, 10000);
+  });
+}
+
+main().catch((error) => {
+  process.exit(1);
+});
@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+
+/**
+ * Test Context File Summarization
+ *
+ * This script tests the file summarization functionality
+ */
+
+const path = require('path');
+const fs = require('fs');
+
+// Load the helper functions from main.js
+const mainPath = path.join(__dirname, '..', '.obsidian', 'plugins', 'claudian', 'main.js');
+
+// Extract YAML frontmatter
+function extractYAMLFrontmatter(content) {
+  // Normalize line endings to \n
+  const normalized = content.replace(/\r\n/g, "\n");
+  const frontmatterMatch = normalized.match(/^---\n([\s\S]*?)\n---/);
+  if (!frontmatterMatch) return {};
+
+  const yaml = frontmatterMatch[1];
+  const result = {};
+
+  // Simple YAML parsing (key: value)
+  const lines = yaml.split("\n");
+  for (const line of lines) {
+    const match = line.match(/^(\w+):\s*(.+)$/);
+    if (match) {
+      const [, key, value] = match;
+
+      // Handle arrays
+      if (value.startsWith("[") && value.endsWith("]")) {
+        result[key] = value
+          .slice(1, -1)
+          .split(",")
+          .map(v => v.trim().replace(/^["']|["']$/g, ""));
+      } else {
+        result[key] = value.replace(/^["']|["']$/g, "");
+      }
+    }
+  }
+
+  return result;
+}
+
+// Extract markdown headings
+function extractMarkdownHeadings(content) {
+  const headings = [];
+  // Normalize line endings
+  const lines = content.replace(/\r\n/g, "\n").split("\n");
+
+  for (const line of lines) {
+    const match = line.match(/^#{1,6}\s+(.+)$/);
+    if (match) {
+      headings.push(match[1].trim());
+    }
+  }
+
+  return headings;
+}
+
+// Generate file summary
+async function generateFileSummary(filePath) {
+  const content = fs.readFileSync(filePath, "utf-8");
+  const fileName = path.basename(filePath);
+
+  // Extract YAML frontmatter
+  const frontmatter = extractYAMLFrontmatter(content);
+
+  // Extract markdown headings
+  const headings = extractMarkdownHeadings(content);
+
+  // Extract first paragraph (skip frontmatter)
+  // Normalize line endings first
+  const normalized = content.replace(/\r\n/g, "\n");
+  const textContent = normalized.replace(/^---\n[\s\S]*?\n---\n/, "");
+  const firstPara = textContent.trim().split("\n\n")[0] || "";
+  const preview = firstPara.slice(0, 300);
+
+  // Format summary
+  const parts = [];
+  parts.push(`📄 **${fileName}** _(Summary - ${(content.length / 1024).toFixed(1)}KB)_`);
+  parts.push("");
+
+  if (frontmatter.type || frontmatter.tags || frontmatter.created) {
+    parts.push("**Metadata:**");
+    if (frontmatter.type) parts.push(`- Type: ${frontmatter.type}`);
+    if (frontmatter.tags && frontmatter.tags.length > 0) {
+      parts.push(`- Tags: ${frontmatter.tags.join(", ")}`);
+    }
+    if (frontmatter.created) parts.push(`- Created: ${frontmatter.created}`);
+    parts.push("");
+  }
+
+  if (headings.length > 0) {
+    parts.push(`**Structure:** ${headings.slice(0, 5).join(" > ")}`);
+    if (headings.length > 5) parts.push(`_(+${headings.length - 5} more sections)_`);
+    parts.push("");
+  }
+
+  if (preview) {
+    parts.push("**Preview:**");
+    parts.push(preview + (firstPara.length > 300 ? "..." : ""));
+    parts.push("");
+  }
+
+  parts.push(`_📖 Use Read tool to view full content: \`${filePath}\`_`);
+
+  return parts.join("\n");
+}
+
+// Main test
+async function main() {
+  console.log("=== Testing Context File Summarization ===\n");
+
+  const testFile = path.join(__dirname, '..', 'test-large-file.md');
+
+  console.log(`Test file: ${testFile}`);
+
+  const stats = fs.statSync(testFile);
+  console.log(`File size: ${stats.size} bytes (${(stats.size / 1024).toFixed(1)}KB)\n`);
+
+  // Debug: Check frontmatter extraction
+  const content = fs.readFileSync(testFile, "utf-8");
+  const frontmatter = extractYAMLFrontmatter(content);
+  const headings = extractMarkdownHeadings(content);
+
+  console.log("DEBUG - Extracted frontmatter:", JSON.stringify(frontmatter, null, 2));
+  console.log("DEBUG - Extracted headings (first 10):", headings.slice(0, 10));
+  console.log("");
+
+  console.log("Generating summary...\n");
+  const summary = await generateFileSummary(testFile);
+
+  console.log("=== SUMMARY OUTPUT ===\n");
+  console.log(summary);
+  console.log("\n=== END SUMMARY ===");
+
+  // Calculate token savings
+  const originalTokens = Math.ceil(stats.size / 4);
+  const summaryTokens = Math.ceil(summary.length / 4);
+  const savings = ((1 - summaryTokens / originalTokens) * 100).toFixed(1);
+
+  console.log(`\n=== Token Analysis ===`);
+  console.log(`Original size: ~${originalTokens} tokens`);
+  console.log(`Summary size: ~${summaryTokens} tokens`);
+  console.log(`Token savings: ${savings}%`);
+}
+
+main().catch(console.error);