my-vault/.claude/mcp-servers/gemini-vision.mjs

#!/usr/bin/env node
import { GoogleGenerativeAI } from '@google/generative-ai'
import { GoogleAIFileManager } from '@google/generative-ai/server'
import { Server } from '@modelcontextprotocol/sdk/server/index.js'
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
import {
  CallToolRequestSchema,
  ListToolsRequestSchema,
} from '@modelcontextprotocol/sdk/types.js'
import fs from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'

const apiKey = process.env.GEMINI_API_KEY
if (!apiKey) {
  console.error('❌ GEMINI_API_KEY environment variable is required')
  console.error('')
  console.error('To fix this:')
  console.error('')
  console.error('1. Get your API key from: https://aistudio.google.com/apikey')
  console.error('')
  console.error('2. Add to your shell profile:')
  console.error('   For macOS/Linux (add to ~/.zshrc or ~/.bashrc):')
  console.error("   export GEMINI_API_KEY='your-actual-api-key-here'")
  console.error('')
  console.error('   For Windows PowerShell:')
  console.error(
    "   [System.Environment]::SetEnvironmentVariable('GEMINI_API_KEY', 'your-key', 'User')",
  )
  console.error('')
  console.error('3. Reload your terminal:')
  console.error('   source ~/.zshrc  (or source ~/.bashrc)')
  console.error('')
  console.error('4. Restart Claude Code')
  console.error('')
  console.error('For detailed instructions, see GEMINI_VISION_SETUP.md')
  process.exit(1)
}

const genAI = new GoogleGenerativeAI(apiKey)
const fileManager = new GoogleAIFileManager(apiKey)
const model = genAI.getGenerativeModel({ model: 'gemini-2.5-flash' })

// Expand home directory in paths
function expandPath(filepath) {
  if (filepath.startsWith('~/')) {
    return path.join(os.homedir(), filepath.slice(2))
  }
  return filepath
}

// Helper function to wait/sleep
function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms))
}

// Upload file to Gemini
async function uploadFile(filePath) {
  const expandedPath = expandPath(filePath)

  try {
    await fs.access(expandedPath)
  } catch {
    throw new Error(`File not found: ${filePath}`)
  }

  const ext = path.extname(expandedPath).toLowerCase()
  const mimeTypes = {
    '.bmp': 'image/bmp',
    '.doc': 'application/msword',
    '.docx':
      'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
    '.gif': 'image/gif',
    '.jpeg': 'image/jpeg',
    '.jpg': 'image/jpeg',
    '.odt': 'application/vnd.oasis.opendocument.text',
    '.pdf': 'application/pdf',
    '.png': 'image/png',
    '.rtf': 'application/rtf',
    '.txt': 'text/plain',
    '.webp': 'image/webp',
    // Video formats
    '.3gp': 'video/3gpp',
    '.avi': 'video/x-msvideo',
    '.flv': 'video/x-flv',
    '.m4v': 'video/x-m4v',
    '.mkv': 'video/x-matroska',
    '.mov': 'video/quicktime',
    '.mp4': 'video/mp4',
    '.webm': 'video/webm',
    '.wmv': 'video/x-ms-wmv',
  }

  const uploadResult = await fileManager.uploadFile(expandedPath, {
    mimeType: mimeTypes[ext] || 'application/octet-stream',
  })

  let file = uploadResult.file

  // For video files, poll until the file is in ACTIVE state
  const videoExtensions = [
    '.mp4',
    '.avi',
    '.mov',
    '.webm',
    '.mkv',
    '.wmv',
    '.flv',
    '.3gp',
    '.m4v',
  ]
  if (videoExtensions.includes(ext)) {
    console.error(
      `Waiting for video file to process: ${path.basename(filePath)}`,
    )
    let attempts = 0
    const maxAttempts = 60 // Max 5 minutes (60 * 5 seconds)

    while (file.state !== 'ACTIVE' && attempts < maxAttempts) {
      await sleep(5000) // Wait 5 seconds
      attempts++

      // Get updated file status
      const fileStatus = await fileManager.getFile(file.name)
      file = fileStatus

      console.error(
        `Video processing status: ${file.state} (attempt ${attempts}/${maxAttempts})`,
      )

      if (file.state === 'FAILED') {
        throw new Error(`Video processing failed for: ${filePath}`)
      }
    }

    if (file.state !== 'ACTIVE') {
      throw new Error(
        `Video processing timeout for: ${filePath}. File state: ${file.state}`,
      )
    }

    console.error('Video file is ready for analysis')
  }

  return file
}

// Tool handlers
async function analyzeDocument(args) {
  const documentPath = args.document_path
  const prompt =
    args.prompt || 'Analyze this document and provide a comprehensive summary'

  const file = await uploadFile(documentPath)
  const result = await model.generateContent([
    prompt,
    { fileData: { fileUri: file.uri, mimeType: file.mimeType } },
  ])

  return result.response.text()
}

async function analyzeImage(args) {
  const imagePath = args.image_path
  const prompt = args.prompt || 'Describe this image in detail'

  const file = await uploadFile(imagePath)
  const result = await model.generateContent([
    prompt,
    { fileData: { fileUri: file.uri, mimeType: file.mimeType } },
  ])

  return result.response.text()
}

async function analyzeMultiple(args) {
  const imagePaths = args.image_paths
  const prompt = args.prompt || 'Analyze these images'

  const content = [prompt]
  for (const imagePath of imagePaths) {
    const file = await uploadFile(imagePath)
    content.push({ fileData: { fileUri: file.uri, mimeType: file.mimeType } })
  }

  const result = await model.generateContent(content)
  return result.response.text()
}

async function compareImages(args) {
  const image1Path = args.image1_path
  const image2Path = args.image2_path
  const focus = args.focus || 'differences'

  const prompts = {
    changes: 'Describe what has changed between the first and second image.',
    differences:
      'Compare these two images and describe all the differences you can find.',
    similarities:
      'Compare these two images and describe what they have in common.',
  }

  const [file1, file2] = await Promise.all([
    uploadFile(image1Path),
    uploadFile(image2Path),
  ])

  const result = await model.generateContent([
    prompts[focus] || prompts.differences,
    { fileData: { fileUri: file1.uri, mimeType: file1.mimeType } },
    { fileData: { fileUri: file2.uri, mimeType: file2.mimeType } },
  ])

  return result.response.text()
}

async function extractText(args) {
  const imagePath = args.image_path
  const format = args.format || 'plain'

  const prompts = {
    markdown:
      'Extract all text from this image and format it in markdown, preserving structure.',
    plain:
      'Extract and transcribe all text from this image. Return only the text, nothing else.',
    structured:
      'Extract all text from this image and organize it with clear sections and structure.',
  }

  const file = await uploadFile(imagePath)
  const result = await model.generateContent([
    prompts[format] || prompts.plain,
    { fileData: { fileUri: file.uri, mimeType: file.mimeType } },
  ])

  return result.response.text()
}

async function suggestFilename(args) {
  const imagePath = args.image_path
  const maxLength = args.max_length || 60
  const includeDate = args.include_date || false

  const prompt = `Analyze this image and suggest a descriptive filename for it. 
  Requirements:
  - Maximum ${maxLength} characters (not including extension)
  - Use title case with spaces (will be converted to hyphens)
  - Be specific and descriptive about the content
  - ${includeDate ? 'Include YYYY-MM-DD prefix if a date is visible in the image' : 'Do not include date prefix'}
  - Focus on the main subject or purpose of the image
  - For screenshots: include the application or website name
  - For diagrams: include the type and subject
  - For photos: include the subject and context
  - Return ONLY the filename suggestion, no explanation or extension`

  const file = await uploadFile(imagePath)
  const result = await model.generateContent([
    prompt,
    { fileData: { fileUri: file.uri, mimeType: file.mimeType } },
  ])

  // Clean up the suggestion and format it
  let suggestion = result.response.text().trim()
  // Remove any file extension if accidentally included
  suggestion = suggestion.replace(/\.(png|jpg|jpeg|gif|webp|pdf)$/i, '')
  // Replace spaces with hyphens
  suggestion = suggestion.replace(/\s+/g, ' ').replace(/ /g, ' - ')
  // Ensure it doesn't exceed max length
  if (suggestion.length > maxLength) {
    suggestion = suggestion.substring(0, maxLength).replace(/ - $/, '')
  }

  return suggestion
}

// Analyze video files or YouTube URLs
async function analyzeVideo(args) {
  const videoPath = args.video_path
  const youtubeUrl = args.youtube_url
  const prompt =
    args.prompt ||
    'Summarize this video in detail, including key moments and any text or speech content'

  if (!videoPath && !youtubeUrl) {
    throw new Error('Either video_path or youtube_url is required')
  }

  if (videoPath && youtubeUrl) {
    throw new Error('Please provide either video_path or youtube_url, not both')
  }

  let fileData

  if (youtubeUrl) {
    // YouTube URLs can be passed directly to the API
    fileData = { fileUri: youtubeUrl }
  } else {
    // Upload local video file
    const file = await uploadFile(videoPath)
    fileData = { fileUri: file.uri, mimeType: file.mimeType }
  }

  const result = await model.generateContent([prompt, { fileData }])

  return result.response.text()
}

// Create MCP server
const server = new Server(
  { name: 'gemini-vision', version: '1.0.0' },
  { capabilities: { tools: {} } },
)

// List available tools
server.setRequestHandler(ListToolsRequestSchema, async () => ({
  tools: [
    {
      description:
        'Analyze an image - transcribe text, describe content, or answer questions',
      inputSchema: {
        properties: {
          image_path: { description: 'Path to the image file', type: 'string' },
          prompt: {
            default: 'Describe this image',
            description: 'What to do with the image',
            type: 'string',
          },
        },
        required: ['image_path'],
        type: 'object',
      },
      name: 'analyze_image',
    },
    {
      description: 'Analyze multiple images at once',
      inputSchema: {
        properties: {
          image_paths: {
            description: 'List of image paths',
            items: { type: 'string' },
            type: 'array',
          },
          prompt: {
            default: 'Analyze these images',
            description: 'What to do with the images',
            type: 'string',
          },
        },
        required: ['image_paths'],
        type: 'object',
      },
      name: 'analyze_multiple',
    },
    {
      description: 'Extract and transcribe all text from an image (OCR)',
      inputSchema: {
        properties: {
          format: {
            default: 'plain',
            enum: ['plain', 'markdown', 'structured'],
            type: 'string',
          },
          image_path: { description: 'Path to the image file', type: 'string' },
        },
        required: ['image_path'],
        type: 'object',
      },
      name: 'extract_text',
    },
    {
      description:
        'Compare two images and describe differences or similarities',
      inputSchema: {
        properties: {
          focus: {
            default: 'differences',
            enum: ['differences', 'similarities', 'changes'],
            type: 'string',
          },
          image1_path: { description: 'Path to first image', type: 'string' },
          image2_path: { description: 'Path to second image', type: 'string' },
        },
        required: ['image1_path', 'image2_path'],
        type: 'object',
      },
      name: 'compare_images',
    },
    {
      description:
        'Analyze an image and suggest a descriptive filename (without extension)',
      inputSchema: {
        properties: {
          image_path: { description: 'Path to the image file', type: 'string' },
          include_date: {
            default: false,
            description: 'Include date prefix in suggestion',
            type: 'boolean',
          },
          max_length: {
            default: 60,
            description: 'Maximum filename length',
            type: 'number',
          },
        },
        required: ['image_path'],
        type: 'object',
      },
      name: 'suggest_image_filename',
    },
    {
      description:
        'Analyze video files or YouTube URLs - extract content, summarize, transcribe speech, identify objects/text. Provide either video_path OR youtube_url',
      inputSchema: {
        properties: {
          prompt: {
            default: 'Summarize this video in detail',
            description: 'What to analyze in the video',
            type: 'string',
          },
          video_path: {
            description: 'Path to local video file (MP4, AVI, MOV, etc.)',
            type: 'string',
          },
          youtube_url: {
            description:
              'YouTube video URL (e.g., https://www.youtube.com/watch?v=...)',
            type: 'string',
          },
        },
        required: [],
        type: 'object',
      },
      name: 'analyze_video',
    },
    {
      description:
        'Analyze a PDF or document with custom prompts - extract specific information, find mentions of topics, summarize sections, etc.',
      inputSchema: {
        properties: {
          document_path: {
            description:
              'Path to the document file (PDF, DOC, DOCX, ODT, RTF, TXT)',
            type: 'string',
          },
          prompt: {
            default:
              'Analyze this document and provide a comprehensive summary',
            description: 'What to analyze or extract from the document',
            type: 'string',
          },
        },
        required: ['document_path'],
        type: 'object',
      },
      name: 'analyze_document',
    },
  ],
}))

// Handle tool calls
server.setRequestHandler(CallToolRequestSchema, async (request) => {
  const { arguments: args, name } = request.params

  try {
    let result
    switch (name) {
      case 'analyze_document':
        result = await analyzeDocument(args)
        break
      case 'analyze_image':
        result = await analyzeImage(args)
        break
      case 'analyze_multiple':
        result = await analyzeMultiple(args)
        break
      case 'analyze_video':
        result = await analyzeVideo(args)
        break
      case 'compare_images':
        result = await compareImages(args)
        break
      case 'extract_text':
        result = await extractText(args)
        break
      case 'suggest_image_filename':
        result = await suggestFilename(args)
        break
      default:
        throw new Error(`Unknown tool: ${name}`)
    }

    return {
      content: [{ text: result, type: 'text' }],
    }
  } catch (error) {
    throw new Error(`Tool execution failed: ${error.message}`)
  }
})

// Start server
async function main() {
  const transport = new StdioServerTransport()
  await server.connect(transport)
  console.error('🚀 Gemini Vision MCP Server running')
}

main().catch(console.error)