Initial commit - Claudesidian v0.2.0
Claude Code + Obsidian starter kit for AI-powered knowledge management. Features: - PARA method folder structure - Bootstrap initialization system - Pre-configured Claude Code commands and agents - Gemini Vision MCP server with video support - Helper scripts for vault management - Automated release management See README.md for setup instructions.
This commit is contained in:
Noah Brier
@@ -0,0 +1,84 @@
|
||||
# Scripts Directory
|
||||
|
||||
Helper scripts for vault automation and web content capture.
|
||||
|
||||
## Available Scripts
|
||||
|
||||
### Attachment Management
|
||||
These are primarily called via npm/pnpm commands in package.json:
|
||||
- `update-attachment-links.js` - Updates note links after moving attachments
|
||||
- `fix-renamed-links.js` - Fixes links after renaming files
|
||||
|
||||
### Web Content Capture
|
||||
**Note**: These scripts require API keys to function:
|
||||
|
||||
#### firecrawl-scrape.sh
|
||||
Scrapes a single URL and saves as markdown.
|
||||
```bash
|
||||
# Requires FIRECRAWL_API_KEY environment variable
|
||||
.scripts/firecrawl-scrape.sh <url> <output_file>
|
||||
```
|
||||
|
||||
#### firecrawl-batch.sh
|
||||
Scrapes multiple URLs and auto-generates filenames.
|
||||
```bash
|
||||
# Requires FIRECRAWL_API_KEY environment variable
|
||||
.scripts/firecrawl-batch.sh <url1> <url2> <url3>
|
||||
# Files saved to 00_Inbox/Clippings/
|
||||
```
|
||||
|
||||
### Transcript Extraction
|
||||
#### transcript-extract.sh
|
||||
Extracts transcripts from YouTube videos.
|
||||
```bash
|
||||
.scripts/transcript-extract.sh <youtube-url>
|
||||
```
|
||||
|
||||
## NPM Scripts
|
||||
|
||||
Run these from the vault root with `pnpm`:
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `attachments:list` | Show first 20 unprocessed attachments |
|
||||
| `attachments:count` | Count unprocessed attachments |
|
||||
| `attachments:organized` | Count files in Organized folder |
|
||||
| `attachments:unprocessed` | Same as count |
|
||||
| `attachments:refs <file>` | Find references to a specific file |
|
||||
| `attachments:sizes` | Show 20 largest attachment files |
|
||||
| `attachments:orphans` | Find unreferenced attachments |
|
||||
| `attachments:recent` | Show files added in last 7 days |
|
||||
| `attachments:create-organized` | Create the Organized subfolder |
|
||||
|
||||
## Setup Requirements
|
||||
|
||||
### For Web Scraping
|
||||
1. Get a Firecrawl API key from [firecrawl.dev](https://firecrawl.dev)
|
||||
2. Add to your shell profile:
|
||||
```bash
|
||||
export FIRECRAWL_API_KEY="your-key-here"
|
||||
```
|
||||
|
||||
### For Transcript Extraction
|
||||
- Requires `yt-dlp` and `jq` installed:
|
||||
```bash
|
||||
# macOS
|
||||
brew install yt-dlp jq
|
||||
|
||||
# Linux
|
||||
apt-get install yt-dlp jq
|
||||
```
|
||||
|
||||
## Adding Custom Scripts
|
||||
|
||||
1. Create script in `.scripts/`
|
||||
2. Make it executable: `chmod +x .scripts/your-script.sh`
|
||||
3. Add npm script to `package.json` if needed
|
||||
4. Document here
|
||||
|
||||
## Notes
|
||||
|
||||
- Scripts assume Unix-like environment (macOS/Linux)
|
||||
- Windows users may need WSL or Git Bash
|
||||
- All paths are relative to vault root
|
||||
- Check script comments for additional requirements
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Firecrawl batch scraper script
|
||||
# Usage: ./firecrawl-batch.sh <url1> <url2> ...
|
||||
# Automatically generates filenames based on page titles and dates
|
||||
# Requires: FIRECRAWL_API_KEY environment variable
|
||||
|
||||
# Source .zshrc to get the API key
|
||||
source ~/.zshrc
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: $0 <url1> <url2> ..."
|
||||
echo "Scrapes multiple URLs and saves them to 00 Inbox/Clippings/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$FIRECRAWL_API_KEY" ]; then
|
||||
echo "Error: FIRECRAWL_API_KEY environment variable not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get today's date
|
||||
TODAY=$(date +"%Y-%m-%d")
|
||||
CLIPPINGS_DIR="00 Inbox/Clippings"
|
||||
|
||||
# Create clippings directory if it doesn't exist
|
||||
mkdir -p "$CLIPPINGS_DIR"
|
||||
|
||||
# Function to sanitize filename
|
||||
sanitize_filename() {
|
||||
echo "$1" | sed 's/[^a-zA-Z0-9 -]//g' | sed 's/ \+/ /g' | sed 's/^ *//;s/ *$//'
|
||||
}
|
||||
|
||||
# Function to extract domain name for fallback
|
||||
get_domain() {
|
||||
echo "$1" | sed -E 's|https?://([^/]+).*|\1|' | sed 's/www\.//'
|
||||
}
|
||||
|
||||
# Counter for successful scrapes
|
||||
SUCCESS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
|
||||
# Process each URL
|
||||
for URL in "$@"; do
|
||||
echo "Processing: $URL"
|
||||
|
||||
# Make the API call and save to temp file
|
||||
TEMP_FILE=$(mktemp)
|
||||
|
||||
curl -s -X POST https://api.firecrawl.dev/v1/scrape \
|
||||
-H "Authorization: Bearer $FIRECRAWL_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"url\": \"$URL\",
|
||||
\"formats\": [\"markdown\"],
|
||||
\"onlyMainContent\": true
|
||||
}" > "$TEMP_FILE"
|
||||
|
||||
# Extract markdown and title
|
||||
MARKDOWN=$(jq -r '.data.markdown // empty' "$TEMP_FILE")
|
||||
TITLE=$(jq -r '.data.metadata.title // empty' "$TEMP_FILE")
|
||||
|
||||
# If no title, try to extract from markdown or use domain
|
||||
if [ -z "$TITLE" ] || [ "$TITLE" = "null" ]; then
|
||||
# Try to get first heading from markdown
|
||||
TITLE=$(echo "$MARKDOWN" | grep -m1 '^# ' | sed 's/^# //')
|
||||
|
||||
# If still no title, use domain
|
||||
if [ -z "$TITLE" ]; then
|
||||
TITLE=$(get_domain "$URL")
|
||||
fi
|
||||
fi
|
||||
|
||||
# Sanitize title for filename
|
||||
SAFE_TITLE=$(sanitize_filename "$TITLE")
|
||||
|
||||
# Truncate title if too long
|
||||
if [ ${#SAFE_TITLE} -gt 60 ]; then
|
||||
SAFE_TITLE="${SAFE_TITLE:0:60}"
|
||||
fi
|
||||
|
||||
# Create filename
|
||||
OUTPUT_FILE="$CLIPPINGS_DIR/$TODAY - $SAFE_TITLE.md"
|
||||
|
||||
# Check if we got content
|
||||
if [ -n "$MARKDOWN" ] && [ "$MARKDOWN" != "null" ]; then
|
||||
# Add metadata header
|
||||
{
|
||||
echo "---"
|
||||
echo "source: $URL"
|
||||
echo "date: $TODAY"
|
||||
echo "title: \"$TITLE\""
|
||||
echo "---"
|
||||
echo ""
|
||||
echo "$MARKDOWN"
|
||||
} > "$OUTPUT_FILE"
|
||||
|
||||
echo " ✓ Saved to: $OUTPUT_FILE"
|
||||
((SUCCESS_COUNT++))
|
||||
else
|
||||
echo " ✗ Failed to scrape content"
|
||||
((FAIL_COUNT++))
|
||||
fi
|
||||
|
||||
# Clean up temp file
|
||||
rm -f "$TEMP_FILE"
|
||||
|
||||
# Small delay to be nice to the API
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Batch scraping complete!"
|
||||
echo " Successful: $SUCCESS_COUNT"
|
||||
echo " Failed: $FAIL_COUNT"
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Firecrawl scraper script that saves directly to file
|
||||
# Usage: ./firecrawl-scrape.sh <url> <output_file>
|
||||
# Requires: FIRECRAWL_API_KEY environment variable
|
||||
|
||||
URL="$1"
|
||||
OUTPUT_FILE="$2"
|
||||
|
||||
if [ -z "$URL" ] || [ -z "$OUTPUT_FILE" ]; then
|
||||
echo "Usage: $0 <url> <output_file>"
|
||||
echo "Requires FIRECRAWL_API_KEY environment variable to be set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$FIRECRAWL_API_KEY" ]; then
|
||||
echo "Error: FIRECRAWL_API_KEY environment variable not set"
|
||||
echo "Export it in your shell profile or run:"
|
||||
echo " export FIRECRAWL_API_KEY='your-api-key'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Make the API call and extract markdown using jq, save directly to file
|
||||
curl -s -X POST https://api.firecrawl.dev/v1/scrape \
|
||||
-H "Authorization: Bearer $FIRECRAWL_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"url\": \"$URL\",
|
||||
\"formats\": [\"markdown\"],
|
||||
\"onlyMainContent\": true
|
||||
}" | jq -r '.data.markdown // empty' > "$OUTPUT_FILE"
|
||||
|
||||
# Check if file was created and has content
|
||||
if [ -s "$OUTPUT_FILE" ]; then
|
||||
echo "✓ Scraped content saved to: $OUTPUT_FILE"
|
||||
else
|
||||
echo "✗ Failed to scrape content"
|
||||
rm -f "$OUTPUT_FILE" # Remove empty file
|
||||
exit 1
|
||||
fi
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Fixes attachment links after files have been renamed
|
||||
* Usage: node .scripts/fix-renamed-links.js <old-name> <new-name>
|
||||
*
|
||||
* This properly handles the case where files are renamed, not just moved
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length !== 2) {
|
||||
console.log('Usage: node .scripts/fix-renamed-links.js <old-filename> <new-filename>');
|
||||
console.log('Example: node .scripts/fix-renamed-links.js "CleanShot 2025-01-01.png" "Project Screenshot.png"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const [oldName, newName] = args;
|
||||
const newPath = `05 Attachments/Organized/${newName}`;
|
||||
|
||||
console.log(`Fixing links: ${oldName} → ${newName}`);
|
||||
|
||||
// Function to walk directory
|
||||
function walkDir(dir, callback) {
|
||||
fs.readdirSync(dir).forEach(f => {
|
||||
const dirPath = path.join(dir, f);
|
||||
const isDirectory = fs.statSync(dirPath).isDirectory();
|
||||
if (isDirectory) {
|
||||
// Skip node_modules, .git
|
||||
if (!f.includes('node_modules') && !f.includes('.git')) {
|
||||
walkDir(dirPath, callback);
|
||||
}
|
||||
} else {
|
||||
callback(path.join(dir, f));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Process all markdown files
|
||||
let updatedCount = 0;
|
||||
const updatedFiles = [];
|
||||
|
||||
walkDir('.', (filepath) => {
|
||||
if (filepath.endsWith('.md')) {
|
||||
let content = fs.readFileSync(filepath, 'utf8');
|
||||
const originalContent = content;
|
||||
|
||||
// Escape special regex characters in filename
|
||||
const escapedOld = oldName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
// Pattern 1: ![[oldname]] without path
|
||||
const pattern1 = new RegExp(`!\\[\\[${escapedOld}\\]\\]`, 'g');
|
||||
content = content.replace(pattern1, `![[${newPath}]]`);
|
||||
|
||||
// Pattern 2: ![[05 Attachments/oldname]]
|
||||
const pattern2 = new RegExp(`!\\[\\[05 Attachments/${escapedOld}\\]\\]`, 'g');
|
||||
content = content.replace(pattern2, `![[${newPath}]]`);
|
||||
|
||||
// Pattern 3: [[oldname]] without ! (for PDFs and other non-embedded)
|
||||
const pattern3 = new RegExp(`(?<!!)\\[\\[${escapedOld}\\]\\]`, 'g');
|
||||
content = content.replace(pattern3, `[[${newPath}]]`);
|
||||
|
||||
// Pattern 4: [[05 Attachments/oldname]] without !
|
||||
const pattern4 = new RegExp(`(?<!!)\\[\\[05 Attachments/${escapedOld}\\]\\]`, 'g');
|
||||
content = content.replace(pattern4, `[[${newPath}]]`);
|
||||
|
||||
// Write back if changed
|
||||
if (content !== originalContent) {
|
||||
fs.writeFileSync(filepath, content, 'utf8');
|
||||
updatedFiles.push(filepath);
|
||||
updatedCount++;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Report results
|
||||
if (updatedCount > 0) {
|
||||
console.log(`\nUpdated ${updatedCount} files:`);
|
||||
updatedFiles.forEach(file => console.log(` - ${file}`));
|
||||
} else {
|
||||
console.log('\nNo files needed updating');
|
||||
}
|
||||
|
||||
console.log('\nDone!');
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Transcript extraction script for YouTube videos
|
||||
# Usage: .scripts/transcript-extract.sh <youtube-url> [output-path]
|
||||
# Default output: 00 Inbox/Clippings/
|
||||
|
||||
set -e
|
||||
|
||||
URL="$1"
|
||||
OUTPUT_PATH="${2:-00 Inbox/Clippings}"
|
||||
|
||||
if [ -z "$URL" ]; then
|
||||
echo "Usage: $0 <youtube-url> [output-path]"
|
||||
echo "Default output path: 00 Inbox/Clippings/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "🔍 Extracting transcript from: $URL"
|
||||
echo "📁 Output path: $OUTPUT_PATH"
|
||||
|
||||
# Create output directory if it doesn't exist
|
||||
mkdir -p "$OUTPUT_PATH"
|
||||
|
||||
# Extract video ID and title
|
||||
VIDEO_ID=$(yt-dlp --get-id "$URL")
|
||||
TITLE=$(yt-dlp --get-title "$URL")
|
||||
SAFE_TITLE=$(echo "$TITLE" | sed 's/[^a-zA-Z0-9 -]//g' | sed 's/ */ /g' | cut -c1-80)
|
||||
DATE=$(date +%Y-%m-%d)
|
||||
|
||||
echo "📹 Video: $TITLE"
|
||||
echo "🆔 Video ID: $VIDEO_ID"
|
||||
|
||||
# Try to extract captions first (fastest method)
|
||||
echo "🎯 Attempting to extract captions..."
|
||||
if yt-dlp --skip-download --write-subs --write-auto-subs --sub-langs 'en.*' --sub-format json3 -o '%(id)s.%(ext)s' "$URL"; then
|
||||
echo "✅ Captions extracted successfully"
|
||||
|
||||
# Convert to markdown
|
||||
FILENAME="$OUTPUT_PATH/$DATE - $SAFE_TITLE - Transcript.md"
|
||||
|
||||
cat > "$FILENAME" << EOF
|
||||
# $TITLE - Transcript
|
||||
|
||||
**Source:** $URL
|
||||
**Title:** $TITLE
|
||||
**Video ID:** $VIDEO_ID
|
||||
**Extracted:** $DATE
|
||||
**Method:** YouTube captions via yt-dlp
|
||||
|
||||
---
|
||||
|
||||
EOF
|
||||
|
||||
# Process JSON3 captions to clean text
|
||||
jq -r '.events[] | select(.segs) | .segs | map(.utf8) | join("")' *.json3 | \
|
||||
sed -E 's/\s+/ /g; s/♪//g; s/^\s*//; s/\s*$//' | \
|
||||
grep -v '^$' >> "$FILENAME"
|
||||
|
||||
# Cleanup temporary files
|
||||
rm -f *.json3
|
||||
|
||||
echo "✅ Transcript saved to: $FILENAME"
|
||||
echo "📝 $(wc -l < "$FILENAME") lines extracted"
|
||||
|
||||
else
|
||||
echo "❌ No captions available for this video"
|
||||
echo "💡 You could try manual transcription tools if needed"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Updates attachment links in markdown files after files are moved to Organized folder
|
||||
* Usage: node .scripts/update-attachment-links.js [specific-file.ext]
|
||||
*
|
||||
* If no argument provided, updates all files in Organized folder
|
||||
* If specific filename provided, only updates references to that file
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
const organizedDir = '05 Attachments/Organized';
|
||||
const args = process.argv.slice(2);
|
||||
const specificFile = args[0];
|
||||
|
||||
// Get list of files to update references for
|
||||
let filesToUpdate = [];
|
||||
|
||||
if (specificFile) {
|
||||
// Update references for a specific file
|
||||
filesToUpdate = [specificFile];
|
||||
console.log(`Updating references for: ${specificFile}`);
|
||||
} else {
|
||||
// Update references for all files in Organized folder
|
||||
if (fs.existsSync(organizedDir)) {
|
||||
filesToUpdate = fs.readdirSync(organizedDir);
|
||||
console.log(`Found ${filesToUpdate.length} files in Organized folder`);
|
||||
} else {
|
||||
console.log('Organized folder does not exist yet');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Function to walk directory
|
||||
function walkDir(dir, callback) {
|
||||
fs.readdirSync(dir).forEach(f => {
|
||||
const dirPath = path.join(dir, f);
|
||||
const isDirectory = fs.statSync(dirPath).isDirectory();
|
||||
if (isDirectory) {
|
||||
// Skip node_modules, .git, and the Organized folder itself
|
||||
if (!f.includes('node_modules') && !f.includes('.git') && dirPath !== organizedDir) {
|
||||
walkDir(dirPath, callback);
|
||||
}
|
||||
} else {
|
||||
callback(path.join(dir, f));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Process all markdown files
|
||||
let updatedCount = 0;
|
||||
const updatedFiles = [];
|
||||
|
||||
walkDir('.', (filepath) => {
|
||||
if (filepath.endsWith('.md')) {
|
||||
let content = fs.readFileSync(filepath, 'utf8');
|
||||
const originalContent = content;
|
||||
|
||||
// For each file to update, fix references
|
||||
filesToUpdate.forEach(filename => {
|
||||
// Escape special regex characters in filename
|
||||
const escapedFile = filename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
// Pattern 1: ![[filename]] without path
|
||||
const pattern1 = new RegExp(`!\\[\\[${escapedFile}\\]\\]`, 'g');
|
||||
content = content.replace(pattern1, `![[05 Attachments/Organized/${filename}]]`);
|
||||
|
||||
// Pattern 2: ![[05 Attachments/filename]] (file in root being moved)
|
||||
const pattern2 = new RegExp(`!\\[\\[05 Attachments/${escapedFile}\\]\\]`, 'g');
|
||||
content = content.replace(pattern2, `![[05 Attachments/Organized/${filename}]]`);
|
||||
|
||||
// Pattern 3: [[filename]] without ! (for PDFs and other non-embedded links)
|
||||
// Only if not already pointing to Organized
|
||||
const pattern3 = new RegExp(`\\[\\[${escapedFile}\\]\\]`, 'g');
|
||||
const pattern3Organized = new RegExp(`\\[\\[05 Attachments/Organized/${escapedFile}\\]\\]`, 'g');
|
||||
|
||||
// Only replace if not already pointing to Organized and not preceded by !
|
||||
if (!pattern3Organized.test(content)) {
|
||||
content = content.replace(pattern3, `[[05 Attachments/Organized/${filename}]]`);
|
||||
}
|
||||
|
||||
// Pattern 4: [[05 Attachments/filename]] without !
|
||||
const pattern4 = new RegExp(`\\[\\[05 Attachments/${escapedFile}\\]\\]`, 'g');
|
||||
content = content.replace(pattern4, `[[05 Attachments/Organized/${filename}]]`);
|
||||
});
|
||||
|
||||
// Write back if changed
|
||||
if (content !== originalContent) {
|
||||
fs.writeFileSync(filepath, content, 'utf8');
|
||||
updatedFiles.push(filepath);
|
||||
updatedCount++;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Report results
|
||||
if (updatedCount > 0) {
|
||||
console.log(`\nUpdated ${updatedCount} files:`);
|
||||
updatedFiles.forEach(file => console.log(` - ${file}`));
|
||||
} else {
|
||||
console.log('\nNo files needed updating');
|
||||
}
|
||||
|
||||
console.log('\nDone!');
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Vault Statistics Script
|
||||
# Shows basic stats about your Obsidian vault
|
||||
|
||||
echo "=== Vault Statistics ==="
|
||||
echo ""
|
||||
|
||||
echo "📝 Note Counts:"
|
||||
echo " Inbox: $(find 00_Inbox -name "*.md" 2>/dev/null | wc -l)"
|
||||
echo " Projects: $(find 01_Projects -name "*.md" 2>/dev/null | wc -l)"
|
||||
echo " Areas: $(find 02_Areas -name "*.md" 2>/dev/null | wc -l)"
|
||||
echo " Resources: $(find 03_Resources -name "*.md" 2>/dev/null | wc -l)"
|
||||
echo " Archive: $(find 04_Archive -name "*.md" 2>/dev/null | wc -l)"
|
||||
echo ""
|
||||
|
||||
echo "📎 Attachments:"
|
||||
echo " Total: $(find 05_Attachments -type f 2>/dev/null | wc -l)"
|
||||
echo " Organized: $(find 05_Attachments/Organized -type f 2>/dev/null | wc -l)"
|
||||
echo ""
|
||||
|
||||
echo "📊 Total Notes: $(find . -name "*.md" | wc -l)"
|
||||
echo ""
|
||||
|
||||
echo "🔄 Recent Activity (last 7 days):"
|
||||
find . -name "*.md" -mtime -7 -type f 2>/dev/null | head -5 | while read file; do
|
||||
echo " - $(basename "$file")"
|
||||
done
|
||||
Reference in New Issue
Block a user