Noah Brier
f609668172
Claude Code + Obsidian starter kit for AI-powered knowledge management. Features: - PARA method folder structure - Bootstrap initialization system - Pre-configured Claude Code commands and agents - Gemini Vision MCP server with video support - Helper scripts for vault management - Automated release management See README.md for setup instructions.
115 lines
3.0 KiB
Bash
Executable File
115 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Firecrawl batch scraper script
|
|
# Usage: ./firecrawl-batch.sh <url1> <url2> ...
|
|
# Automatically generates filenames based on page titles and dates
|
|
# Requires: FIRECRAWL_API_KEY environment variable
|
|
|
|
# Source .zshrc to get the API key
|
|
source ~/.zshrc
|
|
|
|
if [ $# -eq 0 ]; then
|
|
echo "Usage: $0 <url1> <url2> ..."
|
|
echo "Scrapes multiple URLs and saves them to 00 Inbox/Clippings/"
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$FIRECRAWL_API_KEY" ]; then
|
|
echo "Error: FIRECRAWL_API_KEY environment variable not set"
|
|
exit 1
|
|
fi
|
|
|
|
# Get today's date
|
|
TODAY=$(date +"%Y-%m-%d")
|
|
CLIPPINGS_DIR="00 Inbox/Clippings"
|
|
|
|
# Create clippings directory if it doesn't exist
|
|
mkdir -p "$CLIPPINGS_DIR"
|
|
|
|
# Function to sanitize filename
|
|
sanitize_filename() {
|
|
echo "$1" | sed 's/[^a-zA-Z0-9 -]//g' | sed 's/ \+/ /g' | sed 's/^ *//;s/ *$//'
|
|
}
|
|
|
|
# Function to extract domain name for fallback
|
|
get_domain() {
|
|
echo "$1" | sed -E 's|https?://([^/]+).*|\1|' | sed 's/www\.//'
|
|
}
|
|
|
|
# Counter for successful scrapes
|
|
SUCCESS_COUNT=0
|
|
FAIL_COUNT=0
|
|
|
|
# Process each URL
|
|
for URL in "$@"; do
|
|
echo "Processing: $URL"
|
|
|
|
# Make the API call and save to temp file
|
|
TEMP_FILE=$(mktemp)
|
|
|
|
curl -s -X POST https://api.firecrawl.dev/v1/scrape \
|
|
-H "Authorization: Bearer $FIRECRAWL_API_KEY" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"url\": \"$URL\",
|
|
\"formats\": [\"markdown\"],
|
|
\"onlyMainContent\": true
|
|
}" > "$TEMP_FILE"
|
|
|
|
# Extract markdown and title
|
|
MARKDOWN=$(jq -r '.data.markdown // empty' "$TEMP_FILE")
|
|
TITLE=$(jq -r '.data.metadata.title // empty' "$TEMP_FILE")
|
|
|
|
# If no title, try to extract from markdown or use domain
|
|
if [ -z "$TITLE" ] || [ "$TITLE" = "null" ]; then
|
|
# Try to get first heading from markdown
|
|
TITLE=$(echo "$MARKDOWN" | grep -m1 '^# ' | sed 's/^# //')
|
|
|
|
# If still no title, use domain
|
|
if [ -z "$TITLE" ]; then
|
|
TITLE=$(get_domain "$URL")
|
|
fi
|
|
fi
|
|
|
|
# Sanitize title for filename
|
|
SAFE_TITLE=$(sanitize_filename "$TITLE")
|
|
|
|
# Truncate title if too long
|
|
if [ ${#SAFE_TITLE} -gt 60 ]; then
|
|
SAFE_TITLE="${SAFE_TITLE:0:60}"
|
|
fi
|
|
|
|
# Create filename
|
|
OUTPUT_FILE="$CLIPPINGS_DIR/$TODAY - $SAFE_TITLE.md"
|
|
|
|
# Check if we got content
|
|
if [ -n "$MARKDOWN" ] && [ "$MARKDOWN" != "null" ]; then
|
|
# Add metadata header
|
|
{
|
|
echo "---"
|
|
echo "source: $URL"
|
|
echo "date: $TODAY"
|
|
echo "title: \"$TITLE\""
|
|
echo "---"
|
|
echo ""
|
|
echo "$MARKDOWN"
|
|
} > "$OUTPUT_FILE"
|
|
|
|
echo " ✓ Saved to: $OUTPUT_FILE"
|
|
((SUCCESS_COUNT++))
|
|
else
|
|
echo " ✗ Failed to scrape content"
|
|
((FAIL_COUNT++))
|
|
fi
|
|
|
|
# Clean up temp file
|
|
rm -f "$TEMP_FILE"
|
|
|
|
# Small delay to be nice to the API
|
|
sleep 1
|
|
done
|
|
|
|
echo ""
|
|
echo "Batch scraping complete!"
|
|
echo " Successful: $SUCCESS_COUNT"
|
|
echo " Failed: $FAIL_COUNT" |