Noah BrierandClaude
5b00b25240
- Fix transcript-extract.sh: `00 Inbox/Clippings` → `00_Inbox/Clippings/` - Fix firecrawl-batch.sh: `00 Inbox/Clippings` → `00_Inbox/Clippings/` - Add `-o|--output-dir` flag to firecrawl-batch.sh for custom paths - Fix update-attachment-links.js: `05 Attachments` → `05_Attachments` - Fix fix-renamed-links.js: `05 Attachments` → `05_Attachments` - Fix GEMINI_VISION_QUICK_START.md example paths - Update README.md with new flag documentation All folder paths now match actual repo structure (underscores not spaces). This prevents scripts from failing due to incorrect directory references. Fixes #9 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
133 lines
3.3 KiB
Bash
Executable File
133 lines
3.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Firecrawl batch scraper script
|
|
# Usage: ./firecrawl-batch.sh [-o|--output-dir <dir>] <url1> <url2> ...
|
|
# Automatically generates filenames based on page titles and dates
|
|
# Requires: FIRECRAWL_API_KEY environment variable
|
|
|
|
# Default output directory
|
|
OUTPUT_DIR="00_Inbox/Clippings/"
|
|
URLS=()
|
|
|
|
# Parse arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
-o|--output-dir)
|
|
OUTPUT_DIR="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
URLS+=("$1")
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ ${#URLS[@]} -eq 0 ]; then
|
|
echo "Usage: $0 [-o|--output-dir <dir>] <url1> <url2> ..."
|
|
echo "Default output directory: 00_Inbox/Clippings/"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " -o, --output-dir <dir> Specify custom output directory"
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$FIRECRAWL_API_KEY" ]; then
|
|
echo "Error: FIRECRAWL_API_KEY environment variable not set"
|
|
exit 1
|
|
fi
|
|
|
|
# Get today's date
|
|
TODAY=$(date +"%Y-%m-%d")
|
|
CLIPPINGS_DIR="$OUTPUT_DIR"
|
|
|
|
# Create clippings directory if it doesn't exist
|
|
mkdir -p "$CLIPPINGS_DIR"
|
|
|
|
# Function to sanitize filename
|
|
sanitize_filename() {
|
|
echo "$1" | sed 's/[^a-zA-Z0-9 -]//g' | sed 's/ \+/ /g' | sed 's/^ *//;s/ *$//'
|
|
}
|
|
|
|
# Function to extract domain name for fallback
|
|
get_domain() {
|
|
echo "$1" | sed -E 's|https?://([^/]+).*|\1|' | sed 's/www\.//'
|
|
}
|
|
|
|
# Counter for successful scrapes
|
|
SUCCESS_COUNT=0
|
|
FAIL_COUNT=0
|
|
|
|
# Process each URL
|
|
for URL in "${URLS[@]}"; do
|
|
echo "Processing: $URL"
|
|
|
|
# Make the API call and save to temp file
|
|
TEMP_FILE=$(mktemp)
|
|
|
|
curl -s -X POST https://api.firecrawl.dev/v1/scrape \
|
|
-H "Authorization: Bearer $FIRECRAWL_API_KEY" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"url\": \"$URL\",
|
|
\"formats\": [\"markdown\"],
|
|
\"onlyMainContent\": true
|
|
}" > "$TEMP_FILE"
|
|
|
|
# Extract markdown and title
|
|
MARKDOWN=$(jq -r '.data.markdown // empty' "$TEMP_FILE")
|
|
TITLE=$(jq -r '.data.metadata.title // empty' "$TEMP_FILE")
|
|
|
|
# If no title, try to extract from markdown or use domain
|
|
if [ -z "$TITLE" ] || [ "$TITLE" = "null" ]; then
|
|
# Try to get first heading from markdown
|
|
TITLE=$(echo "$MARKDOWN" | grep -m1 '^# ' | sed 's/^# //')
|
|
|
|
# If still no title, use domain
|
|
if [ -z "$TITLE" ]; then
|
|
TITLE=$(get_domain "$URL")
|
|
fi
|
|
fi
|
|
|
|
# Sanitize title for filename
|
|
SAFE_TITLE=$(sanitize_filename "$TITLE")
|
|
|
|
# Truncate title if too long
|
|
if [ ${#SAFE_TITLE} -gt 60 ]; then
|
|
SAFE_TITLE="${SAFE_TITLE:0:60}"
|
|
fi
|
|
|
|
# Create filename
|
|
OUTPUT_FILE="$CLIPPINGS_DIR/$TODAY - $SAFE_TITLE.md"
|
|
|
|
# Check if we got content
|
|
if [ -n "$MARKDOWN" ] && [ "$MARKDOWN" != "null" ]; then
|
|
# Add metadata header
|
|
{
|
|
echo "---"
|
|
echo "source: $URL"
|
|
echo "date: $TODAY"
|
|
echo "title: \"$TITLE\""
|
|
echo "---"
|
|
echo ""
|
|
echo "$MARKDOWN"
|
|
} > "$OUTPUT_FILE"
|
|
|
|
echo " ✓ Saved to: $OUTPUT_FILE"
|
|
((SUCCESS_COUNT++))
|
|
else
|
|
echo " ✗ Failed to scrape content"
|
|
((FAIL_COUNT++))
|
|
fi
|
|
|
|
# Clean up temp file
|
|
rm -f "$TEMP_FILE"
|
|
|
|
# Small delay to be nice to the API
|
|
sleep 1
|
|
done
|
|
|
|
echo ""
|
|
echo "Batch scraping complete!"
|
|
echo " Successful: $SUCCESS_COUNT"
|
|
echo " Failed: $FAIL_COUNT" |