vault backup: 2026-02-26 21:10:47
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
@@ -62,29 +62,33 @@ def _sample_head_mid_tail(content: bytes, span: int = 1200) -> str:
|
||||
return sampled.decode('utf-8', errors='ignore').lower()
|
||||
|
||||
|
||||
def is_excluded(file_path: Path) -> bool:
|
||||
def is_excluded(file_path: Path) -> str | None:
|
||||
"""返回排除原因字符串,未排除则返回 None。"""
|
||||
rel = safe_rel(file_path)
|
||||
if rel is None:
|
||||
return True
|
||||
return 'path:unresolvable'
|
||||
|
||||
parts = set(Path(rel).parts)
|
||||
if parts.intersection(EXCLUDE_DIR_NAMES):
|
||||
return True
|
||||
if parts.intersection(EXCLUDE_PATH_PARTS):
|
||||
return True
|
||||
if any(kw in file_path.name.lower() for kw in EXCLUDE_FILENAME_KEYWORDS):
|
||||
return True
|
||||
for name in parts.intersection(EXCLUDE_DIR_NAMES):
|
||||
return f'dir:{name}'
|
||||
for name in parts.intersection(EXCLUDE_PATH_PARTS):
|
||||
return f'path:{name}'
|
||||
for kw in EXCLUDE_FILENAME_KEYWORDS:
|
||||
if kw in file_path.name.lower():
|
||||
return f'filename:{kw}'
|
||||
|
||||
try:
|
||||
snippet = _sample_head_mid_tail(file_path.read_bytes())
|
||||
if any(marker in snippet for marker in SENSITIVE_LITERAL_MARKERS):
|
||||
return True
|
||||
if any(pattern.search(snippet) for pattern in SENSITIVE_REGEX_PATTERNS):
|
||||
return True
|
||||
for marker in SENSITIVE_LITERAL_MARKERS:
|
||||
if marker in snippet:
|
||||
return f'content:literal:{marker[:20]}'
|
||||
for pattern in SENSITIVE_REGEX_PATTERNS:
|
||||
if pattern.search(snippet):
|
||||
return f'content:regex:{pattern.pattern[:30]}'
|
||||
except Exception:
|
||||
return True
|
||||
return 'content:read_error'
|
||||
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -92,11 +96,20 @@ def index_lock():
|
||||
lock_file = Path(os.getenv('INDEX_LOCK_FILE', str(VAULT_ROOT / '.memory-index.lock')))
|
||||
lock_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(lock_file, 'w', encoding='utf-8') as fh:
|
||||
fcntl.flock(fh, fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(fh, fcntl.LOCK_UN)
|
||||
if sys.platform == 'win32':
|
||||
import msvcrt
|
||||
msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK, 1)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl as _fcntl
|
||||
_fcntl.flock(fh, _fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_fcntl.flock(fh, _fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def embed_text(text: str) -> list[float]:
|
||||
@@ -118,7 +131,7 @@ def embed_text(text: str) -> list[float]:
|
||||
if title:
|
||||
headers['X-OpenRouter-Title'] = title
|
||||
|
||||
payload = {'model': model, 'input': text}
|
||||
payload = {'model': model, 'input': text, 'encoding_format': 'float'}
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, 4):
|
||||
|
||||
Reference in New Issue
Block a user