vault backup: 2026-02-26 21:10:47

2026-02-26 21:10:47 +08:00
parent 1e3905d304
commit 410167d3b4
38 changed files with 3890 additions and 70 deletions
@@ -44,9 +44,9 @@ CLIPPINGS_DIR="$OUTPUT_DIR"
 # Create clippings directory if it doesn't exist
 mkdir -p "$CLIPPINGS_DIR"

-# Function to sanitize filename
+# Function to sanitize filename (preserve CJK characters, only remove filesystem-illegal chars)
 sanitize_filename() {
-    echo "$1" | sed 's/[^a-zA-Z0-9 -]//g' | sed 's/ \+/ /g' | sed 's/^ *//;s/ *$//'
+    echo "$1" | sed 's/[\/\\:*?"<>|]//g' | sed 's/ \+/ /g' | sed 's/^ *//;s/ *$//'
 }

 # Function to extract domain name for fallback
@@ -2,7 +2,13 @@
 set -euo pipefail

 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-VAULT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+ENV_FILE="$SCRIPT_DIR/.env.memory"
+if [[ -f "$ENV_FILE" ]]; then
+  VAULT_DIR="$(grep '^VAULT_DIR=' "$ENV_FILE" | cut -d= -f2- | tr -d '"' | tr -d "'")"
+fi
+if [[ -z "${VAULT_DIR:-}" ]]; then
+  VAULT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+fi
 QUERY="${*:-}"

 if [[ -z "$QUERY" ]]; then
@@ -14,8 +14,6 @@ SENSITIVE_LITERAL_MARKERS = [
    'aws_access_key_id',
    'password:',
    'passwd:',
-    'api_key',
-    'access_key',
    '账号：',
    '账号:',
    '用户名：',
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="$SCRIPT_DIR/../../.env.memory"
+[[ -f "$ENV_FILE" ]] && source <(grep -E '^[A-Z_]+=.+' "$ENV_FILE" | sed 's/^/export /')
+
+PASS=0
+FAIL=0
+
+check() {
+  local label="$1" result="$2"
+  if [[ "$result" == "ok" ]]; then
+    echo "✅ $label"
+    ((PASS++)) || true
+  else
+    echo "❌ $label: $result"
+    ((FAIL++)) || true
+  fi
+}
+
+# 1. .env.memory 存在且关键字段非空
+if [[ -f "$ENV_FILE" ]]; then
+  missing=""
+  for key in PG_DSN VAULT_DIR OPENROUTER_API_KEY OPENROUTER_EMBED_MODEL; do
+    val="$(grep "^${key}=" "$ENV_FILE" | cut -d= -f2- | tr -d '"' | tr -d "'" | xargs)"
+    [[ -z "$val" ]] && missing="$missing $key"
+  done
+  [[ -z "$missing" ]] && check ".env.memory 关键字段" "ok" || check ".env.memory 关键字段" "缺少:$missing"
+else
+  check ".env.memory 存在" "文件不存在: $ENV_FILE"
+fi
+
+# 2. PostgreSQL 可连接
+if command -v psql &>/dev/null; then
+  result="$(psql "$PG_DSN" -c "SELECT 1;" -t 2>&1 | xargs)"
+  [[ "$result" == "1" ]] && check "PostgreSQL 连通" "ok" || check "PostgreSQL 连通" "$result"
+elif command -v docker &>/dev/null; then
+  result="$(docker exec pgvector psql -U postgres -d memory -c "SELECT 1;" -t 2>&1 | xargs)"
+  [[ "$result" == "1" ]] && check "PostgreSQL 连通" "ok" || check "PostgreSQL 连通" "$result"
+else
+  check "PostgreSQL 连通" "psql/docker 均不可用"
+fi
+
+# 3. memory_primary 条数 > 0
+if command -v psql &>/dev/null; then
+  count="$(psql "$PG_DSN" -c "SELECT count(*) FROM memory_primary;" -t 2>&1 | xargs)"
+elif command -v docker &>/dev/null; then
+  count="$(docker exec pgvector psql -U postgres -d memory -c "SELECT count(*) FROM memory_primary;" -t 2>&1 | xargs)"
+else
+  count="0"
+fi
+[[ "$count" -gt 0 ]] 2>/dev/null && check "memory_primary 条数 ($count)" "ok" || check "memory_primary 条数" "为空或查询失败: $count"
+
+# 4. 最近 updated_at 在 24h 内
+if command -v psql &>/dev/null; then
+  fresh="$(psql "$PG_DSN" -c "SELECT count(*) FROM memory_primary WHERE updated_at > now() - interval '24 hours';" -t 2>&1 | xargs)"
+elif command -v docker &>/dev/null; then
+  fresh="$(docker exec pgvector psql -U postgres -d memory -c "SELECT count(*) FROM memory_primary WHERE updated_at > now() - interval '24 hours';" -t 2>&1 | xargs)"
+else
+  fresh="0"
+fi
+[[ "$fresh" -gt 0 ]] 2>/dev/null && check "24h 内有更新 ($fresh 条)" "ok" || check "24h 内有更新" "无近期更新"
+
+echo ""
+echo "结果: ${PASS} 通过 / ${FAIL} 失败"
+[[ "$FAIL" -eq 0 ]] && exit 0 || exit 1
@@ -14,6 +14,7 @@ from index_common import (
 )

 MIN_TEXT_LEN = 50
+PRIMARY_DIRS = {'01_Projects', '02_Areas'}


 def parse_changes(changes_file: Path) -> list[dict]:
@@ -35,11 +36,17 @@ def parse_changes(changes_file: Path) -> list[dict]:


 def upsert_file(cur, rel_path: str) -> None:
+    top_dir = Path(rel_path).parts[0] if Path(rel_path).parts else ''
+    if top_dir not in PRIMARY_DIRS:
+        print(f'[SKIPPED-OUT-OF-SCOPE] {rel_path}')
+        return
+
    abs_path = VAULT_ROOT / rel_path
    if not abs_path.exists() or abs_path.suffix.lower() != '.md':
        return

-    if is_excluded(abs_path):
+    reason = is_excluded(abs_path)
+    if reason:
        cur.execute('DELETE FROM memory_primary WHERE id=%s', (rel_path,))
        cur.execute(
            '''
@@ -49,7 +56,7 @@ def upsert_file(cur, rel_path: str) -> None:
              risk = EXCLUDED.risk,
              updated_at = now()
            ''',
-            (rel_path, rel_path, 'excluded_or_sensitive'),
+            (rel_path, rel_path, reason),
        )
        print(f'[QUARANTINED] {rel_path}')
        return
@@ -61,6 +68,13 @@ def upsert_file(cur, rel_path: str) -> None:
        print(f'[PRUNED] {rel_path}')
        return

+    new_hash = sha256_text(text)
+    cur.execute('SELECT content_hash FROM memory_primary WHERE id=%s', (rel_path,))
+    row = cur.fetchone()
+    if row and row[0] == new_hash:
+        print(f'[SKIPPED] {rel_path}')
+        return
+
    try:
        embedding = embed_text(text)
    except Exception as error:
@@ -1,8 +1,8 @@
 #!/usr/bin/env python3

-import fcntl
 import hashlib
 import os
+import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
@@ -62,29 +62,33 @@ def _sample_head_mid_tail(content: bytes, span: int = 1200) -> str:
    return sampled.decode('utf-8', errors='ignore').lower()


-def is_excluded(file_path: Path) -> bool:
+def is_excluded(file_path: Path) -> str | None:
+    """返回排除原因字符串，未排除则返回 None。"""
    rel = safe_rel(file_path)
    if rel is None:
-        return True
+        return 'path:unresolvable'

    parts = set(Path(rel).parts)
-    if parts.intersection(EXCLUDE_DIR_NAMES):
-        return True
-    if parts.intersection(EXCLUDE_PATH_PARTS):
-        return True
-    if any(kw in file_path.name.lower() for kw in EXCLUDE_FILENAME_KEYWORDS):
-        return True
+    for name in parts.intersection(EXCLUDE_DIR_NAMES):
+        return f'dir:{name}'
+    for name in parts.intersection(EXCLUDE_PATH_PARTS):
+        return f'path:{name}'
+    for kw in EXCLUDE_FILENAME_KEYWORDS:
+        if kw in file_path.name.lower():
+            return f'filename:{kw}'

    try:
        snippet = _sample_head_mid_tail(file_path.read_bytes())
-        if any(marker in snippet for marker in SENSITIVE_LITERAL_MARKERS):
-            return True
-        if any(pattern.search(snippet) for pattern in SENSITIVE_REGEX_PATTERNS):
-            return True
+        for marker in SENSITIVE_LITERAL_MARKERS:
+            if marker in snippet:
+                return f'content:literal:{marker[:20]}'
+        for pattern in SENSITIVE_REGEX_PATTERNS:
+            if pattern.search(snippet):
+                return f'content:regex:{pattern.pattern[:30]}'
    except Exception:
-        return True
+        return 'content:read_error'

-    return False
+    return None


@contextmanager
@@ -92,11 +96,20 @@ def index_lock():
    lock_file = Path(os.getenv('INDEX_LOCK_FILE', str(VAULT_ROOT / '.memory-index.lock')))
    lock_file.parent.mkdir(parents=True, exist_ok=True)
    with open(lock_file, 'w', encoding='utf-8') as fh:
-        fcntl.flock(fh, fcntl.LOCK_EX)
-        try:
-            yield
-        finally:
-            fcntl.flock(fh, fcntl.LOCK_UN)
+        if sys.platform == 'win32':
+            import msvcrt
+            msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK, 1)
+            try:
+                yield
+            finally:
+                msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)
+        else:
+            import fcntl as _fcntl
+            _fcntl.flock(fh, _fcntl.LOCK_EX)
+            try:
+                yield
+            finally:
+                _fcntl.flock(fh, _fcntl.LOCK_UN)


 def embed_text(text: str) -> list[float]:
@@ -118,7 +131,7 @@ def embed_text(text: str) -> list[float]:
    if title:
        headers['X-OpenRouter-Title'] = title

-    payload = {'model': model, 'input': text}
+    payload = {'model': model, 'input': text, 'encoding_format': 'float'}
    last_error: Exception | None = None

    for attempt in range(1, 4):
@@ -49,7 +49,8 @@ def run() -> None:

                    for md_file in target.rglob('*.md'):
                        rel = normalize_rel(md_file)
-                        if is_excluded(md_file):
+                        reason = is_excluded(md_file)
+                        if reason:
                            secure_ids.add(rel)
                            cur.execute(
                                '''
@@ -59,7 +60,7 @@ def run() -> None:
                                  risk = EXCLUDED.risk,
                                  updated_at = now()
                                ''',
-                                (rel, rel, 'excluded_or_sensitive'),
+                                (rel, rel, reason),
                            )
                            cur.execute('DELETE FROM memory_primary WHERE id=%s', (rel,))
                            print(f'[QUARANTINED] {rel}')
@@ -85,7 +86,7 @@ def run() -> None:

                cur.execute('SELECT id FROM memory_primary')
                db_primary_ids = {row[0] for row in cur.fetchall()}
-                stale_primary_ids = sorted(db_primary_ids - valid_ids)
+                stale_primary_ids = sorted(db_primary_ids - valid_ids - set(failed_ids))
                for stale_id in stale_primary_ids:
                    cur.execute('DELETE FROM memory_primary WHERE id=%s', (stale_id,))

@@ -26,8 +26,19 @@ run_memory_async_index() {
  git diff-tree --no-commit-id --name-status -r -M --diff-filter=ACDMRT HEAD -- '*.md' > "$changes_file" 2>/dev/null || true
  [[ -s "$changes_file" ]] || { rm -f "$changes_file"; return 0; }

-  nohup uv run --project "$vault_dir/.scripts/memory" python "$vault_dir/.scripts/memory/incremental_ingest.py" \
-    --changes-file "$changes_file" >> "$log_file" 2>&1 &
+  nohup bash -c "
+    # 日志轮转：超过 5MB 保留最后 1000 行
+    if [[ -f '$log_file' ]] && [[ \$(wc -c < '$log_file') -gt 5242880 ]]; then
+      tail -n 1000 '$log_file' > '$log_file.tmp' && mv '$log_file.tmp' '$log_file'
+    fi
+    uv run --project '$vault_dir/.scripts/memory' python '$vault_dir/.scripts/memory/incremental_ingest.py' \
+      --changes-file '$changes_file' >> '$log_file' 2>&1
+    exit_code=\$?
+    if [[ \$exit_code -ne 0 ]]; then
+      echo \"\$(date '+%Y-%m-%d %H:%M:%S') [ERROR] incremental_ingest 退出码=\$exit_code\" >> '$log_file'
+    fi
+    rm -f '$changes_file'
+  " &
 }
 run_memory_async_index
 # --- memory async index hook end ---
@@ -9,7 +9,7 @@ def sanitize(text: str) -> str:
    return text.replace('```', '` ` `').strip()


-def query(text: str, top_k: int = 5, max_chars: int = 2500) -> str:
+def query(text: str, top_k: int = 5, max_chars: int = 2500, threshold: float = 0.5) -> str:
    embedding = embed_text(text)
    vector = vector_literal(embedding)

@@ -19,10 +19,11 @@ def query(text: str, top_k: int = 5, max_chars: int = 2500) -> str:
                '''
                SELECT source, content
                FROM memory_primary
+                WHERE embedding <=> %s::vector < %s
                ORDER BY embedding <=> %s::vector
                LIMIT %s
                ''',
-                (vector, top_k),
+                (vector, threshold, vector, top_k),
            )
            rows = cur.fetchall()

@@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS memory_primary (
  source TEXT NOT NULL,
  content TEXT NOT NULL,
  content_hash TEXT NOT NULL,
-  embedding VECTOR(1536) NOT NULL,
+  embedding VECTOR(4096) NOT NULL,
  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
 );

@@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS memory_secure_audit (
 );

 CREATE INDEX IF NOT EXISTS memory_primary_embedding_idx
-ON memory_primary USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
+ON memory_primary USING hnsw (embedding vector_cosine_ops);