feat(tagging-system): Add comprehensive tagging system implementation
- Add tagging system Python package with modular architecture (config, core, impl) - Create core interfaces for file discovery, content analysis, and tag generation - Implement content analyzer with language detection and topic extraction - Implement file discovery engine with directory scanning and filtering - Implement tag generator with hierarchical tag creation and consolidation - Add configuration management with example config and validation - Create comprehensive design and requirements documentation in .kiro/specs - Add pytest test suite with unit tests for models, interfaces, and config - Add setup.py, requirements.txt, and pytest.ini for package management - Add README.md with project overview and usage instructions - Update workspace.json with new project structure - Add Excalidraw diagram for system architecture visualization - Establish foundation for automated vault tagging and metadata management
This commit is contained in:
windyboy
@@ -0,0 +1,12 @@
|
||||
"""Configuration system for the tagging system."""
|
||||
|
||||
from .config import TaggingConfig, DirectoryMapping, TagHierarchy, SensitivePatterns, load_config, save_config
|
||||
|
||||
__all__ = [
|
||||
"TaggingConfig",
|
||||
"DirectoryMapping",
|
||||
"TagHierarchy",
|
||||
"SensitivePatterns",
|
||||
"load_config",
|
||||
"save_config"
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,223 @@
|
||||
"""Configuration system for tag hierarchies and rules."""
|
||||
|
||||
import json
|
||||
import yaml
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any, Union
|
||||
|
||||
|
||||
@dataclass
|
||||
class DirectoryMapping:
|
||||
"""Configuration for directory-based tag mapping."""
|
||||
pattern: str # Directory pattern to match
|
||||
primary_tag: str # Primary tag to assign
|
||||
hierarchical_tags: List[str] = field(default_factory=list) # Additional hierarchical tags
|
||||
exclude_patterns: List[str] = field(default_factory=list) # Patterns to exclude
|
||||
|
||||
|
||||
@dataclass
|
||||
class TagHierarchy:
|
||||
"""Configuration for hierarchical tag structures."""
|
||||
root: str # Root tag name
|
||||
children: Dict[str, 'TagHierarchy'] = field(default_factory=dict) # Child hierarchies
|
||||
aliases: List[str] = field(default_factory=list) # Alternative names
|
||||
|
||||
def get_full_path(self, child_path: str = "") -> str:
|
||||
"""Get full hierarchical path."""
|
||||
if child_path:
|
||||
return f"{self.root}/{child_path}"
|
||||
return self.root
|
||||
|
||||
|
||||
@dataclass
|
||||
class SensitivePatterns:
|
||||
"""Configuration for sensitive content detection."""
|
||||
credential_patterns: List[str] = field(default_factory=lambda: [
|
||||
r'api[_-]?key',
|
||||
r'secret[_-]?key',
|
||||
r'password',
|
||||
r'token',
|
||||
r'auth[_-]?token'
|
||||
])
|
||||
personal_patterns: List[str] = field(default_factory=lambda: [
|
||||
r'\b\d{3}-\d{2}-\d{4}\b', # SSN pattern
|
||||
r'\b\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}\b', # Credit card pattern
|
||||
r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b' # Email pattern
|
||||
])
|
||||
financial_patterns: List[str] = field(default_factory=lambda: [
|
||||
r'bank[_-]?account',
|
||||
r'routing[_-]?number',
|
||||
r'credit[_-]?card',
|
||||
r'social[_-]?security'
|
||||
])
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaggingConfig:
|
||||
"""Main configuration for the tagging system."""
|
||||
|
||||
# Directory mappings
|
||||
directory_mappings: List[DirectoryMapping] = field(default_factory=lambda: [
|
||||
DirectoryMapping("100-project", "project"),
|
||||
DirectoryMapping("200-area", "area"),
|
||||
DirectoryMapping("300-resources", "resource"),
|
||||
DirectoryMapping("400-archive", "archive"),
|
||||
DirectoryMapping("Clippings", "clipping"),
|
||||
DirectoryMapping("ReadItLater Inbox", "clipping"),
|
||||
DirectoryMapping("000-inbox", "inbox")
|
||||
])
|
||||
|
||||
# Tag hierarchies
|
||||
tag_hierarchies: Dict[str, TagHierarchy] = field(default_factory=lambda: {
|
||||
"tech": TagHierarchy("tech", {
|
||||
"ai": TagHierarchy("ai", {
|
||||
"llm": TagHierarchy("llm"),
|
||||
"ml": TagHierarchy("ml"),
|
||||
"nlp": TagHierarchy("nlp")
|
||||
}),
|
||||
"infrastructure": TagHierarchy("infrastructure", {
|
||||
"docker": TagHierarchy("docker"),
|
||||
"kubernetes": TagHierarchy("kubernetes"),
|
||||
"cloud": TagHierarchy("cloud")
|
||||
}),
|
||||
"development": TagHierarchy("development", {
|
||||
"python": TagHierarchy("python"),
|
||||
"javascript": TagHierarchy("javascript"),
|
||||
"typescript": TagHierarchy("typescript")
|
||||
})
|
||||
}),
|
||||
"personal": TagHierarchy("personal", {
|
||||
"productivity": TagHierarchy("productivity", {
|
||||
"gtd": TagHierarchy("gtd"),
|
||||
"pkm": TagHierarchy("pkm")
|
||||
}),
|
||||
"health": TagHierarchy("health", {
|
||||
"cycling": TagHierarchy("cycling"),
|
||||
"fitness": TagHierarchy("fitness")
|
||||
}),
|
||||
"finance": TagHierarchy("finance")
|
||||
}),
|
||||
"work": TagHierarchy("work", {
|
||||
"government": TagHierarchy("government"),
|
||||
"enterprise": TagHierarchy("enterprise"),
|
||||
"consulting": TagHierarchy("consulting")
|
||||
})
|
||||
})
|
||||
|
||||
# Sensitive content patterns
|
||||
sensitive_patterns: SensitivePatterns = field(default_factory=SensitivePatterns)
|
||||
|
||||
# File processing settings
|
||||
excluded_directories: List[str] = field(default_factory=lambda: [
|
||||
".obsidian",
|
||||
".git",
|
||||
".smart-env",
|
||||
"node_modules",
|
||||
"__pycache__"
|
||||
])
|
||||
|
||||
excluded_file_patterns: List[str] = field(default_factory=lambda: [
|
||||
"*.pyc",
|
||||
"*.log",
|
||||
"*.tmp",
|
||||
".DS_Store"
|
||||
])
|
||||
|
||||
# Tag formatting rules
|
||||
tag_format_rules: Dict[str, Any] = field(default_factory=lambda: {
|
||||
"case": "kebab", # kebab-case for tags
|
||||
"max_length": 50,
|
||||
"allowed_chars": "abcdefghijklmnopqrstuvwxyz0123456789-/",
|
||||
"hierarchy_separator": "/"
|
||||
})
|
||||
|
||||
# Language detection settings
|
||||
language_detection: Dict[str, Any] = field(default_factory=lambda: {
|
||||
"chinese_threshold": 0.1, # Minimum ratio of Chinese characters
|
||||
"mixed_threshold": 0.3, # Threshold for mixed language detection
|
||||
"min_content_length": 50 # Minimum content length for reliable detection
|
||||
})
|
||||
|
||||
def get_directory_mapping(self, directory_path: str) -> Optional[DirectoryMapping]:
|
||||
"""Get directory mapping for a given path."""
|
||||
for mapping in self.directory_mappings:
|
||||
if mapping.pattern in directory_path:
|
||||
return mapping
|
||||
return None
|
||||
|
||||
def get_tag_hierarchy(self, root_tag: str) -> Optional[TagHierarchy]:
|
||||
"""Get tag hierarchy for a root tag."""
|
||||
return self.tag_hierarchies.get(root_tag)
|
||||
|
||||
def is_excluded_directory(self, directory: str) -> bool:
|
||||
"""Check if directory should be excluded."""
|
||||
return any(excluded in directory for excluded in self.excluded_directories)
|
||||
|
||||
def is_excluded_file(self, filename: str) -> bool:
|
||||
"""Check if file should be excluded based on patterns."""
|
||||
import fnmatch
|
||||
return any(fnmatch.fnmatch(filename, pattern) for pattern in self.excluded_file_patterns)
|
||||
|
||||
|
||||
def load_config(config_path: Optional[Union[str, Path]] = None) -> TaggingConfig:
|
||||
"""Load configuration from file or return default configuration."""
|
||||
if config_path is None:
|
||||
return TaggingConfig()
|
||||
|
||||
config_path = Path(config_path)
|
||||
if not config_path.exists():
|
||||
# Create default config file
|
||||
default_config = TaggingConfig()
|
||||
save_config(default_config, config_path)
|
||||
return default_config
|
||||
|
||||
try:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
if config_path.suffix.lower() == '.json':
|
||||
data = json.load(f)
|
||||
else: # Assume YAML
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
# Convert dict to TaggingConfig (simplified conversion)
|
||||
# In a full implementation, you'd want more robust deserialization
|
||||
config = TaggingConfig()
|
||||
|
||||
# Update with loaded data
|
||||
if 'excluded_directories' in data:
|
||||
config.excluded_directories = data['excluded_directories']
|
||||
if 'excluded_file_patterns' in data:
|
||||
config.excluded_file_patterns = data['excluded_file_patterns']
|
||||
if 'tag_format_rules' in data:
|
||||
config.tag_format_rules.update(data['tag_format_rules'])
|
||||
if 'language_detection' in data:
|
||||
config.language_detection.update(data['language_detection'])
|
||||
|
||||
return config
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error loading config from {config_path}: {e}")
|
||||
return TaggingConfig()
|
||||
|
||||
|
||||
def save_config(config: TaggingConfig, config_path: Union[str, Path]) -> None:
|
||||
"""Save configuration to file."""
|
||||
config_path = Path(config_path)
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Convert to dict for serialization (simplified)
|
||||
config_dict = {
|
||||
'excluded_directories': config.excluded_directories,
|
||||
'excluded_file_patterns': config.excluded_file_patterns,
|
||||
'tag_format_rules': config.tag_format_rules,
|
||||
'language_detection': config.language_detection
|
||||
}
|
||||
|
||||
try:
|
||||
with open(config_path, 'w', encoding='utf-8') as f:
|
||||
if config_path.suffix.lower() == '.json':
|
||||
json.dump(config_dict, f, indent=2)
|
||||
else: # Save as YAML
|
||||
yaml.dump(config_dict, f, default_flow_style=False, allow_unicode=True)
|
||||
except Exception as e:
|
||||
print(f"Error saving config to {config_path}: {e}")
|
||||
Reference in New Issue
Block a user