feat(tagging-system): Add comprehensive tagging system implementation
- Add tagging system Python package with modular architecture (config, core, impl) - Create core interfaces for file discovery, content analysis, and tag generation - Implement content analyzer with language detection and topic extraction - Implement file discovery engine with directory scanning and filtering - Implement tag generator with hierarchical tag creation and consolidation - Add configuration management with example config and validation - Create comprehensive design and requirements documentation in .kiro/specs - Add pytest test suite with unit tests for models, interfaces, and config - Add setup.py, requirements.txt, and pytest.ini for package management - Add README.md with project overview and usage instructions - Update workspace.json with new project structure - Add Excalidraw diagram for system architecture visualization - Establish foundation for automated vault tagging and metadata management
This commit is contained in:
windyboy
@@ -0,0 +1,270 @@
|
||||
# Design Document: Comprehensive Tagging System
|
||||
|
||||
## Overview
|
||||
|
||||
The comprehensive tagging system will analyze and tag all files in the Obsidian vault based on their location, content, and existing metadata. The system uses a multi-layered approach combining directory-based tags, content analysis, and hierarchical topic classification to create a consistent and discoverable knowledge base.
|
||||
|
||||
## Architecture
|
||||
|
||||
The system follows a pipeline architecture with these main components:
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[File Discovery] --> B[Content Analysis]
|
||||
B --> C[Tag Generation]
|
||||
C --> D[Tag Validation]
|
||||
D --> E[Frontmatter Update]
|
||||
E --> F[Batch Processing]
|
||||
|
||||
G[Tag Registry] --> C
|
||||
G --> D
|
||||
H[Directory Mapping] --> C
|
||||
I[Language Detection] --> C
|
||||
J[Content Classification] --> C
|
||||
```
|
||||
|
||||
### Core Components
|
||||
|
||||
1. **File Discovery Engine**: Recursively scans vault directories
|
||||
2. **Content Analyzer**: Extracts topics, language, and content type
|
||||
3. **Tag Generator**: Creates hierarchical tags based on analysis
|
||||
4. **Tag Validator**: Ensures consistency and prevents duplication
|
||||
5. **Frontmatter Manager**: Updates YAML metadata safely
|
||||
6. **Batch Processor**: Handles large-scale operations with progress tracking
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### FileDiscovery Interface
|
||||
```typescript
|
||||
interface FileDiscovery {
|
||||
scanDirectory(path: string): Promise<FileInfo[]>
|
||||
filterByType(files: FileInfo[], types: string[]): FileInfo[]
|
||||
excludeSensitive(files: FileInfo[]): FileInfo[]
|
||||
}
|
||||
```
|
||||
|
||||
### ContentAnalyzer Interface
|
||||
```typescript
|
||||
interface ContentAnalyzer {
|
||||
analyzeContent(content: string): ContentAnalysis
|
||||
detectLanguage(content: string): LanguageInfo
|
||||
extractTopics(content: string): string[]
|
||||
classifyContentType(content: string, filename: string): ContentType
|
||||
}
|
||||
```
|
||||
|
||||
### TagGenerator Interface
|
||||
```typescript
|
||||
interface TagGenerator {
|
||||
generateDirectoryTags(filepath: string): string[]
|
||||
generateContentTags(analysis: ContentAnalysis): string[]
|
||||
generateHierarchicalTags(topics: string[]): string[]
|
||||
consolidateTags(tags: string[]): string[]
|
||||
}
|
||||
```
|
||||
|
||||
### FrontmatterManager Interface
|
||||
```typescript
|
||||
interface FrontmatterManager {
|
||||
parseFrontmatter(content: string): FrontmatterData
|
||||
updateFrontmatter(content: string, updates: FrontmatterData): string
|
||||
validateFrontmatter(data: FrontmatterData): ValidationResult
|
||||
}
|
||||
```
|
||||
|
||||
## Data Models
|
||||
|
||||
### FileInfo Model
|
||||
```typescript
|
||||
interface FileInfo {
|
||||
path: string
|
||||
name: string
|
||||
directory: string
|
||||
extension: string
|
||||
size: number
|
||||
created: Date
|
||||
modified: Date
|
||||
content?: string
|
||||
}
|
||||
```
|
||||
|
||||
### ContentAnalysis Model
|
||||
```typescript
|
||||
interface ContentAnalysis {
|
||||
language: 'en' | 'zh' | 'mixed' | 'unknown'
|
||||
contentType: ContentType
|
||||
topics: string[]
|
||||
mentions: {
|
||||
tools: string[]
|
||||
technologies: string[]
|
||||
people: string[]
|
||||
organizations: string[]
|
||||
}
|
||||
sentiment?: 'positive' | 'neutral' | 'negative'
|
||||
complexity: 'basic' | 'intermediate' | 'advanced'
|
||||
}
|
||||
```
|
||||
|
||||
### TagStructure Model
|
||||
```typescript
|
||||
interface TagStructure {
|
||||
primary: string[] // Main category tags
|
||||
hierarchical: string[] // Topic/subtopic/detail tags
|
||||
content: string[] // Content-derived tags
|
||||
meta: string[] // Metadata tags (language, type, etc.)
|
||||
custom: string[] // Manually added tags to preserve
|
||||
}
|
||||
```
|
||||
|
||||
### FrontmatterData Model
|
||||
```typescript
|
||||
interface FrontmatterData {
|
||||
title?: string
|
||||
tags: string[]
|
||||
created?: string
|
||||
updated?: string
|
||||
type?: string
|
||||
lang?: string
|
||||
source?: string
|
||||
aliases?: string[]
|
||||
description?: string
|
||||
}
|
||||
```
|
||||
|
||||
## Tag Hierarchy Design
|
||||
|
||||
### Directory-Based Tags
|
||||
- `project/` - Files in 100-project/
|
||||
- `project/ai` - AI-related projects
|
||||
- `project/infrastructure` - Infrastructure projects
|
||||
- `project/work` - Work-related projects
|
||||
- `area/` - Files in 200-area/
|
||||
- `area/productivity` - Productivity area
|
||||
- `area/health` - Health and fitness
|
||||
- `area/finance` - Financial management
|
||||
- `resource/` - Files in 300-resources/
|
||||
- `resource/development` - Development resources
|
||||
- `resource/cooking` - Cooking resources
|
||||
- `archive/` - Files in 400-archive/
|
||||
- `clipping/` - Web clippings and saved articles
|
||||
|
||||
### Content-Based Tags
|
||||
- `tech/` - Technology-related content
|
||||
- `tech/ai/llm` - Large Language Models
|
||||
- `tech/infrastructure/docker` - Docker and containers
|
||||
- `tech/development/python` - Python development
|
||||
- `personal/` - Personal content
|
||||
- `personal/productivity/gtd` - Getting Things Done
|
||||
- `personal/health/cycling` - Cycling and fitness
|
||||
- `work/` - Work-related content
|
||||
- `work/government` - Government projects
|
||||
- `work/enterprise` - Enterprise solutions
|
||||
|
||||
### Meta Tags
|
||||
- `lang/en`, `lang/zh`, `lang/mixed` - Language tags
|
||||
- `type/hub`, `type/note`, `type/clipping`, `type/daily-note` - Content type
|
||||
- `status/draft`, `status/complete`, `status/archived` - Status tags
|
||||
- `sensitive/credentials`, `sensitive/personal` - Security tags
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system—essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
Now I'll analyze the acceptance criteria to determine which ones can be tested as properties:
|
||||
|
||||
### Converting EARS to Properties
|
||||
|
||||
Based on the prework analysis, I'll create consolidated properties that eliminate redundancy while maintaining comprehensive coverage:
|
||||
|
||||
Property 1: Frontmatter standardization
|
||||
*For any* processed file, the frontmatter should contain all required fields (title, tags, created, updated) in the correct YAML format, while preserving any existing data
|
||||
**Validates: Requirements 1.1, 1.2, 1.3, 1.5**
|
||||
|
||||
Property 2: Directory-based tag mapping
|
||||
*For any* file in the vault, the tags should correctly reflect its directory location using the appropriate primary tag (project/area/resource/archive/clipping) and hierarchical subtags based on subdirectory structure
|
||||
**Validates: Requirements 2.1, 2.2, 2.3, 2.4, 2.5, 2.6**
|
||||
|
||||
Property 3: Content type classification
|
||||
*For any* file, the content analysis should correctly identify and tag the content type (hub, clipping, daily-note, meeting, documentation, tutorial, reference, draft) based on content patterns and filename analysis
|
||||
**Validates: Requirements 3.1, 3.2, 3.3, 3.4, 3.5**
|
||||
|
||||
Property 4: Hierarchical topic tagging
|
||||
*For any* file with identifiable topic content, the system should generate appropriate hierarchical tags using forward slash notation that correctly categorize the content into topic hierarchies (tech/, personal/, work/, home-automation/)
|
||||
**Validates: Requirements 4.1, 4.2, 4.3, 4.4, 4.5, 4.6**
|
||||
|
||||
Property 5: Language detection and tagging
|
||||
*For any* file, the language detection should correctly identify the primary language (en/zh/mixed/unknown) based on character analysis and content patterns, and add the appropriate lang/ tag
|
||||
**Validates: Requirements 5.1, 5.2, 5.3, 5.4, 5.5**
|
||||
|
||||
Property 6: Content analysis tag generation
|
||||
*For any* file, the content analysis should extract and tag relevant topics, technical terms, tools, frameworks, technologies, and methodologies mentioned in the content, while preserving any manually added tags
|
||||
**Validates: Requirements 6.1, 6.2, 6.3, 6.4, 6.5**
|
||||
|
||||
Property 7: Tag consistency and validation
|
||||
*For any* generated tag set, all tags should follow kebab-case naming conventions, duplicate tags should be consolidated using canonical forms, and hierarchical relationships should be logically valid
|
||||
**Validates: Requirements 7.1, 7.2, 7.3, 7.4, 7.5**
|
||||
|
||||
Property 8: Data preservation during processing
|
||||
*For any* file being processed, existing tags and metadata should be preserved, backups should be created before modifications, and batch processing should maintain data integrity with proper error handling
|
||||
**Validates: Requirements 8.1, 8.2, 8.3, 8.5**
|
||||
|
||||
Property 9: Sensitive content detection and protection
|
||||
*For any* file containing sensitive information (credentials, personal data, financial/legal information), the system should add appropriate sensitive/ tags while ensuring the sensitive data itself is not exposed or logged during processing
|
||||
**Validates: Requirements 9.1, 9.2, 9.3, 9.5**
|
||||
|
||||
Property 10: Obsidian compatibility
|
||||
*For any* generated tag structure, the tags should be compatible with Obsidian's tag pane, search functionality, Dataview queries, graph view, and common plugins, with hub files containing appropriate Dataview queries for tag-based content listing
|
||||
**Validates: Requirements 10.1, 10.2, 10.3, 10.4, 10.5**
|
||||
|
||||
## Error Handling
|
||||
|
||||
### File Processing Errors
|
||||
- **Invalid file formats**: Skip non-text files with logging
|
||||
- **Corrupted frontmatter**: Attempt repair or create new frontmatter
|
||||
- **Permission errors**: Log and continue with next file
|
||||
- **Large files**: Implement streaming analysis for files over 1MB
|
||||
|
||||
### Content Analysis Errors
|
||||
- **Language detection failures**: Default to 'lang/unknown'
|
||||
- **Topic extraction failures**: Use directory-based fallback tags
|
||||
- **Encoding issues**: Attempt multiple encoding detection methods
|
||||
|
||||
### Tag Generation Errors
|
||||
- **Invalid characters**: Sanitize and convert to kebab-case
|
||||
- **Circular hierarchies**: Detect and break circular references
|
||||
- **Tag conflicts**: Use canonical form resolution
|
||||
|
||||
### Batch Processing Errors
|
||||
- **Interrupted processing**: Resume from last successful batch
|
||||
- **Memory constraints**: Process in smaller batches
|
||||
- **Concurrent access**: Implement file locking mechanism
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Dual Testing Approach
|
||||
The system will use both unit tests and property-based tests for comprehensive coverage:
|
||||
|
||||
**Unit Tests** will focus on:
|
||||
- Specific examples of frontmatter parsing and generation
|
||||
- Edge cases like empty files, malformed YAML, special characters
|
||||
- Integration points between components
|
||||
- Error conditions and recovery mechanisms
|
||||
|
||||
**Property-Based Tests** will focus on:
|
||||
- Universal properties that hold across all file types and content
|
||||
- Comprehensive input coverage through randomized file generation
|
||||
- Tag consistency and validation across large datasets
|
||||
- Data preservation guarantees during batch processing
|
||||
|
||||
### Property Test Configuration
|
||||
- Minimum 100 iterations per property test
|
||||
- Each property test references its design document property
|
||||
- Tag format: **Feature: comprehensive-tagging-system, Property {number}: {property_text}**
|
||||
|
||||
### Test Data Generation
|
||||
- **File generators**: Create files with various directory structures, content types, and languages
|
||||
- **Content generators**: Generate text with different topics, technical terms, and sensitive information patterns
|
||||
- **Frontmatter generators**: Create valid and invalid YAML frontmatter variations
|
||||
- **Directory structure generators**: Create nested directory hierarchies matching vault patterns
|
||||
|
||||
The testing strategy ensures that the tagging system maintains correctness across the diverse content and organizational patterns found in the actual vault structure.
|
||||
@@ -0,0 +1,138 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
This specification defines a comprehensive tagging system for an Obsidian vault with mixed organizational patterns. The system will establish consistent tagging conventions, implement hierarchical tag structures based on actual content and directory structure, and ensure all content is properly categorized for improved discoverability and organization.
|
||||
|
||||
## Glossary
|
||||
|
||||
- **Tag_Hierarchy**: Multi-level tag structure using forward slashes (e.g., tech/ai/llm)
|
||||
- **Frontmatter**: YAML metadata block at the beginning of Markdown files
|
||||
- **Hub_File**: Index file that serves as entry point for a category or area
|
||||
- **Content_Type**: Classification of file purpose (note, project, resource, clipping, etc.)
|
||||
- **Language_Tag**: Identifier for content language (en, zh, mixed)
|
||||
- **Directory_Structure**: Existing folder organization (100-project, 200-area, 300-resources, 400-archive, etc.)
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: Standardized Frontmatter Structure
|
||||
|
||||
**User Story:** As a vault user, I want consistent frontmatter across all files, so that I can rely on predictable metadata structure.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a file is processed, THE Tagging_System SHALL ensure it has standardized frontmatter with required fields
|
||||
2. THE Tagging_System SHALL use YAML array format for tags: `tags: [tag1, tag2, tag3]`
|
||||
3. THE Tagging_System SHALL include these core fields: title, tags, created, updated
|
||||
4. WHERE content has specific metadata needs, THE Tagging_System SHALL include additional fields like type, lang, source
|
||||
5. WHEN frontmatter exists but is incomplete, THE Tagging_System SHALL preserve existing data and add missing fields
|
||||
|
||||
### Requirement 2: Directory-Based Tag Structure
|
||||
|
||||
**User Story:** As a vault user, I want tags that reflect the actual directory organization, so that I can navigate content according to the existing structure.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a file is in 100-project directory, THE Tagging_System SHALL include 'project' tag and specific project subtags
|
||||
2. WHEN a file is in 200-area directory, THE Tagging_System SHALL include 'area' tag and area-specific subtags
|
||||
3. WHEN a file is in 300-resources directory, THE Tagging_System SHALL include 'resource' tag and topic-specific subtags
|
||||
4. WHEN a file is in 400-archive directory, THE Tagging_System SHALL include 'archive' tag
|
||||
5. WHEN a file is in Clippings or ReadItLater directories, THE Tagging_System SHALL include 'clipping' tag
|
||||
6. THE Tagging_System SHALL create hierarchical tags based on subdirectory structure (e.g., project/ai, area/productivity)
|
||||
|
||||
### Requirement 3: Content-Type Classification
|
||||
|
||||
**User Story:** As a content creator, I want files tagged by their content type, so that I can filter and find specific kinds of information.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Tagging_System SHALL identify and tag hub files with 'hub' type
|
||||
2. WHEN a file contains clipped web content, THE Tagging_System SHALL add 'clipping' tag
|
||||
3. WHEN a file is a daily note with date format, THE Tagging_System SHALL add 'daily-note' tag
|
||||
4. THE Tagging_System SHALL detect and tag meeting notes, documentation, tutorials, and reference materials
|
||||
5. WHEN file content indicates it's a draft or work-in-progress, THE Tagging_System SHALL add appropriate status tags
|
||||
|
||||
### Requirement 4: Topic-Based Hierarchical Tags
|
||||
|
||||
**User Story:** As a researcher, I want hierarchical topic tags based on actual content themes, so that I can explore related content at different levels of specificity.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Tagging_System SHALL implement hierarchical tags using forward slash notation (topic/subtopic/detail)
|
||||
2. WHEN content relates to AI/technology, THE Tagging_System SHALL use tech hierarchy (tech/ai, tech/infrastructure, tech/development)
|
||||
3. WHEN content relates to personal topics, THE Tagging_System SHALL use personal hierarchy (personal/health, personal/finance, personal/productivity)
|
||||
4. WHEN content relates to work projects, THE Tagging_System SHALL use work hierarchy (work/airport, work/ali, work/government)
|
||||
5. WHEN content relates to home automation, THE Tagging_System SHALL use home-automation hierarchy
|
||||
6. WHEN content spans multiple topics, THE Tagging_System SHALL include all relevant hierarchical tags
|
||||
|
||||
### Requirement 5: Language and Localization Tags
|
||||
|
||||
**User Story:** As a multilingual user, I want content tagged by language, so that I can filter content by language preference.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN content is primarily in English, THE Tagging_System SHALL add 'lang/en' tag
|
||||
2. WHEN content is primarily in Chinese, THE Tagging_System SHALL add 'lang/zh' tag
|
||||
3. WHEN content mixes languages significantly, THE Tagging_System SHALL add 'lang/mixed' tag
|
||||
4. THE Tagging_System SHALL detect language based on character analysis and content patterns
|
||||
5. WHEN language cannot be determined, THE Tagging_System SHALL default to 'lang/unknown'
|
||||
|
||||
### Requirement 6: Automated Tag Generation
|
||||
|
||||
**User Story:** As a vault maintainer, I want automated tag generation based on content analysis, so that I don't have to manually tag every file.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Tagging_System SHALL analyze file content to suggest relevant topic tags
|
||||
2. WHEN file contains technical terms, THE Tagging_System SHALL add appropriate technology tags
|
||||
3. THE Tagging_System SHALL extract and tag mentioned tools, frameworks, and technologies
|
||||
4. WHEN content discusses specific methodologies or concepts, THE Tagging_System SHALL add conceptual tags
|
||||
5. THE Tagging_System SHALL preserve manually added tags while adding automated suggestions
|
||||
|
||||
### Requirement 7: Tag Consistency and Validation
|
||||
|
||||
**User Story:** As a vault organizer, I want consistent tag naming and validation, so that tags remain useful and don't fragment.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Tagging_System SHALL enforce kebab-case naming for all tags (lowercase with hyphens)
|
||||
2. WHEN duplicate or similar tags exist, THE Tagging_System SHALL consolidate them using canonical forms
|
||||
3. THE Tagging_System SHALL validate tag hierarchies to ensure logical parent-child relationships
|
||||
4. WHEN tags violate naming conventions, THE Tagging_System SHALL suggest corrections
|
||||
5. THE Tagging_System SHALL maintain a tag registry to prevent fragmentation
|
||||
|
||||
### Requirement 8: Batch Processing and Migration
|
||||
|
||||
**User Story:** As a vault administrator, I want to process all existing files in batches, so that I can systematically apply the tagging system to the entire vault.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Tagging_System SHALL process files in logical batches by directory structure
|
||||
2. WHEN processing existing files, THE Tagging_System SHALL preserve existing tags and metadata
|
||||
3. THE Tagging_System SHALL create backup copies before making changes to files
|
||||
4. WHEN conflicts arise between existing and generated tags, THE Tagging_System SHALL prompt for resolution
|
||||
5. THE Tagging_System SHALL provide progress reporting and error handling for batch operations
|
||||
|
||||
### Requirement 9: Security and Sensitive Content Handling
|
||||
|
||||
**User Story:** As a security-conscious user, I want sensitive files properly tagged and handled, so that I can manage access and visibility appropriately.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN files contain credentials or API keys, THE Tagging_System SHALL add 'sensitive/credentials' tag
|
||||
2. WHEN files contain personal information, THE Tagging_System SHALL add 'sensitive/personal' tag
|
||||
3. THE Tagging_System SHALL detect and tag files with financial or legal information
|
||||
4. WHEN sensitive content is detected, THE Tagging_System SHALL suggest security best practices
|
||||
5. THE Tagging_System SHALL respect existing security classifications and not expose sensitive data
|
||||
|
||||
### Requirement 10: Integration with Obsidian Features
|
||||
|
||||
**User Story:** As an Obsidian user, I want tags that work seamlessly with Obsidian's built-in features, so that I can leverage the full power of the application.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Tagging_System SHALL generate tags compatible with Obsidian's tag pane and search
|
||||
2. THE Tagging_System SHALL support Dataview queries and tag-based filtering
|
||||
3. WHEN creating hub files, THE Tagging_System SHALL include appropriate Dataview queries for tag-based content listing
|
||||
4. THE Tagging_System SHALL ensure tags work with graph view and tag-based navigation
|
||||
5. THE Tagging_System SHALL maintain compatibility with Obsidian plugins that use tags
|
||||
@@ -0,0 +1,153 @@
|
||||
# Implementation Plan: Comprehensive Tagging System
|
||||
|
||||
## Overview
|
||||
|
||||
This implementation plan creates a Python-based comprehensive tagging system for the Obsidian vault. The system will analyze files, generate appropriate tags based on directory structure and content analysis, and update frontmatter while preserving existing data. The implementation follows a modular architecture with clear separation of concerns.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Set up project structure and core interfaces
|
||||
- Create Python package structure with proper modules
|
||||
- Define core data models and interfaces using dataclasses and protocols
|
||||
- Set up testing framework with pytest and hypothesis for property-based testing
|
||||
- Create configuration system for tag hierarchies and rules
|
||||
- _Requirements: 1.1, 1.2, 1.3_
|
||||
|
||||
- [x] 2. Implement file discovery and content analysis
|
||||
- [x] 2.1 Create FileDiscovery class for vault scanning
|
||||
- Implement recursive directory scanning with file filtering
|
||||
- Add support for excluding sensitive directories and file types
|
||||
- Handle file encoding detection and content reading
|
||||
- _Requirements: 8.1, 9.5_
|
||||
|
||||
- [ ]* 2.2 Write property test for file discovery
|
||||
- **Property 8: Data preservation during processing**
|
||||
- **Validates: Requirements 8.1, 8.2, 8.3, 8.5**
|
||||
|
||||
- [x] 2.3 Implement ContentAnalyzer class
|
||||
- Create language detection using character frequency analysis
|
||||
- Implement content type classification based on patterns and keywords
|
||||
- Add topic extraction using keyword analysis and NLP techniques
|
||||
- Build entity extraction for tools, technologies, and frameworks
|
||||
- _Requirements: 5.4, 3.4, 6.1, 6.2, 6.3_
|
||||
|
||||
- [ ]* 2.4 Write property tests for content analysis
|
||||
- **Property 5: Language detection and tagging**
|
||||
- **Property 3: Content type classification**
|
||||
- **Property 6: Content analysis tag generation**
|
||||
- **Validates: Requirements 5.1-5.5, 3.1-3.5, 6.1-6.5**
|
||||
|
||||
- [x] 3. Implement tag generation system
|
||||
- [x] 3.1 Create TagGenerator class
|
||||
- Implement directory-based tag generation from file paths
|
||||
- Build hierarchical tag creation using topic analysis
|
||||
- Add tag consolidation and deduplication logic
|
||||
- Create tag validation and naming convention enforcement
|
||||
- _Requirements: 2.6, 4.1, 7.1, 7.2, 7.3_
|
||||
|
||||
- [ ]* 3.2 Write property tests for tag generation
|
||||
- **Property 2: Directory-based tag mapping**
|
||||
- **Property 4: Hierarchical topic tagging**
|
||||
- **Property 7: Tag consistency and validation**
|
||||
- **Validates: Requirements 2.1-2.6, 4.1-4.6, 7.1-7.5**
|
||||
|
||||
- [x] 3.3 Implement sensitive content detection
|
||||
- Create pattern matching for credentials, API keys, and personal information
|
||||
- Add financial and legal content detection
|
||||
- Implement secure handling to avoid exposing sensitive data
|
||||
- _Requirements: 9.1, 9.2, 9.3, 9.5_
|
||||
|
||||
- [ ]* 3.4 Write property test for sensitive content detection
|
||||
- **Property 9: Sensitive content detection and protection**
|
||||
- **Validates: Requirements 9.1, 9.2, 9.3, 9.5**
|
||||
|
||||
- [ ] 4. Checkpoint - Core functionality validation
|
||||
- Ensure all core components work together correctly
|
||||
- Run property tests to validate tag generation logic
|
||||
- Test with sample files from different vault directories
|
||||
- Ask the user if questions arise about tag hierarchies or classification rules
|
||||
|
||||
- [ ] 5. Implement frontmatter management
|
||||
- [ ] 5.1 Create FrontmatterManager class
|
||||
- Implement YAML frontmatter parsing with error handling
|
||||
- Build frontmatter updating while preserving existing data
|
||||
- Add validation for frontmatter structure and required fields
|
||||
- Create backup functionality before file modifications
|
||||
- _Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 8.2, 8.3_
|
||||
|
||||
- [ ]* 5.2 Write property tests for frontmatter management
|
||||
- **Property 1: Frontmatter standardization**
|
||||
- **Validates: Requirements 1.1, 1.2, 1.3, 1.5**
|
||||
|
||||
- [ ] 5.3 Implement batch processing system
|
||||
- Create BatchProcessor class for handling large-scale operations
|
||||
- Add progress tracking and error handling
|
||||
- Implement resume functionality for interrupted processing
|
||||
- Build memory-efficient processing for large vaults
|
||||
- _Requirements: 8.1, 8.2, 8.5_
|
||||
|
||||
- [ ]* 5.4 Write unit tests for batch processing
|
||||
- Test error recovery and resume functionality
|
||||
- Test memory management with large file sets
|
||||
- Test progress reporting accuracy
|
||||
- _Requirements: 8.1, 8.2, 8.5_
|
||||
|
||||
- [ ] 6. Implement Obsidian compatibility features
|
||||
- [ ] 6.1 Create ObsidianCompatibility class
|
||||
- Ensure tag format compatibility with Obsidian tag system
|
||||
- Generate Dataview queries for hub files
|
||||
- Validate tags work with graph view and common plugins
|
||||
- Test tag search and filtering functionality
|
||||
- _Requirements: 10.1, 10.2, 10.3, 10.4, 10.5_
|
||||
|
||||
- [ ]* 6.2 Write property test for Obsidian compatibility
|
||||
- **Property 10: Obsidian compatibility**
|
||||
- **Validates: Requirements 10.1, 10.2, 10.3, 10.4, 10.5**
|
||||
|
||||
- [ ] 7. Create command-line interface and configuration
|
||||
- [ ] 7.1 Build CLI using argparse or click
|
||||
- Add commands for full vault processing, single file processing, and validation
|
||||
- Implement configuration file support for custom tag hierarchies
|
||||
- Add dry-run mode for testing without file modifications
|
||||
- Create verbose logging and progress reporting
|
||||
- _Requirements: 8.1, 8.5_
|
||||
|
||||
- [ ]* 7.2 Write integration tests for CLI
|
||||
- Test full vault processing workflow
|
||||
- Test configuration file loading and validation
|
||||
- Test dry-run mode accuracy
|
||||
- _Requirements: 8.1, 8.5_
|
||||
|
||||
- [ ] 8. Integration and comprehensive testing
|
||||
- [ ] 8.1 Wire all components together in main application
|
||||
- Create main TaggingSystem class that orchestrates all components
|
||||
- Implement error handling and logging throughout the system
|
||||
- Add configuration validation and setup
|
||||
- _Requirements: All requirements_
|
||||
|
||||
- [ ]* 8.2 Write end-to-end property tests
|
||||
- Test complete workflow from file discovery to frontmatter update
|
||||
- Validate all properties work together in integrated system
|
||||
- Test with realistic vault structure and content
|
||||
- _Requirements: All requirements_
|
||||
|
||||
- [ ] 8.3 Create comprehensive test suite
|
||||
- Add performance tests for large vault processing
|
||||
- Create regression tests for edge cases
|
||||
- Build test data generators for various content types
|
||||
- _Requirements: All requirements_
|
||||
|
||||
- [ ] 9. Final checkpoint and documentation
|
||||
- Ensure all property tests pass with 100+ iterations each
|
||||
- Validate system works with actual vault content
|
||||
- Create usage documentation and configuration examples
|
||||
- Ask the user if questions arise about final implementation
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional and can be skipped for faster MVP
|
||||
- Each task references specific requirements for traceability
|
||||
- Property tests validate universal correctness properties with randomized inputs
|
||||
- Unit tests validate specific examples and edge cases
|
||||
- The system preserves existing data and creates backups before modifications
|
||||
- All components are designed to be modular and testable independently
|
||||
Vendored
+14
-14
@@ -4,11 +4,11 @@
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "1fb37ab04f75243e",
|
||||
"id": "7515b9d3e7171289",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "bad2139edeaee088",
|
||||
"id": "d5584fd860b652a6",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "empty",
|
||||
@@ -154,7 +154,7 @@
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-tags",
|
||||
"title": "Tags"
|
||||
"title": "标签"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -168,7 +168,7 @@
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-archive",
|
||||
"title": "All properties"
|
||||
"title": "添加笔记属性"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -177,8 +177,8 @@
|
||||
"state": {
|
||||
"type": "advanced-tables-toolbar",
|
||||
"state": {},
|
||||
"icon": "lucide-ghost",
|
||||
"title": "advanced-tables-toolbar"
|
||||
"icon": "spreadsheet",
|
||||
"title": "Advanced Tables"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -187,8 +187,8 @@
|
||||
"state": {
|
||||
"type": "calendar",
|
||||
"state": {},
|
||||
"icon": "lucide-ghost",
|
||||
"title": "calendar"
|
||||
"icon": "calendar-with-checkmark",
|
||||
"title": "Calendar"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -197,8 +197,8 @@
|
||||
"state": {
|
||||
"type": "smart-connections-view",
|
||||
"state": {},
|
||||
"icon": "lucide-ghost",
|
||||
"title": "smart-connections-view"
|
||||
"icon": "smart-connections",
|
||||
"title": "Connections"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -250,8 +250,11 @@
|
||||
"smart-connections:Smart Connections: Open random connection": false
|
||||
}
|
||||
},
|
||||
"active": "ec72abdaaa8e91a6",
|
||||
"active": "d5584fd860b652a6",
|
||||
"lastOpenFiles": [
|
||||
"未命名看板.md",
|
||||
"Excalidraw/Drawing 2025-12-30 19.55.02.excalidraw.md",
|
||||
"Excalidraw",
|
||||
"200-area/Blog/Writing cheatsheet.md",
|
||||
"200-area/Blog/Hugo Version Change.md",
|
||||
"200-area/Blog/Feedback sessions.md",
|
||||
@@ -276,8 +279,6 @@
|
||||
"400-archive/security-sensitive/Apple App Password.md",
|
||||
"400-archive/security-sensitive/Cookies.md",
|
||||
"400-archive/security-sensitive/Domains.md",
|
||||
"400-archive/security-sensitive/Matrix Server.md",
|
||||
"400-archive/security-sensitive/README.md",
|
||||
"100-project/AI/Agent",
|
||||
"100-project/Home-Automation/Hardware/Matter",
|
||||
"100-project/Home-Automation/南电",
|
||||
@@ -287,7 +288,6 @@
|
||||
"100-project/Infrastructure/Services/PowerDNS Auth",
|
||||
"100-project/Infrastructure/Proxy/Clash",
|
||||
"100-project/Infrastructure/Mail/Config",
|
||||
"100-project/Infrastructure/VPS",
|
||||
"300-resources/Development/Architecture/arc42/images/arc42-logo.png",
|
||||
"300-resources/Development/Architecture/arc42/images/08-Crosscutting-Concepts-Structure-EN.png",
|
||||
"300-resources/Development/Architecture/arc42/images/05_building_blocks-EN.png",
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
|
||||
excalidraw-plugin: parsed
|
||||
tags: [excalidraw]
|
||||
|
||||
---
|
||||
==⚠ Switch to EXCALIDRAW VIEW in the MORE OPTIONS menu of this document. ⚠== You can decompress Drawing data with the command palette: 'Decompress current Excalidraw file'. For more info check in plugin settings under 'Saving'
|
||||
|
||||
|
||||
## Drawing
|
||||
```compressed-json
|
||||
N4IgLgngDgpiBcIYA8DGBDANgSwCYCd0B3EAGhADcZ8BnbAewDsEAmcm+gV31TkQAswYKDXgB6MQHNsYfpwBGAOlT0AtmIBeNCtlQbs6RmPry6uA4wC0KDDgLFLUTJ2lH8MTDHQ0YNMWHRJMRZFAEYADkUAZjIkT1UYRjAaBABtAF1ydCgoAGUAsD5QSXw8XOwNPkZOTExyHRgiACF0VABrEq5GXABhekx6fAQQAGIAMwnJkABfaaA==
|
||||
```
|
||||
%%
|
||||
@@ -0,0 +1,103 @@
|
||||
# Comprehensive Tagging System
|
||||
|
||||
A Python-based comprehensive tagging system for Obsidian vaults that analyzes files, generates appropriate tags based on directory structure and content analysis, and updates frontmatter while preserving existing data.
|
||||
|
||||
## Features
|
||||
|
||||
- **Directory-based tagging**: Automatically generates tags based on vault directory structure
|
||||
- **Content analysis**: Analyzes file content to extract topics, technologies, and entities
|
||||
- **Hierarchical tag structures**: Creates organized tag hierarchies using forward slash notation
|
||||
- **Language detection**: Identifies content language (English, Chinese, mixed, unknown)
|
||||
- **Frontmatter management**: Updates YAML frontmatter while preserving existing data
|
||||
- **Batch processing**: Processes entire vaults efficiently with progress tracking
|
||||
- **Sensitive content detection**: Identifies and appropriately tags sensitive information
|
||||
- **Obsidian compatibility**: Ensures tags work with Obsidian's features and plugins
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Development Setup
|
||||
|
||||
```bash
|
||||
# Install development dependencies
|
||||
pip install -e ".[dev,test]"
|
||||
|
||||
# Run tests
|
||||
pytest
|
||||
|
||||
# Run tests with coverage
|
||||
pytest --cov=tagging_system
|
||||
|
||||
# Format code
|
||||
black tagging_system tests
|
||||
|
||||
# Lint code
|
||||
flake8 tagging_system tests
|
||||
|
||||
# Type checking
|
||||
mypy tagging_system
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
tagging_system/
|
||||
├── __init__.py
|
||||
├── core/
|
||||
│ ├── __init__.py
|
||||
│ ├── models.py # Core data models
|
||||
│ └── interfaces.py # Abstract interfaces and protocols
|
||||
├── config/
|
||||
│ ├── __init__.py
|
||||
│ └── config.py # Configuration system
|
||||
└── implementations/ # Concrete implementations (to be added)
|
||||
|
||||
tests/
|
||||
├── __init__.py
|
||||
├── conftest.py # Pytest configuration and fixtures
|
||||
├── test_models.py # Tests for core models
|
||||
├── test_config.py # Tests for configuration system
|
||||
└── test_interfaces.py # Tests for interfaces and protocols
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The system uses a flexible configuration system that supports both YAML and JSON formats. Configuration includes:
|
||||
|
||||
- Directory mappings for tag generation
|
||||
- Tag hierarchies and structures
|
||||
- Sensitive content detection patterns
|
||||
- File processing settings
|
||||
- Language detection parameters
|
||||
|
||||
## Testing
|
||||
|
||||
The project uses pytest with hypothesis for property-based testing:
|
||||
|
||||
- **Unit tests**: Test specific functionality and edge cases
|
||||
- **Property-based tests**: Test universal properties across randomized inputs
|
||||
- **Integration tests**: Test component interactions
|
||||
|
||||
Run tests with:
|
||||
```bash
|
||||
pytest # Run all tests
|
||||
pytest -m unit # Run only unit tests
|
||||
pytest -m property # Run only property-based tests
|
||||
pytest -v # Verbose output
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
This implementation addresses the following requirements:
|
||||
- 1.1, 1.2, 1.3: Standardized frontmatter structure
|
||||
- Directory-based tag mapping
|
||||
- Content analysis and classification
|
||||
- Tag consistency and validation
|
||||
- Batch processing capabilities
|
||||
|
||||
## License
|
||||
|
||||
MIT License
|
||||
@@ -0,0 +1,30 @@
|
||||
# Example configuration for the comprehensive tagging system
|
||||
# Copy this file to config.yaml and customize as needed
|
||||
|
||||
# Directories to exclude from processing
|
||||
excluded_directories:
|
||||
- ".obsidian"
|
||||
- ".git"
|
||||
- ".smart-env"
|
||||
- "node_modules"
|
||||
- "__pycache__"
|
||||
|
||||
# File patterns to exclude
|
||||
excluded_file_patterns:
|
||||
- "*.pyc"
|
||||
- "*.log"
|
||||
- "*.tmp"
|
||||
- ".DS_Store"
|
||||
|
||||
# Tag formatting rules
|
||||
tag_format_rules:
|
||||
case: "kebab" # Use kebab-case for tags
|
||||
max_length: 50
|
||||
allowed_chars: "abcdefghijklmnopqrstuvwxyz0123456789-/"
|
||||
hierarchy_separator: "/"
|
||||
|
||||
# Language detection settings
|
||||
language_detection:
|
||||
chinese_threshold: 0.1 # Minimum ratio of Chinese characters to detect Chinese
|
||||
mixed_threshold: 0.3 # Threshold for mixed language detection
|
||||
min_content_length: 50 # Minimum content length for reliable detection
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
[tool:pytest]
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
addopts =
|
||||
--verbose
|
||||
--tb=short
|
||||
--strict-markers
|
||||
--disable-warnings
|
||||
--cov=tagging_system
|
||||
--cov-report=term-missing
|
||||
--cov-report=html:htmlcov
|
||||
markers =
|
||||
unit: Unit tests
|
||||
property: Property-based tests
|
||||
integration: Integration tests
|
||||
slow: Slow running tests
|
||||
@@ -0,0 +1,14 @@
|
||||
# Core dependencies
|
||||
pyyaml>=6.0
|
||||
python-frontmatter>=1.0.0
|
||||
chardet>=5.0.0
|
||||
|
||||
# Testing dependencies
|
||||
pytest>=7.0.0
|
||||
hypothesis>=6.0.0
|
||||
pytest-cov>=4.0.0
|
||||
|
||||
# Development dependencies
|
||||
black>=22.0.0
|
||||
flake8>=5.0.0
|
||||
mypy>=1.0.0
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Setup script for the comprehensive tagging system."""
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
with open("README.md", "r", encoding="utf-8") as fh:
|
||||
long_description = fh.read()
|
||||
|
||||
with open("requirements.txt", "r", encoding="utf-8") as fh:
|
||||
requirements = [line.strip() for line in fh if line.strip() and not line.startswith("#")]
|
||||
|
||||
setup(
|
||||
name="comprehensive-tagging-system",
|
||||
version="0.1.0",
|
||||
author="Tagging System",
|
||||
description="A comprehensive tagging system for Obsidian vaults",
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
packages=find_packages(),
|
||||
classifiers=[
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: End Users/Desktop",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
],
|
||||
python_requires=">=3.8",
|
||||
install_requires=requirements,
|
||||
extras_require={
|
||||
"dev": [
|
||||
"black>=22.0.0",
|
||||
"flake8>=5.0.0",
|
||||
"mypy>=1.0.0",
|
||||
],
|
||||
"test": [
|
||||
"pytest>=7.0.0",
|
||||
"hypothesis>=6.0.0",
|
||||
"pytest-cov>=4.0.0",
|
||||
],
|
||||
},
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
"tagging-system=tagging_system.cli:main",
|
||||
],
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
Comprehensive Tagging System for Obsidian Vault
|
||||
|
||||
A Python-based system for analyzing files, generating appropriate tags based on
|
||||
directory structure and content analysis, and updating frontmatter while
|
||||
preserving existing data.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__author__ = "Tagging System"
|
||||
|
||||
from .core.models import (
|
||||
FileInfo,
|
||||
ContentAnalysis,
|
||||
TagStructure,
|
||||
FrontmatterData,
|
||||
ContentType,
|
||||
LanguageInfo,
|
||||
ValidationResult
|
||||
)
|
||||
|
||||
from .core.interfaces import (
|
||||
FileDiscovery,
|
||||
ContentAnalyzer,
|
||||
TagGenerator,
|
||||
FrontmatterManager
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FileInfo",
|
||||
"ContentAnalysis",
|
||||
"TagStructure",
|
||||
"FrontmatterData",
|
||||
"ContentType",
|
||||
"LanguageInfo",
|
||||
"ValidationResult",
|
||||
"FileDiscovery",
|
||||
"ContentAnalyzer",
|
||||
"TagGenerator",
|
||||
"FrontmatterManager"
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,73 @@
|
||||
"""Command-line interface for the tagging system."""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from .config import load_config
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the CLI."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Comprehensive Tagging System for Obsidian Vaults"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"vault_path",
|
||||
type=str,
|
||||
help="Path to the Obsidian vault directory"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
type=str,
|
||||
help="Path to configuration file (YAML or JSON)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would be done without making changes"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
"-v",
|
||||
action="store_true",
|
||||
help="Enable verbose output"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate vault path
|
||||
vault_path = Path(args.vault_path)
|
||||
if not vault_path.exists():
|
||||
print(f"Error: Vault path '{vault_path}' does not exist")
|
||||
sys.exit(1)
|
||||
|
||||
if not vault_path.is_dir():
|
||||
print(f"Error: Vault path '{vault_path}' is not a directory")
|
||||
sys.exit(1)
|
||||
|
||||
# Load configuration
|
||||
config = load_config(args.config)
|
||||
|
||||
if args.verbose:
|
||||
print(f"Vault path: {vault_path}")
|
||||
print(f"Configuration: {args.config or 'default'}")
|
||||
print(f"Dry run: {args.dry_run}")
|
||||
|
||||
# TODO: Implement actual tagging logic in future tasks
|
||||
print("Tagging system setup complete!")
|
||||
print("Note: Core implementation will be added in subsequent tasks.")
|
||||
|
||||
if args.dry_run:
|
||||
print("Dry run mode - no files would be modified")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Configuration system for the tagging system."""
|
||||
|
||||
from .config import TaggingConfig, DirectoryMapping, TagHierarchy, SensitivePatterns, load_config, save_config
|
||||
|
||||
__all__ = [
|
||||
"TaggingConfig",
|
||||
"DirectoryMapping",
|
||||
"TagHierarchy",
|
||||
"SensitivePatterns",
|
||||
"load_config",
|
||||
"save_config"
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,223 @@
|
||||
"""Configuration system for tag hierarchies and rules."""
|
||||
|
||||
import json
|
||||
import yaml
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any, Union
|
||||
|
||||
|
||||
@dataclass
|
||||
class DirectoryMapping:
|
||||
"""Configuration for directory-based tag mapping."""
|
||||
pattern: str # Directory pattern to match
|
||||
primary_tag: str # Primary tag to assign
|
||||
hierarchical_tags: List[str] = field(default_factory=list) # Additional hierarchical tags
|
||||
exclude_patterns: List[str] = field(default_factory=list) # Patterns to exclude
|
||||
|
||||
|
||||
@dataclass
|
||||
class TagHierarchy:
|
||||
"""Configuration for hierarchical tag structures."""
|
||||
root: str # Root tag name
|
||||
children: Dict[str, 'TagHierarchy'] = field(default_factory=dict) # Child hierarchies
|
||||
aliases: List[str] = field(default_factory=list) # Alternative names
|
||||
|
||||
def get_full_path(self, child_path: str = "") -> str:
|
||||
"""Get full hierarchical path."""
|
||||
if child_path:
|
||||
return f"{self.root}/{child_path}"
|
||||
return self.root
|
||||
|
||||
|
||||
@dataclass
|
||||
class SensitivePatterns:
|
||||
"""Configuration for sensitive content detection."""
|
||||
credential_patterns: List[str] = field(default_factory=lambda: [
|
||||
r'api[_-]?key',
|
||||
r'secret[_-]?key',
|
||||
r'password',
|
||||
r'token',
|
||||
r'auth[_-]?token'
|
||||
])
|
||||
personal_patterns: List[str] = field(default_factory=lambda: [
|
||||
r'\b\d{3}-\d{2}-\d{4}\b', # SSN pattern
|
||||
r'\b\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}\b', # Credit card pattern
|
||||
r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b' # Email pattern
|
||||
])
|
||||
financial_patterns: List[str] = field(default_factory=lambda: [
|
||||
r'bank[_-]?account',
|
||||
r'routing[_-]?number',
|
||||
r'credit[_-]?card',
|
||||
r'social[_-]?security'
|
||||
])
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaggingConfig:
|
||||
"""Main configuration for the tagging system."""
|
||||
|
||||
# Directory mappings
|
||||
directory_mappings: List[DirectoryMapping] = field(default_factory=lambda: [
|
||||
DirectoryMapping("100-project", "project"),
|
||||
DirectoryMapping("200-area", "area"),
|
||||
DirectoryMapping("300-resources", "resource"),
|
||||
DirectoryMapping("400-archive", "archive"),
|
||||
DirectoryMapping("Clippings", "clipping"),
|
||||
DirectoryMapping("ReadItLater Inbox", "clipping"),
|
||||
DirectoryMapping("000-inbox", "inbox")
|
||||
])
|
||||
|
||||
# Tag hierarchies
|
||||
tag_hierarchies: Dict[str, TagHierarchy] = field(default_factory=lambda: {
|
||||
"tech": TagHierarchy("tech", {
|
||||
"ai": TagHierarchy("ai", {
|
||||
"llm": TagHierarchy("llm"),
|
||||
"ml": TagHierarchy("ml"),
|
||||
"nlp": TagHierarchy("nlp")
|
||||
}),
|
||||
"infrastructure": TagHierarchy("infrastructure", {
|
||||
"docker": TagHierarchy("docker"),
|
||||
"kubernetes": TagHierarchy("kubernetes"),
|
||||
"cloud": TagHierarchy("cloud")
|
||||
}),
|
||||
"development": TagHierarchy("development", {
|
||||
"python": TagHierarchy("python"),
|
||||
"javascript": TagHierarchy("javascript"),
|
||||
"typescript": TagHierarchy("typescript")
|
||||
})
|
||||
}),
|
||||
"personal": TagHierarchy("personal", {
|
||||
"productivity": TagHierarchy("productivity", {
|
||||
"gtd": TagHierarchy("gtd"),
|
||||
"pkm": TagHierarchy("pkm")
|
||||
}),
|
||||
"health": TagHierarchy("health", {
|
||||
"cycling": TagHierarchy("cycling"),
|
||||
"fitness": TagHierarchy("fitness")
|
||||
}),
|
||||
"finance": TagHierarchy("finance")
|
||||
}),
|
||||
"work": TagHierarchy("work", {
|
||||
"government": TagHierarchy("government"),
|
||||
"enterprise": TagHierarchy("enterprise"),
|
||||
"consulting": TagHierarchy("consulting")
|
||||
})
|
||||
})
|
||||
|
||||
# Sensitive content patterns
|
||||
sensitive_patterns: SensitivePatterns = field(default_factory=SensitivePatterns)
|
||||
|
||||
# File processing settings
|
||||
excluded_directories: List[str] = field(default_factory=lambda: [
|
||||
".obsidian",
|
||||
".git",
|
||||
".smart-env",
|
||||
"node_modules",
|
||||
"__pycache__"
|
||||
])
|
||||
|
||||
excluded_file_patterns: List[str] = field(default_factory=lambda: [
|
||||
"*.pyc",
|
||||
"*.log",
|
||||
"*.tmp",
|
||||
".DS_Store"
|
||||
])
|
||||
|
||||
# Tag formatting rules
|
||||
tag_format_rules: Dict[str, Any] = field(default_factory=lambda: {
|
||||
"case": "kebab", # kebab-case for tags
|
||||
"max_length": 50,
|
||||
"allowed_chars": "abcdefghijklmnopqrstuvwxyz0123456789-/",
|
||||
"hierarchy_separator": "/"
|
||||
})
|
||||
|
||||
# Language detection settings
|
||||
language_detection: Dict[str, Any] = field(default_factory=lambda: {
|
||||
"chinese_threshold": 0.1, # Minimum ratio of Chinese characters
|
||||
"mixed_threshold": 0.3, # Threshold for mixed language detection
|
||||
"min_content_length": 50 # Minimum content length for reliable detection
|
||||
})
|
||||
|
||||
def get_directory_mapping(self, directory_path: str) -> Optional[DirectoryMapping]:
|
||||
"""Get directory mapping for a given path."""
|
||||
for mapping in self.directory_mappings:
|
||||
if mapping.pattern in directory_path:
|
||||
return mapping
|
||||
return None
|
||||
|
||||
def get_tag_hierarchy(self, root_tag: str) -> Optional[TagHierarchy]:
|
||||
"""Get tag hierarchy for a root tag."""
|
||||
return self.tag_hierarchies.get(root_tag)
|
||||
|
||||
def is_excluded_directory(self, directory: str) -> bool:
|
||||
"""Check if directory should be excluded."""
|
||||
return any(excluded in directory for excluded in self.excluded_directories)
|
||||
|
||||
def is_excluded_file(self, filename: str) -> bool:
|
||||
"""Check if file should be excluded based on patterns."""
|
||||
import fnmatch
|
||||
return any(fnmatch.fnmatch(filename, pattern) for pattern in self.excluded_file_patterns)
|
||||
|
||||
|
||||
def load_config(config_path: Optional[Union[str, Path]] = None) -> TaggingConfig:
|
||||
"""Load configuration from file or return default configuration."""
|
||||
if config_path is None:
|
||||
return TaggingConfig()
|
||||
|
||||
config_path = Path(config_path)
|
||||
if not config_path.exists():
|
||||
# Create default config file
|
||||
default_config = TaggingConfig()
|
||||
save_config(default_config, config_path)
|
||||
return default_config
|
||||
|
||||
try:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
if config_path.suffix.lower() == '.json':
|
||||
data = json.load(f)
|
||||
else: # Assume YAML
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
# Convert dict to TaggingConfig (simplified conversion)
|
||||
# In a full implementation, you'd want more robust deserialization
|
||||
config = TaggingConfig()
|
||||
|
||||
# Update with loaded data
|
||||
if 'excluded_directories' in data:
|
||||
config.excluded_directories = data['excluded_directories']
|
||||
if 'excluded_file_patterns' in data:
|
||||
config.excluded_file_patterns = data['excluded_file_patterns']
|
||||
if 'tag_format_rules' in data:
|
||||
config.tag_format_rules.update(data['tag_format_rules'])
|
||||
if 'language_detection' in data:
|
||||
config.language_detection.update(data['language_detection'])
|
||||
|
||||
return config
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error loading config from {config_path}: {e}")
|
||||
return TaggingConfig()
|
||||
|
||||
|
||||
def save_config(config: TaggingConfig, config_path: Union[str, Path]) -> None:
|
||||
"""Save configuration to file."""
|
||||
config_path = Path(config_path)
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Convert to dict for serialization (simplified)
|
||||
config_dict = {
|
||||
'excluded_directories': config.excluded_directories,
|
||||
'excluded_file_patterns': config.excluded_file_patterns,
|
||||
'tag_format_rules': config.tag_format_rules,
|
||||
'language_detection': config.language_detection
|
||||
}
|
||||
|
||||
try:
|
||||
with open(config_path, 'w', encoding='utf-8') as f:
|
||||
if config_path.suffix.lower() == '.json':
|
||||
json.dump(config_dict, f, indent=2)
|
||||
else: # Save as YAML
|
||||
yaml.dump(config_dict, f, default_flow_style=False, allow_unicode=True)
|
||||
except Exception as e:
|
||||
print(f"Error saving config to {config_path}: {e}")
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Core components for the tagging system."""
|
||||
|
||||
from .models import (
|
||||
FileInfo,
|
||||
ContentAnalysis,
|
||||
TagStructure,
|
||||
FrontmatterData,
|
||||
ContentType,
|
||||
LanguageInfo,
|
||||
ValidationResult
|
||||
)
|
||||
|
||||
from .interfaces import (
|
||||
FileDiscovery,
|
||||
ContentAnalyzer,
|
||||
TagGenerator,
|
||||
FrontmatterManager
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FileInfo",
|
||||
"ContentAnalysis",
|
||||
"TagStructure",
|
||||
"FrontmatterData",
|
||||
"ContentType",
|
||||
"LanguageInfo",
|
||||
"ValidationResult",
|
||||
"FileDiscovery",
|
||||
"ContentAnalyzer",
|
||||
"TagGenerator",
|
||||
"FrontmatterManager"
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,167 @@
|
||||
"""Core interfaces and protocols for the tagging system."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Protocol, runtime_checkable
|
||||
from .models import FileInfo, ContentAnalysis, TagStructure, FrontmatterData, ValidationResult
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class FileDiscovery(Protocol):
|
||||
"""Protocol for file discovery operations."""
|
||||
|
||||
def scan_directory(self, path: str) -> List[FileInfo]:
|
||||
"""Recursively scan directory and return file information."""
|
||||
...
|
||||
|
||||
def filter_by_type(self, files: List[FileInfo], types: List[str]) -> List[FileInfo]:
|
||||
"""Filter files by extension types."""
|
||||
...
|
||||
|
||||
def exclude_sensitive(self, files: List[FileInfo]) -> List[FileInfo]:
|
||||
"""Exclude sensitive directories and files."""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ContentAnalyzer(Protocol):
|
||||
"""Protocol for content analysis operations."""
|
||||
|
||||
def analyze_content(self, content: str) -> ContentAnalysis:
|
||||
"""Analyze file content and return analysis results."""
|
||||
...
|
||||
|
||||
def detect_language(self, content: str) -> str:
|
||||
"""Detect the primary language of the content."""
|
||||
...
|
||||
|
||||
def extract_topics(self, content: str) -> List[str]:
|
||||
"""Extract topics from content."""
|
||||
...
|
||||
|
||||
def classify_content_type(self, content: str, filename: str) -> str:
|
||||
"""Classify the type of content."""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class TagGenerator(Protocol):
|
||||
"""Protocol for tag generation operations."""
|
||||
|
||||
def generate_directory_tags(self, filepath: str) -> List[str]:
|
||||
"""Generate tags based on directory structure."""
|
||||
...
|
||||
|
||||
def generate_content_tags(self, analysis: ContentAnalysis) -> List[str]:
|
||||
"""Generate tags based on content analysis."""
|
||||
...
|
||||
|
||||
def generate_hierarchical_tags(self, topics: List[str]) -> List[str]:
|
||||
"""Generate hierarchical tags from topics."""
|
||||
...
|
||||
|
||||
def consolidate_tags(self, tags: List[str]) -> List[str]:
|
||||
"""Consolidate and deduplicate tags."""
|
||||
...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class FrontmatterManager(Protocol):
|
||||
"""Protocol for frontmatter management operations."""
|
||||
|
||||
def parse_frontmatter(self, content: str) -> FrontmatterData:
|
||||
"""Parse YAML frontmatter from content."""
|
||||
...
|
||||
|
||||
def update_frontmatter(self, content: str, updates: FrontmatterData) -> str:
|
||||
"""Update frontmatter in content while preserving existing data."""
|
||||
...
|
||||
|
||||
def validate_frontmatter(self, data: FrontmatterData) -> ValidationResult:
|
||||
"""Validate frontmatter structure and content."""
|
||||
...
|
||||
|
||||
|
||||
class BaseFileDiscovery(ABC):
|
||||
"""Abstract base class for file discovery implementations."""
|
||||
|
||||
@abstractmethod
|
||||
def scan_directory(self, path: str) -> List[FileInfo]:
|
||||
"""Recursively scan directory and return file information."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def filter_by_type(self, files: List[FileInfo], types: List[str]) -> List[FileInfo]:
|
||||
"""Filter files by extension types."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def exclude_sensitive(self, files: List[FileInfo]) -> List[FileInfo]:
|
||||
"""Exclude sensitive directories and files."""
|
||||
pass
|
||||
|
||||
|
||||
class BaseContentAnalyzer(ABC):
|
||||
"""Abstract base class for content analyzer implementations."""
|
||||
|
||||
@abstractmethod
|
||||
def analyze_content(self, content: str) -> ContentAnalysis:
|
||||
"""Analyze file content and return analysis results."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def detect_language(self, content: str) -> str:
|
||||
"""Detect the primary language of the content."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def extract_topics(self, content: str) -> List[str]:
|
||||
"""Extract topics from content."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def classify_content_type(self, content: str, filename: str) -> str:
|
||||
"""Classify the type of content."""
|
||||
pass
|
||||
|
||||
|
||||
class BaseTagGenerator(ABC):
|
||||
"""Abstract base class for tag generator implementations."""
|
||||
|
||||
@abstractmethod
|
||||
def generate_directory_tags(self, filepath: str) -> List[str]:
|
||||
"""Generate tags based on directory structure."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def generate_content_tags(self, analysis: ContentAnalysis) -> List[str]:
|
||||
"""Generate tags based on content analysis."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def generate_hierarchical_tags(self, topics: List[str]) -> List[str]:
|
||||
"""Generate hierarchical tags from topics."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def consolidate_tags(self, tags: List[str]) -> List[str]:
|
||||
"""Consolidate and deduplicate tags."""
|
||||
pass
|
||||
|
||||
|
||||
class BaseFrontmatterManager(ABC):
|
||||
"""Abstract base class for frontmatter manager implementations."""
|
||||
|
||||
@abstractmethod
|
||||
def parse_frontmatter(self, content: str) -> FrontmatterData:
|
||||
"""Parse YAML frontmatter from content."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def update_frontmatter(self, content: str, updates: FrontmatterData) -> str:
|
||||
"""Update frontmatter in content while preserving existing data."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def validate_frontmatter(self, data: FrontmatterData) -> ValidationResult:
|
||||
"""Validate frontmatter structure and content."""
|
||||
pass
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Core data models for the tagging system."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import List, Dict, Optional, Any
|
||||
|
||||
|
||||
class ContentType(Enum):
|
||||
"""Content type classifications."""
|
||||
HUB = "hub"
|
||||
NOTE = "note"
|
||||
CLIPPING = "clipping"
|
||||
DAILY_NOTE = "daily-note"
|
||||
MEETING = "meeting"
|
||||
DOCUMENTATION = "documentation"
|
||||
TUTORIAL = "tutorial"
|
||||
REFERENCE = "reference"
|
||||
DRAFT = "draft"
|
||||
PROJECT = "project"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class LanguageInfo(Enum):
|
||||
"""Language detection results."""
|
||||
ENGLISH = "en"
|
||||
CHINESE = "zh"
|
||||
MIXED = "mixed"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileInfo:
|
||||
"""Information about a file in the vault."""
|
||||
path: str
|
||||
name: str
|
||||
directory: str
|
||||
extension: str
|
||||
size: int
|
||||
created: datetime
|
||||
modified: datetime
|
||||
content: Optional[str] = None
|
||||
|
||||
@property
|
||||
def relative_path(self) -> str:
|
||||
"""Get the relative path from vault root."""
|
||||
return self.path
|
||||
|
||||
@property
|
||||
def is_markdown(self) -> bool:
|
||||
"""Check if file is a markdown file."""
|
||||
return self.extension.lower() in ['.md', '.markdown']
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContentAnalysis:
|
||||
"""Results of content analysis."""
|
||||
language: LanguageInfo
|
||||
content_type: ContentType
|
||||
topics: List[str]
|
||||
mentions: Dict[str, List[str]] = field(default_factory=lambda: {
|
||||
'tools': [],
|
||||
'technologies': [],
|
||||
'people': [],
|
||||
'organizations': []
|
||||
})
|
||||
sentiment: Optional[str] = None
|
||||
complexity: str = "basic"
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate complexity level."""
|
||||
if self.complexity not in ['basic', 'intermediate', 'advanced']:
|
||||
self.complexity = 'basic'
|
||||
|
||||
|
||||
@dataclass
|
||||
class TagStructure:
|
||||
"""Structured representation of tags."""
|
||||
primary: List[str] = field(default_factory=list) # Main category tags
|
||||
hierarchical: List[str] = field(default_factory=list) # Topic/subtopic/detail tags
|
||||
content: List[str] = field(default_factory=list) # Content-derived tags
|
||||
meta: List[str] = field(default_factory=list) # Metadata tags (language, type, etc.)
|
||||
custom: List[str] = field(default_factory=list) # Manually added tags to preserve
|
||||
|
||||
def all_tags(self) -> List[str]:
|
||||
"""Get all tags as a flat list."""
|
||||
all_tags = []
|
||||
all_tags.extend(self.primary)
|
||||
all_tags.extend(self.hierarchical)
|
||||
all_tags.extend(self.content)
|
||||
all_tags.extend(self.meta)
|
||||
all_tags.extend(self.custom)
|
||||
return list(set(all_tags)) # Remove duplicates
|
||||
|
||||
|
||||
@dataclass
|
||||
class FrontmatterData:
|
||||
"""YAML frontmatter data structure."""
|
||||
title: Optional[str] = None
|
||||
tags: List[str] = field(default_factory=list)
|
||||
created: Optional[str] = None
|
||||
updated: Optional[str] = None
|
||||
type: Optional[str] = None
|
||||
lang: Optional[str] = None
|
||||
source: Optional[str] = None
|
||||
aliases: List[str] = field(default_factory=list)
|
||||
description: Optional[str] = None
|
||||
custom_fields: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary for YAML serialization."""
|
||||
result = {}
|
||||
|
||||
if self.title:
|
||||
result['title'] = self.title
|
||||
if self.tags:
|
||||
result['tags'] = self.tags
|
||||
if self.created:
|
||||
result['created'] = self.created
|
||||
if self.updated:
|
||||
result['updated'] = self.updated
|
||||
if self.type:
|
||||
result['type'] = self.type
|
||||
if self.lang:
|
||||
result['lang'] = self.lang
|
||||
if self.source:
|
||||
result['source'] = self.source
|
||||
if self.aliases:
|
||||
result['aliases'] = self.aliases
|
||||
if self.description:
|
||||
result['description'] = self.description
|
||||
|
||||
# Add custom fields
|
||||
result.update(self.custom_fields)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
"""Result of validation operations."""
|
||||
is_valid: bool
|
||||
errors: List[str] = field(default_factory=list)
|
||||
warnings: List[str] = field(default_factory=list)
|
||||
suggestions: List[str] = field(default_factory=list)
|
||||
|
||||
def add_error(self, message: str):
|
||||
"""Add an error message."""
|
||||
self.errors.append(message)
|
||||
self.is_valid = False
|
||||
|
||||
def add_warning(self, message: str):
|
||||
"""Add a warning message."""
|
||||
self.warnings.append(message)
|
||||
|
||||
def add_suggestion(self, message: str):
|
||||
"""Add a suggestion message."""
|
||||
self.suggestions.append(message)
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Implementation modules for the tagging system."""
|
||||
|
||||
from .file_discovery import VaultFileDiscovery
|
||||
from .content_analyzer import VaultContentAnalyzer
|
||||
from .tag_generator import TagGeneratorImpl
|
||||
|
||||
__all__ = ['VaultFileDiscovery', 'VaultContentAnalyzer', 'TagGeneratorImpl']
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,303 @@
|
||||
"""Content analysis implementation for extracting topics, language, and content type."""
|
||||
|
||||
import re
|
||||
from collections import Counter
|
||||
from typing import List, Dict, Set, Tuple
|
||||
from ..core.interfaces import BaseContentAnalyzer
|
||||
from ..core.models import ContentAnalysis, ContentType, LanguageInfo
|
||||
|
||||
|
||||
class VaultContentAnalyzer(BaseContentAnalyzer):
|
||||
"""Implementation of content analyzer for Obsidian vault content."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the content analyzer with patterns and keywords."""
|
||||
|
||||
# Language detection patterns
|
||||
self.chinese_chars = re.compile(r'[\u4e00-\u9fff]')
|
||||
self.english_chars = re.compile(r'[a-zA-Z]')
|
||||
|
||||
# Content type patterns
|
||||
self.content_type_patterns = {
|
||||
ContentType.HUB: [
|
||||
r'# .+\n\n.*(?:index|hub|overview|contents?)',
|
||||
r'## (?:Projects?|Areas?|Resources?|Archive)',
|
||||
r'dataview\s*```',
|
||||
r'!\[\[.*\]\].*!\[\[.*\]\]', # Multiple embeds
|
||||
],
|
||||
ContentType.CLIPPING: [
|
||||
r'source:\s*https?://',
|
||||
r'clipped from:',
|
||||
r'saved from:',
|
||||
r'ReadItLater',
|
||||
r'# .+ - .+\.com',
|
||||
],
|
||||
ContentType.DAILY_NOTE: [
|
||||
r'^\d{4}-\d{2}-\d{2}',
|
||||
r'# \d{4}-\d{2}-\d{2}',
|
||||
r'## Daily Notes?',
|
||||
r'## Today',
|
||||
],
|
||||
ContentType.MEETING: [
|
||||
r'# Meeting:',
|
||||
r'## Attendees?',
|
||||
r'## Action Items?',
|
||||
r'## Minutes',
|
||||
r'meeting notes?',
|
||||
],
|
||||
ContentType.DOCUMENTATION: [
|
||||
r'# (?:API|Documentation|Guide|Manual)',
|
||||
r'## Installation',
|
||||
r'## Usage',
|
||||
r'## Configuration',
|
||||
r'```(?:bash|shell|cmd)',
|
||||
],
|
||||
ContentType.TUTORIAL: [
|
||||
r'# (?:How to|Tutorial|Step by Step)',
|
||||
r'## Step \d+',
|
||||
r'### Prerequisites?',
|
||||
r'## Getting Started',
|
||||
],
|
||||
ContentType.REFERENCE: [
|
||||
r'# (?:Reference|Cheat ?Sheet|Quick Reference)',
|
||||
r'## Commands?',
|
||||
r'## Syntax',
|
||||
r'## Examples?',
|
||||
],
|
||||
ContentType.DRAFT: [
|
||||
r'=Draft=',
|
||||
r'# Draft:',
|
||||
r'status:\s*draft',
|
||||
r'TODO:',
|
||||
r'FIXME:',
|
||||
],
|
||||
}
|
||||
|
||||
# Technology and tool keywords
|
||||
self.tech_keywords = {
|
||||
'ai': ['gpt', 'llm', 'chatgpt', 'openai', 'claude', 'anthropic', 'deepseek', 'ollama'],
|
||||
'infrastructure': ['docker', 'kubernetes', 'aws', 'azure', 'gcp', 'terraform', 'ansible'],
|
||||
'development': ['python', 'javascript', 'typescript', 'react', 'vue', 'node', 'npm', 'yarn'],
|
||||
'database': ['mysql', 'postgresql', 'mongodb', 'redis', 'sqlite', 'oracle'],
|
||||
'web': ['html', 'css', 'http', 'api', 'rest', 'graphql', 'json', 'xml'],
|
||||
'devops': ['ci/cd', 'jenkins', 'github actions', 'gitlab', 'git', 'version control'],
|
||||
'network': ['vpn', 'proxy', 'nginx', 'apache', 'dns', 'ssl', 'tls'],
|
||||
'security': ['encryption', 'authentication', 'authorization', 'oauth', 'jwt', 'ssl'],
|
||||
'mobile': ['ios', 'android', 'react native', 'flutter', 'swift', 'kotlin'],
|
||||
'home-automation': ['home assistant', 'zigbee', 'mqtt', 'esphome', 'tuya'],
|
||||
}
|
||||
|
||||
# Topic extraction patterns
|
||||
self.topic_patterns = {
|
||||
'project_management': ['project', 'task', 'milestone', 'deadline', 'planning'],
|
||||
'productivity': ['gtd', 'productivity', 'workflow', 'automation', 'efficiency'],
|
||||
'health': ['health', 'fitness', 'exercise', 'cycling', 'nutrition'],
|
||||
'finance': ['budget', 'investment', 'money', 'financial', 'ynab'],
|
||||
'cooking': ['recipe', 'cooking', 'ingredient', 'meal', 'food'],
|
||||
'travel': ['travel', 'trip', 'vacation', 'hotel', 'flight'],
|
||||
'gaming': ['game', 'gaming', 'steam', 'console', 'multiplayer'],
|
||||
'writing': ['blog', 'article', 'writing', 'content', 'publish'],
|
||||
}
|
||||
|
||||
# Entity extraction patterns
|
||||
self.entity_patterns = {
|
||||
'tools': re.compile(r'\b(?:obsidian|notion|vscode|cursor|docker|kubernetes|git|npm|yarn|pip)\b', re.IGNORECASE),
|
||||
'technologies': re.compile(r'\b(?:python|javascript|typescript|react|vue|node|html|css|sql|json|yaml)\b', re.IGNORECASE),
|
||||
'organizations': re.compile(r'\b(?:google|microsoft|apple|amazon|meta|openai|anthropic|github|gitlab)\b', re.IGNORECASE),
|
||||
'people': re.compile(r'@[a-zA-Z0-9_]+|(?:by|from|author:)\s+([A-Z][a-z]+\s+[A-Z][a-z]+)'),
|
||||
}
|
||||
|
||||
def analyze_content(self, content: str) -> ContentAnalysis:
|
||||
"""Analyze file content and return comprehensive analysis results."""
|
||||
if not content or not content.strip():
|
||||
return ContentAnalysis(
|
||||
language=LanguageInfo.UNKNOWN,
|
||||
content_type=ContentType.UNKNOWN,
|
||||
topics=[],
|
||||
mentions={'tools': [], 'technologies': [], 'people': [], 'organizations': []},
|
||||
complexity='basic'
|
||||
)
|
||||
|
||||
# Detect language
|
||||
language = self._detect_language_enum(content)
|
||||
|
||||
# Classify content type
|
||||
content_type = self._classify_content_type_enum(content, "")
|
||||
|
||||
# Extract topics
|
||||
topics = self.extract_topics(content)
|
||||
|
||||
# Extract mentions
|
||||
mentions = self._extract_mentions(content)
|
||||
|
||||
# Determine complexity
|
||||
complexity = self._determine_complexity(content)
|
||||
|
||||
return ContentAnalysis(
|
||||
language=language,
|
||||
content_type=content_type,
|
||||
topics=topics,
|
||||
mentions=mentions,
|
||||
complexity=complexity
|
||||
)
|
||||
|
||||
def detect_language(self, content: str) -> str:
|
||||
"""Detect the primary language of the content."""
|
||||
return self._detect_language_enum(content).value
|
||||
|
||||
def extract_topics(self, content: str) -> List[str]:
|
||||
"""Extract topics from content using keyword analysis."""
|
||||
if not content:
|
||||
return []
|
||||
|
||||
content_lower = content.lower()
|
||||
topics = []
|
||||
|
||||
# Check technology topics
|
||||
for tech_category, keywords in self.tech_keywords.items():
|
||||
if any(keyword in content_lower for keyword in keywords):
|
||||
topics.append(f'tech/{tech_category}')
|
||||
|
||||
# Check general topics
|
||||
for topic, keywords in self.topic_patterns.items():
|
||||
if any(keyword in content_lower for keyword in keywords):
|
||||
topics.append(topic)
|
||||
|
||||
# Extract topics from headers
|
||||
header_topics = self._extract_header_topics(content)
|
||||
topics.extend(header_topics)
|
||||
|
||||
# Remove duplicates and return
|
||||
return list(set(topics))
|
||||
|
||||
def classify_content_type(self, content: str, filename: str) -> str:
|
||||
"""Classify the type of content."""
|
||||
return self._classify_content_type_enum(content, filename).value
|
||||
|
||||
def _detect_language_enum(self, content: str) -> LanguageInfo:
|
||||
"""Detect language and return LanguageInfo enum."""
|
||||
if not content:
|
||||
return LanguageInfo.UNKNOWN
|
||||
|
||||
# Count character types
|
||||
chinese_count = len(self.chinese_chars.findall(content))
|
||||
english_count = len(self.english_chars.findall(content))
|
||||
total_chars = chinese_count + english_count
|
||||
|
||||
if total_chars == 0:
|
||||
return LanguageInfo.UNKNOWN
|
||||
|
||||
chinese_ratio = chinese_count / total_chars
|
||||
english_ratio = english_count / total_chars
|
||||
|
||||
# Determine language based on ratios
|
||||
if chinese_ratio > 0.3 and english_ratio > 0.3:
|
||||
return LanguageInfo.MIXED
|
||||
elif chinese_ratio > 0.1:
|
||||
return LanguageInfo.CHINESE
|
||||
elif english_ratio > 0.5:
|
||||
return LanguageInfo.ENGLISH
|
||||
else:
|
||||
return LanguageInfo.UNKNOWN
|
||||
|
||||
def _classify_content_type_enum(self, content: str, filename: str) -> ContentType:
|
||||
"""Classify content type and return ContentType enum."""
|
||||
if not content:
|
||||
return ContentType.UNKNOWN
|
||||
|
||||
# Check filename patterns first
|
||||
filename_lower = filename.lower()
|
||||
if re.match(r'\d{4}-\d{2}-\d{2}', filename_lower):
|
||||
return ContentType.DAILY_NOTE
|
||||
|
||||
# Check content patterns
|
||||
for content_type, patterns in self.content_type_patterns.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, content, re.IGNORECASE | re.MULTILINE):
|
||||
return content_type
|
||||
|
||||
# Default classification based on content characteristics
|
||||
if len(content.split('\n')) < 10:
|
||||
return ContentType.NOTE
|
||||
elif '```' in content and ('##' in content or '###' in content):
|
||||
return ContentType.DOCUMENTATION
|
||||
elif content.count('#') > 3:
|
||||
return ContentType.REFERENCE
|
||||
else:
|
||||
return ContentType.NOTE
|
||||
|
||||
def _extract_mentions(self, content: str) -> Dict[str, List[str]]:
|
||||
"""Extract mentions of tools, technologies, people, and organizations."""
|
||||
mentions = {
|
||||
'tools': [],
|
||||
'technologies': [],
|
||||
'people': [],
|
||||
'organizations': []
|
||||
}
|
||||
|
||||
for entity_type, pattern in self.entity_patterns.items():
|
||||
matches = pattern.findall(content)
|
||||
if entity_type == 'people':
|
||||
# Special handling for people mentions
|
||||
people = []
|
||||
for match in matches:
|
||||
if isinstance(match, tuple):
|
||||
people.extend([m for m in match if m])
|
||||
else:
|
||||
people.append(match)
|
||||
mentions[entity_type] = list(set(people))
|
||||
else:
|
||||
mentions[entity_type] = list(set(matches))
|
||||
|
||||
return mentions
|
||||
|
||||
def _extract_header_topics(self, content: str) -> List[str]:
|
||||
"""Extract topics from markdown headers."""
|
||||
topics = []
|
||||
|
||||
# Find all headers
|
||||
header_pattern = re.compile(r'^#+\s+(.+)$', re.MULTILINE)
|
||||
headers = header_pattern.findall(content)
|
||||
|
||||
for header in headers:
|
||||
header_lower = header.lower().strip()
|
||||
|
||||
# Skip common header words
|
||||
skip_words = {'introduction', 'overview', 'conclusion', 'summary', 'notes', 'todo', 'done'}
|
||||
if header_lower in skip_words:
|
||||
continue
|
||||
|
||||
# Extract meaningful topics from headers
|
||||
words = re.findall(r'\b[a-zA-Z]{3,}\b', header_lower)
|
||||
for word in words:
|
||||
if word not in skip_words and len(word) > 3:
|
||||
topics.append(word)
|
||||
|
||||
return topics[:5] # Limit to top 5 header topics
|
||||
|
||||
def _determine_complexity(self, content: str) -> str:
|
||||
"""Determine content complexity based on various factors."""
|
||||
if not content:
|
||||
return 'basic'
|
||||
|
||||
# Count various complexity indicators
|
||||
code_blocks = content.count('```')
|
||||
links = content.count('http')
|
||||
technical_terms = sum(1 for category in self.tech_keywords.values()
|
||||
for term in category if term in content.lower())
|
||||
word_count = len(content.split())
|
||||
|
||||
# Calculate complexity score
|
||||
complexity_score = 0
|
||||
complexity_score += min(code_blocks * 2, 10) # Code blocks add complexity
|
||||
complexity_score += min(links, 5) # External links add complexity
|
||||
complexity_score += min(technical_terms, 15) # Technical terms add complexity
|
||||
complexity_score += min(word_count // 500, 10) # Length adds complexity
|
||||
|
||||
# Classify based on score
|
||||
if complexity_score >= 20:
|
||||
return 'advanced'
|
||||
elif complexity_score >= 10:
|
||||
return 'intermediate'
|
||||
else:
|
||||
return 'basic'
|
||||
@@ -0,0 +1,234 @@
|
||||
"""File discovery implementation for vault scanning."""
|
||||
|
||||
import os
|
||||
import chardet
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Set
|
||||
from ..core.interfaces import BaseFileDiscovery
|
||||
from ..core.models import FileInfo
|
||||
|
||||
|
||||
class VaultFileDiscovery(BaseFileDiscovery):
|
||||
"""Implementation of file discovery for Obsidian vault scanning."""
|
||||
|
||||
def __init__(self, vault_root: str):
|
||||
"""Initialize with vault root directory."""
|
||||
self.vault_root = Path(vault_root).resolve()
|
||||
|
||||
# Sensitive directories to exclude
|
||||
self.sensitive_dirs = {
|
||||
'.obsidian',
|
||||
'.git',
|
||||
'.smart-env',
|
||||
'.pytest_cache',
|
||||
'__pycache__',
|
||||
'node_modules',
|
||||
'.vscode',
|
||||
'.idea',
|
||||
'400-archive/security-sensitive' # From the vault structure
|
||||
}
|
||||
|
||||
# File extensions to include (primarily text files)
|
||||
self.allowed_extensions = {
|
||||
'.md', '.markdown', '.txt', '.yaml', '.yml', '.json',
|
||||
'.py', '.js', '.ts', '.html', '.css', '.xml'
|
||||
}
|
||||
|
||||
# File patterns to exclude
|
||||
self.excluded_patterns = {
|
||||
'.DS_Store',
|
||||
'Thumbs.db',
|
||||
'.gitignore',
|
||||
'.gitkeep'
|
||||
}
|
||||
|
||||
def scan_directory(self, path: str) -> List[FileInfo]:
|
||||
"""Recursively scan directory and return file information."""
|
||||
scan_path = Path(path)
|
||||
if not scan_path.is_absolute():
|
||||
scan_path = self.vault_root / scan_path
|
||||
|
||||
files = []
|
||||
|
||||
try:
|
||||
for root, dirs, filenames in os.walk(scan_path):
|
||||
root_path = Path(root)
|
||||
|
||||
# Filter out sensitive directories
|
||||
dirs[:] = [d for d in dirs if not self._is_sensitive_dir(root_path / d)]
|
||||
|
||||
for filename in filenames:
|
||||
file_path = root_path / filename
|
||||
|
||||
# Skip excluded patterns
|
||||
if filename in self.excluded_patterns:
|
||||
continue
|
||||
|
||||
# Check if file extension is allowed
|
||||
if not self._is_allowed_file(file_path):
|
||||
continue
|
||||
|
||||
try:
|
||||
file_info = self._create_file_info(file_path)
|
||||
if file_info:
|
||||
files.append(file_info)
|
||||
except (OSError, PermissionError) as e:
|
||||
# Log error but continue processing
|
||||
print(f"Warning: Could not process file {file_path}: {e}")
|
||||
continue
|
||||
|
||||
except (OSError, PermissionError) as e:
|
||||
print(f"Error scanning directory {scan_path}: {e}")
|
||||
|
||||
return files
|
||||
|
||||
def filter_by_type(self, files: List[FileInfo], types: List[str]) -> List[FileInfo]:
|
||||
"""Filter files by extension types."""
|
||||
if not types:
|
||||
return files
|
||||
|
||||
# Normalize extensions (ensure they start with .)
|
||||
normalized_types = set()
|
||||
for ext in types:
|
||||
if not ext.startswith('.'):
|
||||
ext = '.' + ext
|
||||
normalized_types.add(ext.lower())
|
||||
|
||||
return [f for f in files if f.extension.lower() in normalized_types]
|
||||
|
||||
def exclude_sensitive(self, files: List[FileInfo]) -> List[FileInfo]:
|
||||
"""Exclude sensitive directories and files."""
|
||||
filtered_files = []
|
||||
|
||||
for file_info in files:
|
||||
file_path = Path(file_info.path)
|
||||
|
||||
# Check if file is in sensitive directory
|
||||
if self._is_in_sensitive_dir(file_path):
|
||||
continue
|
||||
|
||||
# Check for sensitive content patterns in filename
|
||||
if self._has_sensitive_filename(file_info.name):
|
||||
continue
|
||||
|
||||
filtered_files.append(file_info)
|
||||
|
||||
return filtered_files
|
||||
|
||||
def _create_file_info(self, file_path: Path) -> FileInfo:
|
||||
"""Create FileInfo object from file path."""
|
||||
try:
|
||||
stat = file_path.stat()
|
||||
|
||||
# Get relative path from vault root
|
||||
try:
|
||||
relative_path = file_path.relative_to(self.vault_root)
|
||||
except ValueError:
|
||||
# File is outside vault root, use absolute path
|
||||
relative_path = file_path
|
||||
|
||||
# Read content for text files
|
||||
content = None
|
||||
if file_path.suffix.lower() in {'.md', '.markdown', '.txt', '.yaml', '.yml'}:
|
||||
content = self._read_file_content(file_path)
|
||||
|
||||
return FileInfo(
|
||||
path=str(relative_path),
|
||||
name=file_path.name,
|
||||
directory=str(relative_path.parent) if relative_path.parent != Path('.') else '',
|
||||
extension=file_path.suffix,
|
||||
size=stat.st_size,
|
||||
created=datetime.fromtimestamp(stat.st_ctime),
|
||||
modified=datetime.fromtimestamp(stat.st_mtime),
|
||||
content=content
|
||||
)
|
||||
|
||||
except (OSError, PermissionError):
|
||||
return None
|
||||
|
||||
def _read_file_content(self, file_path: Path) -> str:
|
||||
"""Read file content with encoding detection."""
|
||||
try:
|
||||
# First try UTF-8
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
# Detect encoding
|
||||
with open(file_path, 'rb') as f:
|
||||
raw_data = f.read()
|
||||
|
||||
detected = chardet.detect(raw_data)
|
||||
encoding = detected.get('encoding', 'utf-8')
|
||||
|
||||
# Try detected encoding
|
||||
return raw_data.decode(encoding, errors='replace')
|
||||
|
||||
except Exception:
|
||||
# Fallback to reading as binary and replacing errors
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _is_sensitive_dir(self, dir_path: Path) -> bool:
|
||||
"""Check if directory should be excluded as sensitive."""
|
||||
dir_name = dir_path.name
|
||||
|
||||
# Check exact matches
|
||||
if dir_name in self.sensitive_dirs:
|
||||
return True
|
||||
|
||||
# Check relative path matches
|
||||
try:
|
||||
relative_path = dir_path.relative_to(self.vault_root)
|
||||
if str(relative_path) in self.sensitive_dirs:
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Check for hidden directories (starting with .)
|
||||
if dir_name.startswith('.') and dir_name not in {'.kiro'}:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _is_in_sensitive_dir(self, file_path: Path) -> bool:
|
||||
"""Check if file is in a sensitive directory."""
|
||||
try:
|
||||
relative_path = file_path.relative_to(self.vault_root)
|
||||
path_parts = relative_path.parts
|
||||
|
||||
for part in path_parts[:-1]: # Exclude filename
|
||||
if part in self.sensitive_dirs:
|
||||
return True
|
||||
if part.startswith('.') and part not in {'.kiro'}:
|
||||
return True
|
||||
|
||||
# Check full directory path
|
||||
dir_path = str(relative_path.parent)
|
||||
if dir_path in self.sensitive_dirs:
|
||||
return True
|
||||
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
def _is_allowed_file(self, file_path: Path) -> bool:
|
||||
"""Check if file extension is allowed."""
|
||||
return file_path.suffix.lower() in self.allowed_extensions
|
||||
|
||||
def _has_sensitive_filename(self, filename: str) -> bool:
|
||||
"""Check if filename indicates sensitive content."""
|
||||
sensitive_patterns = {
|
||||
'password', 'secret', 'key', 'token', 'credential',
|
||||
'private', 'confidential', 'sensitive'
|
||||
}
|
||||
|
||||
filename_lower = filename.lower()
|
||||
return any(pattern in filename_lower for pattern in sensitive_patterns)
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Tag generation implementation for the comprehensive tagging system."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Set
|
||||
from ..core.interfaces import BaseTagGenerator
|
||||
from ..core.models import ContentAnalysis, TagStructure
|
||||
|
||||
|
||||
class TagGeneratorImpl(BaseTagGenerator):
|
||||
"""Implementation of tag generation based on directory structure and content analysis."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the tag generator with predefined mappings and patterns."""
|
||||
# Directory-based tag mappings
|
||||
self.directory_mappings = {
|
||||
'100-project': 'project',
|
||||
'200-area': 'area',
|
||||
'300-resources': 'resource',
|
||||
'400-archive': 'archive',
|
||||
'clippings': 'clipping',
|
||||
'readitlater inbox': 'clipping'
|
||||
}
|
||||
|
||||
# Hierarchical topic mappings
|
||||
self.topic_hierarchies = {
|
||||
# Technology hierarchies
|
||||
'ai': 'tech/ai',
|
||||
'llm': 'tech/ai/llm',
|
||||
'machine learning': 'tech/ai/ml',
|
||||
'chatgpt': 'tech/ai/llm',
|
||||
'openai': 'tech/ai/llm',
|
||||
'claude': 'tech/ai/llm',
|
||||
'docker': 'tech/infrastructure/docker',
|
||||
'kubernetes': 'tech/infrastructure/k8s',
|
||||
'python': 'tech/development/python',
|
||||
'javascript': 'tech/development/javascript',
|
||||
'typescript': 'tech/development/typescript',
|
||||
'react': 'tech/development/react',
|
||||
'vue': 'tech/development/vue',
|
||||
'node': 'tech/development/nodejs',
|
||||
'api': 'tech/development/api',
|
||||
'database': 'tech/infrastructure/database',
|
||||
'mysql': 'tech/infrastructure/database',
|
||||
'postgresql': 'tech/infrastructure/database',
|
||||
'mongodb': 'tech/infrastructure/database',
|
||||
'redis': 'tech/infrastructure/database',
|
||||
'nginx': 'tech/infrastructure/web',
|
||||
'apache': 'tech/infrastructure/web',
|
||||
'aws': 'tech/infrastructure/cloud',
|
||||
'azure': 'tech/infrastructure/cloud',
|
||||
'gcp': 'tech/infrastructure/cloud',
|
||||
'linux': 'tech/infrastructure/os',
|
||||
'ubuntu': 'tech/infrastructure/os',
|
||||
'centos': 'tech/infrastructure/os',
|
||||
|
||||
# Personal hierarchies
|
||||
'productivity': 'personal/productivity',
|
||||
'gtd': 'personal/productivity/gtd',
|
||||
'health': 'personal/health',
|
||||
'cycling': 'personal/health/cycling',
|
||||
'fitness': 'personal/health/fitness',
|
||||
'finance': 'personal/finance',
|
||||
'investment': 'personal/finance/investment',
|
||||
'budget': 'personal/finance/budget',
|
||||
'cooking': 'personal/cooking',
|
||||
'recipe': 'personal/cooking/recipe',
|
||||
|
||||
# Work hierarchies
|
||||
'government': 'work/government',
|
||||
'enterprise': 'work/enterprise',
|
||||
'airport': 'work/airport',
|
||||
'ali': 'work/ali',
|
||||
|
||||
# Home automation hierarchies
|
||||
'home assistant': 'home-automation/hass',
|
||||
'esphome': 'home-automation/esphome',
|
||||
'zigbee': 'home-automation/zigbee',
|
||||
'mqtt': 'home-automation/mqtt',
|
||||
'sensor': 'home-automation/sensor',
|
||||
'automation': 'home-automation/automation'
|
||||
}
|
||||
|
||||
# Sensitive content patterns
|
||||
self.sensitive_patterns = {
|
||||
'credentials': [
|
||||
r'password\s*[:=]\s*["\']?[\w\-@#$%^&*()]+["\']?',
|
||||
r'api[_\-]?key\s*[:=]\s*["\']?[\w\-]+["\']?',
|
||||
r'secret\s*[:=]\s*["\']?[\w\-]+["\']?',
|
||||
r'token\s*[:=]\s*["\']?[\w\-\.]+["\']?',
|
||||
r'auth[_\-]?token\s*[:=]\s*["\']?[\w\-\.]+["\']?',
|
||||
r'access[_\-]?key\s*[:=]\s*["\']?[\w\-]+["\']?',
|
||||
r'private[_\-]?key',
|
||||
r'ssh[_\-]?key',
|
||||
r'-----BEGIN.*PRIVATE KEY-----'
|
||||
],
|
||||
'personal': [
|
||||
r'\b\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}\b', # Credit card numbers
|
||||
r'\b\d{3}-\d{2}-\d{4}\b', # SSN format
|
||||
r'\b\d{11}\b', # Phone numbers (simplified)
|
||||
r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}', # Email addresses
|
||||
r'\b(?:home|personal|private)\s+(?:address|phone|email)',
|
||||
r'\bbirthdate\b|\bdate\s+of\s+birth\b'
|
||||
],
|
||||
'financial': [
|
||||
r'\b(?:salary|income|wage)\s*[:=]\s*[\$¥€£]?[\d,]+',
|
||||
r'\b(?:bank|account)\s+(?:number|details)',
|
||||
r'\b(?:routing|swift)\s+(?:number|code)',
|
||||
r'\b(?:tax|invoice|receipt)\s+(?:id|number)',
|
||||
r'\b(?:budget|expense|cost)\s*[:=]\s*[\$¥€£]?[\d,]+',
|
||||
r'\b(?:investment|portfolio|stock)\s+(?:value|amount)'
|
||||
],
|
||||
'legal': [
|
||||
r'\b(?:contract|agreement|legal)\s+(?:document|file)',
|
||||
r'\b(?:confidential|proprietary|classified)',
|
||||
r'\b(?:copyright|trademark|patent)\s+(?:notice|info)',
|
||||
r'\b(?:license|licensing)\s+(?:agreement|terms)',
|
||||
r'\bnda\b|\bnon[_\-]?disclosure\b'
|
||||
]
|
||||
}
|
||||
|
||||
# Tag validation patterns
|
||||
self.valid_tag_pattern = re.compile(r'^[a-z0-9]+(?:[-/][a-z0-9]+)*$')
|
||||
|
||||
def generate_directory_tags(self, filepath: str) -> List[str]:
|
||||
"""Generate tags based on directory structure."""
|
||||
tags = []
|
||||
path = Path(filepath)
|
||||
parts = [p.lower() for p in path.parts]
|
||||
|
||||
# Generate primary directory tags
|
||||
for part in parts:
|
||||
if part in self.directory_mappings:
|
||||
primary_tag = self.directory_mappings[part]
|
||||
tags.append(primary_tag)
|
||||
|
||||
# Add hierarchical subdirectory tags
|
||||
try:
|
||||
part_index = parts.index(part)
|
||||
if part_index + 1 < len(parts):
|
||||
subdirs = parts[part_index + 1:-1] # Exclude filename
|
||||
for subdir in subdirs:
|
||||
# Clean and validate subdirectory name
|
||||
clean_subdir = self._clean_tag_name(subdir)
|
||||
if clean_subdir:
|
||||
hierarchical_tag = f"{primary_tag}/{clean_subdir}"
|
||||
tags.append(hierarchical_tag)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# Handle special cases
|
||||
if any('clipping' in part for part in parts):
|
||||
tags.append('clipping')
|
||||
|
||||
return list(set(tags)) # Remove duplicates
|
||||
|
||||
def generate_content_tags(self, analysis: ContentAnalysis) -> List[str]:
|
||||
"""Generate tags based on content analysis."""
|
||||
tags = []
|
||||
|
||||
# Add language tag
|
||||
if analysis.language:
|
||||
tags.append(f"lang/{analysis.language.value}")
|
||||
|
||||
# Add content type tag
|
||||
if analysis.content_type:
|
||||
tags.append(f"type/{analysis.content_type.value}")
|
||||
|
||||
# Add complexity tag if not basic
|
||||
if analysis.complexity and analysis.complexity != 'basic':
|
||||
tags.append(f"complexity/{analysis.complexity}")
|
||||
|
||||
# Add sentiment tag if available
|
||||
if analysis.sentiment and analysis.sentiment != 'neutral':
|
||||
tags.append(f"sentiment/{analysis.sentiment}")
|
||||
|
||||
# Add mention-based tags
|
||||
for category, items in analysis.mentions.items():
|
||||
for item in items:
|
||||
clean_item = self._clean_tag_name(item)
|
||||
if clean_item:
|
||||
tags.append(f"{category}/{clean_item}")
|
||||
|
||||
return tags
|
||||
|
||||
def generate_hierarchical_tags(self, topics: List[str]) -> List[str]:
|
||||
"""Generate hierarchical tags from topics."""
|
||||
hierarchical_tags = []
|
||||
|
||||
for topic in topics:
|
||||
topic_lower = topic.lower().strip()
|
||||
|
||||
# Check for direct mapping
|
||||
if topic_lower in self.topic_hierarchies:
|
||||
hierarchical_tags.append(self.topic_hierarchies[topic_lower])
|
||||
else:
|
||||
# Try partial matching for compound topics
|
||||
for key, hierarchy in self.topic_hierarchies.items():
|
||||
if key in topic_lower or topic_lower in key:
|
||||
hierarchical_tags.append(hierarchy)
|
||||
break
|
||||
else:
|
||||
# Create a generic hierarchical tag
|
||||
clean_topic = self._clean_tag_name(topic_lower)
|
||||
if clean_topic:
|
||||
# Try to categorize based on common patterns
|
||||
if any(tech_word in topic_lower for tech_word in ['tech', 'software', 'code', 'dev', 'program']):
|
||||
hierarchical_tags.append(f"tech/{clean_topic}")
|
||||
elif any(personal_word in topic_lower for personal_word in ['personal', 'life', 'habit', 'goal']):
|
||||
hierarchical_tags.append(f"personal/{clean_topic}")
|
||||
elif any(work_word in topic_lower for work_word in ['work', 'job', 'career', 'business']):
|
||||
hierarchical_tags.append(f"work/{clean_topic}")
|
||||
else:
|
||||
hierarchical_tags.append(clean_topic)
|
||||
|
||||
return list(set(hierarchical_tags))
|
||||
|
||||
def consolidate_tags(self, tags: List[str]) -> List[str]:
|
||||
"""Consolidate and deduplicate tags."""
|
||||
if not tags:
|
||||
return []
|
||||
|
||||
# Clean and validate all tags
|
||||
cleaned_tags = []
|
||||
for tag in tags:
|
||||
clean_tag = self._clean_tag_name(tag)
|
||||
if clean_tag and self._is_valid_tag(clean_tag):
|
||||
cleaned_tags.append(clean_tag)
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
consolidated = []
|
||||
for tag in cleaned_tags:
|
||||
if tag not in seen:
|
||||
seen.add(tag)
|
||||
consolidated.append(tag)
|
||||
|
||||
# Apply consolidation rules
|
||||
consolidated = self._apply_consolidation_rules(consolidated)
|
||||
|
||||
# Sort tags for consistency (hierarchical tags first, then alphabetical)
|
||||
return self._sort_tags(consolidated)
|
||||
|
||||
def detect_sensitive_content(self, content: str, filepath: str) -> List[str]:
|
||||
"""Detect sensitive content and return appropriate tags."""
|
||||
sensitive_tags = []
|
||||
content_lower = content.lower()
|
||||
|
||||
# Check for sensitive patterns
|
||||
for category, patterns in self.sensitive_patterns.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, content, re.IGNORECASE):
|
||||
sensitive_tags.append(f"sensitive/{category}")
|
||||
break # Only add the category once
|
||||
|
||||
# Check filepath for sensitive indicators
|
||||
filepath_lower = filepath.lower()
|
||||
if any(sensitive_dir in filepath_lower for sensitive_dir in ['security-sensitive', 'private', 'confidential']):
|
||||
if 'sensitive/personal' not in sensitive_tags:
|
||||
sensitive_tags.append('sensitive/personal')
|
||||
|
||||
return list(set(sensitive_tags))
|
||||
|
||||
def _clean_tag_name(self, tag: str) -> str:
|
||||
"""Clean and normalize tag names to kebab-case."""
|
||||
if not tag:
|
||||
return ""
|
||||
|
||||
# Convert to lowercase and replace spaces/underscores with hyphens
|
||||
cleaned = re.sub(r'[_\s]+', '-', tag.lower().strip())
|
||||
|
||||
# Remove special characters except hyphens and forward slashes
|
||||
cleaned = re.sub(r'[^a-z0-9\-/]', '', cleaned)
|
||||
|
||||
# Remove multiple consecutive hyphens
|
||||
cleaned = re.sub(r'-+', '-', cleaned)
|
||||
|
||||
# Remove leading/trailing hyphens
|
||||
cleaned = cleaned.strip('-')
|
||||
|
||||
return cleaned
|
||||
|
||||
def _is_valid_tag(self, tag: str) -> bool:
|
||||
"""Validate tag format."""
|
||||
if not tag:
|
||||
return False
|
||||
|
||||
# Check against valid pattern
|
||||
if not self.valid_tag_pattern.match(tag):
|
||||
return False
|
||||
|
||||
# Additional validation rules
|
||||
if len(tag) > 50: # Reasonable length limit
|
||||
return False
|
||||
|
||||
if tag.startswith('/') or tag.endswith('/'):
|
||||
return False
|
||||
|
||||
if '//' in tag: # No empty hierarchy levels
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _apply_consolidation_rules(self, tags: List[str]) -> List[str]:
|
||||
"""Apply tag consolidation rules to remove redundancy."""
|
||||
consolidated = tags.copy()
|
||||
|
||||
# Remove redundant hierarchical tags
|
||||
# If we have both 'tech' and 'tech/ai', keep only 'tech/ai'
|
||||
hierarchical_tags = [tag for tag in consolidated if '/' in tag]
|
||||
simple_tags = [tag for tag in consolidated if '/' not in tag]
|
||||
|
||||
# Remove simple tags that are covered by hierarchical tags
|
||||
filtered_simple = []
|
||||
for simple_tag in simple_tags:
|
||||
is_covered = any(hier_tag.startswith(f"{simple_tag}/") for hier_tag in hierarchical_tags)
|
||||
if not is_covered:
|
||||
filtered_simple.append(simple_tag)
|
||||
|
||||
return filtered_simple + hierarchical_tags
|
||||
|
||||
def _sort_tags(self, tags: List[str]) -> List[str]:
|
||||
"""Sort tags with hierarchical tags first, then alphabetical."""
|
||||
hierarchical = [tag for tag in tags if '/' in tag]
|
||||
simple = [tag for tag in tags if '/' not in tag]
|
||||
|
||||
# Sort hierarchical tags by depth then alphabetically
|
||||
hierarchical.sort(key=lambda x: (x.count('/'), x))
|
||||
simple.sort()
|
||||
|
||||
return hierarchical + simple
|
||||
@@ -0,0 +1 @@
|
||||
"""Test package for the tagging system."""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,124 @@
|
||||
"""Pytest configuration and fixtures."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
from hypothesis import settings, Verbosity
|
||||
|
||||
from tagging_system.core.models import (
|
||||
FileInfo,
|
||||
ContentAnalysis,
|
||||
TagStructure,
|
||||
FrontmatterData,
|
||||
ContentType,
|
||||
LanguageInfo
|
||||
)
|
||||
from tagging_system.config import TaggingConfig
|
||||
|
||||
|
||||
# Configure hypothesis for property-based testing
|
||||
settings.register_profile("default", max_examples=100, verbosity=Verbosity.normal)
|
||||
settings.register_profile("ci", max_examples=1000, verbosity=Verbosity.verbose)
|
||||
settings.load_profile("default")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_file_info() -> FileInfo:
|
||||
"""Create a sample FileInfo for testing."""
|
||||
return FileInfo(
|
||||
path="100-project/AI/test.md",
|
||||
name="test.md",
|
||||
directory="100-project/AI",
|
||||
extension=".md",
|
||||
size=1024,
|
||||
created=datetime(2024, 1, 1, 12, 0, 0),
|
||||
modified=datetime(2024, 1, 2, 12, 0, 0),
|
||||
content="# Test File\n\nThis is a test file about AI and machine learning."
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_content_analysis() -> ContentAnalysis:
|
||||
"""Create a sample ContentAnalysis for testing."""
|
||||
return ContentAnalysis(
|
||||
language=LanguageInfo.ENGLISH,
|
||||
content_type=ContentType.NOTE,
|
||||
topics=["ai", "machine-learning", "technology"],
|
||||
mentions={
|
||||
'tools': ['python', 'tensorflow'],
|
||||
'technologies': ['ai', 'ml'],
|
||||
'people': [],
|
||||
'organizations': ['openai']
|
||||
},
|
||||
sentiment="neutral",
|
||||
complexity="intermediate"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_tag_structure() -> TagStructure:
|
||||
"""Create a sample TagStructure for testing."""
|
||||
return TagStructure(
|
||||
primary=["project"],
|
||||
hierarchical=["tech/ai", "tech/ml"],
|
||||
content=["python", "tensorflow"],
|
||||
meta=["lang/en", "type/note"],
|
||||
custom=["custom-tag"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_frontmatter_data() -> FrontmatterData:
|
||||
"""Create a sample FrontmatterData for testing."""
|
||||
return FrontmatterData(
|
||||
title="Test File",
|
||||
tags=["project", "tech/ai", "python"],
|
||||
created="2024-01-01",
|
||||
updated="2024-01-02",
|
||||
type="note",
|
||||
lang="en",
|
||||
aliases=["test"],
|
||||
description="A test file for AI projects"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def default_config() -> TaggingConfig:
|
||||
"""Create a default TaggingConfig for testing."""
|
||||
return TaggingConfig()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_vault_structure(tmp_path: Path) -> Path:
|
||||
"""Create a temporary vault structure for testing."""
|
||||
vault_root = tmp_path / "test_vault"
|
||||
|
||||
# Create directory structure
|
||||
directories = [
|
||||
"100-project/AI",
|
||||
"100-project/Infrastructure",
|
||||
"200-area/Productivity",
|
||||
"200-area/Health",
|
||||
"300-resources/Development",
|
||||
"400-archive",
|
||||
"Clippings",
|
||||
"ReadItLater Inbox"
|
||||
]
|
||||
|
||||
for directory in directories:
|
||||
(vault_root / directory).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create sample files
|
||||
sample_files = [
|
||||
("100-project/AI/llm-notes.md", "# LLM Notes\n\nNotes about large language models."),
|
||||
("200-area/Productivity/gtd.md", "# Getting Things Done\n\nProductivity methodology."),
|
||||
("300-resources/Development/python.md", "# Python Resources\n\nPython development resources."),
|
||||
("Clippings/article.md", "# Interesting Article\n\nClipped from web.")
|
||||
]
|
||||
|
||||
for file_path, content in sample_files:
|
||||
file_full_path = vault_root / file_path
|
||||
file_full_path.write_text(content, encoding='utf-8')
|
||||
|
||||
return vault_root
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Unit tests for configuration system."""
|
||||
|
||||
import pytest
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tagging_system.config import (
|
||||
TaggingConfig,
|
||||
DirectoryMapping,
|
||||
TagHierarchy,
|
||||
SensitivePatterns,
|
||||
load_config,
|
||||
save_config
|
||||
)
|
||||
|
||||
|
||||
class TestDirectoryMapping:
|
||||
"""Test DirectoryMapping model."""
|
||||
|
||||
def test_directory_mapping_creation(self):
|
||||
"""Test DirectoryMapping creation."""
|
||||
mapping = DirectoryMapping(
|
||||
pattern="100-project",
|
||||
primary_tag="project",
|
||||
hierarchical_tags=["tech"],
|
||||
exclude_patterns=["*.tmp"]
|
||||
)
|
||||
|
||||
assert mapping.pattern == "100-project"
|
||||
assert mapping.primary_tag == "project"
|
||||
assert mapping.hierarchical_tags == ["tech"]
|
||||
assert mapping.exclude_patterns == ["*.tmp"]
|
||||
|
||||
|
||||
class TestTagHierarchy:
|
||||
"""Test TagHierarchy model."""
|
||||
|
||||
def test_tag_hierarchy_creation(self):
|
||||
"""Test TagHierarchy creation."""
|
||||
hierarchy = TagHierarchy(
|
||||
root="tech",
|
||||
children={"ai": TagHierarchy("ai")},
|
||||
aliases=["technology"]
|
||||
)
|
||||
|
||||
assert hierarchy.root == "tech"
|
||||
assert "ai" in hierarchy.children
|
||||
assert hierarchy.aliases == ["technology"]
|
||||
|
||||
def test_get_full_path(self):
|
||||
"""Test get_full_path method."""
|
||||
hierarchy = TagHierarchy("tech")
|
||||
|
||||
assert hierarchy.get_full_path() == "tech"
|
||||
assert hierarchy.get_full_path("ai") == "tech/ai"
|
||||
assert hierarchy.get_full_path("ai/llm") == "tech/ai/llm"
|
||||
|
||||
|
||||
class TestSensitivePatterns:
|
||||
"""Test SensitivePatterns model."""
|
||||
|
||||
def test_sensitive_patterns_defaults(self):
|
||||
"""Test SensitivePatterns default values."""
|
||||
patterns = SensitivePatterns()
|
||||
|
||||
assert len(patterns.credential_patterns) > 0
|
||||
assert len(patterns.personal_patterns) > 0
|
||||
assert len(patterns.financial_patterns) > 0
|
||||
assert any("api" in pattern for pattern in patterns.credential_patterns)
|
||||
|
||||
|
||||
class TestTaggingConfig:
|
||||
"""Test TaggingConfig model."""
|
||||
|
||||
def test_tagging_config_defaults(self):
|
||||
"""Test TaggingConfig default values."""
|
||||
config = TaggingConfig()
|
||||
|
||||
assert len(config.directory_mappings) > 0
|
||||
assert len(config.tag_hierarchies) > 0
|
||||
assert len(config.excluded_directories) > 0
|
||||
assert config.tag_format_rules["case"] == "kebab"
|
||||
|
||||
def test_get_directory_mapping(self):
|
||||
"""Test get_directory_mapping method."""
|
||||
config = TaggingConfig()
|
||||
|
||||
mapping = config.get_directory_mapping("100-project/AI/test.md")
|
||||
assert mapping is not None
|
||||
assert mapping.primary_tag == "project"
|
||||
|
||||
no_mapping = config.get_directory_mapping("unknown/path")
|
||||
assert no_mapping is None
|
||||
|
||||
def test_get_tag_hierarchy(self):
|
||||
"""Test get_tag_hierarchy method."""
|
||||
config = TaggingConfig()
|
||||
|
||||
tech_hierarchy = config.get_tag_hierarchy("tech")
|
||||
assert tech_hierarchy is not None
|
||||
assert tech_hierarchy.root == "tech"
|
||||
|
||||
unknown_hierarchy = config.get_tag_hierarchy("unknown")
|
||||
assert unknown_hierarchy is None
|
||||
|
||||
def test_is_excluded_directory(self):
|
||||
"""Test is_excluded_directory method."""
|
||||
config = TaggingConfig()
|
||||
|
||||
assert config.is_excluded_directory(".obsidian/plugins") is True
|
||||
assert config.is_excluded_directory("100-project/AI") is False
|
||||
|
||||
def test_is_excluded_file(self):
|
||||
"""Test is_excluded_file method."""
|
||||
config = TaggingConfig()
|
||||
|
||||
assert config.is_excluded_file("test.pyc") is True
|
||||
assert config.is_excluded_file(".DS_Store") is True
|
||||
assert config.is_excluded_file("test.md") is False
|
||||
|
||||
|
||||
class TestConfigLoading:
|
||||
"""Test configuration loading and saving."""
|
||||
|
||||
def test_load_default_config(self):
|
||||
"""Test loading default config when no file exists."""
|
||||
config = load_config()
|
||||
|
||||
assert isinstance(config, TaggingConfig)
|
||||
assert len(config.directory_mappings) > 0
|
||||
|
||||
def test_load_config_creates_default_file(self, tmp_path: Path):
|
||||
"""Test that load_config creates default file when path doesn't exist."""
|
||||
config_path = tmp_path / "config.yaml"
|
||||
|
||||
config = load_config(config_path)
|
||||
|
||||
assert isinstance(config, TaggingConfig)
|
||||
assert config_path.exists()
|
||||
|
||||
def test_save_and_load_config(self, tmp_path: Path):
|
||||
"""Test saving and loading configuration."""
|
||||
config_path = tmp_path / "test_config.yaml"
|
||||
original_config = TaggingConfig()
|
||||
original_config.excluded_directories.append("test_exclude")
|
||||
|
||||
save_config(original_config, config_path)
|
||||
loaded_config = load_config(config_path)
|
||||
|
||||
assert config_path.exists()
|
||||
assert "test_exclude" in loaded_config.excluded_directories
|
||||
|
||||
def test_load_json_config(self, tmp_path: Path):
|
||||
"""Test loading JSON configuration."""
|
||||
config_path = tmp_path / "config.json"
|
||||
config_data = {
|
||||
"excluded_directories": [".test", ".custom"],
|
||||
"tag_format_rules": {"case": "snake"}
|
||||
}
|
||||
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(config_data, f)
|
||||
|
||||
config = load_config(config_path)
|
||||
|
||||
assert ".test" in config.excluded_directories
|
||||
assert ".custom" in config.excluded_directories
|
||||
assert config.tag_format_rules["case"] == "snake"
|
||||
|
||||
def test_load_invalid_config_returns_default(self, tmp_path: Path):
|
||||
"""Test that invalid config file returns default config."""
|
||||
config_path = tmp_path / "invalid.yaml"
|
||||
config_path.write_text("invalid: yaml: content: [")
|
||||
|
||||
config = load_config(config_path)
|
||||
|
||||
# Should return default config on error
|
||||
assert isinstance(config, TaggingConfig)
|
||||
assert len(config.directory_mappings) > 0
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Unit tests for core interfaces and protocols."""
|
||||
|
||||
import pytest
|
||||
from typing import List
|
||||
from tagging_system.core.interfaces import (
|
||||
FileDiscovery,
|
||||
ContentAnalyzer,
|
||||
TagGenerator,
|
||||
FrontmatterManager,
|
||||
BaseFileDiscovery,
|
||||
BaseContentAnalyzer,
|
||||
BaseTagGenerator,
|
||||
BaseFrontmatterManager
|
||||
)
|
||||
from tagging_system.core.models import (
|
||||
FileInfo,
|
||||
ContentAnalysis,
|
||||
TagStructure,
|
||||
FrontmatterData,
|
||||
ValidationResult,
|
||||
ContentType,
|
||||
LanguageInfo
|
||||
)
|
||||
|
||||
|
||||
class MockFileDiscovery(BaseFileDiscovery):
|
||||
"""Mock implementation of FileDiscovery for testing."""
|
||||
|
||||
def scan_directory(self, path: str) -> List[FileInfo]:
|
||||
"""Mock directory scanning."""
|
||||
from datetime import datetime
|
||||
return [
|
||||
FileInfo(
|
||||
path=f"{path}/test.md",
|
||||
name="test.md",
|
||||
directory=path,
|
||||
extension=".md",
|
||||
size=100,
|
||||
created=datetime.now(),
|
||||
modified=datetime.now()
|
||||
)
|
||||
]
|
||||
|
||||
def filter_by_type(self, files: List[FileInfo], types: List[str]) -> List[FileInfo]:
|
||||
"""Mock file filtering."""
|
||||
return [f for f in files if f.extension in types]
|
||||
|
||||
def exclude_sensitive(self, files: List[FileInfo]) -> List[FileInfo]:
|
||||
"""Mock sensitive file exclusion."""
|
||||
return [f for f in files if "sensitive" not in f.path]
|
||||
|
||||
|
||||
class MockContentAnalyzer(BaseContentAnalyzer):
|
||||
"""Mock implementation of ContentAnalyzer for testing."""
|
||||
|
||||
def analyze_content(self, content: str) -> ContentAnalysis:
|
||||
"""Mock content analysis."""
|
||||
return ContentAnalysis(
|
||||
language=LanguageInfo.ENGLISH,
|
||||
content_type=ContentType.NOTE,
|
||||
topics=["test"],
|
||||
mentions={'tools': [], 'technologies': [], 'people': [], 'organizations': []}
|
||||
)
|
||||
|
||||
def detect_language(self, content: str) -> str:
|
||||
"""Mock language detection."""
|
||||
return "en"
|
||||
|
||||
def extract_topics(self, content: str) -> List[str]:
|
||||
"""Mock topic extraction."""
|
||||
return ["test", "mock"]
|
||||
|
||||
def classify_content_type(self, content: str, filename: str) -> str:
|
||||
"""Mock content type classification."""
|
||||
return "note"
|
||||
|
||||
|
||||
class MockTagGenerator(BaseTagGenerator):
|
||||
"""Mock implementation of TagGenerator for testing."""
|
||||
|
||||
def generate_directory_tags(self, filepath: str) -> List[str]:
|
||||
"""Mock directory tag generation."""
|
||||
if "100-project" in filepath:
|
||||
return ["project"]
|
||||
return ["unknown"]
|
||||
|
||||
def generate_content_tags(self, analysis: ContentAnalysis) -> List[str]:
|
||||
"""Mock content tag generation."""
|
||||
return analysis.topics
|
||||
|
||||
def generate_hierarchical_tags(self, topics: List[str]) -> List[str]:
|
||||
"""Mock hierarchical tag generation."""
|
||||
return [f"topic/{topic}" for topic in topics]
|
||||
|
||||
def consolidate_tags(self, tags: List[str]) -> List[str]:
|
||||
"""Mock tag consolidation."""
|
||||
return list(set(tags)) # Remove duplicates
|
||||
|
||||
|
||||
class MockFrontmatterManager(BaseFrontmatterManager):
|
||||
"""Mock implementation of FrontmatterManager for testing."""
|
||||
|
||||
def parse_frontmatter(self, content: str) -> FrontmatterData:
|
||||
"""Mock frontmatter parsing."""
|
||||
return FrontmatterData(
|
||||
title="Test",
|
||||
tags=["test"],
|
||||
created="2024-01-01"
|
||||
)
|
||||
|
||||
def update_frontmatter(self, content: str, updates: FrontmatterData) -> str:
|
||||
"""Mock frontmatter updating."""
|
||||
return f"---\ntitle: {updates.title}\ntags: {updates.tags}\n---\n{content}"
|
||||
|
||||
def validate_frontmatter(self, data: FrontmatterData) -> ValidationResult:
|
||||
"""Mock frontmatter validation."""
|
||||
result = ValidationResult(is_valid=True)
|
||||
if not data.title:
|
||||
result.add_error("Title is required")
|
||||
return result
|
||||
|
||||
|
||||
class TestProtocolCompliance:
|
||||
"""Test that mock implementations comply with protocols."""
|
||||
|
||||
def test_file_discovery_protocol_compliance(self):
|
||||
"""Test that MockFileDiscovery implements FileDiscovery protocol."""
|
||||
mock = MockFileDiscovery()
|
||||
|
||||
assert isinstance(mock, FileDiscovery)
|
||||
|
||||
# Test method calls
|
||||
files = mock.scan_directory("test")
|
||||
assert len(files) == 1
|
||||
assert files[0].name == "test.md"
|
||||
|
||||
filtered = mock.filter_by_type(files, [".md"])
|
||||
assert len(filtered) == 1
|
||||
|
||||
non_sensitive = mock.exclude_sensitive(files)
|
||||
assert len(non_sensitive) == 1
|
||||
|
||||
def test_content_analyzer_protocol_compliance(self):
|
||||
"""Test that MockContentAnalyzer implements ContentAnalyzer protocol."""
|
||||
mock = MockContentAnalyzer()
|
||||
|
||||
assert isinstance(mock, ContentAnalyzer)
|
||||
|
||||
# Test method calls
|
||||
analysis = mock.analyze_content("test content")
|
||||
assert analysis.language == LanguageInfo.ENGLISH
|
||||
assert analysis.content_type == ContentType.NOTE
|
||||
|
||||
language = mock.detect_language("test content")
|
||||
assert language == "en"
|
||||
|
||||
topics = mock.extract_topics("test content")
|
||||
assert "test" in topics
|
||||
|
||||
content_type = mock.classify_content_type("test content", "test.md")
|
||||
assert content_type == "note"
|
||||
|
||||
def test_tag_generator_protocol_compliance(self):
|
||||
"""Test that MockTagGenerator implements TagGenerator protocol."""
|
||||
mock = MockTagGenerator()
|
||||
|
||||
assert isinstance(mock, TagGenerator)
|
||||
|
||||
# Test method calls
|
||||
dir_tags = mock.generate_directory_tags("100-project/test.md")
|
||||
assert "project" in dir_tags
|
||||
|
||||
analysis = ContentAnalysis(
|
||||
language=LanguageInfo.ENGLISH,
|
||||
content_type=ContentType.NOTE,
|
||||
topics=["ai", "ml"]
|
||||
)
|
||||
content_tags = mock.generate_content_tags(analysis)
|
||||
assert "ai" in content_tags
|
||||
|
||||
hierarchical = mock.generate_hierarchical_tags(["ai", "ml"])
|
||||
assert "topic/ai" in hierarchical
|
||||
|
||||
consolidated = mock.consolidate_tags(["tag1", "tag1", "tag2"])
|
||||
assert len(consolidated) == 2
|
||||
|
||||
def test_frontmatter_manager_protocol_compliance(self):
|
||||
"""Test that MockFrontmatterManager implements FrontmatterManager protocol."""
|
||||
mock = MockFrontmatterManager()
|
||||
|
||||
assert isinstance(mock, FrontmatterManager)
|
||||
|
||||
# Test method calls
|
||||
frontmatter = mock.parse_frontmatter("---\ntitle: Test\n---\nContent")
|
||||
assert frontmatter.title == "Test"
|
||||
|
||||
updated = mock.update_frontmatter("Content", frontmatter)
|
||||
assert "title: Test" in updated
|
||||
|
||||
validation = mock.validate_frontmatter(frontmatter)
|
||||
assert validation.is_valid is True
|
||||
|
||||
|
||||
class TestAbstractBaseClasses:
|
||||
"""Test abstract base class behavior."""
|
||||
|
||||
def test_base_classes_cannot_be_instantiated(self):
|
||||
"""Test that abstract base classes cannot be instantiated directly."""
|
||||
with pytest.raises(TypeError):
|
||||
BaseFileDiscovery()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
BaseContentAnalyzer()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
BaseTagGenerator()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
BaseFrontmatterManager()
|
||||
|
||||
def test_concrete_implementations_work(self):
|
||||
"""Test that concrete implementations of base classes work."""
|
||||
file_discovery = MockFileDiscovery()
|
||||
content_analyzer = MockContentAnalyzer()
|
||||
tag_generator = MockTagGenerator()
|
||||
frontmatter_manager = MockFrontmatterManager()
|
||||
|
||||
# All should be instances of their respective base classes
|
||||
assert isinstance(file_discovery, BaseFileDiscovery)
|
||||
assert isinstance(content_analyzer, BaseContentAnalyzer)
|
||||
assert isinstance(tag_generator, BaseTagGenerator)
|
||||
assert isinstance(frontmatter_manager, BaseFrontmatterManager)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Unit tests for core data models."""
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from tagging_system.core.models import (
|
||||
FileInfo,
|
||||
ContentAnalysis,
|
||||
TagStructure,
|
||||
FrontmatterData,
|
||||
ValidationResult,
|
||||
ContentType,
|
||||
LanguageInfo
|
||||
)
|
||||
|
||||
|
||||
class TestFileInfo:
|
||||
"""Test FileInfo model."""
|
||||
|
||||
def test_file_info_creation(self):
|
||||
"""Test FileInfo creation with required fields."""
|
||||
file_info = FileInfo(
|
||||
path="test/file.md",
|
||||
name="file.md",
|
||||
directory="test",
|
||||
extension=".md",
|
||||
size=100,
|
||||
created=datetime.now(),
|
||||
modified=datetime.now()
|
||||
)
|
||||
|
||||
assert file_info.path == "test/file.md"
|
||||
assert file_info.name == "file.md"
|
||||
assert file_info.is_markdown is True
|
||||
assert file_info.relative_path == "test/file.md"
|
||||
|
||||
def test_is_markdown_detection(self):
|
||||
"""Test markdown file detection."""
|
||||
md_file = FileInfo("test.md", "test.md", ".", ".md", 100, datetime.now(), datetime.now())
|
||||
txt_file = FileInfo("test.txt", "test.txt", ".", ".txt", 100, datetime.now(), datetime.now())
|
||||
|
||||
assert md_file.is_markdown is True
|
||||
assert txt_file.is_markdown is False
|
||||
|
||||
|
||||
class TestContentAnalysis:
|
||||
"""Test ContentAnalysis model."""
|
||||
|
||||
def test_content_analysis_creation(self):
|
||||
"""Test ContentAnalysis creation."""
|
||||
analysis = ContentAnalysis(
|
||||
language=LanguageInfo.ENGLISH,
|
||||
content_type=ContentType.NOTE,
|
||||
topics=["ai", "ml"]
|
||||
)
|
||||
|
||||
assert analysis.language == LanguageInfo.ENGLISH
|
||||
assert analysis.content_type == ContentType.NOTE
|
||||
assert analysis.topics == ["ai", "ml"]
|
||||
assert analysis.complexity == "basic" # Default value
|
||||
|
||||
def test_complexity_validation(self):
|
||||
"""Test complexity validation in post_init."""
|
||||
analysis = ContentAnalysis(
|
||||
language=LanguageInfo.ENGLISH,
|
||||
content_type=ContentType.NOTE,
|
||||
topics=[],
|
||||
complexity="invalid"
|
||||
)
|
||||
|
||||
assert analysis.complexity == "basic" # Should default to basic
|
||||
|
||||
|
||||
class TestTagStructure:
|
||||
"""Test TagStructure model."""
|
||||
|
||||
def test_tag_structure_creation(self):
|
||||
"""Test TagStructure creation."""
|
||||
tags = TagStructure(
|
||||
primary=["project"],
|
||||
hierarchical=["tech/ai"],
|
||||
content=["python"],
|
||||
meta=["lang/en"],
|
||||
custom=["custom"]
|
||||
)
|
||||
|
||||
assert tags.primary == ["project"]
|
||||
assert tags.hierarchical == ["tech/ai"]
|
||||
assert tags.content == ["python"]
|
||||
assert tags.meta == ["lang/en"]
|
||||
assert tags.custom == ["custom"]
|
||||
|
||||
def test_all_tags_method(self):
|
||||
"""Test all_tags method returns unique tags."""
|
||||
tags = TagStructure(
|
||||
primary=["project", "duplicate"],
|
||||
hierarchical=["tech/ai"],
|
||||
content=["python", "duplicate"], # Duplicate tag
|
||||
meta=["lang/en"],
|
||||
custom=["custom"]
|
||||
)
|
||||
|
||||
all_tags = tags.all_tags()
|
||||
# Expected unique tags: project, duplicate, tech/ai, python, lang/en, custom = 6 tags
|
||||
assert len(all_tags) == 6
|
||||
assert "duplicate" in all_tags
|
||||
assert "project" in all_tags
|
||||
assert "tech/ai" in all_tags
|
||||
assert "python" in all_tags
|
||||
assert "lang/en" in all_tags
|
||||
assert "custom" in all_tags
|
||||
|
||||
# Test that duplicates are actually removed by checking set behavior
|
||||
unique_tags = set(all_tags)
|
||||
assert len(unique_tags) == len(all_tags) # No duplicates should exist
|
||||
|
||||
# Test with actual duplicates to verify deduplication works
|
||||
tags_with_more_duplicates = TagStructure(
|
||||
primary=["tag1", "tag2"],
|
||||
hierarchical=["tag1"], # Duplicate of primary
|
||||
content=["tag2", "tag3"], # Duplicate of primary
|
||||
meta=["tag3"], # Duplicate of content
|
||||
custom=["tag4"]
|
||||
)
|
||||
deduplicated = tags_with_more_duplicates.all_tags()
|
||||
assert len(deduplicated) == 4 # tag1, tag2, tag3, tag4
|
||||
assert len(set(deduplicated)) == len(deduplicated)
|
||||
|
||||
|
||||
class TestFrontmatterData:
|
||||
"""Test FrontmatterData model."""
|
||||
|
||||
def test_frontmatter_data_creation(self):
|
||||
"""Test FrontmatterData creation."""
|
||||
frontmatter = FrontmatterData(
|
||||
title="Test",
|
||||
tags=["tag1", "tag2"],
|
||||
created="2024-01-01",
|
||||
type="note"
|
||||
)
|
||||
|
||||
assert frontmatter.title == "Test"
|
||||
assert frontmatter.tags == ["tag1", "tag2"]
|
||||
assert frontmatter.created == "2024-01-01"
|
||||
assert frontmatter.type == "note"
|
||||
|
||||
def test_to_dict_method(self):
|
||||
"""Test to_dict method excludes None values."""
|
||||
frontmatter = FrontmatterData(
|
||||
title="Test",
|
||||
tags=["tag1"],
|
||||
created="2024-01-01",
|
||||
updated=None, # Should be excluded
|
||||
custom_fields={"custom": "value"}
|
||||
)
|
||||
|
||||
result = frontmatter.to_dict()
|
||||
|
||||
assert result["title"] == "Test"
|
||||
assert result["tags"] == ["tag1"]
|
||||
assert result["created"] == "2024-01-01"
|
||||
assert "updated" not in result # None values excluded
|
||||
assert result["custom"] == "value" # Custom fields included
|
||||
|
||||
|
||||
class TestValidationResult:
|
||||
"""Test ValidationResult model."""
|
||||
|
||||
def test_validation_result_creation(self):
|
||||
"""Test ValidationResult creation."""
|
||||
result = ValidationResult(is_valid=True)
|
||||
|
||||
assert result.is_valid is True
|
||||
assert result.errors == []
|
||||
assert result.warnings == []
|
||||
assert result.suggestions == []
|
||||
|
||||
def test_add_error_sets_invalid(self):
|
||||
"""Test that adding error sets is_valid to False."""
|
||||
result = ValidationResult(is_valid=True)
|
||||
result.add_error("Test error")
|
||||
|
||||
assert result.is_valid is False
|
||||
assert "Test error" in result.errors
|
||||
|
||||
def test_add_warning_and_suggestion(self):
|
||||
"""Test adding warnings and suggestions."""
|
||||
result = ValidationResult(is_valid=True)
|
||||
result.add_warning("Test warning")
|
||||
result.add_suggestion("Test suggestion")
|
||||
|
||||
assert result.is_valid is True # Warnings don't affect validity
|
||||
assert "Test warning" in result.warnings
|
||||
assert "Test suggestion" in result.suggestions
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Test overall package structure and imports."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestPackageStructure:
|
||||
"""Test that the package structure is correct."""
|
||||
|
||||
def test_main_package_imports(self):
|
||||
"""Test that main package imports work correctly."""
|
||||
from tagging_system import (
|
||||
FileInfo,
|
||||
ContentAnalysis,
|
||||
TagStructure,
|
||||
FrontmatterData,
|
||||
ContentType,
|
||||
LanguageInfo,
|
||||
ValidationResult,
|
||||
FileDiscovery,
|
||||
ContentAnalyzer,
|
||||
TagGenerator,
|
||||
FrontmatterManager
|
||||
)
|
||||
|
||||
# Test that all imports are available
|
||||
assert FileInfo is not None
|
||||
assert ContentAnalysis is not None
|
||||
assert TagStructure is not None
|
||||
assert FrontmatterData is not None
|
||||
assert ContentType is not None
|
||||
assert LanguageInfo is not None
|
||||
assert ValidationResult is not None
|
||||
assert FileDiscovery is not None
|
||||
assert ContentAnalyzer is not None
|
||||
assert TagGenerator is not None
|
||||
assert FrontmatterManager is not None
|
||||
|
||||
def test_core_module_imports(self):
|
||||
"""Test that core module imports work correctly."""
|
||||
from tagging_system.core import (
|
||||
FileInfo,
|
||||
ContentAnalysis,
|
||||
TagStructure,
|
||||
FrontmatterData,
|
||||
ContentType,
|
||||
LanguageInfo,
|
||||
ValidationResult,
|
||||
FileDiscovery,
|
||||
ContentAnalyzer,
|
||||
TagGenerator,
|
||||
FrontmatterManager
|
||||
)
|
||||
|
||||
# All imports should be available
|
||||
assert all([
|
||||
FileInfo, ContentAnalysis, TagStructure, FrontmatterData,
|
||||
ContentType, LanguageInfo, ValidationResult,
|
||||
FileDiscovery, ContentAnalyzer, TagGenerator, FrontmatterManager
|
||||
])
|
||||
|
||||
def test_config_module_imports(self):
|
||||
"""Test that config module imports work correctly."""
|
||||
from tagging_system.config import (
|
||||
TaggingConfig,
|
||||
DirectoryMapping,
|
||||
TagHierarchy,
|
||||
SensitivePatterns,
|
||||
load_config,
|
||||
save_config
|
||||
)
|
||||
|
||||
# All imports should be available
|
||||
assert all([
|
||||
TaggingConfig, DirectoryMapping, TagHierarchy,
|
||||
SensitivePatterns, load_config, save_config
|
||||
])
|
||||
|
||||
def test_cli_module_import(self):
|
||||
"""Test that CLI module can be imported."""
|
||||
from tagging_system import cli
|
||||
|
||||
assert hasattr(cli, 'main')
|
||||
assert callable(cli.main)
|
||||
|
||||
def test_package_version(self):
|
||||
"""Test that package version is available."""
|
||||
import tagging_system
|
||||
|
||||
assert hasattr(tagging_system, '__version__')
|
||||
assert tagging_system.__version__ == "0.1.0"
|
||||
|
||||
def test_package_metadata(self):
|
||||
"""Test that package metadata is available."""
|
||||
import tagging_system
|
||||
|
||||
assert hasattr(tagging_system, '__author__')
|
||||
assert tagging_system.__author__ == "Tagging System"
|
||||
Reference in New Issue
Block a user