chore: snapshot current project updates
This commit is contained in:
+53
-17
@@ -6,6 +6,7 @@ directory structure.
|
||||
"""
|
||||
|
||||
import csv
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -72,26 +73,61 @@ def scan_library(
|
||||
if progress_callback is not None:
|
||||
progress_callback(0, total_paths)
|
||||
|
||||
for index, file_path in enumerate(discovered_paths, start=1):
|
||||
video_file = _create_video_file(
|
||||
file_path,
|
||||
root,
|
||||
config.categories,
|
||||
include_video_metadata=include_video_metadata,
|
||||
metadata_cache=metadata_cache
|
||||
)
|
||||
if video_file is None:
|
||||
max_workers = max(1, min(config.enrichment_max_concurrency, total_paths or 1))
|
||||
use_parallel = include_video_metadata and total_paths > 1 and max_workers > 1
|
||||
|
||||
if use_parallel:
|
||||
indexed_results: list[tuple[int, VideoFile]] = []
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
future_to_index = {
|
||||
pool.submit(
|
||||
_create_video_file,
|
||||
file_path,
|
||||
root,
|
||||
config.categories,
|
||||
include_video_metadata,
|
||||
metadata_cache,
|
||||
): index
|
||||
for index, file_path in enumerate(discovered_paths, start=1)
|
||||
}
|
||||
for completed_count, future in enumerate(as_completed(future_to_index), start=1):
|
||||
index = future_to_index[future]
|
||||
try:
|
||||
video_file = future.result()
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to scan file #{index}: {exc}")
|
||||
video_file = None
|
||||
if video_file is not None:
|
||||
indexed_results.append((index, video_file))
|
||||
file_count += 1
|
||||
if progress_callback is not None:
|
||||
progress_callback(completed_count, total_paths)
|
||||
if file_count % 100 == 0 and file_count > 0:
|
||||
logger.debug(f"Scanned {file_count} files so far...")
|
||||
|
||||
indexed_results.sort(key=lambda item: item[0])
|
||||
video_files = [item[1] for item in indexed_results]
|
||||
else:
|
||||
for index, file_path in enumerate(discovered_paths, start=1):
|
||||
video_file = _create_video_file(
|
||||
file_path,
|
||||
root,
|
||||
config.categories,
|
||||
include_video_metadata=include_video_metadata,
|
||||
metadata_cache=metadata_cache
|
||||
)
|
||||
if video_file is None:
|
||||
if progress_callback is not None:
|
||||
progress_callback(index, total_paths)
|
||||
continue
|
||||
video_files.append(video_file)
|
||||
file_count += 1
|
||||
|
||||
if progress_callback is not None:
|
||||
progress_callback(index, total_paths)
|
||||
continue
|
||||
video_files.append(video_file)
|
||||
file_count += 1
|
||||
|
||||
if progress_callback is not None:
|
||||
progress_callback(index, total_paths)
|
||||
|
||||
if file_count % 100 == 0:
|
||||
logger.debug(f"Scanned {file_count} files so far...")
|
||||
if file_count % 100 == 0:
|
||||
logger.debug(f"Scanned {file_count} files so far...")
|
||||
|
||||
logger.info(f"Scan complete. Found {file_count} video files")
|
||||
return video_files
|
||||
|
||||
Reference in New Issue
Block a user