sam/ai-resume
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ingestion: skip empty files instead of failing the run

Browse Source
This commit is contained in:
Paperclip Builder
2026-10-26 22:30:36 +11:00
parent 4e52015fd2
commit da674b5555
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
corpus/sources/estate.py
View File

@@ -271,6 +271,9 @@ def _file_document(result, client, guide, repo, branch, area, updated, created,
result.add_rejection(exclusions.Rejection(path, "gitea", "binary content", full_name)) result.add_rejection(exclusions.Rejection(path, "gitea", "binary content", full_name))
return None return None
text = raw.decode("utf-8", "replace") text = raw.decode("utf-8", "replace")
if not text.strip():
result.add_rejection(exclusions.Rejection(path, "gitea", "empty file", full_name))
return None
hits = exclusions.scan_text(text) hits = exclusions.scan_text(text)
if hits: if hits:
kinds = sorted({hit[0] for hit in hits}) kinds = sorted({hit[0] for hit in hits})
@@ -506,6 +509,9 @@ def _harvest_obsidian(result, client, guide, repos, vault_repo, tree, cfg, base_
result.add_rejection(exclusions.Rejection(path, "obsidian", "binary content")) result.add_rejection(exclusions.Rejection(path, "obsidian", "binary content"))
continue continue
text = raw.decode("utf-8", "replace") text = raw.decode("utf-8", "replace")
if not text.strip():
result.add_rejection(exclusions.Rejection(path, "obsidian", "empty file"))
continue
hits = exclusions.scan_text(text) hits = exclusions.scan_text(text)
if hits: if hits:
result.add_rejection( result.add_rejection(
@@ -517,6 +523,9 @@ def _harvest_obsidian(result, client, guide, repos, vault_repo, tree, cfg, base_
continue continue
fields, body = _split_note(text, validator) fields, body = _split_note(text, validator)
if not body.strip():
result.add_rejection(exclusions.Rejection(path, "obsidian", "empty note"))
continue
title = str(fields.get("title") or _heading(body) or os.path.splitext(os.path.basename(path))[0]) title = str(fields.get("title") or _heading(body) or os.path.splitext(os.path.basename(path))[0])
tag_values = fields.get("tags") or [] tag_values = fields.get("tags") or []
if isinstance(tag_values, str): if isinstance(tag_values, str):