Initial: fingerprint DB + Takeout fetch + ingest flows

This commit is contained in:
2026-08-07 20:41:32 +10:00
commit 16123d00f1
10 changed files with 614 additions and 0 deletions

59
prefect.yaml Normal file
View File

@@ -0,0 +1,59 @@
---
# Prefect project config for photo-pipeline
name: photo-pipeline
prefect-version: 3.8.1
build: null
push: null
pull:
- prefect.deployments.steps.set_working_directory:
directory: /home/sam/photo-pipeline
deployments:
- name: hello
version: null
tags: [photo-pipeline]
description: "Photo pipeline hello flow"
schedule:
cron: "*/5 * * * *"
timezone: "Australia/Melbourne"
flow_name: null
entrypoint: photo_pipeline_hello.py:hello_pipeline
parameters: {}
work_pool:
name: photo-pool
work_queue_name: null
job_variables: {}
- name: ingest
version: null
tags: [photo-pipeline]
description: "Hash + dedup-check a folder against the persistent fingerprint DB"
schedule: null
flow_name: null
entrypoint: photo_ingest.py:photo_ingest
parameters:
base_dir: /mnt/data/takeout
source: takeout
register: true
work_pool:
name: photo-pool
work_queue_name: null
job_variables: {}
- name: fetch
version: null
tags: [photo-pipeline]
description: "Download + track + extract Takeout archives (resume-capable)"
schedule: null
flow_name: null
entrypoint: takeout_fetch.py:takeout_fetch
parameters:
urls: []
manifest: null
export_id: null
work_pool:
name: photo-pool
work_queue_name: null
job_variables: {}