Initial: fingerprint DB + Takeout fetch + ingest flows
This commit is contained in:
59
prefect.yaml
Normal file
59
prefect.yaml
Normal file
@@ -0,0 +1,59 @@
|
||||
---
|
||||
# Prefect project config for photo-pipeline
|
||||
name: photo-pipeline
|
||||
prefect-version: 3.8.1
|
||||
|
||||
build: null
|
||||
push: null
|
||||
|
||||
pull:
|
||||
- prefect.deployments.steps.set_working_directory:
|
||||
directory: /home/sam/photo-pipeline
|
||||
|
||||
deployments:
|
||||
- name: hello
|
||||
version: null
|
||||
tags: [photo-pipeline]
|
||||
description: "Photo pipeline hello flow"
|
||||
schedule:
|
||||
cron: "*/5 * * * *"
|
||||
timezone: "Australia/Melbourne"
|
||||
flow_name: null
|
||||
entrypoint: photo_pipeline_hello.py:hello_pipeline
|
||||
parameters: {}
|
||||
work_pool:
|
||||
name: photo-pool
|
||||
work_queue_name: null
|
||||
job_variables: {}
|
||||
|
||||
- name: ingest
|
||||
version: null
|
||||
tags: [photo-pipeline]
|
||||
description: "Hash + dedup-check a folder against the persistent fingerprint DB"
|
||||
schedule: null
|
||||
flow_name: null
|
||||
entrypoint: photo_ingest.py:photo_ingest
|
||||
parameters:
|
||||
base_dir: /mnt/data/takeout
|
||||
source: takeout
|
||||
register: true
|
||||
work_pool:
|
||||
name: photo-pool
|
||||
work_queue_name: null
|
||||
job_variables: {}
|
||||
|
||||
- name: fetch
|
||||
version: null
|
||||
tags: [photo-pipeline]
|
||||
description: "Download + track + extract Takeout archives (resume-capable)"
|
||||
schedule: null
|
||||
flow_name: null
|
||||
entrypoint: takeout_fetch.py:takeout_fetch
|
||||
parameters:
|
||||
urls: []
|
||||
manifest: null
|
||||
export_id: null
|
||||
work_pool:
|
||||
name: photo-pool
|
||||
work_queue_name: null
|
||||
job_variables: {}
|
||||
Reference in New Issue
Block a user