137 lines
3.9 KiB
YAML
137 lines
3.9 KiB
YAML
# https://taskfile.dev
|
|
|
|
version: '3'
|
|
|
|
output: 'group'
|
|
|
|
vars:
|
|
DATE: '{{ now | date "20060102_150405"}}'
|
|
|
|
env:
|
|
REFINE_MEMORY: 8g
|
|
REFINE_ENDPOINT: http://localhost:3334
|
|
|
|
tasks:
|
|
default:
|
|
desc: Generierung PICA+
|
|
deps: [bibliotheca]
|
|
cmds:
|
|
- tasks/03-ba-sachsen.sh "output/02-bibliotheca-main"
|
|
sources:
|
|
- output/02-bibliotheca-main/bibliotheca.csv
|
|
generates:
|
|
- output/03-ba-sachsen/ba-sachsen.pic
|
|
- output/03-ba-sachsen/ba-sachsen.openrefine.tar.gz
|
|
env:
|
|
REFINE_WORKDIR: output/03-ba-sachsen
|
|
REFINE_LOGFILE: log/03-ba-sachsen/{{.DATE}}.log
|
|
|
|
bibliotheca:
|
|
desc: Bibliotheca Hauptverarbeitung
|
|
# deps: [bautzen, breitenbrunn, dresden, glauchau, plauen]
|
|
cmds:
|
|
- task: bautzen
|
|
- task: breitenbrunn
|
|
- task: dresden
|
|
- task: glauchau
|
|
- task: plauen
|
|
- tasks/02-bibliotheca-main.sh "output/01-bibliotheca-pre"
|
|
sources:
|
|
- output/01-bibliotheca-pre/*.tsv
|
|
generates:
|
|
- output/02-bibliotheca-main/bibliotheca.csv
|
|
- output/02-bibliotheca-main/bibliotheca.openrefine.tar.gz
|
|
env:
|
|
REFINE_WORKDIR: output/02-bibliotheca-main
|
|
REFINE_LOGFILE: log/02-bibliotheca-main/{{.DATE}}.log
|
|
|
|
bautzen:
|
|
desc: Bibliotheca Vorverarbeitung
|
|
cmds:
|
|
- tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
|
|
sources:
|
|
- '{{.INPUT}}'
|
|
generates:
|
|
- output/01-bibliotheca-pre/bautzen.tsv
|
|
vars:
|
|
INPUT: '{{.INPUT | default "input/bautzen.imp"}}'
|
|
env:
|
|
REFINE_MEMORY: '{{.REFINE_MEMORY | default "6G"}}'
|
|
REFINE_ENDPOINT: http://localhost:3335
|
|
REFINE_WORKDIR: output/01-bibliotheca-pre
|
|
REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_bautzen.log
|
|
|
|
breitenbrunn:
|
|
desc: Bibliotheca Vorverarbeitung
|
|
cmds:
|
|
- tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
|
|
sources:
|
|
- '{{.INPUT}}'
|
|
generates:
|
|
- output/01-bibliotheca-pre/breitenbrunn.tsv
|
|
vars:
|
|
INPUT: '{{.INPUT | default "input/breitenbrunn.imp"}}'
|
|
env:
|
|
REFINE_MEMORY: '{{.REFINE_MEMORY | default "4G"}}'
|
|
REFINE_ENDPOINT: http://localhost:3335
|
|
REFINE_WORKDIR: output/01-bibliotheca-pre
|
|
REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_breitenbrunn.log
|
|
|
|
dresden:
|
|
desc: Bibliotheca Vorverarbeitung
|
|
cmds:
|
|
- tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
|
|
sources:
|
|
- '{{.INPUT}}'
|
|
generates:
|
|
- output/01-bibliotheca-pre/dresden.tsv
|
|
vars:
|
|
INPUT: '{{.INPUT | default "input/dresden.imp"}}'
|
|
env:
|
|
REFINE_MEMORY: '{{.REFINE_MEMORY | default "7G"}}'
|
|
REFINE_ENDPOINT: http://localhost:3336
|
|
REFINE_WORKDIR: output/01-bibliotheca-pre
|
|
REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_dresden.log
|
|
|
|
glauchau:
|
|
desc: Bibliotheca Vorverarbeitung
|
|
cmds:
|
|
- tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
|
|
sources:
|
|
- '{{.INPUT}}'
|
|
generates:
|
|
- output/01-bibliotheca-pre/glauchau.tsv
|
|
vars:
|
|
INPUT: '{{.INPUT | default "input/glauchau.imp"}}'
|
|
env:
|
|
REFINE_MEMORY: '{{.REFINE_MEMORY | default "4G"}}'
|
|
REFINE_ENDPOINT: http://localhost:3337
|
|
REFINE_WORKDIR: output/01-bibliotheca-pre
|
|
REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_glauchau.log
|
|
|
|
plauen:
|
|
desc: Bibliotheca Vorverarbeitung
|
|
cmds:
|
|
- tasks/01-bibliotheca-pre.sh "input/plauen.imp"
|
|
sources:
|
|
- input/plauen.imp
|
|
generates:
|
|
- output/01-bibliotheca-pre/plauen.tsv
|
|
env:
|
|
REFINE_MEMORY: '{{.REFINE_MEMORY | default "2G"}}'
|
|
REFINE_ENDPOINT: http://localhost:3338
|
|
REFINE_WORKDIR: output/01-bibliotheca-pre
|
|
REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_plauen.log
|
|
|
|
clean:
|
|
desc: Alle Daten löschen (reset auf Ausgangszustand)
|
|
cmds:
|
|
- rm -r lib log output
|
|
|
|
mkdir:
|
|
desc: Ordner erstellen
|
|
cmds:
|
|
- mkdir -p output/01-bibliotheca-pre log/01-bibliotheca-pre
|
|
- mkdir -p output/02-bibliotheca-main log/02-bibliotheca-main
|
|
- mkdir -p output/03-ba-sachsen log/03-ba-sachsen
|