ba-sachsen-pica/Taskfile.yml

# https://taskfile.dev

version: '3'

output: 'group'

vars:
  DATE: '{{ now | date "20060102_150405"}}'

env:
  REFINE_MEMORY: 8g
  REFINE_ENDPOINT: http://localhost:3334

tasks:
  default:
    desc: Generierung PICA+
    deps: [bibliotheca, alephino]
    cmds:
      - tasks/03-ba-sachsen.sh "output/02-bibliotheca-main"
    sources:
#     - output/02-alephino-main/alephino.csv
      - output/02-bibliotheca-main/bibliotheca.csv
    generates:
      - output/03-ba-sachsen/ba-sachsen.pic
      - output/03-ba-sachsen/ba-sachsen.openrefine.tar.gz
    env:
      REFINE_WORKDIR: output/03-ba-sachsen
      REFINE_LOGFILE: log/03-ba-sachsen/{{.DATE}}.log

  alephino:
    desc: Alephino Hauptverarbeitung
    cmds:
      - task: leipzig
      - task: riesa
      - tasks/02-alephino-main.sh "output/01-alephino-pre"
    sources:
      - output/01-alephino-pre/*.tsv
    generates:
#     - output/02-alephino-main/alephino.csv
      - output/02-alephino-main/alephino.openrefine.tar.gz
    env:
      REFINE_WORKDIR: output/02-alephino-main
      REFINE_LOGFILE: log/02-alephino-main/{{.DATE}}.log

  bibliotheca:
    desc: Bibliotheca Hauptverarbeitung
#   deps: [bautzen, breitenbrunn, dresden, glauchau, plauen]
    cmds:
      - task: bautzen
      - task: breitenbrunn
      - task: dresden
      - task: glauchau
      - task: plauen
      - tasks/02-bibliotheca-main.sh "output/01-bibliotheca-pre"
    sources:
      - output/01-bibliotheca-pre/*.tsv
    generates:
      - output/02-bibliotheca-main/bibliotheca.csv
      - output/02-bibliotheca-main/bibliotheca.openrefine.tar.gz
    env:
      REFINE_WORKDIR: output/02-bibliotheca-main
      REFINE_LOGFILE: log/02-bibliotheca-main/{{.DATE}}.log

  bautzen:
    desc: Bibliotheca Vorverarbeitung
    cmds:
      - tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
    sources:
      - '{{.INPUT}}'
    generates:
      - output/01-bibliotheca-pre/bautzen.tsv
    vars:
      INPUT: '{{.INPUT | default "input/bautzen.imp"}}'
    env:
      REFINE_MEMORY: '{{.REFINE_MEMORY | default "6G"}}'
      REFINE_ENDPOINT: http://localhost:3335
      REFINE_WORKDIR: output/01-bibliotheca-pre
      REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_bautzen.log

  breitenbrunn:
    desc: Bibliotheca Vorverarbeitung
    cmds:
      - tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
    sources:
      - '{{.INPUT}}'
    generates:
      - output/01-bibliotheca-pre/breitenbrunn.tsv
    vars:
      INPUT: '{{.INPUT | default "input/breitenbrunn.imp"}}'
    env:
      REFINE_MEMORY: '{{.REFINE_MEMORY | default "4G"}}'
      REFINE_ENDPOINT: http://localhost:3335
      REFINE_WORKDIR: output/01-bibliotheca-pre
      REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_breitenbrunn.log

  dresden:
    desc: Bibliotheca Vorverarbeitung
    cmds:
      - tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
    sources:
      - '{{.INPUT}}'
    generates:
      - output/01-bibliotheca-pre/dresden.tsv
    vars:
      INPUT: '{{.INPUT | default "input/dresden.imp"}}'
    env:
      REFINE_MEMORY: '{{.REFINE_MEMORY | default "7G"}}'
      REFINE_ENDPOINT: http://localhost:3336
      REFINE_WORKDIR: output/01-bibliotheca-pre
      REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_dresden.log

  leipzig:
    desc: Alephino Vorverarbeitung
    cmds:
      - tasks/01-alephino-pre.sh "{{.TITEL}}" "{{.EXEMPLARE}}"
    sources:
      - '{{.TITEL}}'
      - '{{.EXEMPLARE}}'
    generates:
      - output/01-alephino-pre/leipzig.tsv
    vars:
      TITEL: '{{.TITEL | default "input/leipzig-titel.txt"}}'
      EXEMPLARE: '{{.EXEMPLARE | default "input/leipzig-exemplare.txt"}}'
    env:
      REFINE_MEMORY: '{{.REFINE_MEMORY | default "7G"}}'
      REFINE_ENDPOINT: http://localhost:3339
      REFINE_WORKDIR: output/01-alephino-pre
      REFINE_LOGFILE: log/01-alephino-pre/{{.DATE}}_leipzig.log

  glauchau:
    desc: Bibliotheca Vorverarbeitung
    cmds:
      - tasks/01-bibliotheca-pre.sh "{{.INPUT}}"
    sources:
      - '{{.INPUT}}'
    generates:
      - output/01-bibliotheca-pre/glauchau.tsv
    vars:
      INPUT: '{{.INPUT | default "input/glauchau.imp"}}'
    env:
      REFINE_MEMORY: '{{.REFINE_MEMORY | default "4G"}}'
      REFINE_ENDPOINT: http://localhost:3337
      REFINE_WORKDIR: output/01-bibliotheca-pre
      REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_glauchau.log

  plauen:
    desc: Bibliotheca Vorverarbeitung
    cmds:
      - tasks/01-bibliotheca-pre.sh "input/plauen.imp"
    sources:
      - input/plauen.imp
    generates:
      - output/01-bibliotheca-pre/plauen.tsv
    env:
      REFINE_MEMORY: '{{.REFINE_MEMORY | default "2G"}}'
      REFINE_ENDPOINT: http://localhost:3338
      REFINE_WORKDIR: output/01-bibliotheca-pre
      REFINE_LOGFILE: log/01-bibliotheca-pre/{{.DATE}}_plauen.log

  riesa:
    desc: Alephino Vorverarbeitung
    cmds:
      - tasks/01-alephino-pre.sh "{{.TITEL}}" "{{.EXEMPLARE}}"
    sources:
      - '{{.TITEL}}'
      - '{{.EXEMPLARE}}'
    generates:
      - output/01-alephino-pre/riesa.tsv
    vars:
      TITEL: '{{.TITEL | default "input/riesa-titel.txt"}}'
      EXEMPLARE: '{{.EXEMPLARE | default "input/riesa-exemplare.txt"}}'
    env:
      REFINE_MEMORY: '{{.REFINE_MEMORY | default "7G"}}'
      REFINE_ENDPOINT: http://localhost:3339
      REFINE_WORKDIR: output/01-alephino-pre
      REFINE_LOGFILE: log/01-alephino-pre/{{.DATE}}_riesa.log

  clean:
    desc: Alle Daten löschen (reset auf Ausgangszustand)
    cmds:
      - rm -r lib log output

  mkdir:
    desc: Ordner erstellen
    cmds:
      - mkdir -p output/01-alephino-pre log/01-alephino-pre
      - mkdir -p output/01-bibliotheca-pre log/01-bibliotheca-pre
      - mkdir -p output/02-alephino-main log/02-alephino-main
      - mkdir -p output/02-bibliotheca-main log/02-bibliotheca-main
      - mkdir -p output/03-ba-sachsen log/03-ba-sachsen