opencultureconsulting
/
openrefine-batch
mirror of https://github.com/opencultureconsulting/openrefine-batch.git
2
0
Fork 0
Code Issues Releases Activity

release v1.9, fix codacy issues

This commit is contained in:
Felix Lohmeier 2017-11-05 16:25:42 +01:00
parent 0d1150c5fb
commit 566cd372af
2 changed files with 58 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
openrefine-batch-docker.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
# openrefine-batch-docker.sh, Felix Lohmeier, v1.8, 2017-10-28 # openrefine-batch-docker.sh, Felix Lohmeier, v1.9, 2017-11-05
# https://github.com/felixlohmeier/openrefine-batch # https://github.com/felixlohmeier/openrefine-batch
# check system requirements # check system requirements
@ -169,17 +169,17 @@ echo ""
# declare additional variables # declare additional variables
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Start process" checkpointname[$((checkpoints + 1))]="Start process"
memoryload=() memoryload=()
# launch server # launch server
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Launch OpenRefine" checkpointname[$((checkpoints + 1))]="Launch OpenRefine"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
sudo docker run -d --name=${uuid} -v ${outputdir}:/data:z felixlohmeier/openrefine:${version} -i 0.0.0.0 -m ${ram} -d /data sudo docker run -d --name=${uuid} -v ${outputdir}:/data:z felixlohmeier/openrefine:${version} -i 0.0.0.0 -m ${ram} -d /data
# wait until server is available # wait until server is available
@ -191,11 +191,11 @@ echo ""
# import all files # import all files
if [ -n "$inputfiles" ]; then if [ -n "$inputfiles" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Import all files" checkpointname[$((checkpoints + 1))]="Import all files"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
for inputfile in "${inputfiles[@]}" ; do for inputfile in "${inputfiles[@]}" ; do
echo "import ${inputfile}..." echo "import ${inputfile}..."
@ -221,18 +221,18 @@ fi
# transform and export files # transform and export files
if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Prepare transform & export" checkpointname[$((checkpoints + 1))]="Prepare transform & export"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
# get project ids # get project ids
echo "get project ids..." echo "get project ids..."
sudo docker run --rm --link ${uuid} felixlohmeier/openrefine-client -H ${uuid} -l > "${outputdir}/projects.tmp" sudo docker run --rm --link ${uuid} felixlohmeier/openrefine-client -H ${uuid} -l > "${outputdir}/projects.tmp"
projectids=($(cat "${outputdir}/projects.tmp" | cut -c 2-14)) projectids=($(cut -c 2-14 "${outputdir}/projects.tmp"))
projectnames=($(cat "${outputdir}/projects.tmp" | cut -c 17-)) projectnames=($(cut -c 17- "${outputdir}/projects.tmp"))
cat "${outputdir}/projects.tmp" && rm "${outputdir:?}/projects.tmp" cat "${outputdir}/projects.tmp" && rm "${outputdir:?}/projects.tmp"
echo "" echo ""
@ -257,11 +257,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
# apply transformation rules # apply transformation rules
if [ -n "$jsonfiles" ]; then if [ -n "$jsonfiles" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Transform ${projectnames[i]}" checkpointname[$((checkpoints + 1))]="Transform ${projectnames[i]}"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
for jsonfile in "${jsonfiles[@]}" ; do for jsonfile in "${jsonfiles[@]}" ; do
echo "transform ${jsonfile}..." echo "transform ${jsonfile}..."
@ -287,11 +287,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
# export project to workspace # export project to workspace
if [ "$export" = "true" ]; then if [ "$export" = "true" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Export ${projectnames[i]}" checkpointname[$((checkpoints + 1))]="Export ${projectnames[i]}"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
# get filename without extension # get filename without extension
filename=${projectnames[i]%.*} filename=${projectnames[i]%.*}
@ -340,11 +340,11 @@ echo ""
echo "=== Statistics ===" echo "=== Statistics ==="
echo "" echo ""
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="End process" checkpointname[$((checkpoints + 1))]="End process"
echo "starting time and run time of each step:" echo "starting time and run time of each step:"
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
for i in $(seq 1 $checkpoints); do for i in $(seq 1 $checkpoints); do
diffsec="$((${checkpointdate[$(($i + 1))]} - ${checkpointdate[$i]}))" diffsec="$((${checkpointdate[$(($i + 1))]} - ${checkpointdate[$i]}))"
printf "%35s $(date --date=@${checkpointdate[$i]}) ($(date -d@${diffsec} -u +%H:%M:%S))\n" "${checkpointname[$i]}" printf "%35s $(date --date=@${checkpointdate[$i]}) ($(date -d@${diffsec} -u +%H:%M:%S))\n" "${checkpointname[$i]}"

60
openrefine-batch.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
# openrefine-batch.sh, Felix Lohmeier, v1.8, 2017-10-28 # openrefine-batch.sh, Felix Lohmeier, v1.9, 2017-11-05
# https://github.com/felixlohmeier/openrefine-batch # https://github.com/felixlohmeier/openrefine-batch
# declare download URLs for OpenRefine and OpenRefine client # declare download URLs for OpenRefine and OpenRefine client
@ -192,17 +192,17 @@ echo ""
# declare additional variables # declare additional variables
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Start process" checkpointname[$((checkpoints + 1))]="Start process"
memoryload=() memoryload=()
# launch server # launch server
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Launch OpenRefine" checkpointname[$((checkpoints + 1))]="Launch OpenRefine"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
openrefine/refine -p ${port} -d "${outputdir}" -m ${ram} & openrefine/refine -p ${port} -d "${outputdir}" -m ${ram} &
pid=$! pid=$!
@ -213,11 +213,11 @@ echo ""
# import all files # import all files
if [ -n "$inputfiles" ]; then if [ -n "$inputfiles" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Import all files" checkpointname[$((checkpoints + 1))]="Import all files"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
for inputfile in "${inputfiles[@]}" ; do for inputfile in "${inputfiles[@]}" ; do
echo "import ${inputfile}..." echo "import ${inputfile}..."
@ -244,18 +244,18 @@ fi
# transform and export files # transform and export files
if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Prepare transform & export" checkpointname[$((checkpoints + 1))]="Prepare transform & export"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
# get project ids # get project ids
echo "get project ids..." echo "get project ids..."
openrefine-client/openrefine-client_0-3-1_linux-64bit -P ${port} -l > "${outputdir}/projects.tmp" openrefine-client/openrefine-client_0-3-1_linux-64bit -P ${port} -l > "${outputdir}/projects.tmp"
projectids=($(cat "${outputdir}/projects.tmp" | cut -c 2-14)) projectids=($(cut -c 2-14 "${outputdir}/projects.tmp"))
projectnames=($(cat "${outputdir}/projects.tmp" | cut -c 17-)) projectnames=($(cut -c 17- "${outputdir}/projects.tmp"))
cat "${outputdir}/projects.tmp" && rm "${outputdir:?}/projects.tmp" cat "${outputdir}/projects.tmp" && rm "${outputdir:?}/projects.tmp"
echo "" echo ""
@ -281,11 +281,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
# apply transformation rules # apply transformation rules
if [ -n "$jsonfiles" ]; then if [ -n "$jsonfiles" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Transform ${projectnames[i]}" checkpointname[$((checkpoints + 1))]="Transform ${projectnames[i]}"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
for jsonfile in "${jsonfiles[@]}" ; do for jsonfile in "${jsonfiles[@]}" ; do
echo "transform ${jsonfile}..." echo "transform ${jsonfile}..."
@ -312,11 +312,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
# export project to workspace # export project to workspace
if [ "$export" = "true" ]; then if [ "$export" = "true" ]; then
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="Export ${projectnames[i]}" checkpointname[$((checkpoints + 1))]="Export ${projectnames[i]}"
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ===" echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
echo "" echo ""
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})" echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
echo "" echo ""
# get filename without extension # get filename without extension
filename=${projectnames[i]%.*} filename=${projectnames[i]%.*}
@ -366,13 +366,13 @@ echo ""
echo "=== Statistics ===" echo "=== Statistics ==="
echo "" echo ""
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
checkpointname[$(($checkpoints + 1))]="End process" checkpointname[$((checkpoints + 1))]="End process"
echo "starting time and run time of each step:" echo "starting time and run time of each step:"
checkpoints=${#checkpointdate[@]} checkpoints=${#checkpointdate[@]}
checkpointdate[$(($checkpoints + 1))]=$(date +%s) checkpointdate[$((checkpoints + 1))]=$(date +%s)
for i in $(seq 1 $checkpoints); do for i in $(seq 1 $checkpoints); do
diffsec="$((${checkpointdate[$(($i + 1))]} - ${checkpointdate[$i]}))" diffsec="$((${checkpointdate[$((i + 1))]} - ${checkpointdate[$i]}))"
printf "%35s $(date --date=@${checkpointdate[$i]}) ($(date -d@${diffsec} -u +%H:%M:%S))\n" "${checkpointname[$i]}" printf "%35s $(date --date=@${checkpointdate[$i]}) ($(date -d@${diffsec} -u +%H:%M:%S))\n" "${checkpointname[$i]}"
done done
echo "" echo ""
@ -384,4 +384,4 @@ max=${memoryload[0]}
for n in "${memoryload[@]}" ; do for n in "${memoryload[@]}" ; do
((n > max)) && max=$n ((n > max)) && max=$n
done done
echo "highest memory load: $(($max / 1024)) MB" echo "highest memory load: $((max / 1024)) MB"