release v1.9, fix codacy issues
This commit is contained in:
Felix Lohmeier
parent
0d1150c5fb
commit
566cd372af
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/bash
|
||||
# openrefine-batch-docker.sh, Felix Lohmeier, v1.8, 2017-10-28
|
||||
# openrefine-batch-docker.sh, Felix Lohmeier, v1.9, 2017-11-05
|
||||
# https://github.com/felixlohmeier/openrefine-batch
|
||||
|
||||
# check system requirements
|
||||
|
|
@ -169,17 +169,17 @@ echo ""
|
|||
|
||||
# declare additional variables
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Start process"
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Start process"
|
||||
memoryload=()
|
||||
|
||||
# launch server
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Launch OpenRefine"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Launch OpenRefine"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
sudo docker run -d --name=${uuid} -v ${outputdir}:/data:z felixlohmeier/openrefine:${version} -i 0.0.0.0 -m ${ram} -d /data
|
||||
# wait until server is available
|
||||
|
|
@ -191,11 +191,11 @@ echo ""
|
|||
# import all files
|
||||
if [ -n "$inputfiles" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Import all files"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Import all files"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
for inputfile in "${inputfiles[@]}" ; do
|
||||
echo "import ${inputfile}..."
|
||||
|
|
@ -221,18 +221,18 @@ fi
|
|||
# transform and export files
|
||||
if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Prepare transform & export"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Prepare transform & export"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
|
||||
# get project ids
|
||||
echo "get project ids..."
|
||||
sudo docker run --rm --link ${uuid} felixlohmeier/openrefine-client -H ${uuid} -l > "${outputdir}/projects.tmp"
|
||||
projectids=($(cat "${outputdir}/projects.tmp" | cut -c 2-14))
|
||||
projectnames=($(cat "${outputdir}/projects.tmp" | cut -c 17-))
|
||||
projectids=($(cut -c 2-14 "${outputdir}/projects.tmp"))
|
||||
projectnames=($(cut -c 17- "${outputdir}/projects.tmp"))
|
||||
cat "${outputdir}/projects.tmp" && rm "${outputdir:?}/projects.tmp"
|
||||
echo ""
|
||||
|
||||
|
|
@ -257,11 +257,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
|
|||
# apply transformation rules
|
||||
if [ -n "$jsonfiles" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Transform ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Transform ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
for jsonfile in "${jsonfiles[@]}" ; do
|
||||
echo "transform ${jsonfile}..."
|
||||
|
|
@ -287,11 +287,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
|
|||
# export project to workspace
|
||||
if [ "$export" = "true" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Export ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Export ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
# get filename without extension
|
||||
filename=${projectnames[i]%.*}
|
||||
|
|
@ -340,11 +340,11 @@ echo ""
|
|||
echo "=== Statistics ==="
|
||||
echo ""
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="End process"
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="End process"
|
||||
echo "starting time and run time of each step:"
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
for i in $(seq 1 $checkpoints); do
|
||||
diffsec="$((${checkpointdate[$(($i + 1))]} - ${checkpointdate[$i]}))"
|
||||
printf "%35s $(date --date=@${checkpointdate[$i]}) ($(date -d@${diffsec} -u +%H:%M:%S))\n" "${checkpointname[$i]}"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/bash
|
||||
# openrefine-batch.sh, Felix Lohmeier, v1.8, 2017-10-28
|
||||
# openrefine-batch.sh, Felix Lohmeier, v1.9, 2017-11-05
|
||||
# https://github.com/felixlohmeier/openrefine-batch
|
||||
|
||||
# declare download URLs for OpenRefine and OpenRefine client
|
||||
|
|
@ -192,17 +192,17 @@ echo ""
|
|||
|
||||
# declare additional variables
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Start process"
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Start process"
|
||||
memoryload=()
|
||||
|
||||
# launch server
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Launch OpenRefine"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Launch OpenRefine"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
openrefine/refine -p ${port} -d "${outputdir}" -m ${ram} &
|
||||
pid=$!
|
||||
|
|
@ -213,11 +213,11 @@ echo ""
|
|||
# import all files
|
||||
if [ -n "$inputfiles" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Import all files"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Import all files"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
for inputfile in "${inputfiles[@]}" ; do
|
||||
echo "import ${inputfile}..."
|
||||
|
|
@ -244,18 +244,18 @@ fi
|
|||
# transform and export files
|
||||
if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Prepare transform & export"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Prepare transform & export"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
|
||||
# get project ids
|
||||
echo "get project ids..."
|
||||
openrefine-client/openrefine-client_0-3-1_linux-64bit -P ${port} -l > "${outputdir}/projects.tmp"
|
||||
projectids=($(cat "${outputdir}/projects.tmp" | cut -c 2-14))
|
||||
projectnames=($(cat "${outputdir}/projects.tmp" | cut -c 17-))
|
||||
projectids=($(cut -c 2-14 "${outputdir}/projects.tmp"))
|
||||
projectnames=($(cut -c 17- "${outputdir}/projects.tmp"))
|
||||
cat "${outputdir}/projects.tmp" && rm "${outputdir:?}/projects.tmp"
|
||||
echo ""
|
||||
|
||||
|
|
@ -281,11 +281,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
|
|||
# apply transformation rules
|
||||
if [ -n "$jsonfiles" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Transform ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Transform ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
for jsonfile in "${jsonfiles[@]}" ; do
|
||||
echo "transform ${jsonfile}..."
|
||||
|
|
@ -312,11 +312,11 @@ if [ -n "$jsonfiles" ] || [ "$export" = "true" ]; then
|
|||
# export project to workspace
|
||||
if [ "$export" = "true" ]; then
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="Export ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$(($checkpoints + 1))]} ==="
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="Export ${projectnames[i]}"
|
||||
echo "=== $checkpoints. ${checkpointname[$((checkpoints + 1))]} ==="
|
||||
echo ""
|
||||
echo "starting time: $(date --date=@${checkpointdate[$(($checkpoints + 1))]})"
|
||||
echo "starting time: $(date --date=@${checkpointdate[$((checkpoints + 1))]})"
|
||||
echo ""
|
||||
# get filename without extension
|
||||
filename=${projectnames[i]%.*}
|
||||
|
|
@ -366,13 +366,13 @@ echo ""
|
|||
echo "=== Statistics ==="
|
||||
echo ""
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$(($checkpoints + 1))]="End process"
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
checkpointname[$((checkpoints + 1))]="End process"
|
||||
echo "starting time and run time of each step:"
|
||||
checkpoints=${#checkpointdate[@]}
|
||||
checkpointdate[$(($checkpoints + 1))]=$(date +%s)
|
||||
checkpointdate[$((checkpoints + 1))]=$(date +%s)
|
||||
for i in $(seq 1 $checkpoints); do
|
||||
diffsec="$((${checkpointdate[$(($i + 1))]} - ${checkpointdate[$i]}))"
|
||||
diffsec="$((${checkpointdate[$((i + 1))]} - ${checkpointdate[$i]}))"
|
||||
printf "%35s $(date --date=@${checkpointdate[$i]}) ($(date -d@${diffsec} -u +%H:%M:%S))\n" "${checkpointname[$i]}"
|
||||
done
|
||||
echo ""
|
||||
|
|
@ -384,4 +384,4 @@ max=${memoryload[0]}
|
|||
for n in "${memoryload[@]}" ; do
|
||||
((n > max)) && max=$n
|
||||
done
|
||||
echo "highest memory load: $(($max / 1024)) MB"
|
||||
echo "highest memory load: $((max / 1024)) MB"
|
||||
|
|
|
|||
Loading…
Reference in New Issue