opencultureconsulting
/
openrefine-batch
mirror of https://github.com/opencultureconsulting/openrefine-batch.git
2
0
Fork 0
Code Issues Releases Activity

improved logging with additional if statement

This commit is contained in:
Felix Lohmeier 2017-03-01 22:59:30 +01:00
parent 7b6b9d528f
commit 676f422d1f
1 changed files with 82 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
openrefine-batch.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
# openrefine-batch.sh, Felix Lohmeier, v0.6.1, 01.03.2017 # openrefine-batch.sh, Felix Lohmeier, v0.6.2, 01.03.2017
# https://github.com/felixlohmeier/openrefine-batch # https://github.com/felixlohmeier/openrefine-batch
# user input # user input
@ -9,7 +9,7 @@ if [ -z "$1" ]
exit 2 exit 2
else else
inputdir=$(readlink -f $1) inputdir=$(readlink -f $1)
if [ ! -z "${inputdir// }" ] ; then if [ -n "${inputdir// }" ] ; then
inputfiles=($(find -L ${inputdir}/* -type f -printf "%f\n" 2>/dev/null)) inputfiles=($(find -L ${inputdir}/* -type f -printf "%f\n" 2>/dev/null))
fi fi
fi fi
@ -19,7 +19,7 @@ if [ -z "$2" ]
exit 2 exit 2
else else
configdir=$(readlink -f $2) configdir=$(readlink -f $2)
if [ ! -z "${configdir// }" ] ; then if [ -n "${configdir// }" ] ; then
jsonfiles=($(find -L ${configdir}/* -type f -printf "%f\n" 2>/dev/null)) jsonfiles=($(find -L ${configdir}/* -type f -printf "%f\n" 2>/dev/null))
fi fi
fi fi
@ -37,7 +37,7 @@ if [ -z "$4" ]
exit 2 exit 2
else else
crossdir=$(readlink -f $4) crossdir=$(readlink -f $4)
if [ ! -z "${crossdir// }" ] ; then if [ -n "${crossdir// }" ] ; then
crossprojects=($(find -L ${crossdir}/* -maxdepth 0 -type d -printf "%f\n" 2>/dev/null)) crossprojects=($(find -L ${crossdir}/* -maxdepth 0 -type d -printf "%f\n" 2>/dev/null))
fi fi
fi fi
@ -118,8 +118,8 @@ echo ""
# import all files # import all files
if [ -n "$inputfiles" ]; then if [ -n "$inputfiles" ]; then
echo "=== IMPORT ===" echo "=== IMPORT ==="
echo "" echo ""
for inputfile in "${inputfiles[@]}" ; do for inputfile in "${inputfiles[@]}" ; do
echo "import ${inputfile}..." echo "import ${inputfile}..."
# run client with input command # run client with input command
@ -140,21 +140,23 @@ echo ""
done done
fi fi
echo "=== TRANSFORM / EXPORT ===" # transform and export files
echo "" if [ -n "$jsonfiles" ] || [ "$export" = "export-true" ]; then
echo "=== TRANSFORM / EXPORT ==="
# get project ids echo ""
echo "get project ids..."
projects=($(docker run --rm --link ${uuid} felixlohmeier/openrefine-client -H ${uuid} -l | tee ${outputdir}/projects.tmp | cut -c 2-14)) # get project ids
cat ${outputdir}/projects.tmp && rm ${outputdir}/projects.tmp echo "get project ids..."
echo "" projects=($(docker run --rm --link ${uuid} felixlohmeier/openrefine-client -H ${uuid} -l | tee ${outputdir}/projects.tmp | cut -c 2-14))
cat ${outputdir}/projects.tmp && rm ${outputdir}/projects.tmp
# provide additional OpenRefine projects for cross function echo ""
if [ -n "$crossprojects" ]; then
echo "provide additional projects for cross function..." # provide additional OpenRefine projects for cross function
# copy given projects to workspace if [ -n "$crossprojects" ]; then
rsync -a --exclude='*.project/history' $crossdir/*.project $outputdir echo "provide additional projects for cross function..."
# restart server to advertise copied projects # copy given projects to workspace
rsync -a --exclude='*.project/history' $crossdir/*.project $outputdir
# restart server to advertise copied projects
echo "restart OpenRefine server to advertise copied projects..." echo "restart OpenRefine server to advertise copied projects..."
docker stop -t=5000 ${uuid} docker stop -t=5000 ${uuid}
docker rm ${uuid} docker rm ${uuid}
@ -162,64 +164,65 @@ if [ -n "$crossprojects" ]; then
until docker run --rm --link ${uuid} --entrypoint /usr/bin/curl felixlohmeier/openrefine-client --silent -N http://${uuid}:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done until docker run --rm --link ${uuid} --entrypoint /usr/bin/curl felixlohmeier/openrefine-client --silent -N http://${uuid}:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done
docker attach ${uuid} & docker attach ${uuid} &
echo "" echo ""
fi fi
# loop for all projects # loop for all projects
for projectid in "${projects[@]}" ; do for projectid in "${projects[@]}" ; do
# time # time
echo "--- begin project $projectid @ $(date) ---" echo "--- begin project $projectid @ $(date) ---"
echo "" echo ""
# apply transformation rules # apply transformation rules
if [ -n "$jsonfiles" ]; then if [ -n "$jsonfiles" ]; then
for jsonfile in "${jsonfiles[@]}" ; do for jsonfile in "${jsonfiles[@]}" ; do
echo "transform ${jsonfile}..." echo "transform ${jsonfile}..."
# run client with apply command # run client with apply command
docker run --rm --link ${uuid} -v ${configdir}:/data felixlohmeier/openrefine-client -H ${uuid} -f ${jsonfile} ${projectid} docker run --rm --link ${uuid} -v ${configdir}:/data felixlohmeier/openrefine-client -H ${uuid} -f ${jsonfile} ${projectid}
# show statistics # show statistics
ps -o start,etime,%mem,%cpu,rss -C java --sort=start ps -o start,etime,%mem,%cpu,rss -C java --sort=start
# restart server to clear memory # restart server to clear memory
if [ "$restarttransform" = "restarttransform-true" ]; then if [ "$restarttransform" = "restarttransform-true" ]; then
echo "save project and restart OpenRefine server..." echo "save project and restart OpenRefine server..."
docker stop -t=5000 ${uuid} docker stop -t=5000 ${uuid}
docker rm ${uuid} docker rm ${uuid}
docker run -d --name=${uuid} -v ${outputdir}:/data felixlohmeier/openrefine:${version} -i 0.0.0.0 -m ${ram} -d /data docker run -d --name=${uuid} -v ${outputdir}:/data felixlohmeier/openrefine:${version} -i 0.0.0.0 -m ${ram} -d /data
until docker run --rm --link ${uuid} --entrypoint /usr/bin/curl felixlohmeier/openrefine-client --silent -N http://${uuid}:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done until docker run --rm --link ${uuid} --entrypoint /usr/bin/curl felixlohmeier/openrefine-client --silent -N http://${uuid}:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done
docker attach ${uuid} & docker attach ${uuid} &
fi fi
echo "" echo ""
done done
fi fi
# export project to workspace # export project to workspace
if [ "$export" = "export-true" ]; then if [ "$export" = "export-true" ]; then
echo "export to file ${projectid}.tsv..." echo "export to file ${projectid}.tsv..."
# run client with export command # run client with export command
docker run --rm --link ${uuid} -v ${outputdir}:/data felixlohmeier/openrefine-client -H ${uuid} -E --output=${projectid}.tsv ${projectid} docker run --rm --link ${uuid} -v ${outputdir}:/data felixlohmeier/openrefine-client -H ${uuid} -E --output=${projectid}.tsv ${projectid}
# show statistics # show statistics
ps -o start,etime,%mem,%cpu,rss -C java --sort=start ps -o start,etime,%mem,%cpu,rss -C java --sort=start
# restart server to clear memory # restart server to clear memory
if [ "$restartfile" = "restartfile-true" ]; then if [ "$restartfile" = "restartfile-true" ]; then
echo "restart OpenRefine server..." echo "restart OpenRefine server..."
docker stop -t=5000 ${uuid} docker stop -t=5000 ${uuid}
docker rm ${uuid} docker rm ${uuid}
docker run -d --name=${uuid} -v ${outputdir}:/data felixlohmeier/openrefine:${version} -i 0.0.0.0 -m ${ram} -d /data docker run -d --name=${uuid} -v ${outputdir}:/data felixlohmeier/openrefine:${version} -i 0.0.0.0 -m ${ram} -d /data
until docker run --rm --link ${uuid} --entrypoint /usr/bin/curl felixlohmeier/openrefine-client --silent -N http://${uuid}:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done until docker run --rm --link ${uuid} --entrypoint /usr/bin/curl felixlohmeier/openrefine-client --silent -N http://${uuid}:3333 | cat | grep -q -o "OpenRefine" ; do sleep 1; done
docker attach ${uuid} & docker attach ${uuid} &
fi fi
echo"" echo""
fi fi
# time # time
echo "--- finished project $projectid @ $(date) ---" echo "--- finished project $projectid @ $(date) ---"
echo "" echo ""
done done
# list output files # list output files
if [ "$export" = "export-true" ]; then if [ "$export" = "export-true" ]; then
echo "output (number of lines / size in bytes):" echo "output (number of lines / size in bytes):"
wc -c -l ${outputdir}/*.tsv wc -c -l ${outputdir}/*.tsv
echo "" echo ""
fi
fi fi
# cleanup # cleanup