#!/bin/bash # bash-refine v1.1.0: bash-refine.sh, Felix Lohmeier, 2020-07-10 # https://gist.github.com/felixlohmeier/d76bd27fbc4b8ab6d683822cdf61f81d # license: MIT License https://choosealicense.com/licenses/mit/ # TODO: support for macOS # ================================== CONFIG ================================== # endpoint="http://localhost:3333" memory="1400M" # increase to available RAM date="$(date +%Y%m%d_%H%M%S)" workspace="output/${date}" logfile="${workspace}/${date}.log" csrf=true # set to false for OpenRefine < 3.3 jq="jq" # path to executable openrefine="openrefine/refine" # path to executable declare -A checkpoints # associative array for stats declare -A pids # associative array for monitoring background jobs declare -A projects # associative array for OpenRefine projects # =============================== REQUIREMENTS =============================== # function requirements { # check existence of java and cURL if [[ -z "$(command -v java 2> /dev/null)" ]] ; then echo 1>&2 "ERROR: OpenRefine requires JAVA runtime environment (jre)" \ "https://openjdk.java.net/install/" exit 1 fi if [[ -z "$(command -v curl 2> /dev/null)" ]] ; then echo 1>&2 "ERROR: This shell script requires cURL" \ "https://curl.haxx.se/download.html" exit 1 fi # download jq and OpenRefine if necessary if [[ -z "$(readlink -e "${jq}")" ]]; then echo "Download jq..." # jq 1.4 has much faster startup time than 1.5 and 1.6 curl -L --output "${jq}" \ "https://github.com/stedolan/jq/releases/download/jq-1.4/jq-linux-x86_64" chmod +x "${jq}"; echo fi if [[ -z "$(readlink -e "${openrefine}")" ]]; then echo "Download OpenRefine..." mkdir -p "$(dirname "${openrefine}")" curl -L --output openrefine.tar.gz \ "https://github.com/OpenRefine/OpenRefine/releases/download/3.3/openrefine-linux-3.3.tar.gz" echo "Install OpenRefine in subdirectory $(dirname "${openrefine}")..." tar -xzf openrefine.tar.gz -C "$(dirname "${openrefine}")" --strip 1 --totals rm -f openrefine.tar.gz # do not try to open OpenRefine in browser sed -i '$ a JAVA_OPTIONS=-Drefine.headless=true' \ "$(dirname "${openrefine}")"/refine.ini # set min java heap space to allocated memory sed -i 's/-Xms$REFINE_MIN_MEMORY/-Xms$REFINE_MEMORY/' \ "$(dirname "${openrefine}")"/refine # set autosave period from 5 minutes to 25 hours sed -i 's/#REFINE_AUTOSAVE_PERIOD=60/REFINE_AUTOSAVE_PERIOD=1500/' \ "$(dirname "${openrefine}")"/refine.ini echo fi } # ============================== OPENREFINE API ============================== # function refine_start() { echo "start OpenRefine server..." local dir dir="$(readlink -f "${workspace}")" ${openrefine} -v warn -m "${memory}" -p "${endpoint##*:}" -d "${dir}" & pid_server=${!} timeout 30s bash -c "until curl -s \"${endpoint}\" \ | cat | grep -q -o 'OpenRefine' ; do sleep 1; done" \ || error "starting OpenRefine server failed!" } function refine_stats() { # print server load ps -o start,etime,%mem,%cpu,rss -p "${pid_server}" } function refine_kill() { # kill OpenRefine immediately; SIGKILL (kill -9) prevents saving projects { kill -9 "${pid_server}" && wait "${pid_server}"; } 2>/dev/null # delete temporary OpenRefine projects (cd "${workspace}" && rm -rf ./*.project* && rm -f workspace.json) } function refine_check() { if grep -i 'exception\|error' "${logfile}"; then error "log contains warnings!" else log "checked log file, all good!" fi } function refine_stop() { echo "stop OpenRefine server and print server load..." refine_stats echo refine_kill echo "check log for any warnings..." refine_check } function refine_csrf() { # get CSRF token (introduced in OpenRefine 3.3) if [[ "${csrf}" = true ]]; then local response response=$(curl -fs "${endpoint}/command/core/get-csrf-token") if [[ "${response}" != '{"token":"'* ]]; then error "getting CSRF token failed!" else echo "?csrf_token=$(echo "$response" | cut -d \" -f 4)" fi fi } function refine_store() { # check and store project id from import in associative array projects if [[ $# = 2 ]]; then projects[$1]=$(cut -d '=' -f 2 "$2") else error "invalid arguments supplied to import function!" fi if [[ "${#projects[$1]}" != 13 ]]; then error "returned project id is not valid!" else rm "$2" fi # check if project contains at least one row (may be skipped to gain ~40ms) local rows rows=$(curl -fs --get \ --data project="${projects[$p]}" \ --data limit=0 \ "${endpoint}/command/core/get-rows" \ | tr "," "\n" | grep total | cut -d ":" -f 2) if [[ "$rows" = "0" ]]; then error "imported project contains 0 rows!" fi } # ============================ SCRIPT ENVIRONMENT ============================ # function log() { # log status message echo "$(date +%H:%M:%S.%3N) [ client] $1" } function error() { # log error message and exit echo 1>&2 "ERROR: $1" refine_kill; pkill -P $$; exit 1 } function monitor() { # store pid of last execution pids[$1]="$!" } function monitoring() { # wait for stored pids, remove them from array and check log for errors for pid in "${!pids[@]}"; do wait "${pids[$pid]}" \ || error "${pid} (${projects[$pid]}) failed!" \ && unset pids["$pid"] done refine_check } function checkpoint { # store timestamp in associative array checkpoints and print checkpoint checkpoints[$1]=$(date +%s.%3N) printf '%*.*s %s %*.*s\n' \ 0 "$(((80-2-${#1})/2))" "$(printf '%0.1s' ={1..40})" \ "${#checkpoints[@]}. $1" \ 0 "$(((80-1-${#1})/2))" "$(printf '%0.1s' ={1..40})" } function checkpoint_stats { # calculate run time based on checkpoints local k keys values i diffsec echo "starting time and run time (hh:mm:ss) of each step..." # sort keys by value and store in array key readarray -t keys < <( for k in "${!checkpoints[@]}"; do echo "${checkpoints[$k]}:::$k" done | sort | awk -F::: '{print $2}') # remove milliseconds from corresponding values and store in array values readarray -t values < <( for k in "${keys[@]}" ; do echo "${checkpoints[$k]%.*}" done) # add final timestamp for calculation values+=("$(date +%s)") # calculate and print run time for each step for i in "${!keys[@]}"; do diffsec=$(( values[$((i + 1))] - values[i] )) printf "%35s %s %s %s\n" "${keys[$i]}" "($((i + 1)))" \ "$(date -d @"${values[$i]}")" \ "($(date -d @${diffsec} -u +%H:%M:%S))" done # calculate and print total run time diffsec=$(( values[${#keys[@]}] - values[0] )) printf "%80s\n%80s\n" "----------" "($(date -d @${diffsec} -u +%H:%M:%S))" } function count_output { # word count on all files in workspace echo "files (number of lines / size in bytes) in ${workspace}..." (cd "${workspace}" && wc -c -l ./*) } function init() { # check requirements and download software if necessary requirements # set trap, create directories and tee to log file trap 'error "script interrupted!"' HUP INT QUIT TERM mkdir -p "${workspace}" exec &> >(tee -a "${logfile}") }