opencultureconsulting
/
orcli
mirror of https://github.com/opencultureconsulting/orcli.git
2
0
Fork 0
Code Issues Releases Activity
orcli/orcli

1886 lines
46 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# This script was generated by bashly 0.8.10 (https://bashly.dannyb.co)
# Modifying it manually is not recommended
# :wrapper.bash3_bouncer
if [[ "${BASH_VERSINFO:-0}" -lt 4 ]]; then
printf "bash version 4 or higher is required\n" >&2
exit 1
fi
# :command.master_script
# :command.version_command
version_command() {
echo "$version"
}
# :command.usage
orcli_usage() {
if [[ -n $long_usage ]]; then
printf "orcli - OpenRefine command-line interface written in Bash\n"
echo
else
printf "orcli - OpenRefine command-line interface written in Bash\n"
echo
fi
printf "Usage:\n"
printf " orcli [OPTIONS] COMMAND\n"
printf " orcli [COMMAND] --help | -h\n"
printf " orcli --version | -v\n"
echo
# :command.usage_commands
printf "Commands:\n"
echo " completions Generate bash completions"
echo " import commands to create OpenRefine projects from files or URLs"
echo " list list projects on OpenRefine server"
echo " info show OpenRefine project's metadata"
echo " transform apply undo/redo JSON file(s) to an OpenRefine project"
echo " export commands to export data from OpenRefine projects to files"
echo " run run tmp OpenRefine workspace and execute shell script(s)"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
echo " --version, -v"
printf " Show version number\n"
echo
# :command.usage_flags
# :flag.usage
echo " --quiet, -q"
printf " suppress log output, print errors only\n"
echo
# :command.usage_environment_variables
printf "Environment Variables:\n"
# :environment_variable.usage
echo " OPENREFINE_URL"
printf " URL to OpenRefine server\n"
printf " Default: http://localhost:3333\n"
echo
# :command.usage_examples
printf "Examples:\n"
printf " orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n"
printf " orcli list\n"
printf " orcli info \"duplicates\"\n"
printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n"
printf " orcli export tsv \"duplicates\"\n"
printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n"
printf " orcli run --interactive\n"
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n"
echo
# :command.footer
printf "https://github.com/opencultureconsulting/orcli\n"
echo
fi
}
# :command.usage
orcli_completions_usage() {
if [[ -n $long_usage ]]; then
printf "orcli completions\n"
echo
printf " Generate bash completions\n Usage: eval \"\$(orcli completions)\"\n"
echo
else
printf "orcli completions - Generate bash completions\n"
echo
fi
printf "Usage:\n"
printf " orcli completions\n"
printf " orcli completions --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
fi
}
# :command.usage
orcli_import_usage() {
if [[ -n $long_usage ]]; then
printf "orcli import - commands to create OpenRefine projects from files or URLs\n"
echo
else
printf "orcli import - commands to create OpenRefine projects from files or URLs\n"
echo
fi
printf "Usage:\n"
printf " orcli import COMMAND\n"
printf " orcli import [COMMAND] --help | -h\n"
echo
# :command.usage_commands
printf "Commands:\n"
echo " csv import comma-separated values (CSV)"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
fi
}
# :command.usage
orcli_import_csv_usage() {
if [[ -n $long_usage ]]; then
printf "orcli import csv - import comma-separated values (CSV)\n"
echo
else
printf "orcli import csv - import comma-separated values (CSV)\n"
echo
fi
printf "Usage:\n"
printf " orcli import csv [FILE...] [OPTIONS]\n"
printf " orcli import csv --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
# :command.usage_flags
# :flag.usage
echo " --separator SEPARATOR"
printf " character(s) that separates columns\n"
printf " Default: ,\n"
echo
# :flag.usage
echo " --encoding ENCODING"
printf " set character encoding\n"
echo
# :flag.usage
echo " --trimStrings"
printf " trim leading & trailing whitespace from strings\n"
echo
# :flag.usage
echo " --projectName PROJECTNAME"
printf " set a name for the OpenRefine project\n"
echo
# :command.usage_args
printf "Arguments:\n"
# :argument.usage
echo " FILE..."
printf " Path to one or more files or URLs. When FILE is -, read standard input.\n"
printf " Default: -\n"
echo
# :command.usage_examples
printf "Examples:\n"
printf " orcli import csv \"file\"\n"
printf " orcli import csv \"file1\" \"file2\"\n"
printf " cat \"file\" | orcli import csv\n"
printf " orcli import csv \"https://git.io/fj5hF\"\n"
printf " orcli import csv \"file\" \\\\\n --separator \";\" \\\\\n --encoding \"ISO-8859-1\" \\\\\n --trimStrings \\\\\n --projectName \"duplicates\"\n"
echo
fi
}
# :command.usage
orcli_list_usage() {
if [[ -n $long_usage ]]; then
printf "orcli list - list projects on OpenRefine server\n"
echo
else
printf "orcli list - list projects on OpenRefine server\n"
echo
fi
printf "Usage:\n"
printf " orcli list\n"
printf " orcli list --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
fi
}
# :command.usage
orcli_info_usage() {
if [[ -n $long_usage ]]; then
printf "orcli info - show OpenRefine project's metadata\n"
echo
else
printf "orcli info - show OpenRefine project's metadata\n"
echo
fi
printf "Usage:\n"
printf " orcli info PROJECT\n"
printf " orcli info --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
# :command.usage_args
printf "Arguments:\n"
# :argument.usage
echo " PROJECT"
printf " project name or id\n"
echo
# :command.usage_examples
printf "Examples:\n"
printf " orcli info \"duplicates\"\n"
printf " orcli info 1234567890123\n"
echo
fi
}
# :command.usage
orcli_transform_usage() {
if [[ -n $long_usage ]]; then
printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n"
echo
else
printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n"
echo
fi
printf "Usage:\n"
printf " orcli transform PROJECT [FILE...]\n"
printf " orcli transform --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
# :command.usage_args
printf "Arguments:\n"
# :argument.usage
echo " PROJECT"
printf " project name or id\n"
echo
# :argument.usage
echo " FILE..."
printf " Path to one or more files or URLs containing OpenRefine's undo/redo\n operation history in JSON format. When FILE is -, read standard input.\n"
printf " Default: -\n"
echo
# :command.usage_examples
printf "Examples:\n"
printf " orcli transform \"duplicates\" \"history.json\"\n"
printf " cat \"history.json\" | orcli transform \"duplicates\"\n"
printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n"
printf " orcli transform 1234567890123 \"history.json\"\n"
echo
fi
}
# :command.usage
orcli_export_usage() {
if [[ -n $long_usage ]]; then
printf "orcli export - commands to export data from OpenRefine projects to files\n"
echo
else
printf "orcli export - commands to export data from OpenRefine projects to files\n"
echo
fi
printf "Usage:\n"
printf " orcli export COMMAND\n"
printf " orcli export [COMMAND] --help | -h\n"
echo
# :command.usage_commands
printf "Commands:\n"
echo " tsv export tab-separated values (TSV)"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
fi
}
# :command.usage
orcli_export_tsv_usage() {
if [[ -n $long_usage ]]; then
printf "orcli export tsv - export tab-separated values (TSV)\n"
echo
else
printf "orcli export tsv - export tab-separated values (TSV)\n"
echo
fi
printf "Usage:\n"
printf " orcli export tsv PROJECT [OPTIONS]\n"
printf " orcli export tsv --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
# :command.usage_flags
# :flag.usage
echo " --output FILE"
printf " Write to file instead of stdout\n"
echo
# :flag.usage
echo " --encoding ENCODING"
printf " set character encoding\n"
printf " Default: UTF-8\n"
echo
# :command.usage_args
printf "Arguments:\n"
# :argument.usage
echo " PROJECT"
printf " project name or id\n"
echo
# :command.usage_examples
printf "Examples:\n"
printf " orcli export tsv \"duplicates\"\n"
printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n"
echo
fi
}
# :command.usage
orcli_run_usage() {
if [[ -n $long_usage ]]; then
printf "orcli run - run tmp OpenRefine workspace and execute shell script(s)\n"
echo
else
printf "orcli run - run tmp OpenRefine workspace and execute shell script(s)\n"
echo
fi
printf "Usage:\n"
printf " orcli run [FILE...] [OPTIONS]\n"
printf " orcli run --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "Options:\n"
# :command.usage_fixed_flags
echo " --help, -h"
printf " Show this help\n"
echo
# :command.usage_flags
# :flag.usage
echo " --memory RAM"
printf " maximum RAM for OpenRefine java heap space\n"
printf " Default: 2048M\n"
echo
# :flag.usage
echo " --port PORT"
printf " PORT on which OpenRefine should listen\n"
printf " Default: 3333\n"
echo
# :flag.usage
echo " --interactive"
printf " do not exit on error and keep bash shell open\n"
echo
# :command.usage_args
printf "Arguments:\n"
# :argument.usage
echo " FILE..."
printf " Path to one or more files. When FILE is -, read standard input.\n"
printf " Default: -\n"
echo
# :command.usage_examples
printf "Examples:\n"
printf " orcli run --interactive\n"
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n"
printf " orcli run --memory \"2000M\" --port \"3334\" << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n wait\n echo \"finished import\"\n orcli export csv \"duplicates\" --output duplicates.csv &\n orcli export tsv \"duplicates\" --output duplicates.tsv &\n wait\n wc duplicates*\n EOF\n"
printf " orcli run --interactive \"file1.sh\" \"file2.sh\" - << EOF\n echo \"finished in \$SECONDS seconds\"\n EOF\n"
echo
fi
}
# :command.normalize_input
normalize_input() {
local arg flags
while [[ $# -gt 0 ]]; do
arg="$1"
if [[ $arg =~ ^(--[a-zA-Z0-9_\-]+)=(.+)$ ]]; then
input+=("${BASH_REMATCH[1]}")
input+=("${BASH_REMATCH[2]}")
elif [[ $arg =~ ^(-[a-zA-Z0-9])=(.+)$ ]]; then
input+=("${BASH_REMATCH[1]}")
input+=("${BASH_REMATCH[2]}")
elif [[ $arg =~ ^-([a-zA-Z0-9][a-zA-Z0-9]+)$ ]]; then
flags="${BASH_REMATCH[1]}"
for (( i=0 ; i < ${#flags} ; i++ )); do
input+=("-${flags:i:1}")
done
else
input+=("$arg")
fi
shift
done
}
# :command.inspect_args
inspect_args() {
readarray -t sorted_keys < <(printf '%s\n' "${!args[@]}" | sort)
if (( ${#args[@]} )); then
echo args:
for k in "${sorted_keys[@]}"; do echo "- \${args[$k]} = ${args[$k]}"; done
else
echo args: none
fi
if (( ${#other_args[@]} )); then
echo
echo other_args:
echo "- \${other_args[*]} = ${other_args[*]}"
for i in "${!other_args[@]}"; do
echo "- \${other_args[$i]} = ${other_args[$i]}"
done
fi
}
# :command.user_lib
# src/lib/get_csrf.sh
# get CSRF token (introduced in OpenRefine 3.3)
# shellcheck shell=bash
function get_csrf() {
local response
if ! response="$(curl -fs "${OPENREFINE_URL}/command/core/get-csrf-token")"; then
if ! response="$(curl -fs "${OPENREFINE_URL}/command/core/get-version")"; then
error "no OpenRefine reachable/running at ${OPENREFINE_URL}"
fi
else
if ! [[ "${response}" == '{"token":"'* ]]; then
error "getting CSRF token failed!"
fi
echo "?csrf_token=$(echo "$response" | cut -d \" -f 4)"
fi
}
# src/lib/get_id.sh
# get project id (derived from project name if needed)
# shellcheck shell=bash
function get_id() {
local response
local projects
local ids
if ! response="$(curl -fs --get "${OPENREFINE_URL}/command/core/get-all-project-metadata")"; then
error "no OpenRefine reachable/running at ${OPENREFINE_URL}"
fi
if ! projects="$(echo "$response" | jq -r '.projects | keys[] as $k | "\($k):\(.[$k] | .name)"' | grep -e ":$1$" -e "^$1:")"; then
error "project $1 not found"
fi
ids=$(echo "$projects" | cut -d : -f 1)
if ! [[ "${#ids}" == 13 ]]; then
error "multiple projects found" "$projects"
fi
echo "$ids"
}
# src/lib/init_import.sh
# common import tasks to support multiple files and URLs
# shellcheck shell=bash
function init_import() {
local files file tmpdir
# catch args, convert the space delimited string to an array
files=()
eval "files=(${args[file]})"
# create tmp directory
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' 0 2 3 15
# download files if name starts with http:// or https://
for i in "${!files[@]}"; do
if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "download of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# read pipes if name starts with /dev/fd
for i in "${!files[@]}"; do
if [[ ${files[$i]} == "/dev/fd"* ]]; then
if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "reading of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# create a zip archive if there are multiple files
if [[ ${#files[@]} -gt 1 ]]; then
file="$tmpdir/Untitled.zip"
if ! zip --quiet --must-match "$file" "${files[@]}"; then
error "creating zip archive with ${files[*]} failed!"
fi
else
file="${files[0]}"
fi
# basic post data
if [[ ${file} == "-" ]]; then
data+=("project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "file ${file} not found!"
fi
data+=("project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
data+=("project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
data+=("project-name=${name}")
fi
}
# src/lib/interactive.sh
# shellcheck shell=bash
function interactive() {
cat <<'EOF'
PS1="(orcli) [\u@\h \W]\$ "
source <(orcli completions)
echo '================================================================'
echo 'Interactive Bash shell with OpenRefine running in the background'
echo 'Use the "orcli" command and tab completion to control OpenRefine'
echo 'Type "history -a FILE" to write out your session history'
echo 'Type "exit" or CTRL-D to destroy temporary OpenRefine workspace'
echo '================================================================'
EOF
}
# src/lib/logging.sh
# print messages to STDERR
# shellcheck shell=bash
function error() {
echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S')] ERROR: $1"
shift
for msg in "$@"; do echo >&2 " $msg"; done
exit 1
}
function log() {
if ! [[ ${args[--quiet]} ]]; then
echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S')] $1"
shift
for msg in "$@"; do echo >&2 " $msg"; done
fi
}
# src/lib/post_import.sh
# post to create-project endpoint and validate
# shellcheck shell=bash disable=SC2154
function post_import() {
local curloptions
local projectid
local projectname
local rows
# post
mapfile -t curloptions < <(for d in "$@"; do
echo "--form"
echo "$d"
done)
if ! redirect_url="$(curl -fs --write-out "%{redirect_url}\n" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/create-project-from-upload$(get_csrf)")"; then
error "importing ${args[file]} failed!"
fi
# validate
projectid=$(cut -d '=' -f 2 <<<"$redirect_url")
if [[ ${#projectid} != 13 ]]; then
error "importing ${args[file]} failed!"
fi
projectname=$(curl -fs --get --data project="$projectid" "${OPENREFINE_URL}/command/core/get-project-metadata" | tr "," "\n" | grep name | cut -d ":" -f 2)
projectname="${projectname:1:${#projectname}-2}"
rows=$(curl -fs --get --data project="$projectid" --data limit=0 "${OPENREFINE_URL}/command/core/get-rows" | tr "," "\n" | grep total | cut -d ":" -f 2)
if [[ "$rows" = "0" ]]; then
error "import of ${args[file]} contains 0 rows!"
else
log "imported ${args[file]}" "${redirect_url}" "name: ${projectname}" "rows: ${rows}"
fi
}
# src/lib/send_completions.sh
send_completions() {
echo $'# orcli completion -*- shell-script -*-'
echo $''
echo $'# This bash completions script was generated by'
echo $'# completely (https://github.com/dannyben/completely)'
echo $'# Modifying it manually is not recommended'
echo $''
echo $'_orcli_completions_filter() {'
echo $' local words="$1"'
echo $' local cur=${COMP_WORDS[COMP_CWORD]}'
echo $' local result=()'
echo $''
echo $' if [[ "${cur:0:1}" == "-" ]]; then'
echo $' echo "$words"'
echo $' '
echo $' else'
echo $' for word in $words; do'
echo $' [[ "${word:0:1}" != "-" ]] && result+=("$word")'
echo $' done'
echo $''
echo $' echo "${result[*]}"'
echo $''
echo $' fi'
echo $'}'
echo $''
echo $'_orcli_completions() {'
echo $' local cur=${COMP_WORDS[COMP_CWORD]}'
echo $' local compwords=("${COMP_WORDS[@]:1:$COMP_CWORD-1}")'
echo $' local compline="${compwords[*]}"'
echo $''
echo $' case "$compline" in'
echo $' \'completions\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'import csv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --separator --trimStrings -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'export tsv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'transform\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'import\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'export\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h tsv")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'list\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'info\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'run\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --interactive --memory --port -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' *)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' esac'
echo $'} &&'
echo $'complete -F _orcli_completions orcli'
echo $''
echo $'# ex: filetype=sh'
}
# :command.command_functions
# :command.function
orcli_completions_command() {
# src/completions_command.sh
# Users can now enable bash completion for this script by running:
#
# $ eval "$(orcli completions)"
#
send_completions
}
# :command.function
orcli_import_csv_command() {
# src/import_csv_command.sh
# shellcheck shell=bash
# call init_import function to eval args and to set basic post data
init_import
# check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
orcli_import_csv_usage
exit 1
fi
fi
# assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv")
options='{ '
options+="\"separator\": \"${args[--separator]}\""
if [[ ${args[--encoding]} ]]; then
options+=', '
options+="\"encoding\": \"${args[--encoding]}\""
fi
if [[ ${args[--trimStrings]} ]]; then
options+=', '
options+="\"trimStrings\": true"
fi
options+=' }'
data+=("options=${options}")
# call post_import function to post data and validate results
post_import "${data[@]}"
}
# :command.function
orcli_list_command() {
# src/list_command.sh
# get all project metadata and reshape json to print a list
# shellcheck shell=bash
if ! response="$(curl -fs --get "${OPENREFINE_URL}/command/core/get-all-project-metadata")"; then
error "no OpenRefine reachable/running at ${OPENREFINE_URL}"
else
if [[ "${response}" == '{"projects":{}}' ]]; then
log "${OPENREFINE_URL} does not contain any projects yet."
else
echo "$response" | jq -r '.projects | keys[] as $k | "\($k):\(.[$k] | .name)"'
fi
fi
}
# :command.function
orcli_info_command() {
# src/info_command.sh
# shellcheck shell=bash disable=SC2154
get_id "${args[project]}"
}
# :command.function
orcli_transform_command() {
# src/transform_command.sh
# shellcheck shell=bash disable=SC2154 disable=SC2155
# check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
orcli_transform_usage
exit 1
fi
fi
# catch args, convert the space delimited string to an array
files=()
eval "files=(${args[file]})"
# create tmp directory
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' 0 2 3 15
# download files if name starts with http:// or https://
for i in "${!files[@]}"; do
if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "download of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# support multiple files
for i in "${!files[@]}"; do
# read each operation into one line
mapfile -t jsonlines < <(jq -c '.[]' "${files[$i]}")
for line in "${jsonlines[@]}"; do
# parse one line/operation into array
declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
# map operation names to command endpoints
# https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js
com="${data[op]#core/}"
if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi
if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi
if [[ $com == "column-addition" ]]; then com="add-column"; fi
if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi
if [[ $com == "column-removal" ]]; then com="remove-column"; fi
if [[ $com == "column-rename" ]]; then com="rename-column"; fi
if [[ $com == "column-move" ]]; then com="move-column"; fi
if [[ $com == "column-split" ]]; then com="split-column"; fi
if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi
if [[ $com == "recon" ]]; then com="reconcile"; fi
if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi
if [[ $com == "row-star" ]]; then com="annotate-rows"; fi
if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi
if [[ $com == "row-removal" ]]; then com="remove-rows"; fi
if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
unset "data[op]"
# rename engineConfig to engine
data[engine]="${data[engineConfig]}"
unset "data[engineConfig]"
# drop description
unset "data[description]"
# prepare curl options
mapfile -t curloptions < <(for K in "${!data[@]}"; do
echo "--data"
echo "$K=${data[$K]}"
done)
# get project id and csrf token; post data to it's individual endpoint
if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
response_code="$(jq -r '.code' <<<"$response")"
if [[ $response_code == "ok" ]]; then
log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")"
else
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")"
fi
else
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!"
fi
unset data
done
done
}
# :command.function
orcli_export_tsv_command() {
# src/export_tsv_command.sh
# shellcheck shell=bash
projectid="$(get_id "${args[project]}")"
separator='\t'
# assemble specific post data (some options require json format)
data+=("project=${projectid}")
data+=("format=tsv")
options='{ '
options+="\"separator\": \"${separator}\""
if [[ ${args[--encoding]} ]]; then
options+=', '
options+="\"encoding\": \"${args[--encoding]}\""
fi
options+=' }'
data+=("options=${options}")
# post
mapfile -t curloptions < <(for d in "${data[@]}"; do
echo "--data"
echo "$d"
done)
if [[ ${args[--output]} ]]; then
if ! mkdir -p "$(dirname "${args[--output]}")"; then
error "unable to create parent directory for ${args[--output]}"
fi
curloptions+=("--output")
curloptions+=("${args[--output]}")
fi
if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then
error "exporting ${args[project]} failed!"
else
if [[ ${args[--output]} ]]; then
log "exported ${args[project]}" "file: ${args[--output]}" "rows: $(wc -l <"${args[--output]}")"
fi
fi
}
# :command.function
orcli_run_command() {
# src/run_command.sh
# shellcheck shell=bash disable=SC2154 source=/dev/null
# catch args, convert the space delimited string to an array
files=()
eval "files=(${args[file]})"
# check if stdin is present if selected
if ! [[ ${args[--interactive]} ]]; then
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
orcli_run_usage
exit 1
fi
fi
fi
# update OPENREFINE_URL env
OPENREFINE_URL="http://localhost:${args[--port]}"
# locate orcli and OpenRefine
scriptpath=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
if [[ -x "${scriptpath}/refine" ]]; then
openrefine="${scriptpath}/refine"
else
error "OpenRefine's startup script (refine) not found!" "Did you put orcli in your OpenRefine app dir?"
fi
# create tmp directory
OPENREFINE_TMPDIR="$(mktemp -d)"
trap '{ rm -rf "$OPENREFINE_TMPDIR"; }' 0 2 3 15
# check if OpenRefine is already running
if curl -fs "${OPENREFINE_URL}" &>/dev/null; then
error "OpenRefine is already running on port ${args[--port]}." "Hint: Stop the other process or use another port."
fi
# start OpenRefine with tmp workspace and autosave period 25 hours
REFINE_AUTOSAVE_PERIOD=1440 $openrefine -d "$OPENREFINE_TMPDIR" -m "${args[--memory]}" -p "${args[--port]}" -x refine.headless=true -v warn &>"$OPENREFINE_TMPDIR/openrefine.log" &
OPENREFINE_PID="$!"
# update trap to kill OpenRefine on error or exit
trap '{ rm -rf "$OPENREFINE_TMPDIR"; kill -9 "$OPENREFINE_PID"; }' 0 2 3 15
# wait until OpenRefine is running (timeout 20s)
if ! curl -fs --retry 20 --retry-connrefused --retry-delay 1 "${OPENREFINE_URL}/command/core/get-version" &>/dev/null; then
error "starting OpenRefine server failed!"
else
log "started OpenRefine" "port: ${args[--port]}" "memory: ${args[--memory]}" "tmpdir: ${OPENREFINE_TMPDIR}" "pid: ${OPENREFINE_PID}"
fi
# execute script(s) in subshell
export OPENREFINE_TMPDIR OPENREFINE_URL OPENREFINE_PID
if [[ ${args[file]} == '-' || ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
# case 1: interactive mode if stdin is selected but not present
bash --rcfile <(
cat ~/.bashrc
if ! command -v orcli &>/dev/null; then
echo "alias orcli=${scriptpath}/orcli"
fi
interactive
) -i </dev/tty
exit
fi
fi
if [[ ${args[--interactive]} ]]; then
# case 2: execute scripts and keep shell running
bash --rcfile <(
cat ~/.bashrc
if ! command -v orcli &>/dev/null; then
echo "alias orcli=${scriptpath}/orcli"
fi
for i in "${!files[@]}"; do
log "executing script ${files[$i]}..."
awk 1 "${files[$i]}"
done
interactive
) -i </dev/tty
else
# case 3: just execute scripts
for i in "${!files[@]}"; do
log "executing script ${files[$i]}..."
bash -e <(
if ! command -v orcli &>/dev/null; then
echo "shopt -s expand_aliases"
echo "alias orcli=${scriptpath}/orcli"
fi
awk 1 "${files[$i]}"
)
done
# print stats
log "used $(($(ps --no-headers -o rss -p "$OPENREFINE_PID") / 1024)) MB RAM and $(ps --no-headers -o cputime -p "$OPENREFINE_PID") CPU time"
fi
}
# :command.parse_requirements
parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--version | -v )
version_command
exit
;;
--help | -h )
long_usage=yes
orcli_usage
exit
;;
# :flag.case
--quiet | -q )
# :flag.case_no_arg
args[--quiet]=1
shift
;;
esac
# :command.environment_variables_filter
# :command.environment_variables_default
export OPENREFINE_URL="${OPENREFINE_URL:-http://localhost:3333}"
# :command.dependencies_filter
if ! [[ -x "$(command -v curl)" ]]; then
printf "missing dependency: curl\n" >&2
exit 1
fi
if ! [[ -x "$(command -v jq)" ]]; then
printf "missing dependency: jq\n" >&2
exit 1
fi
# :command.command_filter
action=${1:-}
case $action in
-* )
;;
completions )
action="completions"
shift
orcli_completions_parse_requirements "$@"
shift $#
;;
import )
action="import"
shift
orcli_import_parse_requirements "$@"
shift $#
;;
list )
action="list"
shift
orcli_list_parse_requirements "$@"
shift $#
;;
info )
action="info"
shift
orcli_info_parse_requirements "$@"
shift $#
;;
transform )
action="transform"
shift
orcli_transform_parse_requirements "$@"
shift $#
;;
export )
action="export"
shift
orcli_export_parse_requirements "$@"
shift $#
;;
run )
action="run"
shift
orcli_run_parse_requirements "$@"
shift $#
;;
# :command.command_fallback
"" )
orcli_usage >&2
exit 1
;;
* )
printf "invalid command: %s\n" "$action" >&2
exit 1
;;
esac
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
printf "invalid argument: %s\n" "$key" >&2
exit 1
;;
esac
done
}
# :command.parse_requirements
orcli_completions_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_completions_usage
exit
;;
esac
# :command.command_filter
action="completions"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
printf "invalid argument: %s\n" "$key" >&2
exit 1
;;
esac
done
}
# :command.parse_requirements
orcli_import_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_import_usage
exit
;;
esac
# :command.command_filter
action=${1:-}
case $action in
-* )
;;
csv )
action="csv"
shift
orcli_import_csv_parse_requirements "$@"
shift $#
;;
# :command.command_fallback
"" )
orcli_import_usage >&2
exit 1
;;
* )
printf "invalid command: %s\n" "$action" >&2
exit 1
;;
esac
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
printf "invalid argument: %s\n" "$key" >&2
exit 1
;;
esac
done
}
# :command.parse_requirements
orcli_import_csv_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_import_csv_usage
exit
;;
esac
# :command.command_filter
action="import csv"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# :flag.case
--separator )
# :flag.case_arg
if [[ -n ${2+x} ]]; then
args[--separator]="$2"
shift
shift
else
printf "%s\n" "--separator requires an argument: --separator SEPARATOR" >&2
exit 1
fi
;;
# :flag.case
--encoding )
# :flag.case_arg
if [[ -n ${2+x} ]]; then
args[--encoding]="$2"
shift
shift
else
printf "%s\n" "--encoding requires an argument: --encoding ENCODING" >&2
exit 1
fi
;;
# :flag.case
--trimStrings )
# :flag.case_no_arg
args[--trimStrings]=1
shift
;;
# :flag.case
--projectName )
# :flag.case_arg
if [[ -n ${2+x} ]]; then
args[--projectName]="$2"
shift
shift
else
printf "%s\n" "--projectName requires an argument: --projectName PROJECTNAME" >&2
exit 1
fi
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_repeatable
if [[ -z ${args[file]+x} ]]; then
args[file]="\"$1\""
shift
else
args[file]="${args[file]} \"$1\""
shift
fi
;;
esac
done
# :command.default_assignments
[[ -n ${args[file]:-} ]] || args[file]="-"
[[ -n ${args[--separator]:-} ]] || args[--separator]=","
}
# :command.parse_requirements
orcli_list_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_list_usage
exit
;;
esac
# :command.command_filter
action="list"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
printf "invalid argument: %s\n" "$key" >&2
exit 1
;;
esac
done
}
# :command.parse_requirements
orcli_info_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_info_usage
exit
;;
esac
# :command.command_filter
action="info"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
if [[ -z ${args[project]+x} ]]; then
args[project]=$1
shift
else
printf "invalid argument: %s\n" "$key" >&2
exit 1
fi
;;
esac
done
# :command.required_args_filter
if [[ -z ${args[project]+x} ]]; then
printf "missing required argument: PROJECT\nusage: orcli info PROJECT\n" >&2
exit 1
fi
}
# :command.parse_requirements
orcli_transform_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_transform_usage
exit
;;
esac
# :command.command_filter
action="transform"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_repeatable
if [[ -z ${args[project]+x} ]]; then
args[project]=$1
shift
elif [[ -z ${args[file]+x} ]]; then
args[file]="\"$1\""
shift
else
args[file]="${args[file]} \"$1\""
shift
fi
;;
esac
done
# :command.required_args_filter
if [[ -z ${args[project]+x} ]]; then
printf "missing required argument: PROJECT\nusage: orcli transform PROJECT [FILE...]\n" >&2
exit 1
fi
# :command.default_assignments
[[ -n ${args[file]:-} ]] || args[file]="-"
}
# :command.parse_requirements
orcli_export_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_export_usage
exit
;;
esac
# :command.command_filter
action=${1:-}
case $action in
-* )
;;
tsv )
action="tsv"
shift
orcli_export_tsv_parse_requirements "$@"
shift $#
;;
# :command.command_fallback
"" )
orcli_export_usage >&2
exit 1
;;
* )
printf "invalid command: %s\n" "$action" >&2
exit 1
;;
esac
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
printf "invalid argument: %s\n" "$key" >&2
exit 1
;;
esac
done
}
# :command.parse_requirements
orcli_export_tsv_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_export_tsv_usage
exit
;;
esac
# :command.command_filter
action="export tsv"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# :flag.case
--output )
# :flag.case_arg
if [[ -n ${2+x} ]]; then
args[--output]="$2"
shift
shift
else
printf "%s\n" "--output requires an argument: --output FILE" >&2
exit 1
fi
;;
# :flag.case
--encoding )
# :flag.case_arg
if [[ -n ${2+x} ]]; then
args[--encoding]="$2"
shift
shift
else
printf "%s\n" "--encoding requires an argument: --encoding ENCODING" >&2
exit 1
fi
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
if [[ -z ${args[project]+x} ]]; then
args[project]=$1
shift
else
printf "invalid argument: %s\n" "$key" >&2
exit 1
fi
;;
esac
done
# :command.required_args_filter
if [[ -z ${args[project]+x} ]]; then
printf "missing required argument: PROJECT\nusage: orcli export tsv PROJECT [OPTIONS]\n" >&2
exit 1
fi
# :command.default_assignments
[[ -n ${args[--encoding]:-} ]] || args[--encoding]="UTF-8"
}
# :command.parse_requirements
orcli_run_parse_requirements() {
# :command.fixed_flags_filter
case "${1:-}" in
--help | -h )
long_usage=yes
orcli_run_usage
exit
;;
esac
# :command.command_filter
action="run"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# :flag.case
--memory )
# :flag.case_arg
if [[ -n ${2+x} ]]; then
args[--memory]="$2"
shift
shift
else
printf "%s\n" "--memory requires an argument: --memory RAM" >&2
exit 1
fi
;;
# :flag.case
--port )
# :flag.case_arg
if [[ -n ${2+x} ]]; then
args[--port]="$2"
shift
shift
else
printf "%s\n" "--port requires an argument: --port PORT" >&2
exit 1
fi
;;
# :flag.case
--interactive )
# :flag.case_no_arg
args[--interactive]=1
shift
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
* )
# :command.parse_requirements_case
# :command.parse_requirements_case_repeatable
if [[ -z ${args[file]+x} ]]; then
args[file]="\"$1\""
shift
else
args[file]="${args[file]} \"$1\""
shift
fi
;;
esac
done
# :command.default_assignments
[[ -n ${args[file]:-} ]] || args[file]="-"
[[ -n ${args[--memory]:-} ]] || args[--memory]="2048M"
[[ -n ${args[--port]:-} ]] || args[--port]="3333"
}
# :command.initialize
initialize() {
version="0.1.0"
long_usage=''
set -e
# :command.environment_variables_default
export OPENREFINE_URL="${OPENREFINE_URL:-http://localhost:3333}"
# src/initialize.sh
}
# :command.run
run() {
declare -A args=()
declare -a other_args=()
declare -a input=()
normalize_input "$@"
parse_requirements "${input[@]}"
if [[ $action == "completions" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_completions_usage
else
orcli_completions_command
fi
elif [[ $action == "import" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_import_usage
else
orcli_import_command
fi
elif [[ $action == "import csv" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_import_csv_usage
else
orcli_import_csv_command
fi
elif [[ $action == "list" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_list_usage
else
orcli_list_command
fi
elif [[ $action == "info" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_info_usage
else
orcli_info_command
fi
elif [[ $action == "transform" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_transform_usage
else
orcli_transform_command
fi
elif [[ $action == "export" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_export_usage
else
orcli_export_command
fi
elif [[ $action == "export tsv" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_export_tsv_usage
else
orcli_export_tsv_command
fi
elif [[ $action == "run" ]]; then
if [[ ${args[--help]:-} ]]; then
long_usage=yes
orcli_run_usage
else
orcli_run_command
fi
elif [[ $action == "root" ]]; then
root_command
fi
}
initialize
run "$@"
View Git Blame Copy Permalink