diff --git a/orcli b/orcli index fcfa58c..90896a8 100755 --- a/orcli +++ b/orcli @@ -34,9 +34,10 @@ orcli_usage() { echo # :command.usage_commands printf "Commands:\n" - echo " info show project metadata" echo " import import commands" echo " list list projects on OpenRefine server" + echo " info show project metadata" + echo " export export commands" echo if [[ -n $long_usage ]]; then @@ -60,11 +61,12 @@ orcli_usage() { # :command.usage_examples printf "Examples:\n" - printf " orcli list\n" printf " orcli import csv file\n" printf " orcli import csv\n \"https://github.com/LibraryCarpentry/lc-open-refine/raw/gh-pages/data/doaj-article-sample.csv\"\n" - printf " orcli info Clipboard\n" - printf " orcli info 1234567890123\n" + printf " orcli list\n" + printf " orcli info \"doaj article sample csv\"\n" + printf " orcli export tsv \"doaj article sample csv\"\n" + printf " orcli export tsv \"doaj article sample csv\" --output doaj.tsv\n" echo # :command.footer printf "https://github.com/opencultureconsulting/orcli\n" @@ -73,48 +75,6 @@ orcli_usage() { fi } -# :command.usage -orcli_info_usage() { - if [[ -n $long_usage ]]; then - printf "orcli info - show project metadata\n" - echo - - else - printf "orcli info - show project metadata\n" - echo - - fi - - printf "Usage:\n" - printf " orcli info PROJECT\n" - printf " orcli info --help | -h\n" - echo - - if [[ -n $long_usage ]]; then - printf "Options:\n" - # :command.usage_fixed_flags - echo " --help, -h" - printf " Show this help\n" - echo - - # :command.usage_args - printf "Arguments:\n" - - # :argument.usage - echo " PROJECT" - printf " project name or id\n" - echo - - # :command.usage_examples - printf "Examples:\n" - - printf " info Clipboard\n" - printf " info 1234567890123\n" - echo - - fi -} - # :command.usage orcli_import_usage() { if [[ -n $long_usage ]]; then @@ -238,6 +198,131 @@ orcli_list_usage() { fi } +# :command.usage +orcli_info_usage() { + if [[ -n $long_usage ]]; then + printf "orcli info - show project metadata\n" + echo + + else + printf "orcli info - show project metadata\n" + echo + + fi + + printf "Usage:\n" + printf " orcli info PROJECT\n" + printf " orcli info --help | -h\n" + echo + + if [[ -n $long_usage ]]; then + printf "Options:\n" + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + # :argument.usage + echo " PROJECT" + printf " project name or id\n" + echo + + # :command.usage_examples + printf "Examples:\n" + + printf " info Clipboard\n" + printf " info 1234567890123\n" + echo + + fi +} + +# :command.usage +orcli_export_usage() { + if [[ -n $long_usage ]]; then + printf "orcli export - export commands\n" + echo + + else + printf "orcli export - export commands\n" + echo + + fi + + printf "Usage:\n" + printf " orcli export [command]\n" + printf " orcli export [command] --help | -h\n" + echo + # :command.usage_commands + printf "Commands:\n" + echo " tsv export tab-separated values (TSV)" + echo + + if [[ -n $long_usage ]]; then + printf "Options:\n" + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +orcli_export_tsv_usage() { + if [[ -n $long_usage ]]; then + printf "orcli export tsv - export tab-separated values (TSV)\n" + echo + + else + printf "orcli export tsv - export tab-separated values (TSV)\n" + echo + + fi + + printf "Usage:\n" + printf " orcli export tsv PROJECT [options]\n" + printf " orcli export tsv --help | -h\n" + echo + + if [[ -n $long_usage ]]; then + printf "Options:\n" + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + # :command.usage_flags + # :flag.usage + echo " --output FILE" + printf " Write to file instead of stdout\n" + echo + + # :flag.usage + echo " --encoding ENCODING" + printf " set character encoding\n" + printf " Default: UTF-8\n" + echo + # :command.usage_args + printf "Arguments:\n" + + # :argument.usage + echo " PROJECT" + printf " project name or id\n" + echo + + # :command.usage_examples + printf "Examples:\n" + + printf " orcli export tsv Clipboard\n" + printf " orcli export tsv Clipboard --output clipboard.tsv\n" + echo + + fi +} + # :command.normalize_input normalize_input() { local arg flags @@ -302,7 +387,7 @@ function get_csrf() { # src/lib/get_id.sh # get project id (derived from project name if needed) -# shellcheck shell=bash disable=SC2154 +# shellcheck shell=bash function get_id() { local response local projects @@ -310,8 +395,8 @@ function get_id() { if ! response="$(curl -fs --get "${OPENREFINE_URL}/command/core/get-all-project-metadata")"; then error "no OpenRefine reachable/running at ${OPENREFINE_URL}" fi - if ! projects="$(echo "$response" | jq -r '.projects | keys[] as $k | "\($k):\(.[$k] | .name)"' | grep ":${args[project]}$")"; then - error "project ${args[project]} not found" + if ! projects="$(echo "$response" | jq -r '.projects | keys[] as $k | "\($k):\(.[$k] | .name)"' | grep -e ":$1$" -e "^$1:")"; then + error "project $1 not found" fi ids=$(echo "$projects" | cut -d : -f 1) if ! [[ "${#ids}" == 13 ]]; then @@ -417,12 +502,6 @@ function post_import() { } # :command.command_functions -# :command.function -orcli_info_command() { - # src/info_command.sh - # shellcheck shell=bash - get_id -} # :command.function orcli_import_csv_command() { @@ -473,6 +552,53 @@ orcli_list_command() { fi } +# :command.function +orcli_info_command() { + # src/info_command.sh + # shellcheck shell=bash disable=SC2154 + get_id "${args[project]}" +} + +# :command.function +orcli_export_tsv_command() { + # src/export_tsv_command.sh + # shellcheck shell=bash + projectid="$(get_id "${args[project]}")" + separator='\t' + + # assemble specific post data (some options require json format) + data+=("project=${projectid}") + data+=("format=tsv") + options='{ ' + options+="\"separator\": \"${separator}\"" + if [[ ${args[--encoding]} ]]; then + options+=', ' + options+="\"encoding\": \"${args[--encoding]}\"" + fi + options+=' }' + data+=("options=${options}") + + # post + mapfile -t curloptions < <(for d in "${data[@]}"; do + echo "--data" + echo "$d" + done) + if [[ ${args[--output]} ]]; then + if ! mkdir -p "$(dirname "${args[--output]}")"; then + error "unable to create parent directory for ${args[--output]}" + fi + curloptions+=("--output") + curloptions+=("${args[--output]}") + fi + if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then + error "export of ${args[project]} failed!" + else + if [[ ${args[--output]} ]]; then + log "export of ${args[project]} successful" "file:${args[--output]}" "rows:$(cat "${args[--output]}" | wc -l )" + fi + fi +} + # :command.parse_requirements parse_requirements() { # :command.fixed_flags_filter @@ -507,13 +633,6 @@ parse_requirements() { -* ) ;; - info ) - action="info" - shift - orcli_info_parse_requirements "$@" - shift $# - ;; - import ) action="import" shift @@ -528,6 +647,20 @@ parse_requirements() { shift $# ;; + info ) + action="info" + shift + orcli_info_parse_requirements "$@" + shift $# + ;; + + export ) + action="export" + shift + orcli_export_parse_requirements "$@" + shift $# + ;; + # :command.command_fallback * ) orcli_usage @@ -561,57 +694,6 @@ parse_requirements() { # :command.user_filter } -# :command.parse_requirements -orcli_info_parse_requirements() { - # :command.fixed_flags_filter - case "${1:-}" in - --help | -h ) - long_usage=yes - orcli_info_usage - exit - ;; - - esac - # :command.environment_variables_filter - # :command.dependencies_filter - # :command.command_filter - action="info" - # :command.parse_requirements_while - while [[ $# -gt 0 ]]; do - key="$1" - case "$key" in - - -?* ) - printf "invalid option: %s\n" "$key" - exit 1 - ;; - - * ) - # :command.parse_requirements_case - if [[ -z ${args[project]+x} ]]; then - # :argument.validations - args[project]=$1 - shift - else - printf "invalid argument: %s\n" "$key" - exit 1 - fi - ;; - - esac - done - # :command.required_args_filter - if [[ -z ${args[project]+x} ]]; then - printf "missing required argument: PROJECT\nusage: orcli info PROJECT\n" - exit 1 - fi - # :command.required_flags_filter - # :command.catch_all_filter - # :command.default_assignments - # :command.whitelist_filter - # :command.user_filter -} - # :command.parse_requirements orcli_import_parse_requirements() { # :command.fixed_flags_filter @@ -810,6 +892,196 @@ orcli_list_parse_requirements() { # :command.user_filter } +# :command.parse_requirements +orcli_info_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + orcli_info_usage + exit + ;; + + esac + # :command.environment_variables_filter + # :command.dependencies_filter + # :command.command_filter + action="info" + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" + exit 1 + ;; + + * ) + # :command.parse_requirements_case + if [[ -z ${args[project]+x} ]]; then + # :argument.validations + args[project]=$1 + shift + else + printf "invalid argument: %s\n" "$key" + exit 1 + fi + ;; + + esac + done + # :command.required_args_filter + if [[ -z ${args[project]+x} ]]; then + printf "missing required argument: PROJECT\nusage: orcli info PROJECT\n" + exit 1 + fi + # :command.required_flags_filter + # :command.catch_all_filter + # :command.default_assignments + # :command.whitelist_filter + # :command.user_filter +} + +# :command.parse_requirements +orcli_export_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + orcli_export_usage + exit + ;; + + esac + # :command.environment_variables_filter + # :command.dependencies_filter + # :command.command_filter + action=${1:-} + + case $action in + -* ) + ;; + + tsv ) + action="tsv" + shift + orcli_export_tsv_parse_requirements "$@" + shift $# + ;; + + # :command.command_fallback + * ) + orcli_export_usage + exit 1 + ;; + + esac + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" + exit 1 + ;; + + * ) + # :command.parse_requirements_case + printf "invalid argument: %s\n" "$key" + exit 1 + ;; + + esac + done + # :command.required_args_filter + # :command.required_flags_filter + # :command.catch_all_filter + # :command.default_assignments + # :command.whitelist_filter + # :command.user_filter +} + +# :command.parse_requirements +orcli_export_tsv_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + orcli_export_tsv_usage + exit + ;; + + esac + # :command.environment_variables_filter + # :command.dependencies_filter + # :command.command_filter + action="export tsv" + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --output ) + # :flag.conflicts + if [[ -n ${2+x} ]]; then + # :flag.validations + args[--output]="$2" + shift + shift + else + printf "%s\n" "--output requires an argument: --output FILE" + exit 1 + fi + ;; + + # :flag.case + --encoding ) + # :flag.conflicts + if [[ -n ${2+x} ]]; then + # :flag.validations + args[--encoding]="$2" + shift + shift + else + printf "%s\n" "--encoding requires an argument: --encoding ENCODING" + exit 1 + fi + ;; + + -?* ) + printf "invalid option: %s\n" "$key" + exit 1 + ;; + + * ) + # :command.parse_requirements_case + if [[ -z ${args[project]+x} ]]; then + # :argument.validations + args[project]=$1 + shift + else + printf "invalid argument: %s\n" "$key" + exit 1 + fi + ;; + + esac + done + # :command.required_args_filter + if [[ -z ${args[project]+x} ]]; then + printf "missing required argument: PROJECT\nusage: orcli export tsv PROJECT [options]\n" + exit 1 + fi + # :command.required_flags_filter + # :command.catch_all_filter + # :command.default_assignments + [[ -n ${args[--encoding]:-} ]] || args[--encoding]="UTF-8" + # :command.whitelist_filter + # :command.user_filter +} + # :command.initialize initialize() { version="0.1.0" @@ -827,15 +1099,7 @@ run() { normalize_input "$@" parse_requirements "${input[@]}" - if [[ $action == "info" ]]; then - if [[ ${args[--help]:-} ]]; then - long_usage=yes - orcli_info_usage - else - orcli_info_command - fi - - elif [[ $action == "import" ]]; then + if [[ $action == "import" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes orcli_import_usage @@ -859,6 +1123,30 @@ run() { orcli_list_command fi + elif [[ $action == "info" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + orcli_info_usage + else + orcli_info_command + fi + + elif [[ $action == "export" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + orcli_export_usage + else + orcli_export_command + fi + + elif [[ $action == "export tsv" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + orcli_export_tsv_usage + else + orcli_export_tsv_command + fi + elif [[ $action == "root" ]]; then root_command fi diff --git a/src/bashly.yml b/src/bashly.yml index a62d62d..c1df0e6 100644 --- a/src/bashly.yml +++ b/src/bashly.yml @@ -13,23 +13,14 @@ environment_variables: default: "http://localhost:3333" examples: - - orcli list - orcli import csv file - - orcli import csv "https://github.com/LibraryCarpentry/lc-open-refine/raw/gh-pages/data/doaj-article-sample.csv" - - orcli info Clipboard - - orcli info 1234567890123 + - orcli import csv "https://github.com/LibraryCarpentry/lc-open-refine/raw/gh-pages/data/doaj-article-sample.csv" + - orcli list + - orcli info "doaj article sample csv" + - orcli export tsv "doaj article sample csv" + - orcli export tsv "doaj article sample csv" --output doaj.tsv commands: - - name: info - help: show project metadata - args: - - name: project - help: project name or id - required: true - examples: - - info Clipboard - - info 1234567890123 - - name: import help: import commands @@ -54,12 +45,71 @@ commands: - long: --projectName arg: projectName help: set a name for the OpenRefine project - examples: - orcli import csv file - cat file | orcli import csv - orcli import csv file --separator ; --encoding ISO-8859-1 --trimStrings --projectName example - - orcli import csv "https://github.com/LibraryCarpentry/lc-open-refine/raw/gh-pages/data/doaj-article-sample.csv" + - orcli import csv "https://github.com/LibraryCarpentry/lc-open-refine/raw/gh-pages/data/doaj-article-sample.csv" - name: list help: list projects on OpenRefine server + + - name: info + help: show project metadata + args: + - name: project + help: project name or id + required: true + examples: + - info Clipboard + - info 1234567890123 + + - name: export + help: export commands + + commands: + - name: tsv + help: export tab-separated values (TSV) + args: + - name: project + help: project name or id + required: true + flags: + - long: --output + help: Write to file instead of stdout + arg: file + #- long: --column + # help: filter result set to one or more columns + # repeatable: true + #- long: --facet + # help: filter result set by providing an OpenRefine facet config in json + # repeatable: true + #- long: --mode + # help: set operation mode + # arg: mode + # default: "row-based" + - long: --encoding + help: set character encoding + arg: encoding + default: "UTF-8" + #- long: --noColumnHeaders + # help: do not output column headers + #- long: --blankRows + # help: output blank rows + #- long: --quoteAll + # help: quote all cells + #- long: --preview + # help: limit export to 10 rows/records + #- long: --separator + # help: character(s) that separates columns + # arg: separator + # default: "\t" + #- long: --lineSeparator + # help: character(s) that separates rows/records + # arg: lineSeparator + # default: "\n" + examples: + - orcli export tsv Clipboard + - orcli export tsv Clipboard --output clipboard.tsv + #- orcli export tsv Clipboard --output clipboard.tsv --facet '{"type":"text","name":"a","columnName":"a","mode":"text","caseSensitive":false,"invert":false,"query":"1"}' + #- orcli export tsv Clipboard --output clipboard.tsv --column a --column b diff --git a/src/export_tsv_command.sh b/src/export_tsv_command.sh new file mode 100644 index 0000000..635c704 --- /dev/null +++ b/src/export_tsv_command.sh @@ -0,0 +1,35 @@ +# shellcheck shell=bash +projectid="$(get_id "${args[project]}")" +separator='\t' + +# assemble specific post data (some options require json format) +data+=("project=${projectid}") +data+=("format=tsv") +options='{ ' +options+="\"separator\": \"${separator}\"" +if [[ ${args[--encoding]} ]]; then + options+=', ' + options+="\"encoding\": \"${args[--encoding]}\"" +fi +options+=' }' +data+=("options=${options}") + +# post +mapfile -t curloptions < <(for d in "${data[@]}"; do + echo "--data" + echo "$d" +done) +if [[ ${args[--output]} ]]; then + if ! mkdir -p "$(dirname "${args[--output]}")"; then + error "unable to create parent directory for ${args[--output]}" + fi + curloptions+=("--output") + curloptions+=("${args[--output]}") +fi +if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then + error "export of ${args[project]} failed!" +else + if [[ ${args[--output]} ]]; then + log "export of ${args[project]} successful" "file:${args[--output]}" "rows:$(cat "${args[--output]}" | wc -l )" + fi +fi diff --git a/src/info_command.sh b/src/info_command.sh index 2421a34..85bbe49 100644 --- a/src/info_command.sh +++ b/src/info_command.sh @@ -1,2 +1,2 @@ -# shellcheck shell=bash -get_id +# shellcheck shell=bash disable=SC2154 +get_id "${args[project]}" diff --git a/src/lib/get_id.sh b/src/lib/get_id.sh index 5c413b0..1567f89 100644 --- a/src/lib/get_id.sh +++ b/src/lib/get_id.sh @@ -1,5 +1,5 @@ # get project id (derived from project name if needed) -# shellcheck shell=bash disable=SC2154 +# shellcheck shell=bash function get_id() { local response local projects @@ -7,8 +7,8 @@ function get_id() { if ! response="$(curl -fs --get "${OPENREFINE_URL}/command/core/get-all-project-metadata")"; then error "no OpenRefine reachable/running at ${OPENREFINE_URL}" fi - if ! projects="$(echo "$response" | jq -r '.projects | keys[] as $k | "\($k):\(.[$k] | .name)"' | grep ":${args[project]}$")"; then - error "project ${args[project]} not found" + if ! projects="$(echo "$response" | jq -r '.projects | keys[] as $k | "\($k):\(.[$k] | .name)"' | grep -e ":$1$" -e "^$1:")"; then + error "project $1 not found" fi ids=$(echo "$projects" | cut -d : -f 1) if ! [[ "${#ids}" == 13 ]]; then