opencultureconsulting
/
orcli
mirror of https://github.com/opencultureconsulting/orcli.git
2
0
Fork 0
Code Issues Releases Activity

Merge pull request #64 from opencultureconsulting:felixlohmeier/batch-use-heredoc-58 

support multiple files
This commit is contained in:
Felix Lohmeier 2022-10-07 11:02:34 +02:00 committed by GitHub
parent 27eae58073 5bcdb97509
commit cdb7600ef7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 62 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitpod.yml
View File

@ -8,7 +8,7 @@ tasks:
rm openrefine.tar.gz
command: |
sudo ln -s "${PWD}/orcli" /usr/local/bin/
eval "$(orcli completions)"
source <(orcli completions)
./refine
ports:

136
orcli
View File

@ -28,7 +28,7 @@ orcli_usage() {
fi
printf "Usage:\n"
printf " orcli COMMAND\n"
printf " orcli [OPTIONS] COMMAND\n"
printf " orcli [COMMAND] --help | -h\n"
printf " orcli --version | -v\n"
echo
@ -54,6 +54,12 @@ orcli_usage() {
printf " Show version number\n"
echo
# :command.usage_flags
# :flag.usage
echo " --quiet, -q"
printf " suppress log output, print errors only\n"
echo
# :command.usage_environment_variables
printf "Environment Variables:\n"
@ -151,11 +157,6 @@ orcli_batch_usage() {
printf " Default: 3333\n"
echo
# :flag.usage
echo " --quiet, -q"
printf " suppress log output, print errors only\n"
echo
# :command.usage_args
printf "Arguments:\n"
@ -167,9 +168,9 @@ orcli_batch_usage() {
# :command.usage_examples
printf "Examples:\n"
printf " orcli batch << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n"
printf " orcli batch --memory \"2000M\" --port \"3334\" << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\"\n orcli info \"duplicates\"\n orcli info \"copy\"\n orcli export tsv \"duplicates\"\n EOF\n"
printf " orcli batch --quiet << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n wait\n echo \"finished import\"\n orcli export csv \"duplicates\" --output duplicates.csv &\n orcli export tsv \"duplicates\" --output duplicates.tsv &\n wait\n wc duplicates*\n echo \"finished in $SECONDS seconds\"\n EOF\n"
printf " orcli batch << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli info \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n"
printf " orcli batch --memory \"2000M\" --port \"3334\" << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n wait\n echo \"finished import\"\n orcli export csv \"duplicates\" --output duplicates.csv &\n orcli export tsv \"duplicates\" --output duplicates.tsv &\n wait\n wc duplicates*\n EOF\n"
printf " orcli batch \"file1.sh\" \"file2.sh\" - << EOF\n echo \"finished in \$SECONDS seconds\"\n EOF\n"
echo
fi
@ -256,11 +257,6 @@ orcli_import_csv_usage() {
printf " set a name for the OpenRefine project\n"
echo
# :flag.usage
echo " --quiet, -q"
printf " suppress log output, print errors only\n"
echo
# :command.usage_args
printf "Arguments:\n"
@ -295,7 +291,7 @@ orcli_list_usage() {
fi
printf "Usage:\n"
printf " orcli list [OPTIONS]\n"
printf " orcli list\n"
printf " orcli list --help | -h\n"
echo
@ -308,12 +304,6 @@ orcli_list_usage() {
printf " Show this help\n"
echo
# :command.usage_flags
# :flag.usage
echo " --quiet, -q"
printf " suppress log output, print errors only\n"
echo
fi
}
@ -330,7 +320,7 @@ orcli_info_usage() {
fi
printf "Usage:\n"
printf " orcli info PROJECT [OPTIONS]\n"
printf " orcli info PROJECT\n"
printf " orcli info --help | -h\n"
echo
@ -343,12 +333,6 @@ orcli_info_usage() {
printf " Show this help\n"
echo
# :command.usage_flags
# :flag.usage
echo " --quiet, -q"
printf " suppress log output, print errors only\n"
echo
# :command.usage_args
printf "Arguments:\n"
@ -437,11 +421,6 @@ orcli_export_tsv_usage() {
printf " Default: UTF-8\n"
echo
# :flag.usage
echo " --quiet, -q"
printf " suppress log output, print errors only\n"
echo
# :command.usage_args
printf "Arguments:\n"
@ -685,11 +664,11 @@ send_completions() {
echo $' ;;'
echo $''
echo $' \'import csv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --quiet --separator --trimStrings -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --separator --trimStrings -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'export tsv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'import\'*)'
@ -701,19 +680,19 @@ send_completions() {
echo $' ;;'
echo $''
echo $' \'batch\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'list\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'info\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' *)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v batch completions export import info list")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v batch completions export import info list")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' esac'
@ -727,10 +706,7 @@ send_completions() {
# :command.function
orcli_completions_command() {
# src/completions_command.sh
# Users can now enable bash completion for this script by running:
#
# $ eval "$(orcli completions)"
#
# shellcheck shell=bash disable=SC2154
send_completions
}
@ -745,6 +721,13 @@ orcli_batch_command() {
exit 1
fi
# catch args, convert the space delimited string to an array
files=()
eval "files=(${args[file]})"
# update OPENREFINE_URL env
OPENREFINE_URL="http://localhost:${args[--port]}"
# locate orcli and OpenRefine
if command -v orcli &>/dev/null; then
orcli="orcli"
@ -763,22 +746,6 @@ orcli_batch_command() {
tmpdir="$(mktemp -d)"
trap '{ rm -rf "$tmpdir"; }' 0 2 3 15
# update OPENREFINE_URL env
OPENREFINE_URL="http://localhost:${args[--port]}"
# catch args, convert the space delimited string to an array
files=()
eval "files=(${args[file]})"
# read pipes if name starts with /dev/fd
for i in "${!files[@]}"; do
if [[ ${files[$i]} == "/dev/fd"* ]]; then
if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "reading of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# check if OpenRefine is already running
if curl -fs "${OPENREFINE_URL}" &>/dev/null; then
error "OpenRefine is already running on port ${args[--port]}." "Hint: Stop the other process or use another port."
@ -800,7 +767,8 @@ orcli_batch_command() {
# execute shell script
export orcli tmpdir OPENREFINE_URL openrefine_pid
bash -e "${files[@]}"
bash -e <(awk 1 "${files[@]}")
}
# :command.function
@ -918,6 +886,14 @@ parse_requirements() {
exit
;;
# :flag.case
--quiet | -q )
# :flag.case_no_arg
args[--quiet]=1
shift
;;
esac
# :command.environment_variables_filter
@ -1106,14 +1082,6 @@ orcli_batch_parse_requirements() {
fi
;;
# :flag.case
--quiet | -q )
# :flag.case_no_arg
args[--quiet]=1
shift
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
@ -1277,14 +1245,6 @@ orcli_import_csv_parse_requirements() {
fi
;;
# :flag.case
--quiet | -q )
# :flag.case_no_arg
args[--quiet]=1
shift
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1
@ -1332,13 +1292,6 @@ orcli_list_parse_requirements() {
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# :flag.case
--quiet | -q )
# :flag.case_no_arg
args[--quiet]=1
shift
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
@ -1377,13 +1330,6 @@ orcli_info_parse_requirements() {
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# :flag.case
--quiet | -q )
# :flag.case_no_arg
args[--quiet]=1
shift
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
@ -1409,7 +1355,7 @@ orcli_info_parse_requirements() {
# :command.required_args_filter
if [[ -z ${args[project]+x} ]]; then
printf "missing required argument: PROJECT\nusage: orcli info PROJECT [OPTIONS]\n" >&2
printf "missing required argument: PROJECT\nusage: orcli info PROJECT\n" >&2
exit 1
fi
@ -1526,14 +1472,6 @@ orcli_export_tsv_parse_requirements() {
fi
;;
# :flag.case
--quiet | -q )
# :flag.case_no_arg
args[--quiet]=1
shift
;;
-?* )
printf "invalid option: %s\n" "$key" >&2
exit 1

36
src/bashly.yml
View File

@ -25,6 +25,11 @@ examples:
orcli export tsv "duplicates"
EOF
flags:
- long: --quiet
short: -q
help: suppress log output, print errors only
commands:
- name: completions
help: |-
@ -47,25 +52,15 @@ commands:
help: PORT on which OpenRefine should listen
arg: port
default: "3333"
- long: --quiet
short: -q
help: suppress log output, print errors only
examples:
- |-
orcli batch << EOF
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
orcli info "duplicates"
orcli export tsv "duplicates"
EOF
- |-
orcli batch --memory "2000M" --port "3334" << EOF
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
orcli import csv "https://git.io/fj5hF" --projectName "copy"
orcli info "duplicates"
orcli info "copy"
orcli export tsv "duplicates"
EOF
- |-
orcli batch --quiet << EOF
orcli import csv "https://git.io/fj5hF" --projectName "duplicates" &
orcli import csv "https://git.io/fj5hF" --projectName "copy" &
wait
@ -74,7 +69,10 @@ commands:
orcli export tsv "duplicates" --output duplicates.tsv &
wait
wc duplicates*
echo "finished in $SECONDS seconds"
EOF
- |-
orcli batch "file1.sh" "file2.sh" - << EOF
echo "finished in \$SECONDS seconds"
EOF
- name: import
@ -101,9 +99,6 @@ commands:
- long: --projectName
arg: projectName
help: set a name for the OpenRefine project
- long: --quiet
short: -q
help: suppress log output, print errors only
examples:
- orcli import csv "file"
- orcli import csv "file1" "file2"
@ -118,10 +113,6 @@ commands:
- name: list
help: list projects on OpenRefine server
flags:
- long: --quiet
short: -q
help: suppress log output, print errors only
- name: info
help: show project metadata
@ -129,10 +120,6 @@ commands:
- name: project
help: project name or id
required: true
flags:
- long: --quiet
short: -q
help: suppress log output, print errors only
examples:
- info "duplicates"
- info 1234567890123
@ -155,9 +142,6 @@ commands:
help: set character encoding
arg: encoding
default: "UTF-8"
- long: --quiet
short: -q
help: suppress log output, print errors only
examples:
- orcli export tsv "duplicates"
- orcli export tsv "duplicates" --output "duplicates.tsv"

25
src/batch_command.sh
View File

@ -6,6 +6,13 @@ if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]] && [ -t 0 ]; then
exit 1
fi
# catch args, convert the space delimited string to an array
files=()
eval "files=(${args[file]})"
# update OPENREFINE_URL env
OPENREFINE_URL="http://localhost:${args[--port]}"
# locate orcli and OpenRefine
if command -v orcli &>/dev/null; then
orcli="orcli"
@ -24,22 +31,6 @@ fi
tmpdir="$(mktemp -d)"
trap '{ rm -rf "$tmpdir"; }' 0 2 3 15
# update OPENREFINE_URL env
OPENREFINE_URL="http://localhost:${args[--port]}"
# catch args, convert the space delimited string to an array
files=()
eval "files=(${args[file]})"
# read pipes if name starts with /dev/fd
for i in "${!files[@]}"; do
if [[ ${files[$i]} == "/dev/fd"* ]]; then
if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "reading of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# check if OpenRefine is already running
if curl -fs "${OPENREFINE_URL}" &>/dev/null; then
error "OpenRefine is already running on port ${args[--port]}." "Hint: Stop the other process or use another port."
@ -61,4 +52,4 @@ fi
# execute shell script
export orcli tmpdir OPENREFINE_URL openrefine_pid
bash -e "${files[@]}"
bash -e <(awk 1 "${files[@]}")

12
src/lib/send_completions.sh
View File

@ -35,11 +35,11 @@ send_completions() {
echo $' ;;'
echo $''
echo $' \'import csv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --quiet --separator --trimStrings -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --separator --trimStrings -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'export tsv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'import\'*)'
@ -51,19 +51,19 @@ send_completions() {
echo $' ;;'
echo $''
echo $' \'batch\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'list\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'info\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' *)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v batch completions export import info list")" -- "$cur" )'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v batch completions export import info list")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' esac'