From 26feff650af10b4fc1c74ad114ea3f0e13134d01 Mon Sep 17 00:00:00 2001 From: felixlohmeier Date: Thu, 3 Nov 2022 21:07:08 +0000 Subject: [PATCH] first draft --- README.md | 6 +- orcli | 176 +++++++++++++++++++++++++++++++++++- src/bashly.yml | 21 ++++- src/lib/send_completions.sh | 6 +- src/transform_command.sh | 36 ++++++++ 5 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 src/transform_command.sh diff --git a/README.md b/README.md index a4a05de..cbd141a 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,11 @@ Optional: orcli import csv "https://git.io/fj5hF" --projectName "duplicates" ``` -3. Remove duplicates (coming soon) +3. Remove duplicates by applying an undo/redo JSON file + + ```sh + orcli transform "duplicates" "https://git.io/fj5hF" + ``` 4. Export data from OpenRefine project to tab-separated-values (TSV) file `duplicates.tsv` diff --git a/orcli b/orcli index 136fc88..fd7055b 100755 --- a/orcli +++ b/orcli @@ -38,6 +38,7 @@ orcli_usage() { echo " import commands to create OpenRefine projects from files or URLs" echo " list list projects on OpenRefine server" echo " info show OpenRefine project's metadata" + echo " transform apply undo/redo JSON file(s) to an OpenRefine project" echo " export commands to export data from OpenRefine projects to files" echo " run run tmp OpenRefine workspace and execute shell script(s)" echo @@ -74,6 +75,7 @@ orcli_usage() { printf " orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n" printf " orcli list\n" printf " orcli info \"duplicates\"\n" + printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n" printf " orcli export tsv \"duplicates\"\n" printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n" printf " orcli run --interactive\n" @@ -286,8 +288,59 @@ orcli_info_usage() { # :command.usage_examples printf "Examples:\n" - printf " info \"duplicates\"\n" - printf " info 1234567890123\n" + printf " orcli info \"duplicates\"\n" + printf " orcli info 1234567890123\n" + echo + + fi +} + +# :command.usage +orcli_transform_usage() { + if [[ -n $long_usage ]]; then + printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n" + echo + + else + printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n" + echo + + fi + + printf "Usage:\n" + printf " orcli transform PROJECT [FILE...]\n" + printf " orcli transform --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + # :argument.usage + echo " PROJECT" + printf " project name or id\n" + echo + + # :argument.usage + echo " FILE..." + printf " Path to one or more files or URLs containing OpenRefine's undo/redo\n operation history in JSON format. When FILE is -, read standard input.\n" + printf " Default: -\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " orcli transform \"duplicates\" \"history.json\"\n" + printf " cat \"history.json\" | orcli transform \"duplicates\"\n" + printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n" + printf " orcli transform 1234567890123 \"history.json\"\n" echo fi @@ -693,6 +746,10 @@ send_completions() { echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )' echo $' ;;' echo $'' + echo $' \'transform\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' + echo $' ;;' + echo $'' echo $' \'import\'*)' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )' echo $' ;;' @@ -714,7 +771,7 @@ send_completions() { echo $' ;;' echo $'' echo $' *)' - echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )' echo $' ;;' echo $'' echo $' esac' @@ -796,6 +853,47 @@ orcli_info_command() { } +# :command.function +orcli_transform_command() { + # src/transform_command.sh + # shellcheck shell=bash disable=SC2154 + #get_id "${args[project]}" + + # check if stdin is present if selected + if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then + if ! read -u 0 -t 0; then + orcli_transform_usage + exit 1 + fi + fi + # catch args, convert the space delimited string to an array + files=() + eval "files=(${args[file]})" + # create tmp directory + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' 0 2 3 15 + # download files if name starts with http:// or https:// + for i in "${!files[@]}"; do + if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then + if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then + error "download of ${files[$i]} failed!" + fi + files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" + fi + done + + # support multiple files and stdin + readarray -t jsonlines < <(cat "${files[@]}" | jq --slurp --compact-output 'add | .[]') + for line in "${jsonlines[@]}"; do + declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))" + echo "${data[op]#core/}" + unset "data[op]" + unset "data[description]" + for K in "${!data[@]}"; do echo "$K" --- "${data[$K]}"; done + unset data + done +} + # :command.function orcli_export_tsv_command() { # src/export_tsv_command.sh @@ -1010,6 +1108,13 @@ parse_requirements() { shift $# ;; + transform ) + action="transform" + shift + orcli_transform_parse_requirements "$@" + shift $# + ;; + export ) action="export" shift @@ -1348,6 +1453,63 @@ orcli_info_parse_requirements() { } +# :command.parse_requirements +orcli_transform_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + orcli_transform_usage + exit + ;; + + esac + + # :command.command_filter + action="transform" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_repeatable + if [[ -z ${args[project]+x} ]]; then + + args[project]=$1 + shift + elif [[ -z ${args[file]+x} ]]; then + + args[file]="\"$1\"" + shift + else + args[file]="${args[file]} \"$1\"" + shift + fi + + ;; + + esac + done + + # :command.required_args_filter + if [[ -z ${args[project]+x} ]]; then + printf "missing required argument: PROJECT\nusage: orcli transform PROJECT [FILE...]\n" >&2 + exit 1 + fi + + # :command.default_assignments + [[ -n ${args[file]:-} ]] || args[file]="-" + +} + # :command.parse_requirements orcli_export_parse_requirements() { # :command.fixed_flags_filter @@ -1639,6 +1801,14 @@ run() { orcli_info_command fi + elif [[ $action == "transform" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + orcli_transform_usage + else + orcli_transform_command + fi + elif [[ $action == "export" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes diff --git a/src/bashly.yml b/src/bashly.yml index 320b860..28f1cae 100644 --- a/src/bashly.yml +++ b/src/bashly.yml @@ -16,6 +16,7 @@ examples: - orcli import csv "https://git.io/fj5hF" --projectName "duplicates" - orcli list - orcli info "duplicates" + - orcli transform "duplicates" "https://git.io/fj5ju" - orcli export tsv "duplicates" - orcli export tsv "duplicates" --output "duplicates.tsv" - orcli run --interactive @@ -83,8 +84,24 @@ commands: help: project name or id required: true examples: - - info "duplicates" - - info 1234567890123 + - orcli info "duplicates" + - orcli info 1234567890123 + + - name: transform + help: apply undo/redo JSON file(s) to an OpenRefine project + args: + - name: project + help: project name or id + required: true + - name: file + help: Path to one or more files or URLs containing OpenRefine's undo/redo operation history in JSON format. When FILE is -, read standard input. + default: "-" + repeatable: true + examples: + - orcli transform "duplicates" "history.json" + - cat "history.json" | orcli transform "duplicates" + - orcli transform "duplicates" "https://git.io/fj5ju" + - orcli transform 1234567890123 "history.json" - name: export help: commands to export data from OpenRefine projects to files diff --git a/src/lib/send_completions.sh b/src/lib/send_completions.sh index 4b5f5ed..3771a63 100644 --- a/src/lib/send_completions.sh +++ b/src/lib/send_completions.sh @@ -42,6 +42,10 @@ send_completions() { echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )' echo $' ;;' echo $'' + echo $' \'transform\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' + echo $' ;;' + echo $'' echo $' \'import\'*)' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )' echo $' ;;' @@ -63,7 +67,7 @@ send_completions() { echo $' ;;' echo $'' echo $' *)' - echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )' echo $' ;;' echo $'' echo $' esac' diff --git a/src/transform_command.sh b/src/transform_command.sh new file mode 100644 index 0000000..31cfd31 --- /dev/null +++ b/src/transform_command.sh @@ -0,0 +1,36 @@ +# shellcheck shell=bash disable=SC2154 +#get_id "${args[project]}" + +# check if stdin is present if selected +if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then + if ! read -u 0 -t 0; then + orcli_transform_usage + exit 1 + fi +fi +# catch args, convert the space delimited string to an array +files=() +eval "files=(${args[file]})" +# create tmp directory +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' 0 2 3 15 +# download files if name starts with http:// or https:// +for i in "${!files[@]}"; do + if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then + if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then + error "download of ${files[$i]} failed!" + fi + files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" + fi +done + +# support multiple files and stdin +readarray -t jsonlines < <(cat "${files[@]}" | jq --slurp --compact-output 'add | .[]') +for line in "${jsonlines[@]}"; do + declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))" + echo "${data[op]#core/}" + unset "data[op]" + unset "data[description]" + for K in "${!data[@]}"; do echo "$K" --- "${data[$K]}"; done + unset data +done \ No newline at end of file