diff --git a/README.md b/README.md index a4a05de..cbd141a 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,11 @@ Optional: orcli import csv "https://git.io/fj5hF" --projectName "duplicates" ``` -3. Remove duplicates (coming soon) +3. Remove duplicates by applying an undo/redo JSON file + + ```sh + orcli transform "duplicates" "https://git.io/fj5hF" + ``` 4. Export data from OpenRefine project to tab-separated-values (TSV) file `duplicates.tsv` diff --git a/orcli b/orcli index 136fc88..8287b81 100755 --- a/orcli +++ b/orcli @@ -38,6 +38,7 @@ orcli_usage() { echo " import commands to create OpenRefine projects from files or URLs" echo " list list projects on OpenRefine server" echo " info show OpenRefine project's metadata" + echo " transform apply undo/redo JSON file(s) to an OpenRefine project" echo " export commands to export data from OpenRefine projects to files" echo " run run tmp OpenRefine workspace and execute shell script(s)" echo @@ -74,10 +75,11 @@ orcli_usage() { printf " orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n" printf " orcli list\n" printf " orcli info \"duplicates\"\n" + printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n" printf " orcli export tsv \"duplicates\"\n" printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n" printf " orcli run --interactive\n" - printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli info \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n" + printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n" echo # :command.footer @@ -286,8 +288,59 @@ orcli_info_usage() { # :command.usage_examples printf "Examples:\n" - printf " info \"duplicates\"\n" - printf " info 1234567890123\n" + printf " orcli info \"duplicates\"\n" + printf " orcli info 1234567890123\n" + echo + + fi +} + +# :command.usage +orcli_transform_usage() { + if [[ -n $long_usage ]]; then + printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n" + echo + + else + printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n" + echo + + fi + + printf "Usage:\n" + printf " orcli transform PROJECT [FILE...]\n" + printf " orcli transform --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + # :argument.usage + echo " PROJECT" + printf " project name or id\n" + echo + + # :argument.usage + echo " FILE..." + printf " Path to one or more files or URLs containing OpenRefine's undo/redo\n operation history in JSON format. When FILE is -, read standard input.\n" + printf " Default: -\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " orcli transform \"duplicates\" \"history.json\"\n" + printf " cat \"history.json\" | orcli transform \"duplicates\"\n" + printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n" + printf " orcli transform 1234567890123 \"history.json\"\n" echo fi @@ -437,7 +490,7 @@ orcli_run_usage() { # :command.usage_examples printf "Examples:\n" printf " orcli run --interactive\n" - printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli info \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n" + printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n" printf " orcli run --memory \"2000M\" --port \"3334\" << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n wait\n echo \"finished import\"\n orcli export csv \"duplicates\" --output duplicates.csv &\n orcli export tsv \"duplicates\" --output duplicates.tsv &\n wait\n wc duplicates*\n EOF\n" printf " orcli run --interactive \"file1.sh\" \"file2.sh\" - << EOF\n echo \"finished in \$SECONDS seconds\"\n EOF\n" echo @@ -693,6 +746,10 @@ send_completions() { echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )' echo $' ;;' echo $'' + echo $' \'transform\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' + echo $' ;;' + echo $'' echo $' \'import\'*)' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )' echo $' ;;' @@ -714,7 +771,7 @@ send_completions() { echo $' ;;' echo $'' echo $' *)' - echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )' echo $' ;;' echo $'' echo $' esac' @@ -796,6 +853,90 @@ orcli_info_command() { } +# :command.function +orcli_transform_command() { + # src/transform_command.sh + # shellcheck shell=bash disable=SC2154 disable=SC2155 + + # check if stdin is present if selected + if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then + if ! read -u 0 -t 0; then + orcli_transform_usage + exit 1 + fi + fi + + # catch args, convert the space delimited string to an array + files=() + eval "files=(${args[file]})" + + # create tmp directory + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' 0 2 3 15 + + # download files if name starts with http:// or https:// + for i in "${!files[@]}"; do + if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then + if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then + error "download of ${files[$i]} failed!" + fi + files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" + fi + done + + # support multiple files + for i in "${!files[@]}"; do + # read each operation into one line + mapfile -t jsonlines < <(jq -c '.[]' "${files[$i]}") + for line in "${jsonlines[@]}"; do + # parse one line/operation into array + declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))" + # map operation names to command endpoints + # https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js + com="${data[op]#core/}" + if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi + if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi + if [[ $com == "column-addition" ]]; then com="add-column"; fi + if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi + if [[ $com == "column-removal" ]]; then com="remove-column"; fi + if [[ $com == "column-rename" ]]; then com="rename-column"; fi + if [[ $com == "column-move" ]]; then com="move-column"; fi + if [[ $com == "column-split" ]]; then com="split-column"; fi + if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi + if [[ $com == "recon" ]]; then com="reconcile"; fi + if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi + if [[ $com == "row-star" ]]; then com="annotate-rows"; fi + if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi + if [[ $com == "row-removal" ]]; then com="remove-rows"; fi + if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi + unset "data[op]" + # rename engineConfig to engine + data[engine]="${data[engineConfig]}" + unset "data[engineConfig]" + # drop description + unset "data[description]" + # prepare curl options + mapfile -t curloptions < <(for K in "${!data[@]}"; do + echo "--data" + echo "$K=${data[$K]}" + done) + # get project id and csrf token; post data to it's individual endpoint + if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then + response_code="$(jq -r '.code' <<<"$response")" + if [[ $response_code == "ok" ]]; then + log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")" + else + error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")" + fi + else + error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" + fi + unset data + done + done + +} + # :command.function orcli_export_tsv_command() { # src/export_tsv_command.sh @@ -1010,6 +1151,13 @@ parse_requirements() { shift $# ;; + transform ) + action="transform" + shift + orcli_transform_parse_requirements "$@" + shift $# + ;; + export ) action="export" shift @@ -1348,6 +1496,63 @@ orcli_info_parse_requirements() { } +# :command.parse_requirements +orcli_transform_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + orcli_transform_usage + exit + ;; + + esac + + # :command.command_filter + action="transform" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_repeatable + if [[ -z ${args[project]+x} ]]; then + + args[project]=$1 + shift + elif [[ -z ${args[file]+x} ]]; then + + args[file]="\"$1\"" + shift + else + args[file]="${args[file]} \"$1\"" + shift + fi + + ;; + + esac + done + + # :command.required_args_filter + if [[ -z ${args[project]+x} ]]; then + printf "missing required argument: PROJECT\nusage: orcli transform PROJECT [FILE...]\n" >&2 + exit 1 + fi + + # :command.default_assignments + [[ -n ${args[file]:-} ]] || args[file]="-" + +} + # :command.parse_requirements orcli_export_parse_requirements() { # :command.fixed_flags_filter @@ -1639,6 +1844,14 @@ run() { orcli_info_command fi + elif [[ $action == "transform" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + orcli_transform_usage + else + orcli_transform_command + fi + elif [[ $action == "export" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes diff --git a/src/bashly.yml b/src/bashly.yml index 320b860..4f05d8b 100644 --- a/src/bashly.yml +++ b/src/bashly.yml @@ -16,13 +16,14 @@ examples: - orcli import csv "https://git.io/fj5hF" --projectName "duplicates" - orcli list - orcli info "duplicates" + - orcli transform "duplicates" "https://git.io/fj5ju" - orcli export tsv "duplicates" - orcli export tsv "duplicates" --output "duplicates.tsv" - orcli run --interactive - |- orcli run << EOF orcli import csv "https://git.io/fj5hF" --projectName "duplicates" - orcli info "duplicates" + orcli transform "duplicates" "https://git.io/fj5ju" orcli export tsv "duplicates" EOF @@ -83,8 +84,24 @@ commands: help: project name or id required: true examples: - - info "duplicates" - - info 1234567890123 + - orcli info "duplicates" + - orcli info 1234567890123 + + - name: transform + help: apply undo/redo JSON file(s) to an OpenRefine project + args: + - name: project + help: project name or id + required: true + - name: file + help: Path to one or more files or URLs containing OpenRefine's undo/redo operation history in JSON format. When FILE is -, read standard input. + default: "-" + repeatable: true + examples: + - orcli transform "duplicates" "history.json" + - cat "history.json" | orcli transform "duplicates" + - orcli transform "duplicates" "https://git.io/fj5ju" + - orcli transform 1234567890123 "history.json" - name: export help: commands to export data from OpenRefine projects to files @@ -131,7 +148,7 @@ commands: - |- orcli run << EOF orcli import csv "https://git.io/fj5hF" --projectName "duplicates" - orcli info "duplicates" + orcli transform "duplicates" "https://git.io/fj5ju" orcli export tsv "duplicates" EOF - |- diff --git a/src/lib/send_completions.sh b/src/lib/send_completions.sh index 4b5f5ed..3771a63 100644 --- a/src/lib/send_completions.sh +++ b/src/lib/send_completions.sh @@ -42,6 +42,10 @@ send_completions() { echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )' echo $' ;;' echo $'' + echo $' \'transform\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' + echo $' ;;' + echo $'' echo $' \'import\'*)' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )' echo $' ;;' @@ -63,7 +67,7 @@ send_completions() { echo $' ;;' echo $'' echo $' *)' - echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )' echo $' ;;' echo $'' echo $' esac' diff --git a/src/transform_command.sh b/src/transform_command.sh new file mode 100644 index 0000000..fde2047 --- /dev/null +++ b/src/transform_command.sh @@ -0,0 +1,78 @@ +# shellcheck shell=bash disable=SC2154 disable=SC2155 + +# check if stdin is present if selected +if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then + if ! read -u 0 -t 0; then + orcli_transform_usage + exit 1 + fi +fi + +# catch args, convert the space delimited string to an array +files=() +eval "files=(${args[file]})" + +# create tmp directory +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' 0 2 3 15 + +# download files if name starts with http:// or https:// +for i in "${!files[@]}"; do + if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then + if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then + error "download of ${files[$i]} failed!" + fi + files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" + fi +done + +# support multiple files +for i in "${!files[@]}"; do + # read each operation into one line + mapfile -t jsonlines < <(jq -c '.[]' "${files[$i]}") + for line in "${jsonlines[@]}"; do + # parse one line/operation into array + declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))" + # map operation names to command endpoints + # https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js + com="${data[op]#core/}" + if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi + if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi + if [[ $com == "column-addition" ]]; then com="add-column"; fi + if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi + if [[ $com == "column-removal" ]]; then com="remove-column"; fi + if [[ $com == "column-rename" ]]; then com="rename-column"; fi + if [[ $com == "column-move" ]]; then com="move-column"; fi + if [[ $com == "column-split" ]]; then com="split-column"; fi + if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi + if [[ $com == "recon" ]]; then com="reconcile"; fi + if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi + if [[ $com == "row-star" ]]; then com="annotate-rows"; fi + if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi + if [[ $com == "row-removal" ]]; then com="remove-rows"; fi + if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi + unset "data[op]" + # rename engineConfig to engine + data[engine]="${data[engineConfig]}" + unset "data[engineConfig]" + # drop description + unset "data[description]" + # prepare curl options + mapfile -t curloptions < <(for K in "${!data[@]}"; do + echo "--data" + echo "$K=${data[$K]}" + done) + # get project id and csrf token; post data to it's individual endpoint + if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then + response_code="$(jq -r '.code' <<<"$response")" + if [[ $response_code == "ok" ]]; then + log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")" + else + error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")" + fi + else + error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" + fi + unset data + done +done