Merge pull request #73 from opencultureconsulting/felixlohmeier/transform-5
first draft
This commit is contained in:
Felix Lohmeier
GitHub
commit
be57685be2
|
|
@ -73,7 +73,11 @@ Optional:
|
|||
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||
```
|
||||
|
||||
3. Remove duplicates (coming soon)
|
||||
3. Remove duplicates by applying an undo/redo JSON file
|
||||
|
||||
```sh
|
||||
orcli transform "duplicates" "https://git.io/fj5hF"
|
||||
```
|
||||
|
||||
4. Export data from OpenRefine project to tab-separated-values (TSV) file `duplicates.tsv`
|
||||
|
||||
|
|
|
|||
223
orcli
223
orcli
|
|
@ -38,6 +38,7 @@ orcli_usage() {
|
|||
echo " import commands to create OpenRefine projects from files or URLs"
|
||||
echo " list list projects on OpenRefine server"
|
||||
echo " info show OpenRefine project's metadata"
|
||||
echo " transform apply undo/redo JSON file(s) to an OpenRefine project"
|
||||
echo " export commands to export data from OpenRefine projects to files"
|
||||
echo " run run tmp OpenRefine workspace and execute shell script(s)"
|
||||
echo
|
||||
|
|
@ -74,10 +75,11 @@ orcli_usage() {
|
|||
printf " orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n"
|
||||
printf " orcli list\n"
|
||||
printf " orcli info \"duplicates\"\n"
|
||||
printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n"
|
||||
printf " orcli export tsv \"duplicates\"\n"
|
||||
printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n"
|
||||
printf " orcli run --interactive\n"
|
||||
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli info \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
||||
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
||||
echo
|
||||
|
||||
# :command.footer
|
||||
|
|
@ -286,8 +288,59 @@ orcli_info_usage() {
|
|||
|
||||
# :command.usage_examples
|
||||
printf "Examples:\n"
|
||||
printf " info \"duplicates\"\n"
|
||||
printf " info 1234567890123\n"
|
||||
printf " orcli info \"duplicates\"\n"
|
||||
printf " orcli info 1234567890123\n"
|
||||
echo
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
# :command.usage
|
||||
orcli_transform_usage() {
|
||||
if [[ -n $long_usage ]]; then
|
||||
printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n"
|
||||
echo
|
||||
|
||||
else
|
||||
printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n"
|
||||
echo
|
||||
|
||||
fi
|
||||
|
||||
printf "Usage:\n"
|
||||
printf " orcli transform PROJECT [FILE...]\n"
|
||||
printf " orcli transform --help | -h\n"
|
||||
echo
|
||||
|
||||
# :command.long_usage
|
||||
if [[ -n $long_usage ]]; then
|
||||
printf "Options:\n"
|
||||
|
||||
# :command.usage_fixed_flags
|
||||
echo " --help, -h"
|
||||
printf " Show this help\n"
|
||||
echo
|
||||
|
||||
# :command.usage_args
|
||||
printf "Arguments:\n"
|
||||
|
||||
# :argument.usage
|
||||
echo " PROJECT"
|
||||
printf " project name or id\n"
|
||||
echo
|
||||
|
||||
# :argument.usage
|
||||
echo " FILE..."
|
||||
printf " Path to one or more files or URLs containing OpenRefine's undo/redo\n operation history in JSON format. When FILE is -, read standard input.\n"
|
||||
printf " Default: -\n"
|
||||
echo
|
||||
|
||||
# :command.usage_examples
|
||||
printf "Examples:\n"
|
||||
printf " orcli transform \"duplicates\" \"history.json\"\n"
|
||||
printf " cat \"history.json\" | orcli transform \"duplicates\"\n"
|
||||
printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n"
|
||||
printf " orcli transform 1234567890123 \"history.json\"\n"
|
||||
echo
|
||||
|
||||
fi
|
||||
|
|
@ -437,7 +490,7 @@ orcli_run_usage() {
|
|||
# :command.usage_examples
|
||||
printf "Examples:\n"
|
||||
printf " orcli run --interactive\n"
|
||||
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli info \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
||||
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
||||
printf " orcli run --memory \"2000M\" --port \"3334\" << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n wait\n echo \"finished import\"\n orcli export csv \"duplicates\" --output duplicates.csv &\n orcli export tsv \"duplicates\" --output duplicates.tsv &\n wait\n wc duplicates*\n EOF\n"
|
||||
printf " orcli run --interactive \"file1.sh\" \"file2.sh\" - << EOF\n echo \"finished in \$SECONDS seconds\"\n EOF\n"
|
||||
echo
|
||||
|
|
@ -693,6 +746,10 @@ send_completions() {
|
|||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' \'transform\'*)'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' \'import\'*)'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
|
|
@ -714,7 +771,7 @@ send_completions() {
|
|||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' *)'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' esac'
|
||||
|
|
@ -796,6 +853,90 @@ orcli_info_command() {
|
|||
|
||||
}
|
||||
|
||||
# :command.function
|
||||
orcli_transform_command() {
|
||||
# src/transform_command.sh
|
||||
# shellcheck shell=bash disable=SC2154 disable=SC2155
|
||||
|
||||
# check if stdin is present if selected
|
||||
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
|
||||
if ! read -u 0 -t 0; then
|
||||
orcli_transform_usage
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# catch args, convert the space delimited string to an array
|
||||
files=()
|
||||
eval "files=(${args[file]})"
|
||||
|
||||
# create tmp directory
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmpdir"' 0 2 3 15
|
||||
|
||||
# download files if name starts with http:// or https://
|
||||
for i in "${!files[@]}"; do
|
||||
if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then
|
||||
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
|
||||
error "download of ${files[$i]} failed!"
|
||||
fi
|
||||
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
|
||||
fi
|
||||
done
|
||||
|
||||
# support multiple files
|
||||
for i in "${!files[@]}"; do
|
||||
# read each operation into one line
|
||||
mapfile -t jsonlines < <(jq -c '.[]' "${files[$i]}")
|
||||
for line in "${jsonlines[@]}"; do
|
||||
# parse one line/operation into array
|
||||
declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
||||
# map operation names to command endpoints
|
||||
# https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js
|
||||
com="${data[op]#core/}"
|
||||
if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi
|
||||
if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi
|
||||
if [[ $com == "column-addition" ]]; then com="add-column"; fi
|
||||
if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi
|
||||
if [[ $com == "column-removal" ]]; then com="remove-column"; fi
|
||||
if [[ $com == "column-rename" ]]; then com="rename-column"; fi
|
||||
if [[ $com == "column-move" ]]; then com="move-column"; fi
|
||||
if [[ $com == "column-split" ]]; then com="split-column"; fi
|
||||
if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi
|
||||
if [[ $com == "recon" ]]; then com="reconcile"; fi
|
||||
if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi
|
||||
if [[ $com == "row-star" ]]; then com="annotate-rows"; fi
|
||||
if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi
|
||||
if [[ $com == "row-removal" ]]; then com="remove-rows"; fi
|
||||
if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
|
||||
unset "data[op]"
|
||||
# rename engineConfig to engine
|
||||
data[engine]="${data[engineConfig]}"
|
||||
unset "data[engineConfig]"
|
||||
# drop description
|
||||
unset "data[description]"
|
||||
# prepare curl options
|
||||
mapfile -t curloptions < <(for K in "${!data[@]}"; do
|
||||
echo "--data"
|
||||
echo "$K=${data[$K]}"
|
||||
done)
|
||||
# get project id and csrf token; post data to it's individual endpoint
|
||||
if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
|
||||
response_code="$(jq -r '.code' <<<"$response")"
|
||||
if [[ $response_code == "ok" ]]; then
|
||||
log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")"
|
||||
else
|
||||
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")"
|
||||
fi
|
||||
else
|
||||
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!"
|
||||
fi
|
||||
unset data
|
||||
done
|
||||
done
|
||||
|
||||
}
|
||||
|
||||
# :command.function
|
||||
orcli_export_tsv_command() {
|
||||
# src/export_tsv_command.sh
|
||||
|
|
@ -1010,6 +1151,13 @@ parse_requirements() {
|
|||
shift $#
|
||||
;;
|
||||
|
||||
transform )
|
||||
action="transform"
|
||||
shift
|
||||
orcli_transform_parse_requirements "$@"
|
||||
shift $#
|
||||
;;
|
||||
|
||||
export )
|
||||
action="export"
|
||||
shift
|
||||
|
|
@ -1348,6 +1496,63 @@ orcli_info_parse_requirements() {
|
|||
|
||||
}
|
||||
|
||||
# :command.parse_requirements
|
||||
orcli_transform_parse_requirements() {
|
||||
# :command.fixed_flags_filter
|
||||
case "${1:-}" in
|
||||
--help | -h )
|
||||
long_usage=yes
|
||||
orcli_transform_usage
|
||||
exit
|
||||
;;
|
||||
|
||||
esac
|
||||
|
||||
# :command.command_filter
|
||||
action="transform"
|
||||
|
||||
# :command.parse_requirements_while
|
||||
while [[ $# -gt 0 ]]; do
|
||||
key="$1"
|
||||
case "$key" in
|
||||
|
||||
-?* )
|
||||
printf "invalid option: %s\n" "$key" >&2
|
||||
exit 1
|
||||
;;
|
||||
|
||||
* )
|
||||
# :command.parse_requirements_case
|
||||
# :command.parse_requirements_case_repeatable
|
||||
if [[ -z ${args[project]+x} ]]; then
|
||||
|
||||
args[project]=$1
|
||||
shift
|
||||
elif [[ -z ${args[file]+x} ]]; then
|
||||
|
||||
args[file]="\"$1\""
|
||||
shift
|
||||
else
|
||||
args[file]="${args[file]} \"$1\""
|
||||
shift
|
||||
fi
|
||||
|
||||
;;
|
||||
|
||||
esac
|
||||
done
|
||||
|
||||
# :command.required_args_filter
|
||||
if [[ -z ${args[project]+x} ]]; then
|
||||
printf "missing required argument: PROJECT\nusage: orcli transform PROJECT [FILE...]\n" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# :command.default_assignments
|
||||
[[ -n ${args[file]:-} ]] || args[file]="-"
|
||||
|
||||
}
|
||||
|
||||
# :command.parse_requirements
|
||||
orcli_export_parse_requirements() {
|
||||
# :command.fixed_flags_filter
|
||||
|
|
@ -1639,6 +1844,14 @@ run() {
|
|||
orcli_info_command
|
||||
fi
|
||||
|
||||
elif [[ $action == "transform" ]]; then
|
||||
if [[ ${args[--help]:-} ]]; then
|
||||
long_usage=yes
|
||||
orcli_transform_usage
|
||||
else
|
||||
orcli_transform_command
|
||||
fi
|
||||
|
||||
elif [[ $action == "export" ]]; then
|
||||
if [[ ${args[--help]:-} ]]; then
|
||||
long_usage=yes
|
||||
|
|
|
|||
|
|
@ -16,13 +16,14 @@ examples:
|
|||
- orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||
- orcli list
|
||||
- orcli info "duplicates"
|
||||
- orcli transform "duplicates" "https://git.io/fj5ju"
|
||||
- orcli export tsv "duplicates"
|
||||
- orcli export tsv "duplicates" --output "duplicates.tsv"
|
||||
- orcli run --interactive
|
||||
- |-
|
||||
orcli run << EOF
|
||||
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||
orcli info "duplicates"
|
||||
orcli transform "duplicates" "https://git.io/fj5ju"
|
||||
orcli export tsv "duplicates"
|
||||
EOF
|
||||
|
||||
|
|
@ -83,8 +84,24 @@ commands:
|
|||
help: project name or id
|
||||
required: true
|
||||
examples:
|
||||
- info "duplicates"
|
||||
- info 1234567890123
|
||||
- orcli info "duplicates"
|
||||
- orcli info 1234567890123
|
||||
|
||||
- name: transform
|
||||
help: apply undo/redo JSON file(s) to an OpenRefine project
|
||||
args:
|
||||
- name: project
|
||||
help: project name or id
|
||||
required: true
|
||||
- name: file
|
||||
help: Path to one or more files or URLs containing OpenRefine's undo/redo operation history in JSON format. When FILE is -, read standard input.
|
||||
default: "-"
|
||||
repeatable: true
|
||||
examples:
|
||||
- orcli transform "duplicates" "history.json"
|
||||
- cat "history.json" | orcli transform "duplicates"
|
||||
- orcli transform "duplicates" "https://git.io/fj5ju"
|
||||
- orcli transform 1234567890123 "history.json"
|
||||
|
||||
- name: export
|
||||
help: commands to export data from OpenRefine projects to files
|
||||
|
|
@ -131,7 +148,7 @@ commands:
|
|||
- |-
|
||||
orcli run << EOF
|
||||
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||
orcli info "duplicates"
|
||||
orcli transform "duplicates" "https://git.io/fj5ju"
|
||||
orcli export tsv "duplicates"
|
||||
EOF
|
||||
- |-
|
||||
|
|
|
|||
|
|
@ -42,6 +42,10 @@ send_completions() {
|
|||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' \'transform\'*)'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' \'import\'*)'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
|
|
@ -63,7 +67,7 @@ send_completions() {
|
|||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' *)'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )'
|
||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )'
|
||||
echo $' ;;'
|
||||
echo $''
|
||||
echo $' esac'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,78 @@
|
|||
# shellcheck shell=bash disable=SC2154 disable=SC2155
|
||||
|
||||
# check if stdin is present if selected
|
||||
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
|
||||
if ! read -u 0 -t 0; then
|
||||
orcli_transform_usage
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# catch args, convert the space delimited string to an array
|
||||
files=()
|
||||
eval "files=(${args[file]})"
|
||||
|
||||
# create tmp directory
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmpdir"' 0 2 3 15
|
||||
|
||||
# download files if name starts with http:// or https://
|
||||
for i in "${!files[@]}"; do
|
||||
if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then
|
||||
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
|
||||
error "download of ${files[$i]} failed!"
|
||||
fi
|
||||
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
|
||||
fi
|
||||
done
|
||||
|
||||
# support multiple files
|
||||
for i in "${!files[@]}"; do
|
||||
# read each operation into one line
|
||||
mapfile -t jsonlines < <(jq -c '.[]' "${files[$i]}")
|
||||
for line in "${jsonlines[@]}"; do
|
||||
# parse one line/operation into array
|
||||
declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
||||
# map operation names to command endpoints
|
||||
# https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js
|
||||
com="${data[op]#core/}"
|
||||
if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi
|
||||
if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi
|
||||
if [[ $com == "column-addition" ]]; then com="add-column"; fi
|
||||
if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi
|
||||
if [[ $com == "column-removal" ]]; then com="remove-column"; fi
|
||||
if [[ $com == "column-rename" ]]; then com="rename-column"; fi
|
||||
if [[ $com == "column-move" ]]; then com="move-column"; fi
|
||||
if [[ $com == "column-split" ]]; then com="split-column"; fi
|
||||
if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi
|
||||
if [[ $com == "recon" ]]; then com="reconcile"; fi
|
||||
if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi
|
||||
if [[ $com == "row-star" ]]; then com="annotate-rows"; fi
|
||||
if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi
|
||||
if [[ $com == "row-removal" ]]; then com="remove-rows"; fi
|
||||
if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
|
||||
unset "data[op]"
|
||||
# rename engineConfig to engine
|
||||
data[engine]="${data[engineConfig]}"
|
||||
unset "data[engineConfig]"
|
||||
# drop description
|
||||
unset "data[description]"
|
||||
# prepare curl options
|
||||
mapfile -t curloptions < <(for K in "${!data[@]}"; do
|
||||
echo "--data"
|
||||
echo "$K=${data[$K]}"
|
||||
done)
|
||||
# get project id and csrf token; post data to it's individual endpoint
|
||||
if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
|
||||
response_code="$(jq -r '.code' <<<"$response")"
|
||||
if [[ $response_code == "ok" ]]; then
|
||||
log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")"
|
||||
else
|
||||
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")"
|
||||
fi
|
||||
else
|
||||
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!"
|
||||
fi
|
||||
unset data
|
||||
done
|
||||
done
|
||||
Loading…
Reference in New Issue