add mappings and improve error handling
This commit is contained in:
parent
c7d5273b1c
commit
cdd92fc181
28
orcli
28
orcli
|
@ -79,7 +79,7 @@ orcli_usage() {
|
||||||
printf " orcli export tsv \"duplicates\"\n"
|
printf " orcli export tsv \"duplicates\"\n"
|
||||||
printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n"
|
printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n"
|
||||||
printf " orcli run --interactive\n"
|
printf " orcli run --interactive\n"
|
||||||
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli info \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
||||||
echo
|
echo
|
||||||
|
|
||||||
# :command.footer
|
# :command.footer
|
||||||
|
@ -490,7 +490,7 @@ orcli_run_usage() {
|
||||||
# :command.usage_examples
|
# :command.usage_examples
|
||||||
printf "Examples:\n"
|
printf "Examples:\n"
|
||||||
printf " orcli run --interactive\n"
|
printf " orcli run --interactive\n"
|
||||||
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli info \"duplicates\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
printf " orcli run << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n orcli export tsv \"duplicates\"\n EOF\n"
|
||||||
printf " orcli run --memory \"2000M\" --port \"3334\" << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n wait\n echo \"finished import\"\n orcli export csv \"duplicates\" --output duplicates.csv &\n orcli export tsv \"duplicates\" --output duplicates.tsv &\n wait\n wc duplicates*\n EOF\n"
|
printf " orcli run --memory \"2000M\" --port \"3334\" << EOF\n orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n wait\n echo \"finished import\"\n orcli export csv \"duplicates\" --output duplicates.csv &\n orcli export tsv \"duplicates\" --output duplicates.tsv &\n wait\n wc duplicates*\n EOF\n"
|
||||||
printf " orcli run --interactive \"file1.sh\" \"file2.sh\" - << EOF\n echo \"finished in \$SECONDS seconds\"\n EOF\n"
|
printf " orcli run --interactive \"file1.sh\" \"file2.sh\" - << EOF\n echo \"finished in \$SECONDS seconds\"\n EOF\n"
|
||||||
echo
|
echo
|
||||||
|
@ -892,7 +892,22 @@ orcli_transform_command() {
|
||||||
# parse one line/operation into array
|
# parse one line/operation into array
|
||||||
declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
||||||
# map operation names to command endpoints
|
# map operation names to command endpoints
|
||||||
|
# https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js
|
||||||
com="${data[op]#core/}"
|
com="${data[op]#core/}"
|
||||||
|
if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi
|
||||||
|
if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi
|
||||||
|
if [[ $com == "column-addition" ]]; then com="add-column"; fi
|
||||||
|
if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi
|
||||||
|
if [[ $com == "column-removal" ]]; then com="remove-column"; fi
|
||||||
|
if [[ $com == "column-rename" ]]; then com="rename-column"; fi
|
||||||
|
if [[ $com == "column-move" ]]; then com="move-column"; fi
|
||||||
|
if [[ $com == "column-split" ]]; then com="split-column"; fi
|
||||||
|
if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi
|
||||||
|
if [[ $com == "recon" ]]; then com="reconcile"; fi
|
||||||
|
if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi
|
||||||
|
if [[ $com == "row-star" ]]; then com="annotate-rows"; fi
|
||||||
|
if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi
|
||||||
|
if [[ $com == "row-removal" ]]; then com="remove-rows"; fi
|
||||||
if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
|
if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
|
||||||
unset "data[op]"
|
unset "data[op]"
|
||||||
# rename engineConfig to engine
|
# rename engineConfig to engine
|
||||||
|
@ -907,9 +922,14 @@ orcli_transform_command() {
|
||||||
done)
|
done)
|
||||||
# get project id and csrf token; post data to it's individual endpoint
|
# get project id and csrf token; post data to it's individual endpoint
|
||||||
if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
|
if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
|
||||||
log "applied ${com} to ${args[project]}" "Response: $(jq '.historyEntry.description' <<< "$response")"
|
response_code="$(jq -r '.code' <<<"$response")"
|
||||||
|
if [[ $response_code == "ok" ]]; then
|
||||||
|
log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")"
|
||||||
|
else
|
||||||
|
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
error "applying ${com} from ${files[$i]} to ${args[project]} failed!"
|
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!"
|
||||||
fi
|
fi
|
||||||
unset data
|
unset data
|
||||||
done
|
done
|
||||||
|
|
|
@ -23,7 +23,7 @@ examples:
|
||||||
- |-
|
- |-
|
||||||
orcli run << EOF
|
orcli run << EOF
|
||||||
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||||
orcli info "duplicates"
|
orcli transform "duplicates" "https://git.io/fj5ju"
|
||||||
orcli export tsv "duplicates"
|
orcli export tsv "duplicates"
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
@ -148,7 +148,7 @@ commands:
|
||||||
- |-
|
- |-
|
||||||
orcli run << EOF
|
orcli run << EOF
|
||||||
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||||
orcli info "duplicates"
|
orcli transform "duplicates" "https://git.io/fj5ju"
|
||||||
orcli export tsv "duplicates"
|
orcli export tsv "duplicates"
|
||||||
EOF
|
EOF
|
||||||
- |-
|
- |-
|
||||||
|
|
|
@ -34,7 +34,22 @@ for i in "${!files[@]}"; do
|
||||||
# parse one line/operation into array
|
# parse one line/operation into array
|
||||||
declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
||||||
# map operation names to command endpoints
|
# map operation names to command endpoints
|
||||||
|
# https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js
|
||||||
com="${data[op]#core/}"
|
com="${data[op]#core/}"
|
||||||
|
if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi
|
||||||
|
if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi
|
||||||
|
if [[ $com == "column-addition" ]]; then com="add-column"; fi
|
||||||
|
if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi
|
||||||
|
if [[ $com == "column-removal" ]]; then com="remove-column"; fi
|
||||||
|
if [[ $com == "column-rename" ]]; then com="rename-column"; fi
|
||||||
|
if [[ $com == "column-move" ]]; then com="move-column"; fi
|
||||||
|
if [[ $com == "column-split" ]]; then com="split-column"; fi
|
||||||
|
if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi
|
||||||
|
if [[ $com == "recon" ]]; then com="reconcile"; fi
|
||||||
|
if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi
|
||||||
|
if [[ $com == "row-star" ]]; then com="annotate-rows"; fi
|
||||||
|
if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi
|
||||||
|
if [[ $com == "row-removal" ]]; then com="remove-rows"; fi
|
||||||
if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
|
if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
|
||||||
unset "data[op]"
|
unset "data[op]"
|
||||||
# rename engineConfig to engine
|
# rename engineConfig to engine
|
||||||
|
@ -49,9 +64,14 @@ for i in "${!files[@]}"; do
|
||||||
done)
|
done)
|
||||||
# get project id and csrf token; post data to it's individual endpoint
|
# get project id and csrf token; post data to it's individual endpoint
|
||||||
if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
|
if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
|
||||||
log "applied ${com} to ${args[project]}" "Response: $(jq '.historyEntry.description' <<< "$response")"
|
response_code="$(jq -r '.code' <<<"$response")"
|
||||||
|
if [[ $response_code == "ok" ]]; then
|
||||||
|
log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")"
|
||||||
|
else
|
||||||
|
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
error "applying ${com} from ${files[$i]} to ${args[project]} failed!"
|
error "transforming ${args[project]} with ${com} from ${files[$i]} failed!"
|
||||||
fi
|
fi
|
||||||
unset data
|
unset data
|
||||||
done
|
done
|
||||||
|
|
Loading…
Reference in New Issue