new command: orcli sort columns

This commit is contained in:
Felix Lohmeier 2023-12-11 10:17:46 +00:00
parent bd7327714c
commit 6895a74e41
7 changed files with 380 additions and 6 deletions

View File

@ -1,4 +1,4 @@
# orcli 0.2.2 # orcli 0.3.0
## command help screens ## command help screens
@ -15,6 +15,7 @@
- [list](list.md) - [list](list.md)
- [run](run.md) - [run](run.md)
- [search](search.md) - [search](search.md)
- [sort columns](sort_columns.md)
- [test](test.md) - [test](test.md)
- [transform](transform.md) - [transform](transform.md)
@ -35,6 +36,7 @@ Commands:
list list projects on OpenRefine server list list projects on OpenRefine server
info show OpenRefine project's metadata info show OpenRefine project's metadata
search apply regex to each column and print matches in flattened tsv format search apply regex to each column and print matches in flattened tsv format
sort commands to sort OpenRefine projects
test run functional tests on tmp OpenRefine workspace test run functional tests on tmp OpenRefine workspace
transform apply undo/redo JSON file(s) to an OpenRefine project transform apply undo/redo JSON file(s) to an OpenRefine project
export commands to export data from OpenRefine projects to files export commands to export data from OpenRefine projects to files
@ -58,6 +60,7 @@ Examples:
orcli info "duplicates" orcli info "duplicates"
orcli transform "duplicates" "https://git.io/fj5ju" orcli transform "duplicates" "https://git.io/fj5ju"
orcli search "duplicates" "^Ben" orcli search "duplicates" "^Ben"
orcli sort columns "duplicates"
orcli export tsv "duplicates" orcli export tsv "duplicates"
orcli export tsv "duplicates" --output "duplicates.tsv" orcli export tsv "duplicates" --output "duplicates.tsv"
orcli delete "duplicates" orcli delete "duplicates"

27
help/sort_columns.md Normal file
View File

@ -0,0 +1,27 @@
# orcli sort columns
```
orcli sort columns - re-order columns alphabetically
Usage:
orcli sort columns PROJECT [OPTIONS]
orcli sort columns --help | -h
Options:
--first COLUMN (repeatable)
set key column(s)
--help, -h
Show this help
Arguments:
PROJECT
project name or id
Examples:
orcli sort columns "duplicates"
orcli sort columns "duplicates" --first name
```
code: [src/sort_columns_command.sh](../src/sort_columns_command.sh)

273
orcli
View File

@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# This script was generated by bashly 1.1.3 (https://bashly.dannyb.co) # This script was generated by bashly 1.1.4 (https://bashly.dannyb.co)
# Modifying it manually is not recommended # Modifying it manually is not recommended
# :wrapper.bash3_bouncer # :wrapper.bash3_bouncer
@ -40,6 +40,7 @@ orcli_usage() {
printf " %s list projects on OpenRefine server\n" "list " printf " %s list projects on OpenRefine server\n" "list "
printf " %s show OpenRefine project's metadata\n" "info " printf " %s show OpenRefine project's metadata\n" "info "
printf " %s apply regex to each column and print matches in flattened tsv format\n" "search " printf " %s apply regex to each column and print matches in flattened tsv format\n" "search "
printf " %s commands to sort OpenRefine projects\n" "sort "
printf " %s run functional tests on tmp OpenRefine workspace\n" "test " printf " %s run functional tests on tmp OpenRefine workspace\n" "test "
printf " %s apply undo/redo JSON file(s) to an OpenRefine project\n" "transform " printf " %s apply undo/redo JSON file(s) to an OpenRefine project\n" "transform "
printf " %s commands to export data from OpenRefine projects to files\n" "export " printf " %s commands to export data from OpenRefine projects to files\n" "export "
@ -74,6 +75,7 @@ orcli_usage() {
printf " orcli info \"duplicates\"\n" printf " orcli info \"duplicates\"\n"
printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n" printf " orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n"
printf " orcli search \"duplicates\" \"^Ben\"\n" printf " orcli search \"duplicates\" \"^Ben\"\n"
printf " orcli sort columns \"duplicates\"\n"
printf " orcli export tsv \"duplicates\"\n" printf " orcli export tsv \"duplicates\"\n"
printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n" printf " orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n"
printf " orcli delete \"duplicates\"\n" printf " orcli delete \"duplicates\"\n"
@ -831,6 +833,88 @@ orcli_search_usage() {
fi fi
} }
# :command.usage
orcli_sort_usage() {
if [[ -n $long_usage ]]; then
printf "orcli sort - commands to sort OpenRefine projects\n"
echo
else
printf "orcli sort - commands to sort OpenRefine projects\n"
echo
fi
printf "%s\n" "Usage:"
printf " orcli sort COMMAND\n"
printf " orcli sort [COMMAND] --help | -h\n"
echo
# :command.usage_commands
printf "%s\n" "Commands:"
printf " %s re-order columns alphabetically\n" "columns"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "%s\n" "Options:"
# :command.usage_fixed_flags
printf " %s\n" "--help, -h"
printf " Show this help\n"
echo
fi
}
# :command.usage
orcli_sort_columns_usage() {
if [[ -n $long_usage ]]; then
printf "orcli sort columns - re-order columns alphabetically\n"
echo
else
printf "orcli sort columns - re-order columns alphabetically\n"
echo
fi
printf "%s\n" "Usage:"
printf " orcli sort columns PROJECT [OPTIONS]\n"
printf " orcli sort columns --help | -h\n"
echo
# :command.long_usage
if [[ -n $long_usage ]]; then
printf "%s\n" "Options:"
# :command.usage_flags
# :flag.usage
printf " %s\n" "--first COLUMN (repeatable)"
printf " set key column(s)\n"
echo
# :command.usage_fixed_flags
printf " %s\n" "--help, -h"
printf " Show this help\n"
echo
# :command.usage_args
printf "%s\n" "Arguments:"
# :argument.usage
printf " %s\n" "PROJECT"
printf " project name or id\n"
echo
# :command.usage_examples
printf "%s\n" "Examples:"
printf " orcli sort columns \"duplicates\"\n"
printf " orcli sort columns \"duplicates\" --first name\n"
echo
fi
}
# :command.usage # :command.usage
orcli_test_usage() { orcli_test_usage() {
if [[ -n $long_usage ]]; then if [[ -n $long_usage ]]; then
@ -1602,6 +1686,10 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur" )'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'sort columns\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--first --help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'export jsonl\'*)' echo $' \'export jsonl\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --separator -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --separator -h -q")" -- "$cur" )'
echo $' ;;' echo $' ;;'
@ -1654,6 +1742,10 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'sort\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h columns")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'test\'*)' echo $' \'test\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;' echo $' ;;'
@ -1663,7 +1755,7 @@ send_completions() {
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' *)' echo $' *)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions delete export import info list run search test transform")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions delete export import info list run search sort test transform")" -- "$cur" )'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' esac' echo $' esac'
@ -2089,6 +2181,31 @@ orcli_search_command() {
} }
# :command.function
orcli_sort_columns_command() {
# src/sort_columns_command.sh
# get columns, sort and transform with re-order columns
# shellcheck shell=bash
# catch args, convert the space delimited string to an array
first=()
eval "first=(${args[--first]})"
# convert to a comma-separated list of elements
columns=$(printf ',"'%s'"' "${first[@]}" | cut -c2-)
# get project id
projectid="$(get_id "${args[project]}")"
csrf="$(get_csrf)"
if ! sorted=$(curl -fs --get --data project="$projectid" "${OPENREFINE_URL}/command/core/get-columns-info" | jq --argjson columns "[ ${columns} ]" '($columns) + ([ .[].name ] | del (.[] | select (. | IN( $columns[] ))) | sort)'); then
error "getting columns in ${args[project]} failed!"
fi
if ! curl -fs -o /dev/null --data project="$projectid" --data "columnNames=${sorted}" "${OPENREFINE_URL}/command/core/reorder-columns${csrf}"; then
error "sorting columns in ${args[project]} failed!"
fi
log "sorted columns in ${args[project]}"
}
# :command.function # :command.function
orcli_test_command() { orcli_test_command() {
# src/test_command.sh # src/test_command.sh
@ -2658,6 +2775,13 @@ parse_requirements() {
shift $# shift $#
;; ;;
sort)
action="sort"
shift
orcli_sort_parse_requirements "$@"
shift $#
;;
test) test)
action="test" action="test"
shift shift
@ -3971,6 +4095,147 @@ orcli_search_parse_requirements() {
} }
# :command.parse_requirements
orcli_sort_parse_requirements() {
# :command.fixed_flags_filter
while [[ $# -gt 0 ]]; do
case "${1:-}" in
--help | -h)
long_usage=yes
orcli_sort_usage
exit
;;
*)
break
;;
esac
done
# :command.command_filter
action=${1:-}
case $action in
-*) ;;
columns)
action="columns"
shift
orcli_sort_columns_parse_requirements "$@"
shift $#
;;
# :command.command_fallback
"")
orcli_sort_usage >&2
exit 1
;;
*)
printf "invalid command: %s\n" "$action" >&2
exit 1
;;
esac
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
-?*)
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
*)
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
printf "invalid argument: %s\n" "$key" >&2
exit 1
;;
esac
done
}
# :command.parse_requirements
orcli_sort_columns_parse_requirements() {
# :command.fixed_flags_filter
while [[ $# -gt 0 ]]; do
case "${1:-}" in
--help | -h)
long_usage=yes
orcli_sort_columns_usage
exit
;;
*)
break
;;
esac
done
# :command.command_filter
action="sort columns"
# :command.parse_requirements_while
while [[ $# -gt 0 ]]; do
key="$1"
case "$key" in
# :flag.case
--first)
# :flag.case_arg
if [[ -n ${2+x} ]]; then
if [[ -z ${args['--first']+x} ]]; then
args['--first']="\"$2\""
else
args['--first']="${args['--first']} \"$2\""
fi
shift
shift
else
printf "%s\n" "--first requires an argument: --first COLUMN" >&2
exit 1
fi
;;
-?*)
printf "invalid option: %s\n" "$key" >&2
exit 1
;;
*)
# :command.parse_requirements_case
# :command.parse_requirements_case_simple
if [[ -z ${args['project']+x} ]]; then
args['project']=$1
shift
else
printf "invalid argument: %s\n" "$key" >&2
exit 1
fi
;;
esac
done
# :command.required_args_filter
if [[ -z ${args['project']+x} ]]; then
printf "missing required argument: PROJECT\nusage: orcli sort columns PROJECT [OPTIONS]\n" >&2
exit 1
fi
}
# :command.parse_requirements # :command.parse_requirements
orcli_test_parse_requirements() { orcli_test_parse_requirements() {
# :command.fixed_flags_filter # :command.fixed_flags_filter
@ -4750,7 +5015,7 @@ orcli_run_parse_requirements() {
# :command.initialize # :command.initialize
initialize() { initialize() {
version="0.2.2" version="0.3.0"
long_usage='' long_usage=''
set -e set -e
@ -4781,6 +5046,8 @@ run() {
"list") orcli_list_command ;; "list") orcli_list_command ;;
"info") orcli_info_command ;; "info") orcli_info_command ;;
"search") orcli_search_command ;; "search") orcli_search_command ;;
"sort") orcli_sort_command ;;
"sort columns") orcli_sort_columns_command ;;
"test") orcli_test_command ;; "test") orcli_test_command ;;
"transform") orcli_transform_command ;; "transform") orcli_transform_command ;;
"export") orcli_export_command ;; "export") orcli_export_command ;;

View File

@ -1,6 +1,6 @@
name: orcli name: orcli
help: OpenRefine command-line interface written in Bash help: OpenRefine command-line interface written in Bash
version: 0.2.2 version: 0.3.0
footer: https://github.com/opencultureconsulting/orcli footer: https://github.com/opencultureconsulting/orcli
dependencies: dependencies:
@ -18,6 +18,7 @@ examples:
- orcli info "duplicates" - orcli info "duplicates"
- orcli transform "duplicates" "https://git.io/fj5ju" - orcli transform "duplicates" "https://git.io/fj5ju"
- orcli search "duplicates" "^Ben" - orcli search "duplicates" "^Ben"
- orcli sort columns "duplicates"
- orcli export tsv "duplicates" - orcli export tsv "duplicates"
- orcli export tsv "duplicates" --output "duplicates.tsv" - orcli export tsv "duplicates" --output "duplicates.tsv"
- orcli delete "duplicates" - orcli delete "duplicates"
@ -297,6 +298,23 @@ commands:
- |- - |-
orcli export tsv "duplicates" --facets '[{ "type": "list", "expression": "grel:filter([\\\\"gender\\\\",\\\\"purchase\\\\"],cn,cells[cn].value.find(/^F/).length()>0).length()>0", "columnName": "", "selection": [{"v": {"v": true}}] }]' orcli export tsv "duplicates" --facets '[{ "type": "list", "expression": "grel:filter([\\\\"gender\\\\",\\\\"purchase\\\\"],cn,cells[cn].value.find(/^F/).length()>0).length()>0", "columnName": "", "selection": [{"v": {"v": true}}] }]'
- name: sort
help: commands to sort OpenRefine projects
commands:
- name: columns
help: re-order columns alphabetically
args:
- *project
flags:
- long: --first
help: set key column(s)
arg: column
repeatable: true
examples:
- orcli sort columns "duplicates"
- orcli sort columns "duplicates" --first name
- name: test - name: test
help: run functional tests on tmp OpenRefine workspace help: run functional tests on tmp OpenRefine workspace

View File

@ -50,6 +50,10 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur" )'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'sort columns\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--first --help -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'export jsonl\'*)' echo $' \'export jsonl\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --separator -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --separator -h -q")" -- "$cur" )'
echo $' ;;' echo $' ;;'
@ -102,6 +106,10 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'sort\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h columns")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'test\'*)' echo $' \'test\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
echo $' ;;' echo $' ;;'
@ -111,7 +119,7 @@ send_completions() {
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' *)' echo $' *)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions delete export import info list run search test transform")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions delete export import info list run search sort test transform")" -- "$cur" )'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' esac' echo $' esac'

View File

@ -0,0 +1,20 @@
# get columns, sort and transform with re-order columns
# shellcheck shell=bash
# catch args, convert the space delimited string to an array
first=()
eval "first=(${args[--first]})"
# convert to a comma-separated list of elements
columns=$(printf ',"'%s'"' "${first[@]}" | cut -c2-)
# get project id
projectid="$(get_id "${args[project]}")"
csrf="$(get_csrf)"
if ! sorted=$(curl -fs --get --data project="$projectid" "${OPENREFINE_URL}/command/core/get-columns-info" | jq --argjson columns "[ ${columns} ]" '($columns) + ([ .[].name ] | del (.[] | select (. | IN( $columns[] ))) | sort)'); then
error "getting columns in ${args[project]} failed!"
fi
if ! curl -fs -o /dev/null --data project="$projectid" --data "columnNames=${sorted}" "${OPENREFINE_URL}/command/core/reorder-columns${csrf}"; then
error "sorting columns in ${args[project]} failed!"
fi
log "sorted columns in ${args[project]}"

31
tests/sort-columns.sh Normal file
View File

@ -0,0 +1,31 @@
#!/bin/bash
t="sort-columns"
# create tmp directory
tmpdir="$(mktemp -d)"
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
# assertion
cat << "DATA" > "${tmpdir}/${t}.assert"
name state email gender purchase
Danny Baron CA danny.baron@example1.com M TV
Melanie White NC melanie.white@example2.edu F iPhone
D. Baron CA danny.baron@example1.com M Winter jacket
Ben Tyler NV ben.tyler@example3.org M Flashlight
Arthur Duff OR arthur.duff@example4.com M Dining table
Daniel Baron CA danny.baron@example1.com M Bike
Jean Griffith WA jean.griffith@example5.org F Power drill
Melanie White NC melanie.white@example2.edu F iPad
Ben Morisson FL ben.morisson@example6.org M Amplifier
Arthur Duff OR arthur.duff@example4.com M Night table
DATA
# action
cd "${tmpdir}" || exit 1
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
orcli sort columns "duplicates" --first name --first state
orcli export tsv "duplicates" --output "${t}.output"
# test
diff -u "${t}.assert" "${t}.output"