Merge pull request #85 from opencultureconsulting/felixlohmeier/functional-tests-7
first draft functional tests
This commit is contained in:
commit
d1486c5929
|
@ -137,3 +137,9 @@ orcli uses [bashly](https://github.com/DannyBen/bashly/) for generating the one-
|
||||||
```sh
|
```sh
|
||||||
bashly generate --upgrade
|
bashly generate --upgrade
|
||||||
```
|
```
|
||||||
|
|
||||||
|
4. Run tests
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./orcli test
|
||||||
|
```
|
285
orcli
285
orcli
|
@ -1,5 +1,5 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# This script was generated by bashly 0.8.10 (https://bashly.dannyb.co)
|
# This script was generated by bashly 0.9.0 (https://bashly.dannyb.co)
|
||||||
# Modifying it manually is not recommended
|
# Modifying it manually is not recommended
|
||||||
|
|
||||||
# :wrapper.bash3_bouncer
|
# :wrapper.bash3_bouncer
|
||||||
|
@ -38,6 +38,7 @@ orcli_usage() {
|
||||||
echo " import commands to create OpenRefine projects from files or URLs"
|
echo " import commands to create OpenRefine projects from files or URLs"
|
||||||
echo " list list projects on OpenRefine server"
|
echo " list list projects on OpenRefine server"
|
||||||
echo " info show OpenRefine project's metadata"
|
echo " info show OpenRefine project's metadata"
|
||||||
|
echo " test run functional tests on tmp OpenRefine workspace"
|
||||||
echo " transform apply undo/redo JSON file(s) to an OpenRefine project"
|
echo " transform apply undo/redo JSON file(s) to an OpenRefine project"
|
||||||
echo " export commands to export data from OpenRefine projects to files"
|
echo " export commands to export data from OpenRefine projects to files"
|
||||||
echo " run run tmp OpenRefine workspace and execute shell script(s)"
|
echo " run run tmp OpenRefine workspace and execute shell script(s)"
|
||||||
|
@ -363,6 +364,35 @@ orcli_info_usage() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# :command.usage
|
||||||
|
orcli_test_usage() {
|
||||||
|
if [[ -n $long_usage ]]; then
|
||||||
|
printf "orcli test - run functional tests on tmp OpenRefine workspace\n"
|
||||||
|
echo
|
||||||
|
|
||||||
|
else
|
||||||
|
printf "orcli test - run functional tests on tmp OpenRefine workspace\n"
|
||||||
|
echo
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf "Usage:\n"
|
||||||
|
printf " orcli test\n"
|
||||||
|
printf " orcli test --help | -h\n"
|
||||||
|
echo
|
||||||
|
|
||||||
|
# :command.long_usage
|
||||||
|
if [[ -n $long_usage ]]; then
|
||||||
|
printf "Options:\n"
|
||||||
|
|
||||||
|
# :command.usage_fixed_flags
|
||||||
|
echo " --help, -h"
|
||||||
|
printf " Show this help\n"
|
||||||
|
echo
|
||||||
|
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# :command.usage
|
# :command.usage
|
||||||
orcli_transform_usage() {
|
orcli_transform_usage() {
|
||||||
if [[ -n $long_usage ]]; then
|
if [[ -n $long_usage ]]; then
|
||||||
|
@ -880,12 +910,16 @@ send_completions() {
|
||||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
||||||
echo $' ;;'
|
echo $' ;;'
|
||||||
echo $''
|
echo $''
|
||||||
|
echo $' \'test\'*)'
|
||||||
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
||||||
|
echo $' ;;'
|
||||||
|
echo $''
|
||||||
echo $' \'run\'*)'
|
echo $' \'run\'*)'
|
||||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --interactive --memory --port --quiet -h -q")" -- "$cur" )'
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --interactive --memory --port --quiet -h -q")" -- "$cur" )'
|
||||||
echo $' ;;'
|
echo $' ;;'
|
||||||
echo $''
|
echo $''
|
||||||
echo $' *)'
|
echo $' *)'
|
||||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions export import info list run transform")" -- "$cur" )'
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions export import info list run test transform")" -- "$cur" )'
|
||||||
echo $' ;;'
|
echo $' ;;'
|
||||||
echo $''
|
echo $''
|
||||||
echo $' esac'
|
echo $' esac'
|
||||||
|
@ -1004,6 +1038,85 @@ orcli_info_command() {
|
||||||
echo "$projectid"
|
echo "$projectid"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# :command.function
|
||||||
|
orcli_test_command() {
|
||||||
|
# src/test_command.sh
|
||||||
|
# shellcheck shell=bash disable=SC2154
|
||||||
|
|
||||||
|
# locate orcli and OpenRefine
|
||||||
|
scriptpath=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
|
||||||
|
if [[ -x "${scriptpath}/refine" ]]; then
|
||||||
|
openrefine="${scriptpath}/refine"
|
||||||
|
else
|
||||||
|
error "OpenRefine's startup script (refine) not found!" "Did you put orcli in your OpenRefine app dir?"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
OPENREFINE_TMPDIR="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "$OPENREFINE_TMPDIR"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# download the test files if needed
|
||||||
|
if ! [[ -f "tests/help.sh" ]]; then
|
||||||
|
cd "$OPENREFINE_TMPDIR"
|
||||||
|
if ! curl -fs -L -o orcli.zip https://github.com/opencultureconsulting/orcli/archive/refs/heads/main.zip; then
|
||||||
|
error "downloading test files failed!" "Please download the tests dir manually from GitHub."
|
||||||
|
fi
|
||||||
|
unzip -q -j orcli.zip "*/tests/*.sh" -d "tests/"
|
||||||
|
unzip -q -j orcli.zip "*/tests/data/*" -d "tests/data/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# check if OpenRefine is already running
|
||||||
|
if curl -fs "${OPENREFINE_URL}" &>/dev/null; then
|
||||||
|
error "OpenRefine is already running on port 3333." "Please stop the other process."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# start OpenRefine with tmp workspace
|
||||||
|
$openrefine -d "$OPENREFINE_TMPDIR" -x refine.headless=true -v warn &>"$OPENREFINE_TMPDIR/openrefine.log" &
|
||||||
|
OPENREFINE_PID="$!"
|
||||||
|
|
||||||
|
# update trap to kill OpenRefine on error or exit
|
||||||
|
trap '{ rm -rf "$OPENREFINE_TMPDIR"; kill -9 "$OPENREFINE_PID"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# wait until OpenRefine is running (timeout 20s)
|
||||||
|
if ! curl -fs --retry 20 --retry-connrefused --retry-delay 1 "${OPENREFINE_URL}/command/core/get-version" &>/dev/null; then
|
||||||
|
error "starting OpenRefine server failed!"
|
||||||
|
else
|
||||||
|
log "started OpenRefine with tmp workspace ${OPENREFINE_TMPDIR}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# execute tests in subshell
|
||||||
|
export OPENREFINE_TMPDIR OPENREFINE_URL OPENREFINE_PID
|
||||||
|
cd "tests"
|
||||||
|
files=(*.sh)
|
||||||
|
results=()
|
||||||
|
for i in "${!files[@]}"; do
|
||||||
|
set +e # do not exit on failed tests
|
||||||
|
bash -e <(
|
||||||
|
if ! command -v orcli &>/dev/null; then
|
||||||
|
echo "shopt -s expand_aliases"
|
||||||
|
echo "alias orcli=${scriptpath}/orcli"
|
||||||
|
fi
|
||||||
|
awk 1 "${files[$i]}"
|
||||||
|
) &>"$OPENREFINE_TMPDIR/test.log"
|
||||||
|
results+=(${?})
|
||||||
|
set -e
|
||||||
|
if [[ "${results[$i]}" =~ [1-9] ]]; then
|
||||||
|
cat "$OPENREFINE_TMPDIR/test.log"
|
||||||
|
log "FAILED ${files[$i]} with exit code ${results[$i]}!"
|
||||||
|
else
|
||||||
|
log "PASSED ${files[$i]}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# print overall result
|
||||||
|
if [[ "${results[*]}" =~ [1-9] ]]; then
|
||||||
|
error "failed tests!"
|
||||||
|
else
|
||||||
|
log "all tests passed!"
|
||||||
|
fi
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
# :command.function
|
# :command.function
|
||||||
orcli_transform_command() {
|
orcli_transform_command() {
|
||||||
# src/transform_command.sh
|
# src/transform_command.sh
|
||||||
|
@ -1253,6 +1366,7 @@ orcli_run_command() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
parse_requirements() {
|
parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--version | -v)
|
--version | -v)
|
||||||
version_command
|
version_command
|
||||||
|
@ -1265,18 +1379,24 @@ parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.environment_variables_filter
|
# :command.environment_variables_filter
|
||||||
# :command.environment_variables_default
|
# :command.environment_variables_default
|
||||||
export OPENREFINE_URL="${OPENREFINE_URL:-http://localhost:3333}"
|
export OPENREFINE_URL="${OPENREFINE_URL:-http://localhost:3333}"
|
||||||
|
|
||||||
# :command.dependencies_filter
|
# :command.dependencies_filter
|
||||||
if ! [[ -x "$(command -v curl)" ]]; then
|
if ! command -v curl >/dev/null 2>&1; then
|
||||||
printf "missing dependency: curl\n" >&2
|
printf "missing dependency: curl\n" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
if ! [[ -x "$(command -v jq)" ]]; then
|
|
||||||
|
if ! command -v jq >/dev/null 2>&1; then
|
||||||
printf "missing dependency: jq\n" >&2
|
printf "missing dependency: jq\n" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
@ -1316,6 +1436,13 @@ parse_requirements() {
|
||||||
shift $#
|
shift $#
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
test)
|
||||||
|
action="test"
|
||||||
|
shift
|
||||||
|
orcli_test_parse_requirements "$@"
|
||||||
|
shift $#
|
||||||
|
;;
|
||||||
|
|
||||||
transform)
|
transform)
|
||||||
action="transform"
|
action="transform"
|
||||||
shift
|
shift
|
||||||
|
@ -1376,6 +1503,7 @@ parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_completions_parse_requirements() {
|
orcli_completions_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1383,7 +1511,12 @@ orcli_completions_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="completions"
|
action="completions"
|
||||||
|
@ -1414,6 +1547,7 @@ orcli_completions_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_import_parse_requirements() {
|
orcli_import_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1421,7 +1555,12 @@ orcli_import_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action=${1:-}
|
action=${1:-}
|
||||||
|
@ -1483,6 +1622,7 @@ orcli_import_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_import_csv_parse_requirements() {
|
orcli_import_csv_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1490,7 +1630,12 @@ orcli_import_csv_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="import csv"
|
action="import csv"
|
||||||
|
@ -1591,6 +1736,7 @@ orcli_import_csv_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_import_tsv_parse_requirements() {
|
orcli_import_tsv_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1598,7 +1744,12 @@ orcli_import_tsv_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="import tsv"
|
action="import tsv"
|
||||||
|
@ -1683,6 +1834,7 @@ orcli_import_tsv_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_list_parse_requirements() {
|
orcli_list_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1690,7 +1842,12 @@ orcli_list_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="list"
|
action="list"
|
||||||
|
@ -1721,6 +1878,7 @@ orcli_list_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_info_parse_requirements() {
|
orcli_info_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1728,7 +1886,12 @@ orcli_info_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="info"
|
action="info"
|
||||||
|
@ -1768,9 +1931,54 @@ orcli_info_parse_requirements() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# :command.parse_requirements
|
||||||
|
orcli_test_parse_requirements() {
|
||||||
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "${1:-}" in
|
||||||
|
--help | -h)
|
||||||
|
long_usage=yes
|
||||||
|
orcli_test_usage
|
||||||
|
exit
|
||||||
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# :command.command_filter
|
||||||
|
action="test"
|
||||||
|
|
||||||
|
# :command.parse_requirements_while
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
key="$1"
|
||||||
|
case "$key" in
|
||||||
|
|
||||||
|
-?*)
|
||||||
|
printf "invalid option: %s\n" "$key" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
# :command.parse_requirements_case
|
||||||
|
# :command.parse_requirements_case_simple
|
||||||
|
printf "invalid argument: %s\n" "$key" >&2
|
||||||
|
exit 1
|
||||||
|
|
||||||
|
;;
|
||||||
|
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_transform_parse_requirements() {
|
orcli_transform_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1778,7 +1986,12 @@ orcli_transform_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="transform"
|
action="transform"
|
||||||
|
@ -1835,6 +2048,7 @@ orcli_transform_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_export_parse_requirements() {
|
orcli_export_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1842,7 +2056,12 @@ orcli_export_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action=${1:-}
|
action=${1:-}
|
||||||
|
@ -1897,6 +2116,7 @@ orcli_export_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_export_tsv_parse_requirements() {
|
orcli_export_tsv_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1904,7 +2124,12 @@ orcli_export_tsv_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="export tsv"
|
action="export tsv"
|
||||||
|
@ -1987,6 +2212,7 @@ orcli_export_tsv_parse_requirements() {
|
||||||
# :command.parse_requirements
|
# :command.parse_requirements
|
||||||
orcli_run_parse_requirements() {
|
orcli_run_parse_requirements() {
|
||||||
# :command.fixed_flags_filter
|
# :command.fixed_flags_filter
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
--help | -h)
|
--help | -h)
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
|
@ -1994,7 +2220,12 @@ orcli_run_parse_requirements() {
|
||||||
exit
|
exit
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
|
||||||
esac
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# :command.command_filter
|
# :command.command_filter
|
||||||
action="run"
|
action="run"
|
||||||
|
@ -2099,89 +2330,111 @@ run() {
|
||||||
normalize_input "$@"
|
normalize_input "$@"
|
||||||
parse_requirements "${input[@]}"
|
parse_requirements "${input[@]}"
|
||||||
|
|
||||||
if [[ $action == "completions" ]]; then
|
case "$action" in
|
||||||
|
"completions")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_completions_usage
|
orcli_completions_usage
|
||||||
else
|
else
|
||||||
orcli_completions_command
|
orcli_completions_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "import" ]]; then
|
"import")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_import_usage
|
orcli_import_usage
|
||||||
else
|
else
|
||||||
orcli_import_command
|
orcli_import_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "import csv" ]]; then
|
"import csv")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_import_csv_usage
|
orcli_import_csv_usage
|
||||||
else
|
else
|
||||||
orcli_import_csv_command
|
orcli_import_csv_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "import tsv" ]]; then
|
"import tsv")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_import_tsv_usage
|
orcli_import_tsv_usage
|
||||||
else
|
else
|
||||||
orcli_import_tsv_command
|
orcli_import_tsv_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "list" ]]; then
|
"list")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_list_usage
|
orcli_list_usage
|
||||||
else
|
else
|
||||||
orcli_list_command
|
orcli_list_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "info" ]]; then
|
"info")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_info_usage
|
orcli_info_usage
|
||||||
else
|
else
|
||||||
orcli_info_command
|
orcli_info_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "transform" ]]; then
|
"test")
|
||||||
|
if [[ ${args[--help]:-} ]]; then
|
||||||
|
long_usage=yes
|
||||||
|
orcli_test_usage
|
||||||
|
else
|
||||||
|
orcli_test_command
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
|
"transform")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_transform_usage
|
orcli_transform_usage
|
||||||
else
|
else
|
||||||
orcli_transform_command
|
orcli_transform_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "export" ]]; then
|
"export")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_export_usage
|
orcli_export_usage
|
||||||
else
|
else
|
||||||
orcli_export_command
|
orcli_export_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "export tsv" ]]; then
|
"export tsv")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_export_tsv_usage
|
orcli_export_tsv_usage
|
||||||
else
|
else
|
||||||
orcli_export_tsv_command
|
orcli_export_tsv_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "run" ]]; then
|
"run")
|
||||||
if [[ ${args[--help]:-} ]]; then
|
if [[ ${args[--help]:-} ]]; then
|
||||||
long_usage=yes
|
long_usage=yes
|
||||||
orcli_run_usage
|
orcli_run_usage
|
||||||
else
|
else
|
||||||
orcli_run_command
|
orcli_run_command
|
||||||
fi
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
elif [[ $action == "root" ]]; then
|
"root")
|
||||||
root_command
|
root_command
|
||||||
fi
|
;;
|
||||||
|
|
||||||
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
initialize
|
initialize
|
||||||
|
|
|
@ -116,6 +116,9 @@ commands:
|
||||||
- orcli info "duplicates"
|
- orcli info "duplicates"
|
||||||
- orcli info 1234567890123
|
- orcli info 1234567890123
|
||||||
|
|
||||||
|
- name: test
|
||||||
|
help: run functional tests on tmp OpenRefine workspace
|
||||||
|
|
||||||
- name: transform
|
- name: transform
|
||||||
help: apply undo/redo JSON file(s) to an OpenRefine project
|
help: apply undo/redo JSON file(s) to an OpenRefine project
|
||||||
args:
|
args:
|
||||||
|
|
|
@ -66,12 +66,16 @@ send_completions() {
|
||||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
||||||
echo $' ;;'
|
echo $' ;;'
|
||||||
echo $''
|
echo $''
|
||||||
|
echo $' \'test\'*)'
|
||||||
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
|
||||||
|
echo $' ;;'
|
||||||
|
echo $''
|
||||||
echo $' \'run\'*)'
|
echo $' \'run\'*)'
|
||||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --interactive --memory --port --quiet -h -q")" -- "$cur" )'
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --interactive --memory --port --quiet -h -q")" -- "$cur" )'
|
||||||
echo $' ;;'
|
echo $' ;;'
|
||||||
echo $''
|
echo $''
|
||||||
echo $' *)'
|
echo $' *)'
|
||||||
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions export import info list run transform")" -- "$cur" )'
|
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v completions export import info list run test transform")" -- "$cur" )'
|
||||||
echo $' ;;'
|
echo $' ;;'
|
||||||
echo $''
|
echo $''
|
||||||
echo $' esac'
|
echo $' esac'
|
||||||
|
|
|
@ -0,0 +1,73 @@
|
||||||
|
# shellcheck shell=bash disable=SC2154
|
||||||
|
|
||||||
|
# locate orcli and OpenRefine
|
||||||
|
scriptpath=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
|
||||||
|
if [[ -x "${scriptpath}/refine" ]]; then
|
||||||
|
openrefine="${scriptpath}/refine"
|
||||||
|
else
|
||||||
|
error "OpenRefine's startup script (refine) not found!" "Did you put orcli in your OpenRefine app dir?"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
OPENREFINE_TMPDIR="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "$OPENREFINE_TMPDIR"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# download the test files if needed
|
||||||
|
if ! [[ -f "tests/help.sh" ]]; then
|
||||||
|
cd "$OPENREFINE_TMPDIR"
|
||||||
|
if ! curl -fs -L -o orcli.zip https://github.com/opencultureconsulting/orcli/archive/refs/heads/main.zip; then
|
||||||
|
error "downloading test files failed!" "Please download the tests dir manually from GitHub."
|
||||||
|
fi
|
||||||
|
unzip -q -j orcli.zip "*/tests/*.sh" -d "tests/"
|
||||||
|
unzip -q -j orcli.zip "*/tests/data/*" -d "tests/data/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# check if OpenRefine is already running
|
||||||
|
if curl -fs "${OPENREFINE_URL}" &>/dev/null; then
|
||||||
|
error "OpenRefine is already running on port 3333." "Please stop the other process."
|
||||||
|
fi
|
||||||
|
|
||||||
|
# start OpenRefine with tmp workspace
|
||||||
|
$openrefine -d "$OPENREFINE_TMPDIR" -x refine.headless=true -v warn &>"$OPENREFINE_TMPDIR/openrefine.log" &
|
||||||
|
OPENREFINE_PID="$!"
|
||||||
|
|
||||||
|
# update trap to kill OpenRefine on error or exit
|
||||||
|
trap '{ rm -rf "$OPENREFINE_TMPDIR"; kill -9 "$OPENREFINE_PID"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# wait until OpenRefine is running (timeout 20s)
|
||||||
|
if ! curl -fs --retry 20 --retry-connrefused --retry-delay 1 "${OPENREFINE_URL}/command/core/get-version" &>/dev/null; then
|
||||||
|
error "starting OpenRefine server failed!"
|
||||||
|
else
|
||||||
|
log "started OpenRefine with tmp workspace ${OPENREFINE_TMPDIR}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# execute tests in subshell
|
||||||
|
export OPENREFINE_TMPDIR OPENREFINE_URL OPENREFINE_PID
|
||||||
|
cd "tests"
|
||||||
|
files=(*.sh)
|
||||||
|
results=()
|
||||||
|
for i in "${!files[@]}"; do
|
||||||
|
set +e # do not exit on failed tests
|
||||||
|
bash -e <(
|
||||||
|
if ! command -v orcli &>/dev/null; then
|
||||||
|
echo "shopt -s expand_aliases"
|
||||||
|
echo "alias orcli=${scriptpath}/orcli"
|
||||||
|
fi
|
||||||
|
awk 1 "${files[$i]}"
|
||||||
|
) &>"$OPENREFINE_TMPDIR/test.log"
|
||||||
|
results+=(${?})
|
||||||
|
set -e
|
||||||
|
if [[ "${results[$i]}" =~ [1-9] ]]; then
|
||||||
|
cat "$OPENREFINE_TMPDIR/test.log"
|
||||||
|
log "FAILED ${files[$i]} with exit code ${results[$i]}!"
|
||||||
|
else
|
||||||
|
log "PASSED ${files[$i]}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# print overall result
|
||||||
|
if [[ "${results[*]}" =~ [1-9] ]]; then
|
||||||
|
error "failed tests!"
|
||||||
|
else
|
||||||
|
log "all tests passed!"
|
||||||
|
fi
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="completions"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# assertion
|
||||||
|
cat << "DATA" > "${tmpdir}/${t}.assert"
|
||||||
|
# orcli completion -*- shell-script -*-
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli completions | head -n1 > "${t}.output"
|
||||||
|
|
||||||
|
# test
|
||||||
|
diff -u "${t}.assert" "${t}.output"
|
|
@ -0,0 +1,7 @@
|
||||||
|
email count name state gender purchase
|
||||||
|
arthur.duff@example4.com 2 Arthur Duff OR M Dining table
|
||||||
|
ben.morisson@example6.org 1 Ben Morisson FL M Amplifier
|
||||||
|
ben.tyler@example3.org 1 Ben Tyler NV M Flashlight
|
||||||
|
danny.baron@example1.com 3 Danny Baron CA M TV
|
||||||
|
jean.griffith@example5.org 1 Jean Griffith WA F Power drill
|
||||||
|
melanie.white@example2.edu 2 Melanie White NC F iPhone
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
email,name,state,gender,purchase
|
||||||
|
danny.baron@example1.com,Danny Baron,CA,M,TV
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPhone
|
||||||
|
danny.baron@example1.com,D. Baron,CA,M,Winter jacket
|
||||||
|
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table
|
||||||
|
danny.baron@example1.com,Daniel Baron,CA,M,Bike
|
||||||
|
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPad
|
||||||
|
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Night table
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
email name state gender purchase
|
||||||
|
danny.baron@example1.com Danny Baron CA M TV
|
||||||
|
melanie.white@example2.edu Melanie White NC F iPhone
|
||||||
|
danny.baron@example1.com D. Baron CA M Winter jacket
|
||||||
|
ben.tyler@example3.org Ben Tyler NV M Flashlight
|
||||||
|
arthur.duff@example4.com Arthur Duff OR M Dining table
|
||||||
|
danny.baron@example1.com Daniel Baron CA M Bike
|
||||||
|
jean.griffith@example5.org Jean Griffith WA F Power drill
|
||||||
|
melanie.white@example2.edu Melanie White NC F iPad
|
||||||
|
ben.morisson@example6.org Ben Morisson FL M Amplifier
|
||||||
|
arthur.duff@example4.com Arthur Duff OR M Night table
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
0,0,0
|
||||||
|
$,\,'
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
a b c
|
||||||
|
1 2 3
|
||||||
|
0 0 0
|
||||||
|
$ \ '
|
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="export-tsv"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# input
|
||||||
|
cp data/example.csv "${tmpdir}/${t}.csv"
|
||||||
|
|
||||||
|
# assertion
|
||||||
|
cp data/example.tsv "${tmpdir}/${t}.assert"
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli import csv "${t}.csv"
|
||||||
|
orcli export tsv "${t} csv" --output "${t}.output"
|
||||||
|
|
||||||
|
# test
|
||||||
|
diff -u "${t}.assert" "${t}.output"
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="help"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# assertion
|
||||||
|
cat << "DATA" > "${tmpdir}/${t}.assert"
|
||||||
|
orcli - OpenRefine command-line interface written in Bash
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli --help | head -n1 > "${t}.output"
|
||||||
|
|
||||||
|
# test
|
||||||
|
diff -u "${t}.assert" "${t}.output"
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="import-csv"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# input
|
||||||
|
cp data/example.csv "${tmpdir}/${t}.csv"
|
||||||
|
|
||||||
|
# assertion
|
||||||
|
cp data/example.tsv "${tmpdir}/${t}.assert"
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli import csv "${t}.csv"
|
||||||
|
orcli export tsv "${t} csv" > "${t}.output"
|
||||||
|
|
||||||
|
# test
|
||||||
|
diff -u "${t}.assert" "${t}.output"
|
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="info"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# input
|
||||||
|
cp data/example.csv "${tmpdir}/${t}.csv"
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli import csv "${t}.csv"
|
||||||
|
orcli info "${t} csv"
|
||||||
|
|
||||||
|
# test
|
||||||
|
# grep "${t}" "${t}.output"
|
|
@ -0,0 +1,18 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="list"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# input
|
||||||
|
cp data/example.csv "${tmpdir}/${t}.csv"
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli import csv "${t}.csv"
|
||||||
|
orcli list > "${t}.output"
|
||||||
|
|
||||||
|
# test
|
||||||
|
grep "${t}" "${t}.output"
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="run"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# assertion
|
||||||
|
cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert"
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli run --memory "2000M" --port "3334" << EOF
|
||||||
|
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||||
|
orcli transform "duplicates" "https://git.io/fj5ju"
|
||||||
|
orcli export tsv "duplicates" --output "${t}.output"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# test
|
||||||
|
diff -u "${t}.assert" "${t}.output"
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
t="transform"
|
||||||
|
|
||||||
|
# create tmp directory
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
|
|
||||||
|
# assertion
|
||||||
|
cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert"
|
||||||
|
|
||||||
|
# action
|
||||||
|
cd "${tmpdir}" || exit 1
|
||||||
|
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||||
|
orcli transform "duplicates" "https://git.io/fj5ju"
|
||||||
|
orcli export tsv "duplicates" --output "${t}.output"
|
||||||
|
|
||||||
|
# test
|
||||||
|
diff -u "${t}.assert" "${t}.output"
|
Loading…
Reference in New Issue