From dbc63da7b20fba2a5c043f44baa6b611f5805328 Mon Sep 17 00:00:00 2001 From: Felix Lohmeier Date: Thu, 16 Nov 2023 21:42:20 +0000 Subject: [PATCH] transform: allow \n in expression --- help/README.md | 2 +- orcli | 15 ++++--- src/bashly.yml | 2 +- src/transform_command.sh | 5 ++- tests/data/duplicates-history.json | 70 ++++++++++++++++++++++++++++++ tests/transform.sh | 5 ++- 6 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 tests/data/duplicates-history.json diff --git a/help/README.md b/help/README.md index 9c5d375..72fbe9e 100644 --- a/help/README.md +++ b/help/README.md @@ -1,4 +1,4 @@ -# orcli 0.2.0 +# orcli 0.2.1 ## command help screens diff --git a/orcli b/orcli index e3033e3..51d5bf6 100755 --- a/orcli +++ b/orcli @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# This script was generated by bashly 1.1.1 (https://bashly.dannyb.co) +# This script was generated by bashly 1.1.2 (https://bashly.dannyb.co) # Modifying it manually is not recommended # :wrapper.bash3_bouncer @@ -2231,7 +2231,8 @@ orcli_transform_command() { fi for line in "${jsonlines[@]}"; do # parse one line/operation into array - declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))" + filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"' + declare -A array=$(jq --join-output "${filter}" <<< "$line") if [[ ! ${array[op]} ]]; then error "parsing ${files[$i]} failed!" fi @@ -2259,6 +2260,8 @@ orcli_transform_command() { unset "array[engineConfig]" # drop description unset "array[description]" + # remove line breaks in expression + array[expression]="${array[expression]//$'\n'/}" # prepare curl options mapfile -t curloptions < <(for K in "${!array[@]}"; do echo "--data" @@ -4305,7 +4308,7 @@ orcli_export_jsonl_parse_requirements() { [[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8" # :command.whitelist_filter - if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then + if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then printf "%s\n" "--mode must be one of: rows, records" >&2 exit 1 fi @@ -4454,7 +4457,7 @@ orcli_export_tsv_parse_requirements() { [[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8" # :command.whitelist_filter - if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then + if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then printf "%s\n" "--mode must be one of: rows, records" >&2 exit 1 fi @@ -4638,7 +4641,7 @@ orcli_export_template_parse_requirements() { [[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8" # :command.whitelist_filter - if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then + if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then printf "%s\n" "--mode must be one of: rows, records" >&2 exit 1 fi @@ -4747,7 +4750,7 @@ orcli_run_parse_requirements() { # :command.initialize initialize() { - version="0.2.0" + version="0.2.1" long_usage='' set -e diff --git a/src/bashly.yml b/src/bashly.yml index ec74453..cd497d4 100644 --- a/src/bashly.yml +++ b/src/bashly.yml @@ -1,6 +1,6 @@ name: orcli help: OpenRefine command-line interface written in Bash -version: 0.2.0 +version: 0.2.1 footer: https://github.com/opencultureconsulting/orcli dependencies: diff --git a/src/transform_command.sh b/src/transform_command.sh index 50d7dd0..3fec410 100644 --- a/src/transform_command.sh +++ b/src/transform_command.sh @@ -55,7 +55,8 @@ for i in "${!files[@]}"; do fi for line in "${jsonlines[@]}"; do # parse one line/operation into array - declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))" + filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"' + declare -A array=$(jq --join-output "${filter}" <<< "$line") if [[ ! ${array[op]} ]]; then error "parsing ${files[$i]} failed!" fi @@ -83,6 +84,8 @@ for i in "${!files[@]}"; do unset "array[engineConfig]" # drop description unset "array[description]" + # remove line breaks in expression + array[expression]="${array[expression]//$'\n'/}" # prepare curl options mapfile -t curloptions < <(for K in "${!array[@]}"; do echo "--data" diff --git a/tests/data/duplicates-history.json b/tests/data/duplicates-history.json new file mode 100644 index 0000000..cea73d9 --- /dev/null +++ b/tests/data/duplicates-history.json @@ -0,0 +1,70 @@ +[ + { + "op": "core/row-reorder", + "description": "Reorder rows", + "mode": "record-based", + "sorting": { + "criteria": [ + { + "errorPosition": 1, + "caseSensitive": false, + "valueType": "string", + "column": "email", + "blankPosition": 2, + "reverse": false + } + ] + } + }, + { + "op": "core/column-addition", + "description": "Create column count at index 1 based on column email using expression grel:facetCount(value, \"value\", \"email\")", + "engineConfig": { + "mode": "row-based", + "facets": [] + }, + "newColumnName": "count", + "columnInsertIndex": 1, + "baseColumnName": "email", + "expression": "grel:facetCount(\nvalue,\n \"value\",\n \"email\"\n)", + "onError": "set-to-blank" + }, + { + "op": "core/blank-down", + "description": "Blank down cells in column email", + "engineConfig": { + "mode": "row-based", + "facets": [] + }, + "columnName": "email" + }, + { + "op": "core/row-removal", + "description": "Remove rows", + "engineConfig": { + "mode": "row-based", + "facets": [ + { + "omitError": false, + "expression": "isBlank(value)", + "selectBlank": false, + "selection": [ + { + "v": { + "v": true, + "l": "true" + } + } + ], + "selectError": false, + "invert": false, + "name": "email", + "omitBlank": false, + "type": "list", + "columnName": "email" + } + ] + } + } + ] + \ No newline at end of file diff --git a/tests/transform.sh b/tests/transform.sh index a6ee492..d260b6f 100644 --- a/tests/transform.sh +++ b/tests/transform.sh @@ -9,10 +9,13 @@ trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15 # assertion cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert" +# transform +cp data/duplicates-history.json "${tmpdir}/${t}.history" + # action cd "${tmpdir}" || exit 1 orcli import csv "https://git.io/fj5hF" --projectName "duplicates" -orcli transform "duplicates" "https://git.io/fj5ju" +orcli transform "duplicates" "${t}.history" orcli export tsv "duplicates" --output "${t}.output" # test