Merge pull request #120 from opencultureconsulting:119-transform-line-breaks-n-in-expression-lead-to-parsing-abort

transform: allow \n in expression
This commit is contained in:
Felix Lohmeier 2023-11-16 22:42:44 +01:00 committed by GitHub
commit 779a6e655b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 89 additions and 10 deletions

View File

@ -1,4 +1,4 @@
# orcli 0.2.0
# orcli 0.2.1
## command help screens

15
orcli
View File

@ -1,5 +1,5 @@
#!/usr/bin/env bash
# This script was generated by bashly 1.1.1 (https://bashly.dannyb.co)
# This script was generated by bashly 1.1.2 (https://bashly.dannyb.co)
# Modifying it manually is not recommended
# :wrapper.bash3_bouncer
@ -2231,7 +2231,8 @@ orcli_transform_command() {
fi
for line in "${jsonlines[@]}"; do
# parse one line/operation into array
declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
declare -A array=$(jq --join-output "${filter}" <<< "$line")
if [[ ! ${array[op]} ]]; then
error "parsing ${files[$i]} failed!"
fi
@ -2259,6 +2260,8 @@ orcli_transform_command() {
unset "array[engineConfig]"
# drop description
unset "array[description]"
# remove line breaks in expression
array[expression]="${array[expression]//$'\n'/}"
# prepare curl options
mapfile -t curloptions < <(for K in "${!array[@]}"; do
echo "--data"
@ -4305,7 +4308,7 @@ orcli_export_jsonl_parse_requirements() {
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
# :command.whitelist_filter
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
printf "%s\n" "--mode must be one of: rows, records" >&2
exit 1
fi
@ -4454,7 +4457,7 @@ orcli_export_tsv_parse_requirements() {
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
# :command.whitelist_filter
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
printf "%s\n" "--mode must be one of: rows, records" >&2
exit 1
fi
@ -4638,7 +4641,7 @@ orcli_export_template_parse_requirements() {
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
# :command.whitelist_filter
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
printf "%s\n" "--mode must be one of: rows, records" >&2
exit 1
fi
@ -4747,7 +4750,7 @@ orcli_run_parse_requirements() {
# :command.initialize
initialize() {
version="0.2.0"
version="0.2.1"
long_usage=''
set -e

View File

@ -1,6 +1,6 @@
name: orcli
help: OpenRefine command-line interface written in Bash
version: 0.2.0
version: 0.2.1
footer: https://github.com/opencultureconsulting/orcli
dependencies:

View File

@ -55,7 +55,8 @@ for i in "${!files[@]}"; do
fi
for line in "${jsonlines[@]}"; do
# parse one line/operation into array
declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
declare -A array=$(jq --join-output "${filter}" <<< "$line")
if [[ ! ${array[op]} ]]; then
error "parsing ${files[$i]} failed!"
fi
@ -83,6 +84,8 @@ for i in "${!files[@]}"; do
unset "array[engineConfig]"
# drop description
unset "array[description]"
# remove line breaks in expression
array[expression]="${array[expression]//$'\n'/}"
# prepare curl options
mapfile -t curloptions < <(for K in "${!array[@]}"; do
echo "--data"

View File

@ -0,0 +1,70 @@
[
{
"op": "core/row-reorder",
"description": "Reorder rows",
"mode": "record-based",
"sorting": {
"criteria": [
{
"errorPosition": 1,
"caseSensitive": false,
"valueType": "string",
"column": "email",
"blankPosition": 2,
"reverse": false
}
]
}
},
{
"op": "core/column-addition",
"description": "Create column count at index 1 based on column email using expression grel:facetCount(value, \"value\", \"email\")",
"engineConfig": {
"mode": "row-based",
"facets": []
},
"newColumnName": "count",
"columnInsertIndex": 1,
"baseColumnName": "email",
"expression": "grel:facetCount(\nvalue,\n \"value\",\n \"email\"\n)",
"onError": "set-to-blank"
},
{
"op": "core/blank-down",
"description": "Blank down cells in column email",
"engineConfig": {
"mode": "row-based",
"facets": []
},
"columnName": "email"
},
{
"op": "core/row-removal",
"description": "Remove rows",
"engineConfig": {
"mode": "row-based",
"facets": [
{
"omitError": false,
"expression": "isBlank(value)",
"selectBlank": false,
"selection": [
{
"v": {
"v": true,
"l": "true"
}
}
],
"selectError": false,
"invert": false,
"name": "email",
"omitBlank": false,
"type": "list",
"columnName": "email"
}
]
}
}
]

View File

@ -9,10 +9,13 @@ trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
# assertion
cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert"
# transform
cp data/duplicates-history.json "${tmpdir}/${t}.history"
# action
cd "${tmpdir}" || exit 1
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
orcli transform "duplicates" "https://git.io/fj5ju"
orcli transform "duplicates" "${t}.history"
orcli export tsv "duplicates" --output "${t}.output"
# test