Merge pull request #120 from opencultureconsulting:119-transform-line-breaks-n-in-expression-lead-to-parsing-abort
transform: allow \n in expression
This commit is contained in:
commit
779a6e655b
|
@ -1,4 +1,4 @@
|
||||||
# orcli 0.2.0
|
# orcli 0.2.1
|
||||||
|
|
||||||
## command help screens
|
## command help screens
|
||||||
|
|
||||||
|
|
15
orcli
15
orcli
|
@ -1,5 +1,5 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# This script was generated by bashly 1.1.1 (https://bashly.dannyb.co)
|
# This script was generated by bashly 1.1.2 (https://bashly.dannyb.co)
|
||||||
# Modifying it manually is not recommended
|
# Modifying it manually is not recommended
|
||||||
|
|
||||||
# :wrapper.bash3_bouncer
|
# :wrapper.bash3_bouncer
|
||||||
|
@ -2231,7 +2231,8 @@ orcli_transform_command() {
|
||||||
fi
|
fi
|
||||||
for line in "${jsonlines[@]}"; do
|
for line in "${jsonlines[@]}"; do
|
||||||
# parse one line/operation into array
|
# parse one line/operation into array
|
||||||
declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
|
||||||
|
declare -A array=$(jq --join-output "${filter}" <<< "$line")
|
||||||
if [[ ! ${array[op]} ]]; then
|
if [[ ! ${array[op]} ]]; then
|
||||||
error "parsing ${files[$i]} failed!"
|
error "parsing ${files[$i]} failed!"
|
||||||
fi
|
fi
|
||||||
|
@ -2259,6 +2260,8 @@ orcli_transform_command() {
|
||||||
unset "array[engineConfig]"
|
unset "array[engineConfig]"
|
||||||
# drop description
|
# drop description
|
||||||
unset "array[description]"
|
unset "array[description]"
|
||||||
|
# remove line breaks in expression
|
||||||
|
array[expression]="${array[expression]//$'\n'/}"
|
||||||
# prepare curl options
|
# prepare curl options
|
||||||
mapfile -t curloptions < <(for K in "${!array[@]}"; do
|
mapfile -t curloptions < <(for K in "${!array[@]}"; do
|
||||||
echo "--data"
|
echo "--data"
|
||||||
|
@ -4305,7 +4308,7 @@ orcli_export_jsonl_parse_requirements() {
|
||||||
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
||||||
|
|
||||||
# :command.whitelist_filter
|
# :command.whitelist_filter
|
||||||
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
|
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
|
||||||
printf "%s\n" "--mode must be one of: rows, records" >&2
|
printf "%s\n" "--mode must be one of: rows, records" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
@ -4454,7 +4457,7 @@ orcli_export_tsv_parse_requirements() {
|
||||||
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
||||||
|
|
||||||
# :command.whitelist_filter
|
# :command.whitelist_filter
|
||||||
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
|
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
|
||||||
printf "%s\n" "--mode must be one of: rows, records" >&2
|
printf "%s\n" "--mode must be one of: rows, records" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
@ -4638,7 +4641,7 @@ orcli_export_template_parse_requirements() {
|
||||||
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
||||||
|
|
||||||
# :command.whitelist_filter
|
# :command.whitelist_filter
|
||||||
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
|
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
|
||||||
printf "%s\n" "--mode must be one of: rows, records" >&2
|
printf "%s\n" "--mode must be one of: rows, records" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
@ -4747,7 +4750,7 @@ orcli_run_parse_requirements() {
|
||||||
|
|
||||||
# :command.initialize
|
# :command.initialize
|
||||||
initialize() {
|
initialize() {
|
||||||
version="0.2.0"
|
version="0.2.1"
|
||||||
long_usage=''
|
long_usage=''
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
name: orcli
|
name: orcli
|
||||||
help: OpenRefine command-line interface written in Bash
|
help: OpenRefine command-line interface written in Bash
|
||||||
version: 0.2.0
|
version: 0.2.1
|
||||||
footer: https://github.com/opencultureconsulting/orcli
|
footer: https://github.com/opencultureconsulting/orcli
|
||||||
|
|
||||||
dependencies:
|
dependencies:
|
||||||
|
|
|
@ -55,7 +55,8 @@ for i in "${!files[@]}"; do
|
||||||
fi
|
fi
|
||||||
for line in "${jsonlines[@]}"; do
|
for line in "${jsonlines[@]}"; do
|
||||||
# parse one line/operation into array
|
# parse one line/operation into array
|
||||||
declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
|
||||||
|
declare -A array=$(jq --join-output "${filter}" <<< "$line")
|
||||||
if [[ ! ${array[op]} ]]; then
|
if [[ ! ${array[op]} ]]; then
|
||||||
error "parsing ${files[$i]} failed!"
|
error "parsing ${files[$i]} failed!"
|
||||||
fi
|
fi
|
||||||
|
@ -83,6 +84,8 @@ for i in "${!files[@]}"; do
|
||||||
unset "array[engineConfig]"
|
unset "array[engineConfig]"
|
||||||
# drop description
|
# drop description
|
||||||
unset "array[description]"
|
unset "array[description]"
|
||||||
|
# remove line breaks in expression
|
||||||
|
array[expression]="${array[expression]//$'\n'/}"
|
||||||
# prepare curl options
|
# prepare curl options
|
||||||
mapfile -t curloptions < <(for K in "${!array[@]}"; do
|
mapfile -t curloptions < <(for K in "${!array[@]}"; do
|
||||||
echo "--data"
|
echo "--data"
|
||||||
|
|
|
@ -0,0 +1,70 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"op": "core/row-reorder",
|
||||||
|
"description": "Reorder rows",
|
||||||
|
"mode": "record-based",
|
||||||
|
"sorting": {
|
||||||
|
"criteria": [
|
||||||
|
{
|
||||||
|
"errorPosition": 1,
|
||||||
|
"caseSensitive": false,
|
||||||
|
"valueType": "string",
|
||||||
|
"column": "email",
|
||||||
|
"blankPosition": 2,
|
||||||
|
"reverse": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"op": "core/column-addition",
|
||||||
|
"description": "Create column count at index 1 based on column email using expression grel:facetCount(value, \"value\", \"email\")",
|
||||||
|
"engineConfig": {
|
||||||
|
"mode": "row-based",
|
||||||
|
"facets": []
|
||||||
|
},
|
||||||
|
"newColumnName": "count",
|
||||||
|
"columnInsertIndex": 1,
|
||||||
|
"baseColumnName": "email",
|
||||||
|
"expression": "grel:facetCount(\nvalue,\n \"value\",\n \"email\"\n)",
|
||||||
|
"onError": "set-to-blank"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"op": "core/blank-down",
|
||||||
|
"description": "Blank down cells in column email",
|
||||||
|
"engineConfig": {
|
||||||
|
"mode": "row-based",
|
||||||
|
"facets": []
|
||||||
|
},
|
||||||
|
"columnName": "email"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"op": "core/row-removal",
|
||||||
|
"description": "Remove rows",
|
||||||
|
"engineConfig": {
|
||||||
|
"mode": "row-based",
|
||||||
|
"facets": [
|
||||||
|
{
|
||||||
|
"omitError": false,
|
||||||
|
"expression": "isBlank(value)",
|
||||||
|
"selectBlank": false,
|
||||||
|
"selection": [
|
||||||
|
{
|
||||||
|
"v": {
|
||||||
|
"v": true,
|
||||||
|
"l": "true"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"selectError": false,
|
||||||
|
"invert": false,
|
||||||
|
"name": "email",
|
||||||
|
"omitBlank": false,
|
||||||
|
"type": "list",
|
||||||
|
"columnName": "email"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
|
@ -9,10 +9,13 @@ trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
||||||
# assertion
|
# assertion
|
||||||
cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert"
|
cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert"
|
||||||
|
|
||||||
|
# transform
|
||||||
|
cp data/duplicates-history.json "${tmpdir}/${t}.history"
|
||||||
|
|
||||||
# action
|
# action
|
||||||
cd "${tmpdir}" || exit 1
|
cd "${tmpdir}" || exit 1
|
||||||
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||||
orcli transform "duplicates" "https://git.io/fj5ju"
|
orcli transform "duplicates" "${t}.history"
|
||||||
orcli export tsv "duplicates" --output "${t}.output"
|
orcli export tsv "duplicates" --output "${t}.output"
|
||||||
|
|
||||||
# test
|
# test
|
||||||
|
|
Loading…
Reference in New Issue