transform: allow \n in expression
This commit is contained in:
parent
f1ae2ae865
commit
dbc63da7b2
|
@ -1,4 +1,4 @@
|
|||
# orcli 0.2.0
|
||||
# orcli 0.2.1
|
||||
|
||||
## command help screens
|
||||
|
||||
|
|
15
orcli
15
orcli
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
# This script was generated by bashly 1.1.1 (https://bashly.dannyb.co)
|
||||
# This script was generated by bashly 1.1.2 (https://bashly.dannyb.co)
|
||||
# Modifying it manually is not recommended
|
||||
|
||||
# :wrapper.bash3_bouncer
|
||||
|
@ -2231,7 +2231,8 @@ orcli_transform_command() {
|
|||
fi
|
||||
for line in "${jsonlines[@]}"; do
|
||||
# parse one line/operation into array
|
||||
declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
||||
filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
|
||||
declare -A array=$(jq --join-output "${filter}" <<< "$line")
|
||||
if [[ ! ${array[op]} ]]; then
|
||||
error "parsing ${files[$i]} failed!"
|
||||
fi
|
||||
|
@ -2259,6 +2260,8 @@ orcli_transform_command() {
|
|||
unset "array[engineConfig]"
|
||||
# drop description
|
||||
unset "array[description]"
|
||||
# remove line breaks in expression
|
||||
array[expression]="${array[expression]//$'\n'/}"
|
||||
# prepare curl options
|
||||
mapfile -t curloptions < <(for K in "${!array[@]}"; do
|
||||
echo "--data"
|
||||
|
@ -4305,7 +4308,7 @@ orcli_export_jsonl_parse_requirements() {
|
|||
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
||||
|
||||
# :command.whitelist_filter
|
||||
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
|
||||
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
|
||||
printf "%s\n" "--mode must be one of: rows, records" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
@ -4454,7 +4457,7 @@ orcli_export_tsv_parse_requirements() {
|
|||
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
||||
|
||||
# :command.whitelist_filter
|
||||
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
|
||||
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
|
||||
printf "%s\n" "--mode must be one of: rows, records" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
@ -4638,7 +4641,7 @@ orcli_export_template_parse_requirements() {
|
|||
[[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
|
||||
|
||||
# :command.whitelist_filter
|
||||
if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
|
||||
if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
|
||||
printf "%s\n" "--mode must be one of: rows, records" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
@ -4747,7 +4750,7 @@ orcli_run_parse_requirements() {
|
|||
|
||||
# :command.initialize
|
||||
initialize() {
|
||||
version="0.2.0"
|
||||
version="0.2.1"
|
||||
long_usage=''
|
||||
set -e
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
name: orcli
|
||||
help: OpenRefine command-line interface written in Bash
|
||||
version: 0.2.0
|
||||
version: 0.2.1
|
||||
footer: https://github.com/opencultureconsulting/orcli
|
||||
|
||||
dependencies:
|
||||
|
|
|
@ -55,7 +55,8 @@ for i in "${!files[@]}"; do
|
|||
fi
|
||||
for line in "${jsonlines[@]}"; do
|
||||
# parse one line/operation into array
|
||||
declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
|
||||
filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
|
||||
declare -A array=$(jq --join-output "${filter}" <<< "$line")
|
||||
if [[ ! ${array[op]} ]]; then
|
||||
error "parsing ${files[$i]} failed!"
|
||||
fi
|
||||
|
@ -83,6 +84,8 @@ for i in "${!files[@]}"; do
|
|||
unset "array[engineConfig]"
|
||||
# drop description
|
||||
unset "array[description]"
|
||||
# remove line breaks in expression
|
||||
array[expression]="${array[expression]//$'\n'/}"
|
||||
# prepare curl options
|
||||
mapfile -t curloptions < <(for K in "${!array[@]}"; do
|
||||
echo "--data"
|
||||
|
|
|
@ -0,0 +1,70 @@
|
|||
[
|
||||
{
|
||||
"op": "core/row-reorder",
|
||||
"description": "Reorder rows",
|
||||
"mode": "record-based",
|
||||
"sorting": {
|
||||
"criteria": [
|
||||
{
|
||||
"errorPosition": 1,
|
||||
"caseSensitive": false,
|
||||
"valueType": "string",
|
||||
"column": "email",
|
||||
"blankPosition": 2,
|
||||
"reverse": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"op": "core/column-addition",
|
||||
"description": "Create column count at index 1 based on column email using expression grel:facetCount(value, \"value\", \"email\")",
|
||||
"engineConfig": {
|
||||
"mode": "row-based",
|
||||
"facets": []
|
||||
},
|
||||
"newColumnName": "count",
|
||||
"columnInsertIndex": 1,
|
||||
"baseColumnName": "email",
|
||||
"expression": "grel:facetCount(\nvalue,\n \"value\",\n \"email\"\n)",
|
||||
"onError": "set-to-blank"
|
||||
},
|
||||
{
|
||||
"op": "core/blank-down",
|
||||
"description": "Blank down cells in column email",
|
||||
"engineConfig": {
|
||||
"mode": "row-based",
|
||||
"facets": []
|
||||
},
|
||||
"columnName": "email"
|
||||
},
|
||||
{
|
||||
"op": "core/row-removal",
|
||||
"description": "Remove rows",
|
||||
"engineConfig": {
|
||||
"mode": "row-based",
|
||||
"facets": [
|
||||
{
|
||||
"omitError": false,
|
||||
"expression": "isBlank(value)",
|
||||
"selectBlank": false,
|
||||
"selection": [
|
||||
{
|
||||
"v": {
|
||||
"v": true,
|
||||
"l": "true"
|
||||
}
|
||||
}
|
||||
],
|
||||
"selectError": false,
|
||||
"invert": false,
|
||||
"name": "email",
|
||||
"omitBlank": false,
|
||||
"type": "list",
|
||||
"columnName": "email"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -9,10 +9,13 @@ trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
|
|||
# assertion
|
||||
cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert"
|
||||
|
||||
# transform
|
||||
cp data/duplicates-history.json "${tmpdir}/${t}.history"
|
||||
|
||||
# action
|
||||
cd "${tmpdir}" || exit 1
|
||||
orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
|
||||
orcli transform "duplicates" "https://git.io/fj5ju"
|
||||
orcli transform "duplicates" "${t}.history"
|
||||
orcli export tsv "duplicates" --output "${t}.output"
|
||||
|
||||
# test
|
||||
|
|
Loading…
Reference in New Issue