tests for file formats

This commit is contained in:
Felix Lohmeier 2020-08-04 13:50:56 +02:00
parent ca359925fe
commit 1011c0903d
56 changed files with 1442 additions and 160 deletions

View File

@ -89,7 +89,7 @@ results=()
for t in tests/*.sh; do for t in tests/*.sh; do
tests+=("${t}") tests+=("${t}")
echo "========================= ${t} =========================" &>> tests.log echo "========================= ${t} =========================" &>> tests.log
${t} "${cmd}" "${version}" &>> tests.log bash ${t} "${cmd}" "${version}" &>> tests.log
results+=(${?}) results+=(${?})
done done
echo echo

4
tests/apply-utf8.sh Executable file → Normal file
View File

@ -31,7 +31,7 @@ cat << "DATA" > "tmp/${t}/${t}.transform"
"newColumnName": "apply", "newColumnName": "apply",
"columnInsertIndex": 2, "columnInsertIndex": 2,
"baseColumnName": "b", "baseColumnName": "b",
"expression": "grel:value.replace('2','٩(͡๏̯͡๏)۶')", "expression": "grel:value.replace('2','')",
"onError": "set-to-blank" "onError": "set-to-blank"
} }
] ]
@ -41,7 +41,7 @@ DATA
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
a b apply c a b apply c
1 2 ٩(͡๏̯͡๏)۶ 3 1 2 3
0 0 0 0 0 0 0 0
$ \ \ ' $ \ \ '
DATA DATA

0
tests/apply.sh Executable file → Normal file
View File

40
tests/create-csv-utf8.sh Normal file
View File

@ -0,0 +1,40 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨ code meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc ⛲"
${cmd} --export "${t} biểu tượng cảm xúc ⛲" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

0
tests/create-csv.sh Executable file → Normal file
View File

53
tests/create-json-utf8.sh Normal file
View File

@ -0,0 +1,53 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.json"
[
{
"⌨": "⛲",
"code": "1F347",
"meaning": "FOUNTAIN"
},
{
"⌨": "⛳",
"code": "1F349",
"meaning": "FLAG IN HOLE"
},
{
"⌨": "⛵",
"code": "1F352",
"meaning": "SAILBOAT"
}
]
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
_ - ⌨ _ - code _ - meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.json"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

53
tests/create-json.sh Normal file
View File

@ -0,0 +1,53 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.json"
[
{
"a": 1,
"b": 2,
"c": 3
},
{
"a": 0,
"b": 0,
"c": 0
},
{
"a": "$",
"b": "\\",
"c": "\""
}
]
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
_ - a _ - b _ - c
1 2 3
0 0 0
$ \ """"
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.json"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

View File

@ -0,0 +1,44 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cp "data/example.ods" "tmp/${t}/${t}.ods"
# ================================= ASSERTION ================================ #
if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨ code meaning Column Column 5 Column 6 Column 7 Column 8
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.ods" --sheets 1
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

48
tests/create-ods.sh Normal file
View File

@ -0,0 +1,48 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cp "data/example.ods" "tmp/${t}/${t}.ods"
#a b c
#1 2 3
#0 0 0
#$ \ '
# ================================= ASSERTION ================================ #
if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
a b c Column Column 5 Column 6 Column 7 Column 8
1.0 2.0 3.0
0.0 0.0 0.0
$ \ '
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.ods"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

40
tests/create-tsv-utf8.sh Normal file
View File

@ -0,0 +1,40 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.tsv"
⌨ code meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.tsv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.csv"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.csv"

4
tests/export-filename.sh → tests/create-tsv.sh Executable file → Normal file
View File

@ -17,6 +17,8 @@ mkdir -p "tmp/${t}"
cat << "DATA" > "tmp/${t}/${t}.tsv" cat << "DATA" > "tmp/${t}/${t}.tsv"
a b c a b c
1 2 3 1 2 3
0 0 0
$ \ '
DATA DATA
# ================================= ASSERTION ================================ # # ================================= ASSERTION ================================ #
@ -24,6 +26,8 @@ DATA
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
a,b,c a,b,c
1,2,3 1,2,3
0,0,0
$,\,'
DATA DATA
# ================================== ACTION ================================== # # ================================== ACTION ================================== #

View File

@ -0,0 +1,81 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.txt"
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
cat << "DATA" > "tmp/${t}/${t}.transform"
[
{
"op": "core/text-transform",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"columnName": "Column 1",
"expression": "grel:value.trim()",
"onError": "keep-original",
"repeat": false,
"repeatCount": 10
},
{
"op": "core/text-transform",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"columnName": "Column 2",
"expression": "grel:value.trim()",
"onError": "keep-original",
"repeat": false,
"repeatCount": 10
},
{
"op": "core/text-transform",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"columnName": "Column 3",
"expression": "grel:value.trim()",
"onError": "keep-original",
"repeat": false,
"repeatCount": 10
}
]
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
Column 1 Column 2 Column 3
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.txt" --columnWidths "6" --columnWidths "6" --columnWidths "60"
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

81
tests/create-txt-fixed.sh Normal file
View File

@ -0,0 +1,81 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.txt"
1 2 3
mon tue wed
$2 $300 $1
DATA
cat << "DATA" > "tmp/${t}/${t}.transform"
[
{
"op": "core/text-transform",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"columnName": "Column 1",
"expression": "grel:value.trim()",
"onError": "keep-original",
"repeat": false,
"repeatCount": 10
},
{
"op": "core/text-transform",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"columnName": "Column 2",
"expression": "grel:value.trim()",
"onError": "keep-original",
"repeat": false,
"repeatCount": 10
},
{
"op": "core/text-transform",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"columnName": "Column 3",
"expression": "grel:value.trim()",
"onError": "keep-original",
"repeat": false,
"repeatCount": 10
}
]
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
Column 1 Column 2 Column 3
1 2 3
mon tue wed
$2 $300 $1
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.txt" --columnWidths "6" --columnWidths "6" --columnWidths "6"
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

39
tests/create-txt.sh Normal file
View File

@ -0,0 +1,39 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.txt"
1 2 3
mon tue wed
$2 $300 $1
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
Column 1
1 2 3
mon tue wed
$2 $300 $1
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.txt"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

View File

@ -0,0 +1,44 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cp "data/example.xls" "tmp/${t}/${t}.xls"
# ================================= ASSERTION ================================ #
if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨ code meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.xls" --sheets 1
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

48
tests/create-xls.sh Normal file
View File

@ -0,0 +1,48 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cp "data/example.xls" "tmp/${t}/${t}.xls"
#a b c
#1 2 3
#0 0 0
#$ \ '
# ================================= ASSERTION ================================ #
if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1.0 2.0 3.0
0.0 0.0 0.0
$ \ '
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.xls"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

View File

@ -0,0 +1,44 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cp "data/example.xlsx" "tmp/${t}/${t}.xlsx"
# ================================= ASSERTION ================================ #
if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨ code meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.xlsx" --sheets 1
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

48
tests/create-xlsx.sh Normal file
View File

@ -0,0 +1,48 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cp "data/example.xlsx" "tmp/${t}/${t}.xlsx"
#a b c
#1 2 3
#0 0 0
#$ \ '
# ================================= ASSERTION ================================ #
if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1.0 2.0 3.0
0.0 0.0 0.0
$ \ '
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.xlsx"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

95
tests/create-xml-utf8.sh Normal file
View File

@ -0,0 +1,95 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.xml"
<?xml version="1.0" encoding="UTF-8"?>
<root>
<record>
<icon>⛲</icon>
<code>1F347</code>
<meaning>FOUNTAIN</meaning>
</record>
<record>
<icon>⛳</icon>
<code>1F349</code>
<meaning>FLAG IN HOLE</meaning>
</record>
<record>
<icon>⛵</icon>
<code>1F352</code>
<meaning>SAILBOAT</meaning>
</record>
</root>
DATA
cat << "DATA" > "tmp/${t}/${t}.transform"
[
{
"op": "core/column-removal",
"columnName": "root"
},
{
"op": "core/column-removal",
"columnName": "root - record"
},
{
"op": "core/row-removal",
"engineConfig": {
"facets": [
{
"type": "list",
"name": "Blank Rows",
"expression": "(filter(row.columnNames,cn,isNonBlank(cells[cn].value)).length()==0).toString()",
"columnName": "",
"invert": false,
"omitBlank": false,
"omitError": false,
"selection": [
{
"v": {
"v": "true",
"l": "true"
}
}
],
"selectBlank": false,
"selectError": false
}
],
"mode": "record-based"
}
}
]
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
root - record - icon root - record - code root - record - meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.xml"
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

95
tests/create-xml.sh Normal file
View File

@ -0,0 +1,95 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.xml"
<?xml version="1.0" encoding="UTF-8"?>
<root>
<record>
<a>1</a>
<b>2</b>
<c>3</c>
</record>
<record>
<a>0</a>
<b>0</b>
<c>0</c>
</record>
<record>
<a>$</a>
<b>\</b>
<c>'</c>
</record>
</root>
DATA
cat << "DATA" > "tmp/${t}/${t}.transform"
[
{
"op": "core/column-removal",
"columnName": "root"
},
{
"op": "core/column-removal",
"columnName": "root - record"
},
{
"op": "core/row-removal",
"engineConfig": {
"facets": [
{
"type": "list",
"name": "Blank Rows",
"expression": "(filter(row.columnNames,cn,isNonBlank(cells[cn].value)).length()==0).toString()",
"columnName": "",
"invert": false,
"omitBlank": false,
"omitError": false,
"selection": [
{
"v": {
"v": "true",
"l": "true"
}
}
],
"selectBlank": false,
"selectError": false
}
],
"mode": "record-based"
}
}
]
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
root - record - a root - record - b root - record - c
1 2 3
0 0 0
$ \ '
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.xml"
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

BIN
tests/data/example.ods Normal file

Binary file not shown.

BIN
tests/data/example.xls Normal file

Binary file not shown.

BIN
tests/data/example.xlsx Normal file

Binary file not shown.

4
tests/delete-utf8.sh Executable file → Normal file
View File

@ -26,8 +26,8 @@ DATA
# ================================== ACTION ================================== # # ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc 🍉" ${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc "
${cmd} --delete "${t} biểu tượng cảm xúc 🍉" ${cmd} --delete "${t} biểu tượng cảm xúc "
${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output" ${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output"
# =================================== TEST =================================== # # =================================== TEST =================================== #

0
tests/delete.sh Executable file → Normal file
View File

0
tests/download.sh Executable file → Normal file
View File

44
tests/export-csv-utf8.sh Normal file
View File

@ -0,0 +1,44 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.tsv"
🔣 code meaning
🍇 1F347 GRAPES
🍉 1F349 WATERMELON
🍒 1F352 CHERRIES
🍓 1F353 STRAWBERRY
🍍 1F34D PINEAPPLE
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
🔣,code,meaning
🍇,1F347,GRAPES
🍉,1F349,WATERMELON
🍒,1F352,CHERRIES
🍓,1F353,STRAWBERRY
🍍,1F34D,PINEAPPLE
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.tsv"
${cmd} --export "${t}" --output "tmp/${t}/${t} biểu tượng cảm xúc 🍉.csv"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t} biểu tượng cảm xúc 🍉.csv"

8
tests/export-filename-utf8.sh → tests/export-csv.sh Executable file → Normal file
View File

@ -17,6 +17,8 @@ mkdir -p "tmp/${t}"
cat << "DATA" > "tmp/${t}/${t}.tsv" cat << "DATA" > "tmp/${t}/${t}.tsv"
a b c a b c
1 2 3 1 2 3
0 0 0
$ \ '
DATA DATA
# ================================= ASSERTION ================================ # # ================================= ASSERTION ================================ #
@ -24,13 +26,15 @@ DATA
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
a,b,c a,b,c
1,2,3 1,2,3
0,0,0
$,\,'
DATA DATA
# ================================== ACTION ================================== # # ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.tsv" ${cmd} --create "tmp/${t}/${t}.tsv"
${cmd} --export "${t}" --output "tmp/${t}/biểu tượng cảm xúc 🍉.csv" ${cmd} --export "${t}" --output "tmp/${t}/${t}.csv"
# =================================== TEST =================================== # # =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/biểu tượng cảm xúc 🍉.csv" diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.csv"

72
tests/export-html-utf8.sh Normal file
View File

@ -0,0 +1,72 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
majorversion="${2%%.*}"
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================= ASSERTION ================================ #
if [[ "$majorversion" = 2 ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
<html>
<head>
<title>export-html-utf8</title>
<meta charset="utf-8" />
</head>
<body>
<table>
<tr><th>⌨</th><th>code</th><th>meaning</th></tr>
<tr><td>&#9970;</td><td>1F347</td><td>FOUNTAIN</td></tr>
<tr><td>&#9971;</td><td>1F349</td><td>FLAG IN HOLE</td></tr>
<tr><td>&#9973;</td><td>1F352</td><td>SAILBOAT</td></tr>
</table>
</body>
</html>
DATA
else
cat << "DATA" > "tmp/${t}/${t}.assert"
<html>
<head>
<title>export-html-utf8</title>
<meta charset="utf-8" />
</head>
<body>
<table>
<tr><th>⌨</th><th>code</th><th>meaning</th></tr>
<tr><td>⛲</td><td>1F347</td><td>FOUNTAIN</td></tr>
<tr><td>⛳</td><td>1F349</td><td>FLAG IN HOLE</td></tr>
<tr><td>⛵</td><td>1F352</td><td>SAILBOAT</td></tr>
</table>
</body>
</html>
DATA
fi
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.html"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.html"

50
tests/export-html.sh Normal file
View File

@ -0,0 +1,50 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
<html>
<head>
<title>export-html</title>
<meta charset="utf-8" />
</head>
<body>
<table>
<tr><th>a</th><th>b</th><th>c</th></tr>
<tr><td>1</td><td>2</td><td>3</td></tr>
<tr><td>0</td><td>0</td><td>0</td></tr>
<tr><td>$</td><td>\</td><td>&apos;</td></tr>
</table>
</body>
</html>
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.html"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.html"

43
tests/export-ods-utf8.sh Normal file
View File

@ -0,0 +1,43 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,"FLAG IN HOLE"
⛵,1F352,SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.ods"
(cd tmp/${t} &&
ssconvert -S ${t}.ods ${t}.csv &&
mv ${t}.csv.1 ${t}.output)
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

47
tests/export-ods.sh Normal file
View File

@ -0,0 +1,47 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
if [[ -z "$(command -v ssconvert 2> /dev/null)" ]] ; then
echo 1>&2 "ERROR: This test requires ssconvert (gnumeric)"
exit 127
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.ods"
(cd tmp/${t} &&
ssconvert -S ${t}.ods ${t}.csv &&
mv ${t}.csv.1 ${t}.output)
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

44
tests/export-tsv-utf8.sh Normal file
View File

@ -0,0 +1,44 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
🔣,code,meaning
🍇,1F347,GRAPES
🍉,1F349,WATERMELON
🍒,1F352,CHERRIES
🍓,1F353,STRAWBERRY
🍍,1F34D,PINEAPPLE
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
🔣 code meaning
🍇 1F347 GRAPES
🍉 1F349 WATERMELON
🍒 1F352 CHERRIES
🍓 1F353 STRAWBERRY
🍍 1F34D PINEAPPLE
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.tsv"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.tsv"

40
tests/export-tsv.sh Normal file
View File

@ -0,0 +1,40 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 2 3
0 0 0
$ \ '
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.tsv"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.tsv"

0
tests/export-utf8.sh Executable file → Normal file
View File

43
tests/export-xls-utf8.sh Normal file
View File

@ -0,0 +1,43 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.xls"
(cd tmp/${t} &&
ssconvert -S ${t}.xls ${t}.csv &&
mv ${t}.csv ${t}.output)
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

17
tests/create-utf8.sh → tests/export-xls.sh Executable file → Normal file
View File

@ -16,20 +16,27 @@ mkdir -p "tmp/${t}"
cat << "DATA" > "tmp/${t}/${t}.csv" cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c a,b,c
🍉,🍒,🍓 1,2,3
0,0,0
$,\,'
DATA DATA
# ================================= ASSERTION ================================ # # ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
a b c a,b,c
🍉 🍒 🍓 1,2,3
0,0,0
$,\,'
DATA DATA
# ================================== ACTION ================================== # # ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc 🍉" ${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t} biểu tượng cảm xúc 🍉" --output "tmp/${t}/${t}.output" ${cmd} --export "${t}" --output "tmp/${t}/${t}.xls"
(cd tmp/${t} &&
ssconvert -S ${t}.xls ${t}.csv &&
mv ${t}.csv ${t}.output)
# =================================== TEST =================================== # # =================================== TEST =================================== #

45
tests/export-xlsx-utf8.sh Normal file
View File

@ -0,0 +1,45 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.xlsx"
(cd tmp/${t} &&
ssconvert -S ${t}.xlsx ${t}.csv &&
mv ${t}.csv ${t}.output)
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

43
tests/export-xlsx.sh Normal file
View File

@ -0,0 +1,43 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.xlsx"
(cd tmp/${t} &&
ssconvert -S ${t}.xlsx ${t}.csv &&
mv ${t}.csv ${t}.output)
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

4
tests/export.sh Executable file → Normal file
View File

@ -17,6 +17,8 @@ mkdir -p "tmp/${t}"
cat << "DATA" > "tmp/${t}/${t}.csv" cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c a,b,c
1,2,3 1,2,3
0,0,0
$,\,'
DATA DATA
# ================================= ASSERTION ================================ # # ================================= ASSERTION ================================ #
@ -24,6 +26,8 @@ DATA
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
a b c a b c
1 2 3 1 2 3
0 0 0
$ \ '
DATA DATA
# ================================== ACTION ================================== # # ================================== ACTION ================================== #

41
tests/format-create.sh Normal file
View File

@ -0,0 +1,41 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
Column 1
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv" --format "line-based"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

40
tests/format-export.sh Normal file
View File

@ -0,0 +1,40 @@
#!/bin/bash
# =============================== ENVIRONMENT ================================ #
if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi
t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"
# =================================== DATA =================================== #
cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert"
a,b,c
1,2,3
0,0,0
$,\,'
DATA
# ================================== ACTION ================================== #
${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --format "csv" > "tmp/${t}/${t}.output"
# =================================== TEST =================================== #
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

144
tests/help.sh Executable file → Normal file
View File

@ -15,154 +15,12 @@ mkdir -p "tmp/${t}"
# ================================= ASSERTION ================================ # # ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
Usage: refine.py [--help | OPTIONS]
Script to provide a command line interface to an OpenRefine server. Script to provide a command line interface to an OpenRefine server.
Options:
-h, --help show this help message and exit
Connection options:
-H 127.0.0.1, --host=127.0.0.1
OpenRefine hostname (default: 127.0.0.1)
-P 3333, --port=3333
OpenRefine port (default: 3333)
Commands:
-c [FILE], --create=[FILE]
Create project from file. The filename ending (e.g.
.csv) defines the input format
(csv,tsv,xml,json,txt,xls,xlsx,ods)
-l, --list List projects
--download=[URL] Download file from URL (e.g. example data). Combine
with --output to specify a filename.
Commands with argument [PROJECTID/PROJECTNAME]:
-d, --delete Delete project
-f [FILE], --apply=[FILE]
Apply JSON rules to OpenRefine project
-E, --export Export project in tsv format to stdout.
-o [FILE], --output=[FILE]
Export project to file. The filename ending (e.g.
.tsv) defines the output format
(csv,tsv,xls,xlsx,html)
--template=[STRING]
Export project with templating. Provide (big) text
string that you enter in the *row template* textfield
in the export/templating menu in the browser app)
--info show project metadata
General options:
--format=FILE_FORMAT
Override file detection (import:
csv,tsv,xml,json,line-based,fixed-width,xls,xlsx,ods;
export: csv,tsv,html,xls,xlsx,ods)
Create options:
--columnWidths=COLUMNWIDTHS
(txt/fixed-width), please provide widths in multiple
arguments, e.g. --columnWidths=7 --columnWidths=5
--encoding=ENCODING
(csv,tsv,txt), please provide short encoding name
(e.g. UTF-8)
--guessCellValueTypes=true/false
(xml,csv,tsv,txt,json, default: false)
--headerLines=HEADERLINES
(csv,tsv,txt/fixed-width,xls,xlsx,ods), default: 1,
default txt/fixed-width: 0
--ignoreLines=IGNORELINES
(csv,tsv,txt,xls,xlsx,ods), default: -1
--includeFileSources=true/false
(all formats), default: false
--limit=LIMIT (all formats), default: -1
--linesPerRow=LINESPERROW
(txt/line-based), default: 1
--processQuotes=true/false
(csv,tsv), default: true
--projectName=PROJECTNAME
(all formats), default: filename
--projectTags=PROJECTTAGS
(all formats), please provide tags in multiple
arguments, e.g. --projectTags=beta
--projectTags=client1
--recordPath=RECORDPATH
(xml,json), please provide path in multiple arguments,
e.g. /collection/record/ should be entered:
--recordPath=collection --recordPath=record, default
xml: root element, default json: _ _
--separator=SEPARATOR
(csv,tsv), default csv: , default tsv: \t
--sheets=SHEETS (xls,xlsx,ods), please provide sheets in multiple
arguments, e.g. --sheets=0 --sheets=1, default: 0
(first sheet)
--skipDataLines=SKIPDATALINES
(csv,tsv,txt,xls,xlsx,ods), default: 0, default line-
based: -1
--storeBlankCellsAsNulls=true/false
(csv,tsv,txt,xls,xlsx,ods), default: true
--storeBlankRows=true/false
(csv,tsv,txt,xls,xlsx,ods), default: true
--storeEmptyStrings=true/false
(xml,json), default: true
--trimStrings=true/false
(xml,json), default: false
Templating options:
--mode=row-based/record-based
engine mode (default: row-based)
--prefix=PREFIX text string that you enter in the *prefix* textfield
in the browser app
--rowSeparator=ROWSEPARATOR
text string that you enter in the *row separator*
textfield in the browser app
--suffix=SUFFIX text string that you enter in the *suffix* textfield
in the browser app
--filterQuery=REGEX
Simple RegEx text filter on filterColumn, e.g. ^12015$
--filterColumn=COLUMNNAME
column name for filterQuery (default: name of first
column)
--facets=FACETS facets config in json format (may be extracted with
browser dev tools in browser app)
--splitToFiles=true/false
will split each row/record into a single file; it
specifies a presumably unique character series for
splitting; --prefix and --suffix will be applied to
all files; filename-prefix can be specified with
--output (default: %Y%m%d)
--suffixById=true/false
enhancement option for --splitToFiles; will generate
filename-suffix from values in key column
Example data:
--download "https://git.io/fj5hF" --output=duplicates.csv
--download "https://git.io/fj5ju" --output=duplicates-deletion.json
Basic commands:
--list # list all projects
--list -H 127.0.0.1 -P 80 # specify hostname and port
--create duplicates.csv # create new project from file
--info "duplicates" # show project metadata
--apply duplicates-deletion.json "duplicates" # apply rules in file to project
--export "duplicates" # export project to terminal in tsv format
--export --output=deduped.xls "duplicates" # export project to file in xls format
--delete "duplicates" # delete project
Some more examples:
--info 1234567890123 # specify project by id
--create example.tsv --encoding=UTF-8
--create example.xml --recordPath=collection --recordPath=record
--create example.json --recordPath=_ --recordPath=_
--create example.xlsx --sheets=0
--create example.ods --sheets=0
Example for Templating Export:
Cf. https://github.com/opencultureconsulting/openrefine-client#advanced-templating
DATA DATA
# ================================== ACTION ================================== # # ================================== ACTION ================================== #
${cmd} --help > "tmp/${t}/${t}.output" ${cmd} --help | sed '3q;d' > "tmp/${t}/${t}.output"
# =================================== TEST =================================== # # =================================== TEST =================================== #

0
tests/info-utf8.sh Executable file → Normal file
View File

0
tests/info.sh Executable file → Normal file
View File

0
tests/list.sh Executable file → Normal file
View File

View File

@ -8,7 +8,8 @@ else
echo 1>&2 "execute tests.sh to run all tests"; exit 1 echo 1>&2 "execute tests.sh to run all tests"; exit 1
fi fi
#if [[ ${2} ]]; then #if [[ ${2} ]]; then
# version="${2##*.}" # version="${2}"
# majorversion="${2%%.*}"
#fi #fi
t="$(basename "${BASH_SOURCE[0]}" .sh)" t="$(basename "${BASH_SOURCE[0]}" .sh)"
@ -42,7 +43,7 @@ DATA
# ================================= ASSERTION ================================ # # ================================= ASSERTION ================================ #
#if [[ "$version" >= 2 ]]; then #if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
a b apply c a b apply c
1 2 TEST 3 1 2 TEST 3

0
tests/templating-filter-utf8.sh Executable file → Normal file
View File

0
tests/templating-filter.sh Executable file → Normal file
View File

0
tests/templating-split-id-utf8.sh Executable file → Normal file
View File

0
tests/templating-split-id.sh Executable file → Normal file
View File

0
tests/templating-split-utf8.sh Executable file → Normal file
View File

0
tests/templating-split.sh Executable file → Normal file
View File

0
tests/templating-utf8.sh Executable file → Normal file
View File

0
tests/templating.sh Executable file → Normal file
View File

5
tests/usage.sh Executable file → Normal file
View File

@ -15,13 +15,12 @@ mkdir -p "tmp/${t}"
# ================================= ASSERTION ================================ # # ================================= ASSERTION ================================ #
cat << "DATA" > "tmp/${t}/${t}.assert" cat << "DATA" > "tmp/${t}/${t}.assert"
Usage: refine.py [--help | OPTIONS] Usage:
DATA DATA
# ================================== ACTION ================================== # # ================================== ACTION ================================== #
${cmd} > "tmp/${t}/${t}.output" ${cmd} | head -n 1 | cut -c 1-6 > "tmp/${t}/${t}.output"
# =================================== TEST =================================== # # =================================== TEST =================================== #