Browse Source

add functional tests

all CLI options
replaces manual tests with jupyter notebook
tags/0.3.10
Felix Lohmeier 3 months ago
parent
commit
82da3f7b4e
83 changed files with 3763 additions and 3650 deletions
  1. +4
    -1
      .gitignore
  2. +8
    -6
      README.md
  3. +123
    -0
      tests-cli.sh
  4. +57
    -0
      tests/apply-utf8.sh
  5. +57
    -0
      tests/apply.sh
  6. +0
    -819
      tests/cli_bash.ipynb
  7. +0
    -2824
      tests/cli_python2.ipynb
  8. +41
    -0
      tests/create-csv-encoding.sh
  9. +40
    -0
      tests/create-csv-guessCellValueTypes.sh
  10. +41
    -0
      tests/create-csv-headerLines.sh
  11. +39
    -0
      tests/create-csv-ignoreLines.sh
  12. +39
    -0
      tests/create-csv-limit.sh
  13. +40
    -0
      tests/create-csv-processQuotes.sh
  14. +45
    -0
      tests/create-csv-projectTags.sh
  15. +40
    -0
      tests/create-csv-separator.sh
  16. +39
    -0
      tests/create-csv-skipDataLines.sh
  17. +58
    -0
      tests/create-csv-storeBlankCellsAsNulls.sh
  18. +39
    -0
      tests/create-csv-storeBlankRows.sh
  19. +40
    -0
      tests/create-csv-utf8.sh
  20. +40
    -0
      tests/create-csv.sh
  21. +55
    -0
      tests/create-json-recordPath.sh
  22. +52
    -0
      tests/create-json-storeEmptyStrings.sh
  23. +62
    -0
      tests/create-json-trimStrings.sh
  24. +53
    -0
      tests/create-json-utf8.sh
  25. +53
    -0
      tests/create-json.sh
  26. +44
    -0
      tests/create-ods-sheets-utf8.sh
  27. +48
    -0
      tests/create-ods.sh
  28. +40
    -0
      tests/create-tsv-utf8.sh
  29. +40
    -0
      tests/create-tsv.sh
  30. +80
    -0
      tests/create-txt-fixed-width-headerLines.sh
  31. +81
    -0
      tests/create-txt-fixed-width-utf8.sh
  32. +81
    -0
      tests/create-txt-fixed-width.sh
  33. +39
    -0
      tests/create-txt-linesPerRow.sh
  34. +39
    -0
      tests/create-txt.sh
  35. +44
    -0
      tests/create-xls-sheets-utf8.sh
  36. +48
    -0
      tests/create-xls.sh
  37. +44
    -0
      tests/create-xlsx-sheets-utf8.sh
  38. +48
    -0
      tests/create-xlsx.sh
  39. +91
    -0
      tests/create-xml-recordPath.sh
  40. +95
    -0
      tests/create-xml-utf8.sh
  41. +95
    -0
      tests/create-xml.sh
  42. +44
    -0
      tests/create-zip-includeFileSources.sh
  43. +44
    -0
      tests/create-zip.sh
  44. BIN
      tests/data/example.ods
  45. BIN
      tests/data/example.xls
  46. BIN
      tests/data/example.xlsx
  47. +35
    -0
      tests/delete-utf8.sh
  48. +35
    -0
      tests/delete.sh
  49. +21
    -0
      tests/download.sh
  50. +44
    -0
      tests/export-csv-utf8.sh
  51. +40
    -0
      tests/export-csv.sh
  52. +72
    -0
      tests/export-html-utf8.sh
  53. +50
    -0
      tests/export-html.sh
  54. +43
    -0
      tests/export-ods-utf8.sh
  55. +47
    -0
      tests/export-ods.sh
  56. +44
    -0
      tests/export-tsv-utf8.sh
  57. +40
    -0
      tests/export-tsv.sh
  58. +44
    -0
      tests/export-utf8.sh
  59. +43
    -0
      tests/export-xls-utf8.sh
  60. +43
    -0
      tests/export-xls.sh
  61. +45
    -0
      tests/export-xlsx-utf8.sh
  62. +43
    -0
      tests/export-xlsx.sh
  63. +40
    -0
      tests/export.sh
  64. +40
    -0
      tests/format-create-separator.sh
  65. +41
    -0
      tests/format-create.sh
  66. +40
    -0
      tests/format-export-output.sh
  67. +40
    -0
      tests/format-export.sh
  68. +27
    -0
      tests/help.sh
  69. +39
    -0
      tests/info-utf8.sh
  70. +35
    -0
      tests/info.sh
  71. +35
    -0
      tests/list-utf8.sh
  72. +35
    -0
      tests/list.sh
  73. +58
    -0
      tests/template-facets.sh
  74. +54
    -0
      tests/template-filterQuery-utf8.sh
  75. +59
    -0
      tests/template-filterQuery.sh
  76. +58
    -0
      tests/template-splitToFiles-mode.sh
  77. +52
    -0
      tests/template-splitToFiles-suffixById-utf8.sh
  78. +58
    -0
      tests/template-splitToFiles-suffixById.sh
  79. +51
    -0
      tests/template-splitToFiles-utf8.sh
  80. +57
    -0
      tests/template-splitToFiles.sh
  81. +54
    -0
      tests/template-utf8.sh
  82. +64
    -0
      tests/template.sh
  83. +27
    -0
      tests/usage.sh

+ 4
- 1
.gitignore View File

@@ -4,4 +4,7 @@ dist
.*
openrefine_client.egg-info
refine.spec
README.html
openrefine-2.*
openrefine-3.*
openrefine-client_*
tests-cli.log

+ 8
- 6
README.md View File

@@ -679,11 +679,12 @@ There is also a script that uses docker images to run the unit tests with differ

Note to myself: When releasing a new version...

1. Run tests
1. Run functional tests

```sh
./tests.sh -a
jupyter notebook tests/cli_python2.ipynb
for v in 2.7 2.8 3.0 3.1 3.2; do
./tests-cli.sh $v
done
```

2. Make final changes in Git
@@ -710,11 +711,12 @@ Note to myself: When releasing a new version...
python2 -m PyInstaller --onefile refine.py --hidden-import google.refine.__main__
```

4. Run test with Linux executable
4. Run functional tests with Linux executable

```sh
./tests.sh -a
jupyter notebook tests/cli_bash.ipynb
for v in 2.7 2.8 3.0 3.1 3.2; do
./tests-cli.sh $v openrefine-client_0-3-7_linux
done
```

5. Create release in GitHub


+ 123
- 0
tests-cli.sh View File

@@ -0,0 +1,123 @@
#!/bin/bash
# Script for running functional tests against the CLI

# Copyright (c) 2011 Paul Makepeace, Real Programmers. All rights reserved.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>

# ================================== CONFIG ================================== #

cd "${BASH_SOURCE%/*}/" || exit 1

port=3334

if [[ ${1} ]]; then
version="${1}"
else
version="3.2"
fi
refine="openrefine-${version}/refine"

if [[ ${2} ]]; then
client="$(readlink -e "${2}")"
else
client="python2 $(readlink -e refine.py)"
fi
cmd="${client} -H localhost -P ${port}"

if [[ ${3} ]]; then
filename="${3%%.*}"
else
filename=""
fi
cmd="${client} -H localhost -P ${port}"

# =============================== REQUIREMENTS =============================== #

# check existence of java and cURL
if [[ -z "$(command -v java 2> /dev/null)" ]] ; then
echo 1>&2 "ERROR: OpenRefine requires JAVA runtime environment (jre)" \
"https://openjdk.java.net/install/"
exit 1
fi
if [[ -z "$(command -v curl 2> /dev/null)" ]] ; then
echo 1>&2 "ERROR: This shell script requires cURL" \
"https://curl.haxx.se/download.html"
exit 1
fi
# download OpenRefine
if [[ -z "$(readlink -e "${refine}")" ]]; then
echo "Download OpenRefine ${version}..."
mkdir -p "$(dirname "${refine}")"
curl -L --output openrefine.tar.gz \
"https://github.com/OpenRefine/OpenRefine/releases/download/${version}/openrefine-linux-${version}.tar.gz"
echo "Install OpenRefine ${version} in subdirectory $(dirname "${refine}")..."
tar -xzf openrefine.tar.gz -C "$(dirname "${refine}")" --strip 1 --totals
rm -f openrefine.tar.gz
# do not try to open OpenRefine in browser
sed -i '$ a JAVA_OPTIONS=-Drefine.headless=true' \
"$(dirname "${refine}")"/refine.ini
# set autosave period from 5 minutes to 25 hours
sed -i 's/#REFINE_AUTOSAVE_PERIOD=60/REFINE_AUTOSAVE_PERIOD=1500/' \
"$(dirname "${refine}")"/refine.ini
echo
fi

# ================================== SETUP =================================== #

dir="$(readlink -f "tests/tmp")"
mkdir -p "${dir}"
rm -f tests-cli.log

echo "start OpenRefine ${version}..."
${refine} -v warn -p ${port} -d "${dir}" &>> tests-cli.log &
pid_server=${!}
timeout 30s bash -c "until curl -s 'http://localhost:3334' \
| cat | grep -q -o 'OpenRefine' ; do sleep 1; done" \
|| error "starting OpenRefine server failed!"
echo

# ================================== TESTS =================================== #

echo "running tests, please wait..."
tests=()
results=()
for t in tests/*${filename}*.sh; do
tests+=("${t}")
echo "======================= ${t} =======================" &>> tests-cli.log
bash "${t}" "${cmd}" "${version}" &>> tests-cli.log
results+=(${?})
done
echo

# ================================= TEARDOWN ================================= #

echo "cleanup..."
{ kill -9 "${pid_server}" && wait "${pid_server}"; } 2>/dev/null
rm -rf "${dir}"
echo

# ================================= SUMMARY ================================== #

printf "%s\t%s\n" "code" "test"
printf "%s\t%s\n" "----" "----------------"
for i in "${!tests[@]}"; do
printf "%s\t%s\n" "${results[$i]}" "${tests[$i]}"
done
echo
if [[ " ${results[*]} " =~ [1-9] ]]; then
echo "failed tests! check tests-cli.log for debugging"; echo
else
echo "all tests passed!"; echo
fi

+ 57
- 0
tests/apply-utf8.sh View File

@@ -0,0 +1,57 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

cat << "DATA" > "tmp/${t}/${t}.transform"
[
{
"op": "core/column-addition",
"engineConfig": {
"mode": "row-based"
},
"newColumnName": "apply",
"columnInsertIndex": 2,
"baseColumnName": "b",
"expression": "grel:value.replace('2','⛲')",
"onError": "set-to-blank"
}
]
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b apply c
1 2 ⛲ 3
0 0 0 0
$ \ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 57
- 0
tests/apply.sh View File

@@ -0,0 +1,57 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

cat << "DATA" > "tmp/${t}/${t}.transform"
[
{
"op": "core/column-addition",
"engineConfig": {
"mode": "row-based"
},
"newColumnName": "apply",
"columnInsertIndex": 2,
"baseColumnName": "b",
"expression": "grel:value.replace('2','TEST')",
"onError": "set-to-blank"
}
]
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b apply c
1 2 TEST 3
0 0 0 0
$ \ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 0
- 819
tests/cli_bash.ipynb View File

@@ -1,819 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Test executable in a Linux Bash environment"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Install\n",
"\n",
"This notebook requires a [Bash kernel](https://github.com/takluyver/bash_kernel) environment and an OpenRefine server running at http://127.0.0.1:3333."
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/tmp/20190822_013937\n"
]
}
],
"source": [
"workspace=$(date +%Y%m%d_%H%M%S)\n",
"mkdir -p /tmp/$workspace\n",
"cp -r data /tmp/$workspace\n",
"cd /tmp/$workspace && pwd"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux:\n",
"2019-08-22 01:39:40 ERROR 404: Not Found.\n"
]
}
],
"source": [
"wget -nv https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux -O openrefine-client\n",
"chmod +x openrefine-client"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## README.MD"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Download"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Download to file duplicates.csv complete\n"
]
}
],
"source": [
"./openrefine-client --download \"https://git.io/fj5hF\" --output=duplicates.csv"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Create"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"id: 2334935475634\n",
"rows: 10\n"
]
}
],
"source": [
"./openrefine-client --create duplicates.csv"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### List"
]
},
{
"cell_type": "code",
"execution_count": 53,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" 2334935475634: duplicates\n"
]
}
],
"source": [
"./openrefine-client --list"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Info"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" id: 2334935475634\n",
" url: http://127.0.0.1:3333/project?project=2334935475634\n",
" name: duplicates\n",
" modified: 2019-08-21T23:40:30Z\n",
" created: 2019-08-21T23:40:30Z\n",
" rowCount: 10\n",
"importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n",
" column 001: email\n",
" column 002: name\n",
" column 003: state\n",
" column 004: gender\n",
" column 005: purchase\n"
]
}
],
"source": [
"./openrefine-client --info \"duplicates\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Export"
]
},
{
"cell_type": "code",
"execution_count": 55,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"email\tname\tstate\tgender\tpurchase\n",
"danny.baron@example1.com\tDanny Baron\tCA\tM\tTV\n",
"melanie.white@example2.edu\tMelanie White\tNC\tF\tiPhone\n",
"danny.baron@example1.com\tD. Baron\tCA\tM\tWinter jacket\n",
"ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\n",
"arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\n",
"danny.baron@example1.com\tDaniel Baron\tCA\tM\tBike\n",
"jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\n",
"melanie.white@example2.edu\tMelanie White\tNC\tF\tiPad\n",
"ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\n",
"arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\n"
]
}
],
"source": [
"./openrefine-client --export \"duplicates\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Apply"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Download to file duplicates-deletion.json complete\n"
]
}
],
"source": [
"./openrefine-client --download \"https://git.io/fj5ju\" --output=duplicates-deletion.json"
]
},
{
"cell_type": "code",
"execution_count": 57,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"File duplicates-deletion.json has been successfully applied to project 2334935475634\n"
]
}
],
"source": [
"./openrefine-client --apply duplicates-deletion.json \"duplicates\""
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"email\tcount\tname\tstate\tgender\tpurchase\n",
"arthur.duff@example4.com\t2\tArthur Duff\tOR\tM\tDining table\n",
"ben.morisson@example6.org\t1\tBen Morisson\tFL\tM\tAmplifier\n",
"ben.tyler@example3.org\t1\tBen Tyler\tNV\tM\tFlashlight\n",
"danny.baron@example1.com\t3\tDanny Baron\tCA\tM\tTV\n",
"jean.griffith@example5.org\t1\tJean Griffith\tWA\tF\tPower drill\n",
"melanie.white@example2.edu\t2\tMelanie White\tNC\tF\tiPhone\n"
]
}
],
"source": [
"./openrefine-client --export \"duplicates\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Export XLS"
]
},
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Export to file deduped.xls complete\n"
]
}
],
"source": [
"./openrefine-client --export \"duplicates\" --output deduped.xls"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Delete"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Project 2334935475634 has been successfully deleted\n"
]
}
],
"source": [
"./openrefine-client --delete \"duplicates\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Templating"
]
},
{
"cell_type": "code",
"execution_count": 61,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"id: 1633409429491\n",
"rows: 10\n"
]
}
],
"source": [
"./openrefine-client --create duplicates.csv --projectName=advanced"
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{ \"events\" : [\n",
" { \"name\" : \"Melanie White\", \"purchase\" : \"iPhone\" },\n",
" { \"name\" : \"Jean Griffith\", \"purchase\" : \"Power drill\" },\n",
" { \"name\" : \"Melanie White\", \"purchase\" : \"iPad\" }\n",
"] }"
]
}
],
"source": [
"./openrefine-client --export \"advanced\" \\\n",
"--prefix='{ \"events\" : [\n",
"' \\\n",
"--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n",
"--rowSeparator=',\n",
"' \\\n",
"--suffix='\n",
"] }' \\\n",
"--filterQuery='^F$' \\\n",
"--filterColumn='gender'"
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Export to files complete. Last file: advanced_3.json\n"
]
}
],
"source": [
"./openrefine-client --export \"advanced\" \\\n",
"--prefix='{ \"events\" : [\n",
"' \\\n",
"--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n",
"--rowSeparator=',\n",
"' \\\n",
"--suffix='\n",
"] }' \\\n",
"--filterQuery='^F$' \\\n",
"--filterColumn='gender' \\\n",
"--output=advanced.json \\\n",
"--splitToFiles=true"
]
},
{
"cell_type": "code",
"execution_count": 64,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Export to files complete. Last file: advanced_melanie.white@example2.edu.json\n"
]
}
],
"source": [
"./openrefine-client --export \"advanced\" \\\n",
"--prefix='{ \"events\" : [\n",
"' \\\n",
"--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n",
"--rowSeparator=',\n",
"' \\\n",
"--suffix='\n",
"] }' \\\n",
"--filterQuery='^F$' \\\n",
"--filterColumn='gender' \\\n",
"--output=advanced.json \\\n",
"--splitToFiles=true \\\n",
"--suffixById=true"
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"advanced_1.json \u001b[0m\u001b[38;5;33mdata\u001b[0m\n",
"advanced_2.json deduped.xls\n",
"advanced_3.json duplicates.csv\n",
"advanced_jean.griffith@example5.org.json duplicates-deletion.json\n",
"advanced_melanie.white@example2.edu.json \u001b[38;5;40mopenrefine-client\u001b[0m\n"
]
}
],
"source": [
"ls"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Delete"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Project 1633409429491 has been successfully deleted\n"
]
}
],
"source": [
"./openrefine-client --delete \"advanced\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Unicode"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### fruits"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"id: 2280962953279\n",
"rows: 5\n",
" id: 2280962953279\n",
" url: http://127.0.0.1:3333/project?project=2280962953279\n",
" name: evil-fruits\n",
" modified: 2019-08-21T23:40:43Z\n",
" created: 2019-08-21T23:40:43Z\n",
" rowCount: 5\n",
"importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/evil-fruits.tsv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'evil-fruits', u'processQuotes': True, u'limit': -1, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n",
" column 001: 🔣\n",
" column 002: code\n",
" column 003: meaning\n",
"🔣\tcode\tmeaning\n",
"🍇\t1F347\tGRAPES\n",
"🍉\t1F349\tWATERMELON\n",
"🍒\t1F352\tCHERRIES\n",
"🍓\t1F353\tSTRAWBERRY\n",
"🍍\t1F34D\tPINEAPPLE\n"
]
}
],
"source": [
"./openrefine-client --create data/cli/evil-fruits.tsv\n",
"./openrefine-client --info \"evil-fruits\"\n",
"./openrefine-client --export \"evil-fruits\""
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Export to file emojis.csv complete\n",
"🔣,code,meaning\n",
"🍇,1F347,GRAPES\n",
"🍉,1F349,WATERMELON\n",
"🍒,1F352,CHERRIES\n",
"🍓,1F353,STRAWBERRY\n",
"🍍,1F34D,PINEAPPLE\n"
]
}
],
"source": [
"./openrefine-client --export \"evil-fruits\" --output emojis.csv\n",
"cat emojis.csv"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{ \"emojis\" : [\n",
" { \"symbol\" : \"🍇\", \"meaning\" : \"GRAPES\" },\n",
" { \"symbol\" : \"🍉\", \"meaning\" : \"WATERMELON\" },\n",
" { \"symbol\" : \"🍍\", \"meaning\" : \"PINEAPPLE\" }\n",
"] }"
]
}
],
"source": [
"./openrefine-client --export \"evil-fruits\" \\\n",
"--prefix='{ \"emojis\" : [\n",
"' \\\n",
"--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n",
"--rowSeparator=',\n",
"' \\\n",
"--suffix='\n",
"] }' \\\n",
"--filterQuery='^1F34' \\\n",
"--filterColumn='code'"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Export to files complete. Last file: trái cây_3.json\n"
]
}
],
"source": [
"./openrefine-client --export \"evil-fruits\" \\\n",
"--prefix='{ \"emojis\" : [\n",
"' \\\n",
"--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n",
"--rowSeparator=',\n",
"' \\\n",
"--suffix='\n",
"] }' \\\n",
"--filterQuery='^1F34' \\\n",
"--filterColumn='code' \\\n",
"--output='trái cây.json' \\\n",
"--splitToFiles=true"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Export to files complete. Last file: trái cây_🍍.json\n"
]
}
],
"source": [
"./openrefine-client --export \"evil-fruits\" \\\n",
"--prefix='{ \"emojis\" : [\n",
"' \\\n",
"--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n",
"--rowSeparator=',\n",
"' \\\n",
"--suffix='\n",
"] }' \\\n",
"--filterQuery='^1F34' \\\n",
"--filterColumn='code' \\\n",
"--output='trái cây.json' \\\n",
"--splitToFiles=true \\\n",
"--suffixById=true"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" advanced_1.json emojis.csv\n",
" advanced_2.json \u001b[0m\u001b[38;5;40mopenrefine-client\u001b[0m\n",
" advanced_3.json 'trái cây_1.json'\n",
" advanced_jean.griffith@example5.org.json 'trái cây_2.json'\n",
" advanced_melanie.white@example2.edu.json 'trái cây_3.json'\n",
" \u001b[38;5;33mdata\u001b[0m 'trái cây_🍇.json'\n",
" deduped.xls 'trái cây_🍉.json'\n",
" duplicates.csv 'trái cây_🍍.json'\n",
" duplicates-deletion.json\n"
]
}
],
"source": [
"ls"
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Project 2280962953279 has been successfully deleted\n"
]
}
],
"source": [
"./openrefine-client --delete \"evil-fruits\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### emoji-data"
]
},
{
"cell_type": "code",
"execution_count": 74,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"id: 2019865211741\n",
"rows: 20\n",
" id: 2019865211741\n",
" url: http://127.0.0.1:3333/project?project=2019865211741\n",
" name: dữ liệu biểu tượng cảm xúc\n",
" modified: 2019-08-21T23:41:06Z\n",
" created: 2019-08-21T23:41:06Z\n",
" rowCount: 20\n",
"importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/d\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac.txt', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'd\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac', u'processQuotes': True, u'skipDataLines': 34, u'limit': 20, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False, u'headerLines': 0}]\n",
" column 001: Column 1\n",
" column 002: Column 2\n",
" column 003: Column 3\n",
" column 004: Column 4\n",
" column 005: Column 5\n",
" column 006: Column 6\n",
"Column 1\tColumn 2\tColumn 3\tColumn 4\tColumn 5\tColumn 6\n",
"00A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (©) COPYRIGHT SIGN\n",
"00AE ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (®) REGISTERED SIGN\n",
"203C ;\ttext ;\tL1 ;\tnone ;\ta j\t# V1.1 (‼) DOUBLE EXCLAMATION MARK\n",
"2049 ;\ttext ;\tL1 ;\tnone ;\ta j\t# V3.0 (⁉) EXCLAMATION QUESTION MARK\n",
"2122 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (™) TRADE MARK SIGN\n",
"2139 ;\ttext ;\tL1 ;\tnone ;\tj\t# V3.0 (ℹ) INFORMATION SOURCE\n",
"2194 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↔) LEFT RIGHT ARROW\n",
"2195 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↕) UP DOWN ARROW\n",
"2196 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↖) NORTH WEST ARROW\n",
"2197 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↗) NORTH EAST ARROW\n",
"2198 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↘) SOUTH EAST ARROW\n",
"2199 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↙) SOUTH WEST ARROW\n",
"21A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↩) LEFTWARDS ARROW WITH HOOK\n",
"21AA ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↪) RIGHTWARDS ARROW WITH HOOK\n",
"231A ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌚) WATCH\n",
"231B ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌛) HOURGLASS\n",
"2328 ;\ttext ;\tL2 ;\tnone ;\tx\t# V1.1 (⌨) KEYBOARD\n",
"23CF ;\ttext ;\tL2 ;\tnone ;\tx\t# V4.0 (⏏) EJECT SYMBOL\n",
"23E9 ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏩) BLACK RIGHT-POINTING DOUBLE TRIANGLE\n",
"23EA ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏪) BLACK LEFT-POINTING DOUBLE TRIANGLE\n"
]
}
],
"source": [
"./openrefine-client --create \"data/cli/dữ liệu biểu tượng cảm xúc.txt\" \\\n",
"--format=tsv \\\n",
"--headerLines=0 \\\n",
"--skipDataLines=34 \\\n",
"--limit=20\n",
"./openrefine-client --info \"dữ liệu biểu tượng cảm xúc\"\n",
"./openrefine-client --export \"dữ liệu biểu tượng cảm xúc\""
]
},
{
"cell_type": "code",
"execution_count": 75,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" 2019865211741: dữ liệu biểu tượng cảm xúc\n"
]
}
],
"source": [
"./openrefine-client --list"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Delete"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Project 2019865211741 has been successfully deleted\n"
]
}
],
"source": [
"./openrefine-client --delete \"dữ liệu biểu tượng cảm xúc\""
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Bash",
"language": "bash",
"name": "bash"
},
"language_info": {
"codemirror_mode": "shell",
"file_extension": ".sh",
"mimetype": "text/x-sh",
"name": "bash"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

+ 0
- 2824
tests/cli_python2.ipynb
File diff suppressed because it is too large
View File


+ 41
- 0
tests/create-csv-encoding.sh View File

@@ -0,0 +1,41 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}-utf8.csv"
a,b,c
1,2,3
ä,é,ß
$,\,'
DATA
iconv -f UTF-8 -t ISO-8859-1 "tmp/${t}/${t}-utf8.csv" > "tmp/${t}/${t}.csv"

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 2 3
ä é ß
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --encoding "ISO-8859-1"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 40
- 0
tests/create-csv-guessCellValueTypes.sh View File

@@ -0,0 +1,40 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
01,02,03
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 2 3
1 2 3
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --guessCellValueTypes "true"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 41
- 0
tests/create-csv-headerLines.sh View File

@@ -0,0 +1,41 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
Column 1 Column 2 Column 3
a b c
1 2 3
0 0 0
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --headerLines "0"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 39
- 0
tests/create-csv-ignoreLines.sh View File

@@ -0,0 +1,39 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
1 2 3
0 0 0
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --ignoreLines "1"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 39
- 0
tests/create-csv-limit.sh View File

@@ -0,0 +1,39 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 2 3
0 0 0
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --limit "2"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 40
- 0
tests/create-csv-processQuotes.sh View File

@@ -0,0 +1,40 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,"2,0",3
0,0,0
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c Column 4
1 2 0 3
0 0 0
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --processQuotes "false"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 45
- 0
tests/create-csv-projectTags.sh View File

@@ -0,0 +1,45 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

# ================================= ASSERTION ================================ #

if [[ "${version:0:1}" = "2" ]]; then
echo "projectTags were introduced in OpenRefine 3.0"
exit 200
else
cat << "DATA" > "tmp/${t}/${t}.assert"
tags: [u'beta', u'client1']
DATA
fi

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --projectTags "beta" --projectTags "client1"
${cmd} --info "${t}" | grep ' tags: ' > "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 40
- 0
tests/create-csv-separator.sh View File

@@ -0,0 +1,40 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a;b;c
1;2;3
0;0;0
$;\;'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 2 3
0 0 0
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --separator ";"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 39
- 0
tests/create-csv-skipDataLines.sh View File

@@ -0,0 +1,39 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
0 0 0
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --skipDataLines "1"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 58
- 0
tests/create-csv-storeBlankCellsAsNulls.sh View File

@@ -0,0 +1,58 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,,0
$,\,'
DATA

cat << "DATA" > "tmp/${t}/${t}.transform"
[
{
"op": "core/text-transform",
"engineConfig": {
"facets": [],
"mode": "row-based"
},
"columnName": "b",
"expression": "grel:isNull(value)",
"onError": "keep-original",
"repeat": false,
"repeatCount": 10
}
]
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 false 3
0 false 0
$ false '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --storeBlankCellsAsNulls "false"
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 39
- 0
tests/create-csv-storeBlankRows.sh View File

@@ -0,0 +1,39 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
,,
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 2 3
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --storeBlankRows "false"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 40
- 0
tests/create-csv-utf8.sh View File

@@ -0,0 +1,40 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
⌨,code,meaning
⛲,1F347,FOUNTAIN
⛳,1F349,FLAG IN HOLE
⛵,1F352,SAILBOAT
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
⌨ code meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc ⛲"
${cmd} --export "${t} biểu tượng cảm xúc ⛲" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 40
- 0
tests/create-csv.sh View File

@@ -0,0 +1,40 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.csv"
a,b,c
1,2,3
0,0,0
$,\,'
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
a b c
1 2 3
0 0 0
$ \ '
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.csv"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 55
- 0
tests/create-json-recordPath.sh View File

@@ -0,0 +1,55 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.json"
{
"rows":[
{
"a":1,
"b":2,
"c":3
},
{
"a":0,
"b":0,
"c":0
},
{
"a":"$",
"b":"\\",
"c":"\""
}
]
}
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
_ - a _ - b _ - c
1 2 3
0 0 0
$ \ """"
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.json" --recordPath "_" --recordPath "rows" --recordPath "_"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 52
- 0
tests/create-json-storeEmptyStrings.sh View File

@@ -0,0 +1,52 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.json"
[
{
"a": 1,
"b": 2,
"c": 3
},
{
"a": "",
"b": "",
"c": ""
},
{
"a": "$",
"b": "\\",
"c": "\""
}
]
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
_ - a _ - b _ - c
1 2 3
$ \ """"
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.json" --storeEmptyStrings "false"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 62
- 0
tests/create-json-trimStrings.sh View File

@@ -0,0 +1,62 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.json"
[
{
"a": 1,
"b": 2,
"c": 3
},
{
"a": "0",
"b": " 0",
"c": "0 "
},
{
"a": "$",
"b": "\\",
"c": "\""
}
]
DATA

# ================================= ASSERTION ================================ #

if [[ "${version:0:1}" = "2" || "${version}" = "3.0" || "${version}" = "3.1" || "${version}" = "3.2" || "${version}" = "3.3" ]]; then
echo "trimStrings option does not work in OpenRefine <=3.3"
echo "https://github.com/OpenRefine/OpenRefine/issues/2409"
exit 200
else
cat << "DATA" > "tmp/${t}/${t}.assert"
_ - a _ - b _ - c
1 2 3
0 0 0
$ \ """"
DATA
fi

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.json"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 53
- 0
tests/create-json-utf8.sh View File

@@ -0,0 +1,53 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.json"
[
{
"⌨": "⛲",
"code": "1F347",
"meaning": "FOUNTAIN"
},
{
"⌨": "⛳",
"code": "1F349",
"meaning": "FLAG IN HOLE"
},
{
"⌨": "⛵",
"code": "1F352",
"meaning": "SAILBOAT"
}
]
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
_ - ⌨ _ - code _ - meaning
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.json"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 53
- 0
tests/create-json.sh View File

@@ -0,0 +1,53 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cat << "DATA" > "tmp/${t}/${t}.json"
[
{
"a": 1,
"b": 2,
"c": 3
},
{
"a": 0,
"b": 0,
"c": 0
},
{
"a": "$",
"b": "\\",
"c": "\""
}
]
DATA

# ================================= ASSERTION ================================ #

cat << "DATA" > "tmp/${t}/${t}.assert"
_ - a _ - b _ - c
1 2 3
0 0 0
$ \ """"
DATA

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.json"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 44
- 0
tests/create-ods-sheets-utf8.sh View File

@@ -0,0 +1,44 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cp "data/example.ods" "tmp/${t}/${t}.ods"

# ================================= ASSERTION ================================ #

if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
⌨ code meaning Column Column 5 Column 6 Column 7 Column 8
⛲ 1F347 FOUNTAIN
⛳ 1F349 FLAG IN HOLE
⛵ 1F352 SAILBOAT
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.ods" --sheets 1
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 48
- 0
tests/create-ods.sh View File

@@ -0,0 +1,48 @@
#!/bin/bash

# =============================== ENVIRONMENT ================================ #

if [[ ${1} ]]; then
cmd="${1}"
else
echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1
fi
if [[ ${2} ]]; then
version="${2}"
fi

t="$(basename "${BASH_SOURCE[0]}" .sh)"
cd "${BASH_SOURCE%/*}/" || exit 1
mkdir -p "tmp/${t}"

# =================================== DATA =================================== #

cp "data/example.ods" "tmp/${t}/${t}.ods"
#a b c
#1 2 3
#0 0 0
#$ \ '

# ================================= ASSERTION ================================ #

if [[ "${version}" = "2.7" ]]; then
cat << "DATA" > "tmp/${t}/${t}.assert"
a b c Column Column 5 Column 6 Column 7 Column 8
1.0 2.0 3.0
0.0 0.0 0.0
$ \ '
DATA
else
#TODO
echo "https://github.com/opencultureconsulting/openrefine-client/issues/4"
exit 200
fi

# ================================== ACTION ================================== #

${cmd} --create "tmp/${t}/${t}.ods"
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"

# =================================== TEST =================================== #

diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"

+ 40
- 0
tests/create-tsv-utf8.sh View File

@@ -0,0 +1,40 @@