tests for commands
This commit is contained in:
Felix Lohmeier
parent
ddc8d31380
commit
ca359925fe
|
|
@ -4,5 +4,7 @@ dist
|
||||||
.*
|
.*
|
||||||
openrefine_client.egg-info
|
openrefine_client.egg-info
|
||||||
refine.spec
|
refine.spec
|
||||||
|
openrefine-2.*
|
||||||
openrefine-3.*
|
openrefine-3.*
|
||||||
|
openrefine-client_*
|
||||||
tests.log
|
tests.log
|
||||||
|
|
|
||||||
29
tests.sh
29
tests.sh
|
|
@ -20,14 +20,22 @@
|
||||||
|
|
||||||
cd "${BASH_SOURCE%/*}/" || exit 1
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
|
||||||
|
port=3334
|
||||||
|
|
||||||
if [[ ${1} ]]; then
|
if [[ ${1} ]]; then
|
||||||
version="${1}"
|
version="${1}"
|
||||||
else
|
else
|
||||||
version="3.2"
|
version="3.2"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
refine="openrefine-${version}/refine"
|
refine="openrefine-${version}/refine"
|
||||||
|
|
||||||
|
if [[ ${2} ]]; then
|
||||||
|
client="$(readlink -e "${2}")"
|
||||||
|
else
|
||||||
|
client="python3 $(readlink -e refine.py)"
|
||||||
|
fi
|
||||||
|
cmd="${client} -H localhost -P ${port}"
|
||||||
|
|
||||||
# =============================== REQUIREMENTS =============================== #
|
# =============================== REQUIREMENTS =============================== #
|
||||||
|
|
||||||
# check existence of java and cURL
|
# check existence of java and cURL
|
||||||
|
|
@ -66,7 +74,7 @@ mkdir -p "${dir}"
|
||||||
rm -f tests.log
|
rm -f tests.log
|
||||||
|
|
||||||
echo "start OpenRefine server..."
|
echo "start OpenRefine server..."
|
||||||
${refine} -v warn -p 3334 -d "${dir}" &>> tests.log &
|
${refine} -v warn -p ${port} -d "${dir}" &>> tests.log &
|
||||||
pid_server=${!}
|
pid_server=${!}
|
||||||
timeout 30s bash -c "until curl -s 'http://localhost:3334' \
|
timeout 30s bash -c "until curl -s 'http://localhost:3334' \
|
||||||
| cat | grep -q -o 'OpenRefine' ; do sleep 1; done" \
|
| cat | grep -q -o 'OpenRefine' ; do sleep 1; done" \
|
||||||
|
|
@ -81,16 +89,23 @@ results=()
|
||||||
for t in tests/*.sh; do
|
for t in tests/*.sh; do
|
||||||
tests+=("${t}")
|
tests+=("${t}")
|
||||||
echo "========================= ${t} =========================" &>> tests.log
|
echo "========================= ${t} =========================" &>> tests.log
|
||||||
${t} ${version} >> tests.log
|
${t} "${cmd}" "${version}" &>> tests.log
|
||||||
results+=(${?})
|
results+=(${?})
|
||||||
done
|
done
|
||||||
echo
|
echo
|
||||||
|
|
||||||
|
# ================================= TEARDOWN ================================= #
|
||||||
|
|
||||||
|
echo "cleanup..."
|
||||||
|
{ kill -9 "${pid_server}" && wait "${pid_server}"; } 2>/dev/null
|
||||||
|
rm -rf "${dir}"
|
||||||
|
echo
|
||||||
|
|
||||||
# ================================= SUMMARY ================================== #
|
# ================================= SUMMARY ================================== #
|
||||||
|
|
||||||
for i in "${!tests[@]}"; do
|
|
||||||
printf "%s\t%s\n" "code" "test"
|
printf "%s\t%s\n" "code" "test"
|
||||||
printf "%s\t%s\n" "----" "----------------"
|
printf "%s\t%s\n" "----" "----------------"
|
||||||
|
for i in "${!tests[@]}"; do
|
||||||
printf "%s\t%s\n" "${results[$i]}" "${tests[$i]}"
|
printf "%s\t%s\n" "${results[$i]}" "${tests[$i]}"
|
||||||
done
|
done
|
||||||
echo
|
echo
|
||||||
|
|
@ -99,9 +114,3 @@ if [[ " ${results[*]} " =~ [1-9] ]]; then
|
||||||
else
|
else
|
||||||
echo "all tests passed!"; echo
|
echo "all tests passed!"; echo
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ================================= TEARDOWN ================================= #
|
|
||||||
|
|
||||||
echo "cleanup..."
|
|
||||||
{ kill -9 "${pid_server}" && wait "${pid_server}"; } 2>/dev/null
|
|
||||||
rm -rf "${dir}"
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,57 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
0,0,0
|
||||||
|
$,\,'
|
||||||
|
DATA
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.transform"
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"op": "core/column-addition",
|
||||||
|
"engineConfig": {
|
||||||
|
"mode": "row-based"
|
||||||
|
},
|
||||||
|
"newColumnName": "apply",
|
||||||
|
"columnInsertIndex": 2,
|
||||||
|
"baseColumnName": "b",
|
||||||
|
"expression": "grel:value.replace('2','٩(͡๏̯͡๏)۶')",
|
||||||
|
"onError": "set-to-blank"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
a b apply c
|
||||||
|
1 2 ٩(͡๏̯͡๏)۶ 3
|
||||||
|
0 0 0 0
|
||||||
|
$ \ \ '
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
|
||||||
|
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,57 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
0,0,0
|
||||||
|
$,\,'
|
||||||
|
DATA
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.transform"
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"op": "core/column-addition",
|
||||||
|
"engineConfig": {
|
||||||
|
"mode": "row-based"
|
||||||
|
},
|
||||||
|
"newColumnName": "apply",
|
||||||
|
"columnInsertIndex": 2,
|
||||||
|
"baseColumnName": "b",
|
||||||
|
"expression": "grel:value.replace('2','TEST')",
|
||||||
|
"onError": "set-to-blank"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
a b apply c
|
||||||
|
1 2 TEST 3
|
||||||
|
0 0 0 0
|
||||||
|
$ \ \ '
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
|
||||||
|
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -1,819 +0,0 @@
|
||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"# Test executable in a Linux Bash environment"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Install\n",
|
|
||||||
"\n",
|
|
||||||
"This notebook requires a [Bash kernel](https://github.com/takluyver/bash_kernel) environment and an OpenRefine server running at http://127.0.0.1:3333."
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 49,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"/tmp/20190822_013937\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"workspace=$(date +%Y%m%d_%H%M%S)\n",
|
|
||||||
"mkdir -p /tmp/$workspace\n",
|
|
||||||
"cp -r data /tmp/$workspace\n",
|
|
||||||
"cd /tmp/$workspace && pwd"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 50,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux:\n",
|
|
||||||
"2019-08-22 01:39:40 ERROR 404: Not Found.\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"wget -nv https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux -O openrefine-client\n",
|
|
||||||
"chmod +x openrefine-client"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## README.MD"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Download"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 51,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Download to file duplicates.csv complete\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --download \"https://git.io/fj5hF\" --output=duplicates.csv"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Create"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 52,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"id: 2334935475634\n",
|
|
||||||
"rows: 10\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --create duplicates.csv"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### List"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 53,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
" 2334935475634: duplicates\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --list"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Info"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 54,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
" id: 2334935475634\n",
|
|
||||||
" url: http://127.0.0.1:3333/project?project=2334935475634\n",
|
|
||||||
" name: duplicates\n",
|
|
||||||
" modified: 2019-08-21T23:40:30Z\n",
|
|
||||||
" created: 2019-08-21T23:40:30Z\n",
|
|
||||||
" rowCount: 10\n",
|
|
||||||
"importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n",
|
|
||||||
" column 001: email\n",
|
|
||||||
" column 002: name\n",
|
|
||||||
" column 003: state\n",
|
|
||||||
" column 004: gender\n",
|
|
||||||
" column 005: purchase\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --info \"duplicates\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Export"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 55,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"email\tname\tstate\tgender\tpurchase\n",
|
|
||||||
"danny.baron@example1.com\tDanny Baron\tCA\tM\tTV\n",
|
|
||||||
"melanie.white@example2.edu\tMelanie White\tNC\tF\tiPhone\n",
|
|
||||||
"danny.baron@example1.com\tD. Baron\tCA\tM\tWinter jacket\n",
|
|
||||||
"ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\n",
|
|
||||||
"arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\n",
|
|
||||||
"danny.baron@example1.com\tDaniel Baron\tCA\tM\tBike\n",
|
|
||||||
"jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\n",
|
|
||||||
"melanie.white@example2.edu\tMelanie White\tNC\tF\tiPad\n",
|
|
||||||
"ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\n",
|
|
||||||
"arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"duplicates\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Apply"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 56,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Download to file duplicates-deletion.json complete\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --download \"https://git.io/fj5ju\" --output=duplicates-deletion.json"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 57,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"File duplicates-deletion.json has been successfully applied to project 2334935475634\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --apply duplicates-deletion.json \"duplicates\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 58,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"email\tcount\tname\tstate\tgender\tpurchase\n",
|
|
||||||
"arthur.duff@example4.com\t2\tArthur Duff\tOR\tM\tDining table\n",
|
|
||||||
"ben.morisson@example6.org\t1\tBen Morisson\tFL\tM\tAmplifier\n",
|
|
||||||
"ben.tyler@example3.org\t1\tBen Tyler\tNV\tM\tFlashlight\n",
|
|
||||||
"danny.baron@example1.com\t3\tDanny Baron\tCA\tM\tTV\n",
|
|
||||||
"jean.griffith@example5.org\t1\tJean Griffith\tWA\tF\tPower drill\n",
|
|
||||||
"melanie.white@example2.edu\t2\tMelanie White\tNC\tF\tiPhone\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"duplicates\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Export XLS"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 59,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Export to file deduped.xls complete\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"duplicates\" --output deduped.xls"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Delete"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 60,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Project 2334935475634 has been successfully deleted\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --delete \"duplicates\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Templating"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 61,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"id: 1633409429491\n",
|
|
||||||
"rows: 10\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --create duplicates.csv --projectName=advanced"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 62,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"{ \"events\" : [\n",
|
|
||||||
" { \"name\" : \"Melanie White\", \"purchase\" : \"iPhone\" },\n",
|
|
||||||
" { \"name\" : \"Jean Griffith\", \"purchase\" : \"Power drill\" },\n",
|
|
||||||
" { \"name\" : \"Melanie White\", \"purchase\" : \"iPad\" }\n",
|
|
||||||
"] }"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"advanced\" \\\n",
|
|
||||||
"--prefix='{ \"events\" : [\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n",
|
|
||||||
"--rowSeparator=',\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--suffix='\n",
|
|
||||||
"] }' \\\n",
|
|
||||||
"--filterQuery='^F$' \\\n",
|
|
||||||
"--filterColumn='gender'"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 63,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Export to files complete. Last file: advanced_3.json\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"advanced\" \\\n",
|
|
||||||
"--prefix='{ \"events\" : [\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n",
|
|
||||||
"--rowSeparator=',\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--suffix='\n",
|
|
||||||
"] }' \\\n",
|
|
||||||
"--filterQuery='^F$' \\\n",
|
|
||||||
"--filterColumn='gender' \\\n",
|
|
||||||
"--output=advanced.json \\\n",
|
|
||||||
"--splitToFiles=true"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 64,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Export to files complete. Last file: advanced_melanie.white@example2.edu.json\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"advanced\" \\\n",
|
|
||||||
"--prefix='{ \"events\" : [\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n",
|
|
||||||
"--rowSeparator=',\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--suffix='\n",
|
|
||||||
"] }' \\\n",
|
|
||||||
"--filterQuery='^F$' \\\n",
|
|
||||||
"--filterColumn='gender' \\\n",
|
|
||||||
"--output=advanced.json \\\n",
|
|
||||||
"--splitToFiles=true \\\n",
|
|
||||||
"--suffixById=true"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 65,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"advanced_1.json \u001b[0m\u001b[38;5;33mdata\u001b[0m\n",
|
|
||||||
"advanced_2.json deduped.xls\n",
|
|
||||||
"advanced_3.json duplicates.csv\n",
|
|
||||||
"advanced_jean.griffith@example5.org.json duplicates-deletion.json\n",
|
|
||||||
"advanced_melanie.white@example2.edu.json \u001b[38;5;40mopenrefine-client\u001b[0m\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"ls"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Delete"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 66,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": false
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Project 1633409429491 has been successfully deleted\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --delete \"advanced\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## Unicode"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### fruits"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 67,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"id: 2280962953279\n",
|
|
||||||
"rows: 5\n",
|
|
||||||
" id: 2280962953279\n",
|
|
||||||
" url: http://127.0.0.1:3333/project?project=2280962953279\n",
|
|
||||||
" name: evil-fruits\n",
|
|
||||||
" modified: 2019-08-21T23:40:43Z\n",
|
|
||||||
" created: 2019-08-21T23:40:43Z\n",
|
|
||||||
" rowCount: 5\n",
|
|
||||||
"importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/evil-fruits.tsv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'evil-fruits', u'processQuotes': True, u'limit': -1, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n",
|
|
||||||
" column 001: 🔣\n",
|
|
||||||
" column 002: code\n",
|
|
||||||
" column 003: meaning\n",
|
|
||||||
"🔣\tcode\tmeaning\n",
|
|
||||||
"🍇\t1F347\tGRAPES\n",
|
|
||||||
"🍉\t1F349\tWATERMELON\n",
|
|
||||||
"🍒\t1F352\tCHERRIES\n",
|
|
||||||
"🍓\t1F353\tSTRAWBERRY\n",
|
|
||||||
"🍍\t1F34D\tPINEAPPLE\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --create data/cli/evil-fruits.tsv\n",
|
|
||||||
"./openrefine-client --info \"evil-fruits\"\n",
|
|
||||||
"./openrefine-client --export \"evil-fruits\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 68,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Export to file emojis.csv complete\n",
|
|
||||||
"🔣,code,meaning\n",
|
|
||||||
"🍇,1F347,GRAPES\n",
|
|
||||||
"🍉,1F349,WATERMELON\n",
|
|
||||||
"🍒,1F352,CHERRIES\n",
|
|
||||||
"🍓,1F353,STRAWBERRY\n",
|
|
||||||
"🍍,1F34D,PINEAPPLE\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"evil-fruits\" --output emojis.csv\n",
|
|
||||||
"cat emojis.csv"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 69,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"{ \"emojis\" : [\n",
|
|
||||||
" { \"symbol\" : \"🍇\", \"meaning\" : \"GRAPES\" },\n",
|
|
||||||
" { \"symbol\" : \"🍉\", \"meaning\" : \"WATERMELON\" },\n",
|
|
||||||
" { \"symbol\" : \"🍍\", \"meaning\" : \"PINEAPPLE\" }\n",
|
|
||||||
"] }"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"evil-fruits\" \\\n",
|
|
||||||
"--prefix='{ \"emojis\" : [\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n",
|
|
||||||
"--rowSeparator=',\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--suffix='\n",
|
|
||||||
"] }' \\\n",
|
|
||||||
"--filterQuery='^1F34' \\\n",
|
|
||||||
"--filterColumn='code'"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 70,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Export to files complete. Last file: trái cây_3.json\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"evil-fruits\" \\\n",
|
|
||||||
"--prefix='{ \"emojis\" : [\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n",
|
|
||||||
"--rowSeparator=',\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--suffix='\n",
|
|
||||||
"] }' \\\n",
|
|
||||||
"--filterQuery='^1F34' \\\n",
|
|
||||||
"--filterColumn='code' \\\n",
|
|
||||||
"--output='trái cây.json' \\\n",
|
|
||||||
"--splitToFiles=true"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 71,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Export to files complete. Last file: trái cây_🍍.json\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --export \"evil-fruits\" \\\n",
|
|
||||||
"--prefix='{ \"emojis\" : [\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n",
|
|
||||||
"--rowSeparator=',\n",
|
|
||||||
"' \\\n",
|
|
||||||
"--suffix='\n",
|
|
||||||
"] }' \\\n",
|
|
||||||
"--filterQuery='^1F34' \\\n",
|
|
||||||
"--filterColumn='code' \\\n",
|
|
||||||
"--output='trái cây.json' \\\n",
|
|
||||||
"--splitToFiles=true \\\n",
|
|
||||||
"--suffixById=true"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 72,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
" advanced_1.json emojis.csv\n",
|
|
||||||
" advanced_2.json \u001b[0m\u001b[38;5;40mopenrefine-client\u001b[0m\n",
|
|
||||||
" advanced_3.json 'trái cây_1.json'\n",
|
|
||||||
" advanced_jean.griffith@example5.org.json 'trái cây_2.json'\n",
|
|
||||||
" advanced_melanie.white@example2.edu.json 'trái cây_3.json'\n",
|
|
||||||
" \u001b[38;5;33mdata\u001b[0m 'trái cây_🍇.json'\n",
|
|
||||||
" deduped.xls 'trái cây_🍉.json'\n",
|
|
||||||
" duplicates.csv 'trái cây_🍍.json'\n",
|
|
||||||
" duplicates-deletion.json\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"ls"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 73,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Project 2280962953279 has been successfully deleted\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --delete \"evil-fruits\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### emoji-data"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 74,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"id: 2019865211741\n",
|
|
||||||
"rows: 20\n",
|
|
||||||
" id: 2019865211741\n",
|
|
||||||
" url: http://127.0.0.1:3333/project?project=2019865211741\n",
|
|
||||||
" name: dữ liệu biểu tượng cảm xúc\n",
|
|
||||||
" modified: 2019-08-21T23:41:06Z\n",
|
|
||||||
" created: 2019-08-21T23:41:06Z\n",
|
|
||||||
" rowCount: 20\n",
|
|
||||||
"importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/d\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac.txt', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'd\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac', u'processQuotes': True, u'skipDataLines': 34, u'limit': 20, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False, u'headerLines': 0}]\n",
|
|
||||||
" column 001: Column 1\n",
|
|
||||||
" column 002: Column 2\n",
|
|
||||||
" column 003: Column 3\n",
|
|
||||||
" column 004: Column 4\n",
|
|
||||||
" column 005: Column 5\n",
|
|
||||||
" column 006: Column 6\n",
|
|
||||||
"Column 1\tColumn 2\tColumn 3\tColumn 4\tColumn 5\tColumn 6\n",
|
|
||||||
"00A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (©) COPYRIGHT SIGN\n",
|
|
||||||
"00AE ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (®) REGISTERED SIGN\n",
|
|
||||||
"203C ;\ttext ;\tL1 ;\tnone ;\ta j\t# V1.1 (‼) DOUBLE EXCLAMATION MARK\n",
|
|
||||||
"2049 ;\ttext ;\tL1 ;\tnone ;\ta j\t# V3.0 (⁉) EXCLAMATION QUESTION MARK\n",
|
|
||||||
"2122 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (™) TRADE MARK SIGN\n",
|
|
||||||
"2139 ;\ttext ;\tL1 ;\tnone ;\tj\t# V3.0 (ℹ) INFORMATION SOURCE\n",
|
|
||||||
"2194 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↔) LEFT RIGHT ARROW\n",
|
|
||||||
"2195 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↕) UP DOWN ARROW\n",
|
|
||||||
"2196 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↖) NORTH WEST ARROW\n",
|
|
||||||
"2197 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↗) NORTH EAST ARROW\n",
|
|
||||||
"2198 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↘) SOUTH EAST ARROW\n",
|
|
||||||
"2199 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↙) SOUTH WEST ARROW\n",
|
|
||||||
"21A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↩) LEFTWARDS ARROW WITH HOOK\n",
|
|
||||||
"21AA ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↪) RIGHTWARDS ARROW WITH HOOK\n",
|
|
||||||
"231A ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌚) WATCH\n",
|
|
||||||
"231B ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌛) HOURGLASS\n",
|
|
||||||
"2328 ;\ttext ;\tL2 ;\tnone ;\tx\t# V1.1 (⌨) KEYBOARD\n",
|
|
||||||
"23CF ;\ttext ;\tL2 ;\tnone ;\tx\t# V4.0 (⏏) EJECT SYMBOL\n",
|
|
||||||
"23E9 ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏩) BLACK RIGHT-POINTING DOUBLE TRIANGLE\n",
|
|
||||||
"23EA ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏪) BLACK LEFT-POINTING DOUBLE TRIANGLE\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --create \"data/cli/dữ liệu biểu tượng cảm xúc.txt\" \\\n",
|
|
||||||
"--format=tsv \\\n",
|
|
||||||
"--headerLines=0 \\\n",
|
|
||||||
"--skipDataLines=34 \\\n",
|
|
||||||
"--limit=20\n",
|
|
||||||
"./openrefine-client --info \"dữ liệu biểu tượng cảm xúc\"\n",
|
|
||||||
"./openrefine-client --export \"dữ liệu biểu tượng cảm xúc\""
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 75,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
" 2019865211741: dữ liệu biểu tượng cảm xúc\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --list"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Delete"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 76,
|
|
||||||
"metadata": {
|
|
||||||
"scrolled": false
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Project 2019865211741 has been successfully deleted\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"./openrefine-client --delete \"dữ liệu biểu tượng cảm xúc\""
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Bash",
|
|
||||||
"language": "bash",
|
|
||||||
"name": "bash"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": "shell",
|
|
||||||
"file_extension": ".sh",
|
|
||||||
"mimetype": "text/x-sh",
|
|
||||||
"name": "bash"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -2,10 +2,14 @@
|
||||||
|
|
||||||
# =============================== ENVIRONMENT ================================ #
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
t="create-csv"
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
cd "${BASH_SOURCE%/*}/" || exit 1
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
client="python3 ../refine.py -H localhost -P 3334"
|
|
||||||
mkdir -p "tmp/${t}"
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
# =================================== DATA =================================== #
|
# =================================== DATA =================================== #
|
||||||
|
|
@ -28,8 +32,8 @@ DATA
|
||||||
|
|
||||||
# ================================== ACTION ================================== #
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
${client} --create "tmp/${t}/${t}.csv"
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
${client} --export "${t}" --output "tmp/${t}/${t}.output"
|
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
# =================================== TEST =================================== #
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
🍉,🍒,🍓
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
a b c
|
||||||
|
🍉 🍒 🍓
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc 🍉"
|
||||||
|
${cmd} --export "${t} biểu tượng cảm xúc 🍉" --output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
email,name,state,gender,purchase,count,date
|
email,name,state,gender,purchase
|
||||||
danny.baron@example1.com,Danny Baron,CA,M,TV (UTF-8: 📺),1,"Wed, 4 Jul 2001"
|
danny.baron@example1.com,Danny Baron,CA,M,TV
|
||||||
melanie.white@example2.edu,Melanie White,NC,F,<iPhone>,1,2001-07-04T12:08:56
|
melanie.white@example2.edu,Melanie White,NC,F,iPhone
|
||||||
danny.baron@example1.com, D. ("Tab") Baron,CA,M,Winter jacket,1,2001-07-04
|
danny.baron@example1.com,D. Baron,CA,M,Winter jacket
|
||||||
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight,1,2001/07/04
|
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight
|
||||||
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table,1,2001-07
|
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table
|
||||||
danny.baron@example1.com,Daniel Baron,,,Bike,1,2001
|
danny.baron@example1.com,Daniel Baron,CA,M,Bike
|
||||||
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill,1,2000
|
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill
|
||||||
melanie.white@example2.edu,Melanie White,NC,F,'iPad',1,1999
|
melanie.white@example2.edu,Melanie White,NC,F,iPad
|
||||||
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier,1,1998
|
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier
|
||||||
arthur.duff@example4.com,Arthur Duff,OR,M,Night table,1,1997
|
arthur.duff@example4.com,Arthur Duff,OR,M,Night table
|
||||||
|
Can't render this file because it contains an unexpected character in line 4 and column 33.
|
|
|
@ -1,92 +0,0 @@
|
||||||
[
|
|
||||||
{
|
|
||||||
"email": "danny.baron@example1.com",
|
|
||||||
"name": "Danny Baron",
|
|
||||||
"state": "CA",
|
|
||||||
"gender": "M",
|
|
||||||
"purchase": "TV (UTF-8: 📺)",
|
|
||||||
"count": 1,
|
|
||||||
"date": "Wed, 4 Jul 2001"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "melanie.white@example2.edu",
|
|
||||||
"name": "Melanie White",
|
|
||||||
"state": "NC",
|
|
||||||
"gender": "F",
|
|
||||||
"purchase": "<iPhone>",
|
|
||||||
"count": 1,
|
|
||||||
"date": "2001-07-04T12:08:56"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "danny.baron@example1.com",
|
|
||||||
"name": " D.\t(\"Tab\") Baron",
|
|
||||||
"state": "CA",
|
|
||||||
"gender": "M",
|
|
||||||
"purchase": "Winter jacket",
|
|
||||||
"count": 1,
|
|
||||||
"date": "2001-07-04"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "ben.tyler@example3.org",
|
|
||||||
"name": "Ben Tyler",
|
|
||||||
"state": "NV",
|
|
||||||
"gender": "M",
|
|
||||||
"purchase": "Flashlight",
|
|
||||||
"count": 1,
|
|
||||||
"date": "2001/07/04"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "arthur.duff@example4.com",
|
|
||||||
"name": "Arthur Duff",
|
|
||||||
"state": "OR",
|
|
||||||
"gender": "M",
|
|
||||||
"purchase": "Dining table",
|
|
||||||
"count": 1,
|
|
||||||
"date": "2001-07"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "danny.baron@example1.com",
|
|
||||||
"name": "Daniel Baron",
|
|
||||||
"state": "",
|
|
||||||
"gender": "",
|
|
||||||
"purchase": "Bike",
|
|
||||||
"count": 1,
|
|
||||||
"date": 2001
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "jean.griffith@example5.org",
|
|
||||||
"name": "Jean Griffith",
|
|
||||||
"state": "WA",
|
|
||||||
"gender": "F",
|
|
||||||
"purchase": "Power drill",
|
|
||||||
"count": 1,
|
|
||||||
"date": 2000
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "melanie.white@example2.edu",
|
|
||||||
"name": "Melanie White",
|
|
||||||
"state": "NC",
|
|
||||||
"gender": "F",
|
|
||||||
"purchase": "'iPad'",
|
|
||||||
"count": 1,
|
|
||||||
"date": 1999
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "ben.morisson@example6.org",
|
|
||||||
"name": "Ben Morisson",
|
|
||||||
"state": "FL",
|
|
||||||
"gender": "M",
|
|
||||||
"purchase": "Amplifier",
|
|
||||||
"count": 1,
|
|
||||||
"date": 1998
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"email": "arthur.duff@example4.com",
|
|
||||||
"name": "Arthur Duff",
|
|
||||||
"state": "OR",
|
|
||||||
"gender": "M",
|
|
||||||
"purchase": "Night table",
|
|
||||||
"count": 1,
|
|
||||||
"date": 1997
|
|
||||||
}
|
|
||||||
]
|
|
||||||
Binary file not shown.
|
|
@ -1,11 +0,0 @@
|
||||||
email name state gender purchase count date
|
|
||||||
danny.baron@example1.com Danny Baron CA M TV (UTF-8: 📺) 1 Wed, 4 Jul 2001
|
|
||||||
melanie.white@example2.edu Melanie White NC F <iPhone> 1 2001-07-04T12:08:56
|
|
||||||
danny.baron@example1.com "D. (""Tab"") Baron" CA M Winter jacket 1 2001-07-04
|
|
||||||
ben.tyler@example3.org Ben Tyler NV M Flashlight 1 2001/07/04
|
|
||||||
arthur.duff@example4.com Arthur Duff OR M Dining table 1 2001-07
|
|
||||||
danny.baron@example1.com Daniel Baron Bike 1 2001
|
|
||||||
jean.griffith@example5.org Jean Griffith WA F Power drill 1 2000
|
|
||||||
melanie.white@example2.edu Melanie White NC F 'iPad' 1 1999
|
|
||||||
ben.morisson@example6.org Ben Morisson FL M Amplifier 1 1998
|
|
||||||
arthur.duff@example4.com Arthur Duff OR M Night table 1 1997
|
|
||||||
|
|
|
@ -1,11 +0,0 @@
|
||||||
email name state gender purchase count date
|
|
||||||
danny.baron@example1.com Danny Baron CA M TV (UTF-8: 📺) 1 Wed, 4 Jul 2001
|
|
||||||
melanie.white@example2.edu Melanie White NC F <iPhone> 1 2001-07-04T12:08:5
|
|
||||||
danny.baron@example1.com D. ("Tab") Baron CA M Winter jacket 1 2001-07-04
|
|
||||||
ben.tyler@example3.org Ben Tyler NV M Flashlight 1 2001/07/04
|
|
||||||
arthur.duff@example4.com Arthur Duff OR M Dining table 1 2001-07
|
|
||||||
danny.baron@example1.com Daniel Baron Bike 1 2001
|
|
||||||
jean.griffith@example5.org Jean Griffith WA F Power drill 1 2000
|
|
||||||
melanie.white@example2.edu Melanie White NC F 'iPad' 1 1999
|
|
||||||
ben.morisson@example6.org Ben Morisson FL M Amplifier 1 1998
|
|
||||||
arthur.duff@example4.com Arthur Duff OR M Night table 1 1997
|
|
||||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,93 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<root>
|
|
||||||
<record>
|
|
||||||
<email>danny.baron@example1.com</email>
|
|
||||||
<name>Danny Baron</name>
|
|
||||||
<state>CA</state>
|
|
||||||
<gender>M</gender>
|
|
||||||
<purchase>TV (UTF-8: 📺)</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>Wed, 4 Jul 2001</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>melanie.white@example2.edu</email>
|
|
||||||
<name>Melanie White</name>
|
|
||||||
<state>NC</state>
|
|
||||||
<gender>F</gender>
|
|
||||||
<purchase><iPhone></purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>2001-07-04T12:08:56</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>danny.baron@example1.com</email>
|
|
||||||
<name> D. ("Tab") Baron</name>
|
|
||||||
<state>CA</state>
|
|
||||||
<gender>M</gender>
|
|
||||||
<purchase>Winter jacket</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>2001-07-04</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>ben.tyler@example3.org</email>
|
|
||||||
<name>Ben Tyler</name>
|
|
||||||
<state>NV</state>
|
|
||||||
<gender>M</gender>
|
|
||||||
<purchase>Flashlight</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>2001/07/04</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>arthur.duff@example4.com</email>
|
|
||||||
<name>Arthur Duff</name>
|
|
||||||
<state>OR</state>
|
|
||||||
<gender>M</gender>
|
|
||||||
<purchase>Dining table</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>2001-07</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>danny.baron@example1.com</email>
|
|
||||||
<name>Daniel Baron</name>
|
|
||||||
<state></state>
|
|
||||||
<gender></gender>
|
|
||||||
<purchase>Bike</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>2001</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>jean.griffith@example5.org</email>
|
|
||||||
<name>Jean Griffith</name>
|
|
||||||
<state>WA</state>
|
|
||||||
<gender>F</gender>
|
|
||||||
<purchase>Power drill</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>2000</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>melanie.white@example2.edu</email>
|
|
||||||
<name>Melanie White</name>
|
|
||||||
<state>NC</state>
|
|
||||||
<gender>F</gender>
|
|
||||||
<purchase>'iPad'</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>1999</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>ben.morisson@example6.org</email>
|
|
||||||
<name>Ben Morisson</name>
|
|
||||||
<state>FL</state>
|
|
||||||
<gender>M</gender>
|
|
||||||
<purchase>Amplifier</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>1998</date>
|
|
||||||
</record>
|
|
||||||
<record>
|
|
||||||
<email>arthur.duff@example4.com</email>
|
|
||||||
<name>Arthur Duff</name>
|
|
||||||
<state>OR</state>
|
|
||||||
<gender>M</gender>
|
|
||||||
<purchase>Night table</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>1997</date>
|
|
||||||
</record>
|
|
||||||
</root>
|
|
||||||
Binary file not shown.
|
|
@ -1,10 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<record>
|
|
||||||
<email>danny.baron@example1.com</email>
|
|
||||||
<name>Danny Baron</name>
|
|
||||||
<state>CA</state>
|
|
||||||
<gender>M</gender>
|
|
||||||
<purchase>TV (UTF-8: 📺)</purchase>
|
|
||||||
<count>1</count>
|
|
||||||
<date>Wed, 4 Jul 2001</date>
|
|
||||||
</record>
|
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,11 +0,0 @@
|
||||||
email,name,state,gender,purchase
|
|
||||||
danny.baron@example1.com,Danny Baron,CA,M,TV
|
|
||||||
melanie.white@example2.edu,Melanie White,NC,F,iPhone
|
|
||||||
danny.baron@example1.com,D. Baron,CA,M,Winter jacket
|
|
||||||
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight
|
|
||||||
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table
|
|
||||||
danny.baron@example1.com,Daniel Baron,CA,M,Bike
|
|
||||||
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill
|
|
||||||
melanie.white@example2.edu,Melanie White,NC,F,iPad
|
|
||||||
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier
|
|
||||||
arthur.duff@example4.com,Arthur Duff,OR,M,Night table
|
|
||||||
|
File diff suppressed because it is too large
Load Diff
|
|
@ -1,6 +0,0 @@
|
||||||
🔣 code meaning
|
|
||||||
🍇 1F347 GRAPES
|
|
||||||
🍉 1F349 WATERMELON
|
|
||||||
🍒 1F352 CHERRIES
|
|
||||||
🍓 1F353 STRAWBERRY
|
|
||||||
🍍 1F34D PINEAPPLE
|
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << DATA > "tmp/${t}/${t}.assert"
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc 🍉"
|
||||||
|
${cmd} --delete "${t} biểu tượng cảm xúc 🍉"
|
||||||
|
${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << DATA > "tmp/${t}/${t}.assert"
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --delete "${t}"
|
||||||
|
${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --download "https://git.io/fj5ju" --output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "data/duplicates-deletion.json" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.tsv"
|
||||||
|
a b c
|
||||||
|
1 2 3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.tsv"
|
||||||
|
${cmd} --export "${t}" --output "tmp/${t}/biểu tượng cảm xúc 🍉.csv"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/biểu tượng cảm xúc 🍉.csv"
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.tsv"
|
||||||
|
a b c
|
||||||
|
1 2 3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.tsv"
|
||||||
|
${cmd} --export "${t}" --output "tmp/${t}/${t}.csv"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.csv"
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
🔣,code,meaning
|
||||||
|
🍇,1F347,GRAPES
|
||||||
|
🍉,1F349,WATERMELON
|
||||||
|
🍒,1F352,CHERRIES
|
||||||
|
🍓,1F353,STRAWBERRY
|
||||||
|
🍍,1F34D,PINEAPPLE
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
🔣 code meaning
|
||||||
|
🍇 1F347 GRAPES
|
||||||
|
🍉 1F349 WATERMELON
|
||||||
|
🍒 1F352 CHERRIES
|
||||||
|
🍓 1F353 STRAWBERRY
|
||||||
|
🍍 1F34D PINEAPPLE
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
a b c
|
||||||
|
1 2 3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,169 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
Usage: refine.py [--help | OPTIONS]
|
||||||
|
|
||||||
|
Script to provide a command line interface to an OpenRefine server.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
|
||||||
|
Connection options:
|
||||||
|
-H 127.0.0.1, --host=127.0.0.1
|
||||||
|
OpenRefine hostname (default: 127.0.0.1)
|
||||||
|
-P 3333, --port=3333
|
||||||
|
OpenRefine port (default: 3333)
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
-c [FILE], --create=[FILE]
|
||||||
|
Create project from file. The filename ending (e.g.
|
||||||
|
.csv) defines the input format
|
||||||
|
(csv,tsv,xml,json,txt,xls,xlsx,ods)
|
||||||
|
-l, --list List projects
|
||||||
|
--download=[URL] Download file from URL (e.g. example data). Combine
|
||||||
|
with --output to specify a filename.
|
||||||
|
|
||||||
|
Commands with argument [PROJECTID/PROJECTNAME]:
|
||||||
|
-d, --delete Delete project
|
||||||
|
-f [FILE], --apply=[FILE]
|
||||||
|
Apply JSON rules to OpenRefine project
|
||||||
|
-E, --export Export project in tsv format to stdout.
|
||||||
|
-o [FILE], --output=[FILE]
|
||||||
|
Export project to file. The filename ending (e.g.
|
||||||
|
.tsv) defines the output format
|
||||||
|
(csv,tsv,xls,xlsx,html)
|
||||||
|
--template=[STRING]
|
||||||
|
Export project with templating. Provide (big) text
|
||||||
|
string that you enter in the *row template* textfield
|
||||||
|
in the export/templating menu in the browser app)
|
||||||
|
--info show project metadata
|
||||||
|
|
||||||
|
General options:
|
||||||
|
--format=FILE_FORMAT
|
||||||
|
Override file detection (import:
|
||||||
|
csv,tsv,xml,json,line-based,fixed-width,xls,xlsx,ods;
|
||||||
|
export: csv,tsv,html,xls,xlsx,ods)
|
||||||
|
|
||||||
|
Create options:
|
||||||
|
--columnWidths=COLUMNWIDTHS
|
||||||
|
(txt/fixed-width), please provide widths in multiple
|
||||||
|
arguments, e.g. --columnWidths=7 --columnWidths=5
|
||||||
|
--encoding=ENCODING
|
||||||
|
(csv,tsv,txt), please provide short encoding name
|
||||||
|
(e.g. UTF-8)
|
||||||
|
--guessCellValueTypes=true/false
|
||||||
|
(xml,csv,tsv,txt,json, default: false)
|
||||||
|
--headerLines=HEADERLINES
|
||||||
|
(csv,tsv,txt/fixed-width,xls,xlsx,ods), default: 1,
|
||||||
|
default txt/fixed-width: 0
|
||||||
|
--ignoreLines=IGNORELINES
|
||||||
|
(csv,tsv,txt,xls,xlsx,ods), default: -1
|
||||||
|
--includeFileSources=true/false
|
||||||
|
(all formats), default: false
|
||||||
|
--limit=LIMIT (all formats), default: -1
|
||||||
|
--linesPerRow=LINESPERROW
|
||||||
|
(txt/line-based), default: 1
|
||||||
|
--processQuotes=true/false
|
||||||
|
(csv,tsv), default: true
|
||||||
|
--projectName=PROJECTNAME
|
||||||
|
(all formats), default: filename
|
||||||
|
--projectTags=PROJECTTAGS
|
||||||
|
(all formats), please provide tags in multiple
|
||||||
|
arguments, e.g. --projectTags=beta
|
||||||
|
--projectTags=client1
|
||||||
|
--recordPath=RECORDPATH
|
||||||
|
(xml,json), please provide path in multiple arguments,
|
||||||
|
e.g. /collection/record/ should be entered:
|
||||||
|
--recordPath=collection --recordPath=record, default
|
||||||
|
xml: root element, default json: _ _
|
||||||
|
--separator=SEPARATOR
|
||||||
|
(csv,tsv), default csv: , default tsv: \t
|
||||||
|
--sheets=SHEETS (xls,xlsx,ods), please provide sheets in multiple
|
||||||
|
arguments, e.g. --sheets=0 --sheets=1, default: 0
|
||||||
|
(first sheet)
|
||||||
|
--skipDataLines=SKIPDATALINES
|
||||||
|
(csv,tsv,txt,xls,xlsx,ods), default: 0, default line-
|
||||||
|
based: -1
|
||||||
|
--storeBlankCellsAsNulls=true/false
|
||||||
|
(csv,tsv,txt,xls,xlsx,ods), default: true
|
||||||
|
--storeBlankRows=true/false
|
||||||
|
(csv,tsv,txt,xls,xlsx,ods), default: true
|
||||||
|
--storeEmptyStrings=true/false
|
||||||
|
(xml,json), default: true
|
||||||
|
--trimStrings=true/false
|
||||||
|
(xml,json), default: false
|
||||||
|
|
||||||
|
Templating options:
|
||||||
|
--mode=row-based/record-based
|
||||||
|
engine mode (default: row-based)
|
||||||
|
--prefix=PREFIX text string that you enter in the *prefix* textfield
|
||||||
|
in the browser app
|
||||||
|
--rowSeparator=ROWSEPARATOR
|
||||||
|
text string that you enter in the *row separator*
|
||||||
|
textfield in the browser app
|
||||||
|
--suffix=SUFFIX text string that you enter in the *suffix* textfield
|
||||||
|
in the browser app
|
||||||
|
--filterQuery=REGEX
|
||||||
|
Simple RegEx text filter on filterColumn, e.g. ^12015$
|
||||||
|
--filterColumn=COLUMNNAME
|
||||||
|
column name for filterQuery (default: name of first
|
||||||
|
column)
|
||||||
|
--facets=FACETS facets config in json format (may be extracted with
|
||||||
|
browser dev tools in browser app)
|
||||||
|
--splitToFiles=true/false
|
||||||
|
will split each row/record into a single file; it
|
||||||
|
specifies a presumably unique character series for
|
||||||
|
splitting; --prefix and --suffix will be applied to
|
||||||
|
all files; filename-prefix can be specified with
|
||||||
|
--output (default: %Y%m%d)
|
||||||
|
--suffixById=true/false
|
||||||
|
enhancement option for --splitToFiles; will generate
|
||||||
|
filename-suffix from values in key column
|
||||||
|
|
||||||
|
Example data:
|
||||||
|
--download "https://git.io/fj5hF" --output=duplicates.csv
|
||||||
|
--download "https://git.io/fj5ju" --output=duplicates-deletion.json
|
||||||
|
|
||||||
|
Basic commands:
|
||||||
|
--list # list all projects
|
||||||
|
--list -H 127.0.0.1 -P 80 # specify hostname and port
|
||||||
|
--create duplicates.csv # create new project from file
|
||||||
|
--info "duplicates" # show project metadata
|
||||||
|
--apply duplicates-deletion.json "duplicates" # apply rules in file to project
|
||||||
|
--export "duplicates" # export project to terminal in tsv format
|
||||||
|
--export --output=deduped.xls "duplicates" # export project to file in xls format
|
||||||
|
--delete "duplicates" # delete project
|
||||||
|
|
||||||
|
Some more examples:
|
||||||
|
--info 1234567890123 # specify project by id
|
||||||
|
--create example.tsv --encoding=UTF-8
|
||||||
|
--create example.xml --recordPath=collection --recordPath=record
|
||||||
|
--create example.json --recordPath=_ --recordPath=_
|
||||||
|
--create example.xlsx --sheets=0
|
||||||
|
--create example.ods --sheets=0
|
||||||
|
|
||||||
|
Example for Templating Export:
|
||||||
|
Cf. https://github.com/opencultureconsulting/openrefine-client#advanced-templating
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --help > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.tsv"
|
||||||
|
🔣 code meaning
|
||||||
|
🍇 1F347 GRAPES
|
||||||
|
🍉 1F349 WATERMELON
|
||||||
|
🍒 1F352 CHERRIES
|
||||||
|
🍓 1F353 STRAWBERRY
|
||||||
|
🍍 1F34D PINEAPPLE
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << DATA > "tmp/${t}/${t}.assert"
|
||||||
|
column 001: 🔣
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.tsv"
|
||||||
|
${cmd} --info "${t}" | grep 'column 001' > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << DATA > "tmp/${t}/${t}.assert"
|
||||||
|
column 002: b
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --info "${t}" | grep 'column 002' > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
a,b,c
|
||||||
|
1,2,3
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << DATA > "tmp/${t}/${t}.assert"
|
||||||
|
${t}
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -2,14 +2,17 @@
|
||||||
|
|
||||||
# =============================== ENVIRONMENT ================================ #
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
t="test"
|
|
||||||
|
|
||||||
if [[ ${1} ]]; then
|
if [[ ${1} ]]; then
|
||||||
version="${1##*.}"
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
fi
|
fi
|
||||||
|
#if [[ ${2} ]]; then
|
||||||
|
# version="${2##*.}"
|
||||||
|
#fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
cd "${BASH_SOURCE%/*}/" || exit 1
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
client="python3 ../refine.py -H localhost -P 3334"
|
|
||||||
mkdir -p "tmp/${t}"
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
# =================================== DATA =================================== #
|
# =================================== DATA =================================== #
|
||||||
|
|
@ -51,9 +54,9 @@ DATA
|
||||||
|
|
||||||
# ================================== ACTION ================================== #
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
${client} --create "tmp/${t}/${t}.csv"
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
${client} --apply "tmp/${t}/${t}.transform" "${t}"
|
${cmd} --apply "tmp/${t}/${t}.transform" "${t}"
|
||||||
${client} --export "${t}" --output "tmp/${t}/${t}.output"
|
${cmd} --export "${t}" --output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
# =================================== TEST =================================== #
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
🔣,code,meaning
|
||||||
|
🍇,1F347,GRAPES
|
||||||
|
🍉,1F349,WATERMELON
|
||||||
|
🍒,1F352,CHERRIES
|
||||||
|
🍓,1F353,STRAWBERRY
|
||||||
|
🍍,1F34D,PINEAPPLE
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "emojis" : [
|
||||||
|
{ "symbol" : "🍇", "meaning" : "GRAPES" },
|
||||||
|
{ "symbol" : "🍉", "meaning" : "WATERMELON" },
|
||||||
|
{ "symbol" : "🍍", "meaning" : "PINEAPPLE" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "emojis" : [
|
||||||
|
' \
|
||||||
|
--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--filterQuery '^1F34' \
|
||||||
|
--filterColumn 'code' \
|
||||||
|
--output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,59 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
email,name,state,gender,purchase
|
||||||
|
danny.baron@example1.com,Danny Baron,CA,M,TV
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPhone
|
||||||
|
danny.baron@example1.com,D. Baron,CA,M,Winter jacket
|
||||||
|
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table
|
||||||
|
danny.baron@example1.com,Daniel Baron,CA,M,Bike
|
||||||
|
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPad
|
||||||
|
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Night table
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "events" : [
|
||||||
|
{ "name" : "Melanie White", "purchase" : "iPhone" },
|
||||||
|
{ "name" : "Jean Griffith", "purchase" : "Power drill" },
|
||||||
|
{ "name" : "Melanie White", "purchase" : "iPad" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "events" : [
|
||||||
|
' \
|
||||||
|
--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--filterQuery '^F$' \
|
||||||
|
--filterColumn 'gender' \
|
||||||
|
--output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
🔣,code,meaning
|
||||||
|
🍇,1F347,GRAPES
|
||||||
|
🍉,1F349,WATERMELON
|
||||||
|
🍒,1F352,CHERRIES
|
||||||
|
🍓,1F353,STRAWBERRY
|
||||||
|
🍍,1F34D,PINEAPPLE
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "emojis" : [
|
||||||
|
{ "symbol" : "🍍", "meaning" : "PINEAPPLE" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "emojis" : [
|
||||||
|
' \
|
||||||
|
--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--splitToFiles true \
|
||||||
|
--suffixById true \
|
||||||
|
--output "tmp/${t}/trái cây.json"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/trái cây_🍍.json"
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
email,name,state,gender,purchase
|
||||||
|
danny.baron@example1.com,Danny Baron,CA,M,TV
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPhone
|
||||||
|
danny.baron@example1.com,D. Baron,CA,M,Winter jacket
|
||||||
|
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table
|
||||||
|
danny.baron@example1.com,Daniel Baron,CA,M,Bike
|
||||||
|
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPad
|
||||||
|
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Night table
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "events" : [
|
||||||
|
{ "name" : "Arthur Duff", "purchase" : "Night table" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "events" : [
|
||||||
|
' \
|
||||||
|
--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--splitToFiles true \
|
||||||
|
--suffixById true \
|
||||||
|
--output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
ls "tmp/${t}"
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}_arthur.duff@example4.com.output"
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
🔣,code,meaning
|
||||||
|
🍇,1F347,GRAPES
|
||||||
|
🍉,1F349,WATERMELON
|
||||||
|
🍒,1F352,CHERRIES
|
||||||
|
🍓,1F353,STRAWBERRY
|
||||||
|
🍍,1F34D,PINEAPPLE
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "emojis" : [
|
||||||
|
{ "symbol" : "🍍", "meaning" : "PINEAPPLE" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "emojis" : [
|
||||||
|
' \
|
||||||
|
--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--splitToFiles true \
|
||||||
|
--output "tmp/${t}/trái cây.json"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/trái cây_5.json"
|
||||||
|
|
@ -0,0 +1,57 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
email,name,state,gender,purchase
|
||||||
|
danny.baron@example1.com,Danny Baron,CA,M,TV
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPhone
|
||||||
|
danny.baron@example1.com,D. Baron,CA,M,Winter jacket
|
||||||
|
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table
|
||||||
|
danny.baron@example1.com,Daniel Baron,CA,M,Bike
|
||||||
|
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPad
|
||||||
|
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Night table
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "events" : [
|
||||||
|
{ "name" : "Arthur Duff", "purchase" : "Night table" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "events" : [
|
||||||
|
' \
|
||||||
|
--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--splitToFiles true \
|
||||||
|
--output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
ls "tmp/${t}"
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}_10.output"
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
🔣,code,meaning
|
||||||
|
🍇,1F347,GRAPES
|
||||||
|
🍉,1F349,WATERMELON
|
||||||
|
🍒,1F352,CHERRIES
|
||||||
|
🍓,1F353,STRAWBERRY
|
||||||
|
🍍,1F34D,PINEAPPLE
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "emojis" : [
|
||||||
|
{ "symbol" : "🍇", "meaning" : "GRAPES" },
|
||||||
|
{ "symbol" : "🍉", "meaning" : "WATERMELON" },
|
||||||
|
{ "symbol" : "🍒", "meaning" : "CHERRIES" },
|
||||||
|
{ "symbol" : "🍓", "meaning" : "STRAWBERRY" },
|
||||||
|
{ "symbol" : "🍍", "meaning" : "PINEAPPLE" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "emojis" : [
|
||||||
|
' \
|
||||||
|
--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# =================================== DATA =================================== #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.csv"
|
||||||
|
email,name,state,gender,purchase
|
||||||
|
danny.baron@example1.com,Danny Baron,CA,M,TV
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPhone
|
||||||
|
danny.baron@example1.com,D. Baron,CA,M,Winter jacket
|
||||||
|
ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Dining table
|
||||||
|
danny.baron@example1.com,Daniel Baron,CA,M,Bike
|
||||||
|
jean.griffith@example5.org,Jean Griffith,WA,F,Power drill
|
||||||
|
melanie.white@example2.edu,Melanie White,NC,F,iPad
|
||||||
|
ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier
|
||||||
|
arthur.duff@example4.com,Arthur Duff,OR,M,Night table
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
{ "events" : [
|
||||||
|
{ "name" : "Danny Baron", "purchase" : "TV" },
|
||||||
|
{ "name" : "Melanie White", "purchase" : "iPhone" },
|
||||||
|
{ "name" : "D. Baron", "purchase" : "Winter jacket" },
|
||||||
|
{ "name" : "Ben Tyler", "purchase" : "Flashlight" },
|
||||||
|
{ "name" : "Arthur Duff", "purchase" : "Dining table" },
|
||||||
|
{ "name" : "Daniel Baron", "purchase" : "Bike" },
|
||||||
|
{ "name" : "Jean Griffith", "purchase" : "Power drill" },
|
||||||
|
{ "name" : "Melanie White", "purchase" : "iPad" },
|
||||||
|
{ "name" : "Ben Morisson", "purchase" : "Amplifier" },
|
||||||
|
{ "name" : "Arthur Duff", "purchase" : "Night table" }
|
||||||
|
] }
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} --create "tmp/${t}/${t}.csv"
|
||||||
|
${cmd} --export "${t}" \
|
||||||
|
--prefix '{ "events" : [
|
||||||
|
' \
|
||||||
|
--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \
|
||||||
|
--rowSeparator ',
|
||||||
|
' \
|
||||||
|
--suffix '
|
||||||
|
] }
|
||||||
|
' \
|
||||||
|
--output "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# =============================== ENVIRONMENT ================================ #
|
||||||
|
|
||||||
|
if [[ ${1} ]]; then
|
||||||
|
cmd="${1}"
|
||||||
|
else
|
||||||
|
echo 1>&2 "execute tests.sh to run all tests"; exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
t="$(basename "${BASH_SOURCE[0]}" .sh)"
|
||||||
|
cd "${BASH_SOURCE%/*}/" || exit 1
|
||||||
|
mkdir -p "tmp/${t}"
|
||||||
|
|
||||||
|
# ================================= ASSERTION ================================ #
|
||||||
|
|
||||||
|
cat << "DATA" > "tmp/${t}/${t}.assert"
|
||||||
|
Usage: refine.py [--help | OPTIONS]
|
||||||
|
|
||||||
|
DATA
|
||||||
|
|
||||||
|
# ================================== ACTION ================================== #
|
||||||
|
|
||||||
|
${cmd} > "tmp/${t}/${t}.output"
|
||||||
|
|
||||||
|
# =================================== TEST =================================== #
|
||||||
|
|
||||||
|
diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"
|
||||||
Loading…
Reference in New Issue