diff --git a/.gitignore b/.gitignore index f2e4460..7c12cbe 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ dist .* openrefine_client.egg-info refine.spec -README.html +openrefine-2.* +openrefine-3.* +openrefine-client_* +tests-cli.log diff --git a/README.md b/README.md index 378104c..d657672 100644 --- a/README.md +++ b/README.md @@ -679,11 +679,12 @@ There is also a script that uses docker images to run the unit tests with differ Note to myself: When releasing a new version... -1. Run tests +1. Run functional tests ```sh - ./tests.sh -a - jupyter notebook tests/cli_python2.ipynb + for v in 2.7 2.8 3.0 3.1 3.2; do + ./tests-cli.sh $v + done ``` 2. Make final changes in Git @@ -710,11 +711,12 @@ Note to myself: When releasing a new version... python2 -m PyInstaller --onefile refine.py --hidden-import google.refine.__main__ ``` -4. Run test with Linux executable +4. Run functional tests with Linux executable ```sh - ./tests.sh -a - jupyter notebook tests/cli_bash.ipynb + for v in 2.7 2.8 3.0 3.1 3.2; do + ./tests-cli.sh $v openrefine-client_0-3-7_linux + done ``` 5. Create release in GitHub diff --git a/tests-cli.sh b/tests-cli.sh new file mode 100755 index 0000000..deef082 --- /dev/null +++ b/tests-cli.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Script for running functional tests against the CLI + +# Copyright (c) 2011 Paul Makepeace, Real Programmers. All rights reserved. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +# ================================== CONFIG ================================== # + +cd "${BASH_SOURCE%/*}/" || exit 1 + +port=3334 + +if [[ ${1} ]]; then + version="${1}" +else + version="3.2" +fi +refine="openrefine-${version}/refine" + +if [[ ${2} ]]; then + client="$(readlink -e "${2}")" +else + client="python2 $(readlink -e refine.py)" +fi +cmd="${client} -H localhost -P ${port}" + +if [[ ${3} ]]; then + filename="${3%%.*}" +else + filename="" +fi +cmd="${client} -H localhost -P ${port}" + +# =============================== REQUIREMENTS =============================== # + +# check existence of java and cURL +if [[ -z "$(command -v java 2> /dev/null)" ]] ; then + echo 1>&2 "ERROR: OpenRefine requires JAVA runtime environment (jre)" \ + "https://openjdk.java.net/install/" + exit 1 +fi +if [[ -z "$(command -v curl 2> /dev/null)" ]] ; then + echo 1>&2 "ERROR: This shell script requires cURL" \ + "https://curl.haxx.se/download.html" + exit 1 +fi +# download OpenRefine +if [[ -z "$(readlink -e "${refine}")" ]]; then + echo "Download OpenRefine ${version}..." + mkdir -p "$(dirname "${refine}")" + curl -L --output openrefine.tar.gz \ + "https://github.com/OpenRefine/OpenRefine/releases/download/${version}/openrefine-linux-${version}.tar.gz" + echo "Install OpenRefine ${version} in subdirectory $(dirname "${refine}")..." + tar -xzf openrefine.tar.gz -C "$(dirname "${refine}")" --strip 1 --totals + rm -f openrefine.tar.gz + # do not try to open OpenRefine in browser + sed -i '$ a JAVA_OPTIONS=-Drefine.headless=true' \ + "$(dirname "${refine}")"/refine.ini + # set autosave period from 5 minutes to 25 hours + sed -i 's/#REFINE_AUTOSAVE_PERIOD=60/REFINE_AUTOSAVE_PERIOD=1500/' \ + "$(dirname "${refine}")"/refine.ini + echo +fi + +# ================================== SETUP =================================== # + +dir="$(readlink -f "tests/tmp")" +mkdir -p "${dir}" +rm -f tests-cli.log + +echo "start OpenRefine ${version}..." +${refine} -v warn -p ${port} -d "${dir}" &>> tests-cli.log & +pid_server=${!} +timeout 30s bash -c "until curl -s 'http://localhost:3334' \ + | cat | grep -q -o 'OpenRefine' ; do sleep 1; done" \ + || error "starting OpenRefine server failed!" +echo + +# ================================== TESTS =================================== # + +echo "running tests, please wait..." +tests=() +results=() +for t in tests/*${filename}*.sh; do + tests+=("${t}") + echo "======================= ${t} =======================" &>> tests-cli.log + bash "${t}" "${cmd}" "${version}" &>> tests-cli.log + results+=(${?}) +done +echo + +# ================================= TEARDOWN ================================= # + +echo "cleanup..." +{ kill -9 "${pid_server}" && wait "${pid_server}"; } 2>/dev/null +rm -rf "${dir}" +echo + +# ================================= SUMMARY ================================== # + +printf "%s\t%s\n" "code" "test" +printf "%s\t%s\n" "----" "----------------" +for i in "${!tests[@]}"; do + printf "%s\t%s\n" "${results[$i]}" "${tests[$i]}" +done +echo +if [[ " ${results[*]} " =~ [1-9] ]]; then + echo "failed tests! check tests-cli.log for debugging"; echo +else + echo "all tests passed!"; echo +fi diff --git a/tests/apply-utf8.sh b/tests/apply-utf8.sh new file mode 100644 index 0000000..bf58766 --- /dev/null +++ b/tests/apply-utf8.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/column-addition", + "engineConfig": { + "mode": "row-based" + }, + "newColumnName": "apply", + "columnInsertIndex": 2, + "baseColumnName": "b", + "expression": "grel:value.replace('2','⛲')", + "onError": "set-to-blank" + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b apply c +1 2 ⛲ 3 +0 0 0 0 +$ \ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/apply.sh b/tests/apply.sh new file mode 100644 index 0000000..ffbaa91 --- /dev/null +++ b/tests/apply.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/column-addition", + "engineConfig": { + "mode": "row-based" + }, + "newColumnName": "apply", + "columnInsertIndex": 2, + "baseColumnName": "b", + "expression": "grel:value.replace('2','TEST')", + "onError": "set-to-blank" + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b apply c +1 2 TEST 3 +0 0 0 0 +$ \ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/cli_bash.ipynb b/tests/cli_bash.ipynb deleted file mode 100644 index 5e4ca86..0000000 --- a/tests/cli_bash.ipynb +++ /dev/null @@ -1,819 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test executable in a Linux Bash environment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Install\n", - "\n", - "This notebook requires a [Bash kernel](https://github.com/takluyver/bash_kernel) environment and an OpenRefine server running at http://127.0.0.1:3333." - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/tmp/20190822_013937\n" - ] - } - ], - "source": [ - "workspace=$(date +%Y%m%d_%H%M%S)\n", - "mkdir -p /tmp/$workspace\n", - "cp -r data /tmp/$workspace\n", - "cd /tmp/$workspace && pwd" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux:\n", - "2019-08-22 01:39:40 ERROR 404: Not Found.\n" - ] - } - ], - "source": [ - "wget -nv https://github.com/opencultureconsulting/openrefine-client/releases/download/v0.3.8/openrefine-client_0-3-8_linux -O openrefine-client\n", - "chmod +x openrefine-client" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## README.MD" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Download" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Download to file duplicates.csv complete\n" - ] - } - ], - "source": [ - "./openrefine-client --download \"https://git.io/fj5hF\" --output=duplicates.csv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2334935475634\n", - "rows: 10\n" - ] - } - ], - "source": [ - "./openrefine-client --create duplicates.csv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### List" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 2334935475634: duplicates\n" - ] - } - ], - "source": [ - "./openrefine-client --list" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Info" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " id: 2334935475634\n", - " url: http://127.0.0.1:3333/project?project=2334935475634\n", - " name: duplicates\n", - " modified: 2019-08-21T23:40:30Z\n", - " created: 2019-08-21T23:40:30Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n" - ] - } - ], - "source": [ - "./openrefine-client --info \"duplicates\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Export" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "email\tname\tstate\tgender\tpurchase\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\tiPhone\n", - "danny.baron@example1.com\tD. Baron\tCA\tM\tWinter jacket\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\n", - "danny.baron@example1.com\tDaniel Baron\tCA\tM\tBike\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\tiPad\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"duplicates\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Apply" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Download to file duplicates-deletion.json complete\n" - ] - } - ], - "source": [ - "./openrefine-client --download \"https://git.io/fj5ju\" --output=duplicates-deletion.json" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "File duplicates-deletion.json has been successfully applied to project 2334935475634\n" - ] - } - ], - "source": [ - "./openrefine-client --apply duplicates-deletion.json \"duplicates\"" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "email\tcount\tname\tstate\tgender\tpurchase\n", - "arthur.duff@example4.com\t2\tArthur Duff\tOR\tM\tDining table\n", - "ben.morisson@example6.org\t1\tBen Morisson\tFL\tM\tAmplifier\n", - "ben.tyler@example3.org\t1\tBen Tyler\tNV\tM\tFlashlight\n", - "danny.baron@example1.com\t3\tDanny Baron\tCA\tM\tTV\n", - "jean.griffith@example5.org\t1\tJean Griffith\tWA\tF\tPower drill\n", - "melanie.white@example2.edu\t2\tMelanie White\tNC\tF\tiPhone\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"duplicates\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Export XLS" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to file deduped.xls complete\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"duplicates\" --output deduped.xls" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Delete" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2334935475634 has been successfully deleted\n" - ] - } - ], - "source": [ - "./openrefine-client --delete \"duplicates\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Templating" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1633409429491\n", - "rows: 10\n" - ] - } - ], - "source": [ - "./openrefine-client --create duplicates.csv --projectName=advanced" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{ \"events\" : [\n", - " { \"name\" : \"Melanie White\", \"purchase\" : \"iPhone\" },\n", - " { \"name\" : \"Jean Griffith\", \"purchase\" : \"Power drill\" },\n", - " { \"name\" : \"Melanie White\", \"purchase\" : \"iPad\" }\n", - "] }" - ] - } - ], - "source": [ - "./openrefine-client --export \"advanced\" \\\n", - "--prefix='{ \"events\" : [\n", - "' \\\n", - "--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n", - "--rowSeparator=',\n", - "' \\\n", - "--suffix='\n", - "] }' \\\n", - "--filterQuery='^F$' \\\n", - "--filterColumn='gender'" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: advanced_3.json\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"advanced\" \\\n", - "--prefix='{ \"events\" : [\n", - "' \\\n", - "--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n", - "--rowSeparator=',\n", - "' \\\n", - "--suffix='\n", - "] }' \\\n", - "--filterQuery='^F$' \\\n", - "--filterColumn='gender' \\\n", - "--output=advanced.json \\\n", - "--splitToFiles=true" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: advanced_melanie.white@example2.edu.json\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"advanced\" \\\n", - "--prefix='{ \"events\" : [\n", - "' \\\n", - "--template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }' \\\n", - "--rowSeparator=',\n", - "' \\\n", - "--suffix='\n", - "] }' \\\n", - "--filterQuery='^F$' \\\n", - "--filterColumn='gender' \\\n", - "--output=advanced.json \\\n", - "--splitToFiles=true \\\n", - "--suffixById=true" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "advanced_1.json \u001b[0m\u001b[38;5;33mdata\u001b[0m\n", - "advanced_2.json deduped.xls\n", - "advanced_3.json duplicates.csv\n", - "advanced_jean.griffith@example5.org.json duplicates-deletion.json\n", - "advanced_melanie.white@example2.edu.json \u001b[38;5;40mopenrefine-client\u001b[0m\n" - ] - } - ], - "source": [ - "ls" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Delete" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 1633409429491 has been successfully deleted\n" - ] - } - ], - "source": [ - "./openrefine-client --delete \"advanced\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Unicode" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### fruits" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2280962953279\n", - "rows: 5\n", - " id: 2280962953279\n", - " url: http://127.0.0.1:3333/project?project=2280962953279\n", - " name: evil-fruits\n", - " modified: 2019-08-21T23:40:43Z\n", - " created: 2019-08-21T23:40:43Z\n", - " rowCount: 5\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/evil-fruits.tsv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'evil-fruits', u'processQuotes': True, u'limit': -1, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: 🔣\n", - " column 002: code\n", - " column 003: meaning\n", - "🔣\tcode\tmeaning\n", - "🍇\t1F347\tGRAPES\n", - "🍉\t1F349\tWATERMELON\n", - "🍒\t1F352\tCHERRIES\n", - "🍓\t1F353\tSTRAWBERRY\n", - "🍍\t1F34D\tPINEAPPLE\n" - ] - } - ], - "source": [ - "./openrefine-client --create data/cli/evil-fruits.tsv\n", - "./openrefine-client --info \"evil-fruits\"\n", - "./openrefine-client --export \"evil-fruits\"" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to file emojis.csv complete\n", - "🔣,code,meaning\n", - "🍇,1F347,GRAPES\n", - "🍉,1F349,WATERMELON\n", - "🍒,1F352,CHERRIES\n", - "🍓,1F353,STRAWBERRY\n", - "🍍,1F34D,PINEAPPLE\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"evil-fruits\" --output emojis.csv\n", - "cat emojis.csv" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{ \"emojis\" : [\n", - " { \"symbol\" : \"🍇\", \"meaning\" : \"GRAPES\" },\n", - " { \"symbol\" : \"🍉\", \"meaning\" : \"WATERMELON\" },\n", - " { \"symbol\" : \"🍍\", \"meaning\" : \"PINEAPPLE\" }\n", - "] }" - ] - } - ], - "source": [ - "./openrefine-client --export \"evil-fruits\" \\\n", - "--prefix='{ \"emojis\" : [\n", - "' \\\n", - "--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n", - "--rowSeparator=',\n", - "' \\\n", - "--suffix='\n", - "] }' \\\n", - "--filterQuery='^1F34' \\\n", - "--filterColumn='code'" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: trái cây_3.json\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"evil-fruits\" \\\n", - "--prefix='{ \"emojis\" : [\n", - "' \\\n", - "--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n", - "--rowSeparator=',\n", - "' \\\n", - "--suffix='\n", - "] }' \\\n", - "--filterQuery='^1F34' \\\n", - "--filterColumn='code' \\\n", - "--output='trái cây.json' \\\n", - "--splitToFiles=true" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: trái cây_🍍.json\n" - ] - } - ], - "source": [ - "./openrefine-client --export \"evil-fruits\" \\\n", - "--prefix='{ \"emojis\" : [\n", - "' \\\n", - "--template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }' \\\n", - "--rowSeparator=',\n", - "' \\\n", - "--suffix='\n", - "] }' \\\n", - "--filterQuery='^1F34' \\\n", - "--filterColumn='code' \\\n", - "--output='trái cây.json' \\\n", - "--splitToFiles=true \\\n", - "--suffixById=true" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " advanced_1.json emojis.csv\n", - " advanced_2.json \u001b[0m\u001b[38;5;40mopenrefine-client\u001b[0m\n", - " advanced_3.json 'trái cây_1.json'\n", - " advanced_jean.griffith@example5.org.json 'trái cây_2.json'\n", - " advanced_melanie.white@example2.edu.json 'trái cây_3.json'\n", - " \u001b[38;5;33mdata\u001b[0m 'trái cây_🍇.json'\n", - " deduped.xls 'trái cây_🍉.json'\n", - " duplicates.csv 'trái cây_🍍.json'\n", - " duplicates-deletion.json\n" - ] - } - ], - "source": [ - "ls" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2280962953279 has been successfully deleted\n" - ] - } - ], - "source": [ - "./openrefine-client --delete \"evil-fruits\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### emoji-data" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2019865211741\n", - "rows: 20\n", - " id: 2019865211741\n", - " url: http://127.0.0.1:3333/project?project=2019865211741\n", - " name: dữ liệu biểu tượng cảm xúc\n", - " modified: 2019-08-21T23:41:06Z\n", - " created: 2019-08-21T23:41:06Z\n", - " rowCount: 20\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/d\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac.txt', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'd\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac', u'processQuotes': True, u'skipDataLines': 34, u'limit': 20, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False, u'headerLines': 0}]\n", - " column 001: Column 1\n", - " column 002: Column 2\n", - " column 003: Column 3\n", - " column 004: Column 4\n", - " column 005: Column 5\n", - " column 006: Column 6\n", - "Column 1\tColumn 2\tColumn 3\tColumn 4\tColumn 5\tColumn 6\n", - "00A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (©) COPYRIGHT SIGN\n", - "00AE ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (®) REGISTERED SIGN\n", - "203C ;\ttext ;\tL1 ;\tnone ;\ta j\t# V1.1 (‼) DOUBLE EXCLAMATION MARK\n", - "2049 ;\ttext ;\tL1 ;\tnone ;\ta j\t# V3.0 (⁉) EXCLAMATION QUESTION MARK\n", - "2122 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (™) TRADE MARK SIGN\n", - "2139 ;\ttext ;\tL1 ;\tnone ;\tj\t# V3.0 (ℹ) INFORMATION SOURCE\n", - "2194 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↔) LEFT RIGHT ARROW\n", - "2195 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↕) UP DOWN ARROW\n", - "2196 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↖) NORTH WEST ARROW\n", - "2197 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↗) NORTH EAST ARROW\n", - "2198 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↘) SOUTH EAST ARROW\n", - "2199 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↙) SOUTH WEST ARROW\n", - "21A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↩) LEFTWARDS ARROW WITH HOOK\n", - "21AA ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↪) RIGHTWARDS ARROW WITH HOOK\n", - "231A ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌚) WATCH\n", - "231B ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌛) HOURGLASS\n", - "2328 ;\ttext ;\tL2 ;\tnone ;\tx\t# V1.1 (⌨) KEYBOARD\n", - "23CF ;\ttext ;\tL2 ;\tnone ;\tx\t# V4.0 (⏏) EJECT SYMBOL\n", - "23E9 ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏩) BLACK RIGHT-POINTING DOUBLE TRIANGLE\n", - "23EA ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏪) BLACK LEFT-POINTING DOUBLE TRIANGLE\n" - ] - } - ], - "source": [ - "./openrefine-client --create \"data/cli/dữ liệu biểu tượng cảm xúc.txt\" \\\n", - "--format=tsv \\\n", - "--headerLines=0 \\\n", - "--skipDataLines=34 \\\n", - "--limit=20\n", - "./openrefine-client --info \"dữ liệu biểu tượng cảm xúc\"\n", - "./openrefine-client --export \"dữ liệu biểu tượng cảm xúc\"" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 2019865211741: dữ liệu biểu tượng cảm xúc\n" - ] - } - ], - "source": [ - "./openrefine-client --list" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Delete" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2019865211741 has been successfully deleted\n" - ] - } - ], - "source": [ - "./openrefine-client --delete \"dữ liệu biểu tượng cảm xúc\"" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Bash", - "language": "bash", - "name": "bash" - }, - "language_info": { - "codemirror_mode": "shell", - "file_extension": ".sh", - "mimetype": "text/x-sh", - "name": "bash" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/cli_python2.ipynb b/tests/cli_python2.ipynb deleted file mode 100644 index 933ea3c..0000000 --- a/tests/cli_python2.ipynb +++ /dev/null @@ -1,2824 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test module cli in a Python 2 environment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Install\n", - "\n", - "This notebook requires a Python 2.7 environment and an OpenRefine server running at http://127.0.0.1:3333." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mDEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support\u001b[0m\n", - "Processing /home/felix/git/openrefine-client\n", - "Requirement already satisfied, skipping upgrade: urllib2_file in /home/felix/.local/lib/python2.7/site-packages (from openrefine-client==0.3.7) (0.2.1)\n", - "Installing collected packages: openrefine-client\n", - " Found existing installation: openrefine-client 0.3.7\n", - " Uninstalling openrefine-client-0.3.7:\n", - " Successfully uninstalled openrefine-client-0.3.7\n", - " Running setup.py install for openrefine-client ... \u001b[?25ldone\n", - "\u001b[?25hSuccessfully installed openrefine-client-0.3.7\n" - ] - } - ], - "source": [ - "import sys\n", - "!{sys.executable} -m pip install .. --user --upgrade" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/tmp/tmp24HyYg\n" - ] - } - ], - "source": [ - "import tempfile\n", - "import shutil\n", - "import os\n", - "dirpath = tempfile.mkdtemp()\n", - "shutil.copytree('data',dirpath + '/data')\n", - "print(dirpath)\n", - "os.chdir(dirpath)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from google.refine import cli" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## README.md" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Download" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Download to file duplicates.csv complete\n" - ] - } - ], - "source": [ - "cli.download('https://git.io/fj5hF','duplicates.csv')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2019539621291\n", - "rows: 10\n" - ] - } - ], - "source": [ - "p1 = cli.create('duplicates.csv')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### List" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 2019539621291: duplicates\n" - ] - } - ], - "source": [ - "cli.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Info" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " id: 2019539621291\n", - " url: http://127.0.0.1:3333/project?project=2019539621291\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:03Z\n", - " created: 2019-08-21T23:31:02Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n" - ] - } - ], - "source": [ - "cli.info(p1.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Export" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "email\tname\tstate\tgender\tpurchase\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\tiPhone\n", - "danny.baron@example1.com\tD. Baron\tCA\tM\tWinter jacket\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\n", - "danny.baron@example1.com\tDaniel Baron\tCA\tM\tBike\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\tiPad\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\n" - ] - } - ], - "source": [ - "cli.export(p1.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Apply" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Download to file duplicates-deletion.json complete\n" - ] - } - ], - "source": [ - "cli.download('https://git.io/fj5ju','duplicates-deletion.json')" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "File duplicates-deletion.json has been successfully applied to project 2019539621291\n" - ] - } - ], - "source": [ - "cli.apply(p1.project_id, 'duplicates-deletion.json')" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "email\tcount\tname\tstate\tgender\tpurchase\n", - "arthur.duff@example4.com\t2\tArthur Duff\tOR\tM\tDining table\n", - "ben.morisson@example6.org\t1\tBen Morisson\tFL\tM\tAmplifier\n", - "ben.tyler@example3.org\t1\tBen Tyler\tNV\tM\tFlashlight\n", - "danny.baron@example1.com\t3\tDanny Baron\tCA\tM\tTV\n", - "jean.griffith@example5.org\t1\tJean Griffith\tWA\tF\tPower drill\n", - "melanie.white@example2.edu\t2\tMelanie White\tNC\tF\tiPhone\n" - ] - } - ], - "source": [ - "cli.export(p1.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Export XLS" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "email\tcount\tname\tstate\tgender\tpurchase\n", - "arthur.duff@example4.com\t2\tArthur Duff\tOR\tM\tDining table\n", - "ben.morisson@example6.org\t1\tBen Morisson\tFL\tM\tAmplifier\n", - "ben.tyler@example3.org\t1\tBen Tyler\tNV\tM\tFlashlight\n", - "danny.baron@example1.com\t3\tDanny Baron\tCA\tM\tTV\n", - "jean.griffith@example5.org\t1\tJean Griffith\tWA\tF\tPower drill\n", - "melanie.white@example2.edu\t2\tMelanie White\tNC\tF\tiPhone\n" - ] - } - ], - "source": [ - "cli.export(p1.project_id, 'deduped.xls')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Delete" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2019539621291 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p1.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Templating" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1716843473792\n", - "rows: 10\n" - ] - } - ], - "source": [ - "p2 = cli.create('duplicates.csv')" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{ \"events\" : [\n", - " { \"name\" : \"Melanie White\", \"purchase\" : \"iPhone\" },\n", - " { \"name\" : \"Jean Griffith\", \"purchase\" : \"Power drill\" },\n", - " { \"name\" : \"Melanie White\", \"purchase\" : \"iPad\" }\n", - "] }" - ] - } - ], - "source": [ - "cli.templating(p2.project_id,\n", - "prefix='''{ \"events\" : [\n", - "''',\n", - "template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }',\n", - "rowSeparator=''',\n", - "''',\n", - "suffix='''\n", - "] }''',\n", - "filterQuery='^F$',\n", - "filterColumn='gender')" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: advanced_3.json\n" - ] - } - ], - "source": [ - "cli.templating(p2.project_id,\n", - "prefix='''{ \"events\" : [\n", - "''',\n", - "template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }',\n", - "rowSeparator=''',\n", - "''',\n", - "suffix='''\n", - "] }''',\n", - "filterQuery='^F$',\n", - "filterColumn='gender',\n", - "output_file='advanced.json',\n", - "splitToFiles=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: advanced_melanie.white@example2.edu.json\n" - ] - } - ], - "source": [ - "cli.templating(p2.project_id,\n", - "prefix='''{ \"events\" : [\n", - "''',\n", - "template=' { \"name\" : {{jsonize(cells[\"name\"].value)}}, \"purchase\" : {{jsonize(cells[\"purchase\"].value)}} }',\n", - "rowSeparator=''',\n", - "''',\n", - "suffix='''\n", - "] }''',\n", - "filterQuery='^F$',\n", - "filterColumn='gender',\n", - "output_file='advanced.json',\n", - "splitToFiles=True,\n", - "suffixById=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['advanced_jean.griffith@example5.org.json',\n", - " 'advanced_melanie.white@example2.edu.json',\n", - " 'advanced_3.json',\n", - " 'advanced_2.json',\n", - " 'advanced_1.json',\n", - " 'duplicates-deletion.json',\n", - " 'duplicates.csv',\n", - " 'data']" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.listdir(os.getcwd())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Delete" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 1716843473792 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p2.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Unicode" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### fruits" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1929957235590\n", - "rows: 5\n", - " id: 1929957235590\n", - " url: http://127.0.0.1:3333/project?project=1929957235590\n", - " name: evil-fruits\n", - " modified: 2019-08-21T23:35:47Z\n", - " created: 2019-08-21T23:35:47Z\n", - " rowCount: 5\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/evil-fruits.tsv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'evil-fruits', u'processQuotes': True, u'limit': -1, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: 🔣\n", - " column 002: code\n", - " column 003: meaning\n", - "🔣\tcode\tmeaning\n", - "🍇\t1F347\tGRAPES\n", - "🍉\t1F349\tWATERMELON\n", - "🍒\t1F352\tCHERRIES\n", - "🍓\t1F353\tSTRAWBERRY\n", - "🍍\t1F34D\tPINEAPPLE\n" - ] - } - ], - "source": [ - "p1 = cli.create('data/cli/evil-fruits.tsv')\n", - "cli.info(p1.project_id)\n", - "cli.export(p1.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to file emojis.csv complete\n", - "🔣,code,meaning\n", - "🍇,1F347,GRAPES\n", - "🍉,1F349,WATERMELON\n", - "🍒,1F352,CHERRIES\n", - "🍓,1F353,STRAWBERRY\n", - "🍍,1F34D,PINEAPPLE\n", - "\n" - ] - } - ], - "source": [ - "cli.export(p1.project_id, output_file='emojis.csv')\n", - "with open('emojis.csv', 'r') as f:\n", - " print(f.read())" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{ \"emojis\" : [\n", - " { \"symbol\" : \"🍇\", \"meaning\" : \"GRAPES\" },\n", - " { \"symbol\" : \"🍉\", \"meaning\" : \"WATERMELON\" },\n", - " { \"symbol\" : \"🍍\", \"meaning\" : \"PINEAPPLE\" }\n", - "] }" - ] - } - ], - "source": [ - "cli.templating(p1.project_id,\n", - "prefix='''{ \"emojis\" : [\n", - "''',\n", - "template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }',\n", - "rowSeparator=''',\n", - "''',\n", - "suffix='''\n", - "] }''',\n", - "filterQuery='^1F34',\n", - "filterColumn='code')" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: trái cây_3.json\n" - ] - } - ], - "source": [ - "cli.templating(p1.project_id,\n", - "prefix='''{ \"emojis\" : [\n", - "''',\n", - "template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }',\n", - "rowSeparator=''',\n", - "''',\n", - "suffix='''\n", - "] }''',\n", - "filterQuery='^1F34',\n", - "filterColumn='code',\n", - "output_file='trái cây.json',\n", - "splitToFiles=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Export to files complete. Last file: trái cây_🍍.json\n" - ] - } - ], - "source": [ - "cli.templating(p1.project_id,\n", - "prefix='''{ \"emojis\" : [\n", - "''',\n", - "template=' { \"symbol\" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, \"meaning\" : {{jsonize(cells[\"meaning\"].value)}} }',\n", - "rowSeparator=''',\n", - "''',\n", - "suffix='''\n", - "] }''',\n", - "filterQuery='^1F34',\n", - "filterColumn='code',\n", - "output_file='trái cây.json',\n", - "splitToFiles=True,\n", - "suffixById=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['tr\\xc3\\xa1i c\\xc3\\xa2y_\\xf0\\x9f\\x8d\\x8d.json',\n", - " 'tr\\xc3\\xa1i c\\xc3\\xa2y_\\xf0\\x9f\\x8d\\x89.json',\n", - " 'tr\\xc3\\xa1i c\\xc3\\xa2y_\\xf0\\x9f\\x8d\\x87.json',\n", - " 'tr\\xc3\\xa1i c\\xc3\\xa2y_3.json',\n", - " 'tr\\xc3\\xa1i c\\xc3\\xa2y_2.json',\n", - " 'tr\\xc3\\xa1i c\\xc3\\xa2y_1.json',\n", - " 'emojis.csv',\n", - " 'advanced_jean.griffith@example5.org.json',\n", - " 'advanced_melanie.white@example2.edu.json',\n", - " 'advanced_3.json',\n", - " 'advanced_2.json',\n", - " 'advanced_1.json',\n", - " 'duplicates-deletion.json',\n", - " 'duplicates.csv',\n", - " 'data']" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.listdir(os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2401578251107 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p1.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### emoji data" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2314250240290\n", - "rows: 20\n", - " id: 2314250240290\n", - " url: http://127.0.0.1:3333/project?project=2314250240290\n", - " name: dữ liệu biểu tượng cảm xúc\n", - " modified: 2019-08-21T23:36:05Z\n", - " created: 2019-08-21T23:36:05Z\n", - " rowCount: 20\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/d\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac.txt', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'd\\u1eef li\\u1ec7u bi\\u1ec3u t\\u01b0\\u1ee3ng c\\u1ea3m x\\xfac', u'processQuotes': True, u'skipDataLines': 34, u'limit': 20, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False, u'headerLines': 0}]\n", - " column 001: Column 1\n", - " column 002: Column 2\n", - " column 003: Column 3\n", - " column 004: Column 4\n", - " column 005: Column 5\n", - " column 006: Column 6\n", - "Column 1\tColumn 2\tColumn 3\tColumn 4\tColumn 5\tColumn 6\n", - "00A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (©) COPYRIGHT SIGN\n", - "00AE ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (®) REGISTERED SIGN\n", - "203C ;\ttext ;\tL1 ;\tnone ;\ta j\t# V1.1 (‼) DOUBLE EXCLAMATION MARK\n", - "2049 ;\ttext ;\tL1 ;\tnone ;\ta j\t# V3.0 (⁉) EXCLAMATION QUESTION MARK\n", - "2122 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (™) TRADE MARK SIGN\n", - "2139 ;\ttext ;\tL1 ;\tnone ;\tj\t# V3.0 (ℹ) INFORMATION SOURCE\n", - "2194 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↔) LEFT RIGHT ARROW\n", - "2195 ;\ttext ;\tL1 ;\tnone ;\tz j\t# V1.1 (↕) UP DOWN ARROW\n", - "2196 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↖) NORTH WEST ARROW\n", - "2197 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↗) NORTH EAST ARROW\n", - "2198 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↘) SOUTH EAST ARROW\n", - "2199 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↙) SOUTH WEST ARROW\n", - "21A9 ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↩) LEFTWARDS ARROW WITH HOOK\n", - "21AA ;\ttext ;\tL1 ;\tnone ;\tj\t# V1.1 (↪) RIGHTWARDS ARROW WITH HOOK\n", - "231A ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌚) WATCH\n", - "231B ;\temoji ;\tL1 ;\tnone ;\tj\t# V1.1 (⌛) HOURGLASS\n", - "2328 ;\ttext ;\tL2 ;\tnone ;\tx\t# V1.1 (⌨) KEYBOARD\n", - "23CF ;\ttext ;\tL2 ;\tnone ;\tx\t# V4.0 (⏏) EJECT SYMBOL\n", - "23E9 ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏩) BLACK RIGHT-POINTING DOUBLE TRIANGLE\n", - "23EA ;\temoji ;\tL1 ;\tnone ;\tj w\t# V6.0 (⏪) BLACK LEFT-POINTING DOUBLE TRIANGLE\n" - ] - } - ], - "source": [ - "p1 = cli.create('data/cli/dữ liệu biểu tượng cảm xúc.txt',\n", - " project_format='tsv',\n", - " headerLines=0,\n", - " skipDataLines=34,\n", - " limit=20)\n", - "cli.info(p1.project_id)\n", - "cli.export(p1.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 2314250240290: dữ liệu biểu tượng cảm xúc\n", - " 1929957235590: evil-fruits\n" - ] - } - ], - "source": [ - "cli.ls()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Delete" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 1602939526221 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p1.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## CSV" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1675776970201\n", - "rows: 10\n", - " id: 1675776970201\n", - " url: http://127.0.0.1:3333/project?project=1675776970201\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:05Z\n", - " created: 2019-08-21T23:31:05Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "danny.baron@example1.com\t\" D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "Project 1675776970201 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### encoding\n", - "\n", - "check TV symbol in line 1" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2268199900543\n", - "rows: 10\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "danny.baron@example1.com\t\" D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "Project 2268199900543 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', encoding='ISO-8859-1')\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1798292162864\n", - "rows: 10\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "danny.baron@example1.com\t\" D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "Project 1798292162864 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', encoding='UTF-8')\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### guessCellValueTypes\n", - "\n", - "check OpenRefine GUI at url below: numbers should be green" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2351526371150\n", - "rows: 10\n", - " id: 2351526371150\n", - " url: http://127.0.0.1:3333/project?project=2351526371150\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:05Z\n", - " created: 2019-08-21T23:31:05Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': True, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', guessCellValueTypes=True)\n", - "cli.info(p.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2351526371150 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### headerLines\n", - "\n", - "check column names, should be Column 1..." - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1753036694840\n", - "rows: 11\n", - "Column 1\tColumn 2\tColumn 3\tColumn 4\tColumn 5\tColumn 6\tColumn 7\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "danny.baron@example1.com\t\" D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "Project 1753036694840 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', headerLines=0)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### ignoreLines\n", - "\n", - "check column names, should start with arthur.duff as header" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1567779238383\n", - "rows: 5\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "Project 1567779238383 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', ignoreLines=5)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### limit\n", - "\n", - "should contain 5 rows" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2236287775552\n", - "rows: 5\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "danny.baron@example1.com\t\" D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "Project 2236287775552 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', limit=5)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### separator and processQuotes\n", - "\n", - "should contain 10 rows and 2 columns (Column 2)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2493837924937\n", - "rows: 10\n", - "email,name,state,gender,purchase,count,date\tColumn 2\n", - "\"danny.baron@example1.com,Danny Baron,CA,M,TV (UTF-8: 📺),1,\"\"Wed, 4 Jul 2001\"\t\n", - "melanie.white@example2.edu,Melanie White,NC,F,,1,2001-07-04T12:08:56\t\n", - "danny.baron@example1.com, D.\t\"(\"\"Tab\"\") Baron,CA,M,Winter jacket,1,2001-07-04\"\n", - "ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight,1,2001/07/04\t\n", - "arthur.duff@example4.com,Arthur Duff,OR,M,Dining table,1,2001-07\t\n", - "danny.baron@example1.com,Daniel Baron,,,Bike,1,2001\t\n", - "jean.griffith@example5.org,Jean Griffith,WA,F,Power drill,1,2000\t\n", - "melanie.white@example2.edu,Melanie White,NC,F,'iPad',1,1999\t\n", - "ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier,1,1998\t\n", - "arthur.duff@example4.com,Arthur Duff,OR,M,Night table,1,1997\t\n", - "Project 2493837924937 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', separator=' ', processQuotes=False)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### projectName" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1568868311685\n", - "rows: 10\n", - " id: 1568868311685\n", - " url: http://127.0.0.1:3333/project?project=1568868311685\n", - " name: foo\n", - " modified: 2019-08-21T23:31:06Z\n", - " created: 2019-08-21T23:31:06Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'foo', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n", - "Project 1568868311685 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', projectName='foo')\n", - "cli.info(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### projectTags (introduced in OpenRefine 2.8)\n", - "\n", - "check manually at http://127.0.0.1:3333 > Open Project if tags where stored" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1889306695897\n", - "rows: 10\n", - " id: 1889306695897\n", - " url: http://127.0.0.1:3333/project?project=1889306695897\n", - " name: duplicates\n", - " tags: [u'client1', u'beta']\n", - " modified: 2019-08-21T23:31:06Z\n", - " created: 2019-08-21T23:31:06Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'projectTags': [u'client1', u'beta'], u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', projectTags=['client1', 'beta'])\n", - "cli.info(p.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 1889306695897 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### skipDataLines\n", - "\n", - "should contain 5 rows" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1906416549071\n", - "rows: 5\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "Project 1906416549071 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', skipDataLines=5)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### storeBlankCellsAsNulls\n", - "\n", - "check OpenRefine GUI at url below:\n", - "* All > View > Show/Hide 'null' values in cells\n", - "* row 6 should contain null values in columns state and gender" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1641203332364\n", - "rows: 10\n", - " id: 1641203332364\n", - " url: http://127.0.0.1:3333/project?project=1641203332364\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:06Z\n", - " created: 2019-08-21T23:31:06Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': True, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', guessCellValueTypes=True)\n", - "cli.info(p.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 1641203332364 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## TSV" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2332414205165\n", - "rows: 10\n", - " id: 2332414205165\n", - " url: http://127.0.0.1:3333/project?project=2332414205165\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:06Z\n", - " created: 2019-08-21T23:31:06Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.tsv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'limit': -1, u'trimStrings': False, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "danny.baron@example1.com\t\"D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "Project 2332414205165 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.tsv')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## JSON" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1978993820770\n", - "rows: 10\n", - " id: 1978993820770\n", - " url: http://127.0.0.1:3333/project?project=1978993820770\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:06Z\n", - " created: 2019-08-21T23:31:06Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.json', u'storeBlankRows': True, u'encoding': u'', u'recordPath': [u'_', u'_'], u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: _ - name\n", - " column 002: _ - date\n", - " column 003: _ - email\n", - " column 004: _ - state\n", - " column 005: _ - count\n", - " column 006: _ - gender\n", - " column 007: _ - purchase\n", - "_ - name\t_ - date\t_ - email\t_ - state\t_ - count\t_ - gender\t_ - purchase\n", - "Danny Baron\tWed, 4 Jul 2001\tdanny.baron@example1.com\tCA\t1\tM\tTV (UTF-8: 📺)\n", - "Melanie White\t2001-07-04T12:08:56\tmelanie.white@example2.edu\tNC\t1\tF\t\n", - "\" D.\t(\"\"Tab\"\") Baron\"\t2001-07-04\tdanny.baron@example1.com\tCA\t1\tM\tWinter jacket\n", - "Ben Tyler\t2001/07/04\tben.tyler@example3.org\tNV\t1\tM\tFlashlight\n", - "Arthur Duff\t2001-07\tarthur.duff@example4.com\tOR\t1\tM\tDining table\n", - "Daniel Baron\t2001\tdanny.baron@example1.com\t\t1\t\tBike\n", - "Jean Griffith\t2000\tjean.griffith@example5.org\tWA\t1\tF\tPower drill\n", - "Melanie White\t1999\tmelanie.white@example2.edu\tNC\t1\tF\t'iPad'\n", - "Ben Morisson\t1998\tben.morisson@example6.org\tFL\t1\tM\tAmplifier\n", - "Arthur Duff\t1997\tarthur.duff@example4.com\tOR\t1\tM\tNight table\n", - "Project 1978993820770 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.json')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### trimStrings (broken, does not work in the GUI either)\n", - "\n", - "check row 3 if spaces before `D.` are deleted" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1892692171021\n", - "rows: 10\n", - " id: 1892692171021\n", - " url: http://127.0.0.1:3333/project?project=1892692171021\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:06Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.json', u'storeBlankRows': True, u'encoding': u'', u'recordPath': [u'_', u'_'], u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': True, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: _ - name\n", - " column 002: _ - date\n", - " column 003: _ - email\n", - " column 004: _ - state\n", - " column 005: _ - count\n", - " column 006: _ - gender\n", - " column 007: _ - purchase\n", - "_ - name\t_ - date\t_ - email\t_ - state\t_ - count\t_ - gender\t_ - purchase\n", - "Danny Baron\tWed, 4 Jul 2001\tdanny.baron@example1.com\tCA\t1\tM\tTV (UTF-8: 📺)\n", - "Melanie White\t2001-07-04T12:08:56\tmelanie.white@example2.edu\tNC\t1\tF\t\n", - "\" D.\t(\"\"Tab\"\") Baron\"\t2001-07-04\tdanny.baron@example1.com\tCA\t1\tM\tWinter jacket\n", - "Ben Tyler\t2001/07/04\tben.tyler@example3.org\tNV\t1\tM\tFlashlight\n", - "Arthur Duff\t2001-07\tarthur.duff@example4.com\tOR\t1\tM\tDining table\n", - "Daniel Baron\t2001\tdanny.baron@example1.com\t\t1\t\tBike\n", - "Jean Griffith\t2000\tjean.griffith@example5.org\tWA\t1\tF\tPower drill\n", - "Melanie White\t1999\tmelanie.white@example2.edu\tNC\t1\tF\t'iPad'\n", - "Ben Morisson\t1998\tben.morisson@example6.org\tFL\t1\tM\tAmplifier\n", - "Arthur Duff\t1997\tarthur.duff@example4.com\tOR\t1\tM\tNight table\n", - "Project 1892692171021 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.json', trimStrings=True)\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### recordPath" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1945894618034\n", - "rows: 10\n", - " id: 1945894618034\n", - " url: http://127.0.0.1:3333/project?project=1945894618034\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:07Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.json', u'storeBlankRows': True, u'encoding': u'', u'recordPath': [u'_', u'_', u'purchase'], u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: purchase\n", - "purchase\n", - "TV (UTF-8: 📺)\n", - "\n", - "Winter jacket\n", - "Flashlight\n", - "Dining table\n", - "Bike\n", - "Power drill\n", - "'iPad'\n", - "Amplifier\n", - "Night table\n", - "Project 1945894618034 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.json', recordPath=['_', '_', 'purchase'])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### storeEmptyStrings\n", - "\n", - "default: True; set to False for null values\n", - "\n", - "check OpenRefine GUI at url below:\n", - "* All > View > Show/Hide 'null' values in cells\n", - "* row 6 should contain null values in columns state and gender" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2551263767214\n", - "rows: 10\n", - " id: 2551263767214\n", - " url: http://127.0.0.1:3333/project?project=2551263767214\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:07Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': False, u'fileSource': u'data/cli/duplicates.json', u'storeBlankRows': True, u'encoding': u'', u'recordPath': [u'_', u'_'], u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: _ - name\n", - " column 002: _ - date\n", - " column 003: _ - email\n", - " column 004: _ - count\n", - " column 005: _ - purchase\n", - " column 006: _ - state\n", - " column 007: _ - gender\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.json', storeEmptyStrings=False)\n", - "cli.info(p.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2551263767214 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## XML" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1926835461545\n", - "rows: 80\n", - " id: 1926835461545\n", - " url: http://127.0.0.1:3333/project?project=1926835461545\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:07Z\n", - " rowCount: 80\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.xml', u'storeBlankRows': True, u'encoding': u'', u'recordPath': [u'root'], u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: root\n", - " column 002: root - record\n", - " column 003: root - record - name\n", - " column 004: root - record - date\n", - " column 005: root - record - email\n", - " column 006: root - record - count\n", - " column 007: root - record - purchase\n", - " column 008: root - record - state\n", - " column 009: root - record - gender\n", - "root\troot - record\troot - record - name\troot - record - date\troot - record - email\troot - record - count\troot - record - purchase\troot - record - state\troot - record - gender\n", - "\"\n", - " \"\t\"\n", - " \"\tDanny Baron\tWed, 4 Jul 2001\tdanny.baron@example1.com\t1\tTV (UTF-8: 📺)\tCA\tM\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\tMelanie White\t2001-07-04T12:08:56\tmelanie.white@example2.edu\t1\t\tNC\tF\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - "\"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\" D.\t(\"\"Tab\"\") Baron\"\t2001-07-04\tdanny.baron@example1.com\t1\tWinter jacket\tCA\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tBen Tyler\t2001/07/04\tben.tyler@example3.org\t1\tFlashlight\tNV\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tArthur Duff\t2001-07\tarthur.duff@example4.com\t1\tDining table\tOR\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tDaniel Baron\t2001\tdanny.baron@example1.com\t1\tBike\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tJean Griffith\t2000\tjean.griffith@example5.org\t1\tPower drill\tWA\tF\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tMelanie White\t1999\tmelanie.white@example2.edu\t1\t'iPad'\tNC\tF\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tBen Morisson\t1998\tben.morisson@example6.org\t1\tAmplifier\tFL\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tArthur Duff\t1997\tarthur.duff@example4.com\t1\tNight table\tOR\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "Project 1926835461545 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.xml')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### trimStrings (broken, does not work in the GUI either)\n", - "\n", - "check if spaces before `D.` are deleted" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1615744471501\n", - "rows: 80\n", - " id: 1615744471501\n", - " url: http://127.0.0.1:3333/project?project=1615744471501\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:07Z\n", - " rowCount: 80\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.xml', u'storeBlankRows': True, u'encoding': u'', u'recordPath': [u'root'], u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': True, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: root\n", - " column 002: root - record\n", - " column 003: root - record - name\n", - " column 004: root - record - date\n", - " column 005: root - record - email\n", - " column 006: root - record - count\n", - " column 007: root - record - purchase\n", - " column 008: root - record - state\n", - " column 009: root - record - gender\n", - "root\troot - record\troot - record - name\troot - record - date\troot - record - email\troot - record - count\troot - record - purchase\troot - record - state\troot - record - gender\n", - "\"\n", - " \"\t\"\n", - " \"\tDanny Baron\tWed, 4 Jul 2001\tdanny.baron@example1.com\t1\tTV (UTF-8: 📺)\tCA\tM\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - " \"\t\"\n", - " \"\tMelanie White\t2001-07-04T12:08:56\tmelanie.white@example2.edu\t1\t\tNC\tF\n", - "\"\n", - " \"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\"\n", - "\"\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\" D.\t(\"\"Tab\"\") Baron\"\t2001-07-04\tdanny.baron@example1.com\t1\tWinter jacket\tCA\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tBen Tyler\t2001/07/04\tben.tyler@example3.org\t1\tFlashlight\tNV\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tArthur Duff\t2001-07\tarthur.duff@example4.com\t1\tDining table\tOR\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tDaniel Baron\t2001\tdanny.baron@example1.com\t1\tBike\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tJean Griffith\t2000\tjean.griffith@example5.org\t1\tPower drill\tWA\tF\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tMelanie White\t1999\tmelanie.white@example2.edu\t1\t'iPad'\tNC\tF\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tBen Morisson\t1998\tben.morisson@example6.org\t1\tAmplifier\tFL\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\tArthur Duff\t1997\tarthur.duff@example4.com\t1\tNight table\tOR\tM\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "\t\"\n", - " \"\t\t\t\t\t\t\t\n", - "Project 1615744471501 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.xml', trimStrings=True)\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### recordPath" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1843370951454\n", - "rows: 10\n", - " id: 1843370951454\n", - " url: http://127.0.0.1:3333/project?project=1843370951454\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:07Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.xml', u'storeBlankRows': True, u'encoding': u'', u'recordPath': [u'root', u'record', u'purchase'], u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: purchase\n", - "purchase\n", - "TV (UTF-8: 📺)\n", - "\n", - "Winter jacket\n", - "Flashlight\n", - "Dining table\n", - "Bike\n", - "Power drill\n", - "'iPad'\n", - "Amplifier\n", - "Night table\n", - "Project 1843370951454 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.xml', recordPath=['root', 'record', 'purchase'])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### storeEmptyStrings\n", - "\n", - "default: True; set to False for null values\n", - "\n", - "check OpenRefine GUI at url below:\n", - "* All > View > Show/Hide 'null' values in cells\n", - "* row 6 should contain null values in columns state and gender" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2549624481101\n", - "rows: 10\n", - " id: 2549624481101\n", - " url: http://127.0.0.1:3333/project?project=2549624481101\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:07Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': False, u'fileSource': u'data/cli/duplicates.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.csv', storeEmptyStrings=False)\n", - "cli.info(p.project_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Project 2549624481101 has been successfully deleted\n" - ] - } - ], - "source": [ - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## TXT" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default (line-based)" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2029778313736\n", - "rows: 11\n", - " id: 2029778313736\n", - " url: http://127.0.0.1:3333/project?project=2029778313736\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:07Z\n", - " created: 2019-08-21T23:31:07Z\n", - " rowCount: 11\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.txt', u'storeBlankRows': True, u'encoding': u'', u'ignoreLines': -1, u'projectName': u'duplicates', u'processQuotes': True, u'skipDataLines': -1, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False, u'headerLines': 0}]\n", - " column 001: Column 1\n", - "Column 1\n", - "email name state gender purchase count date \n", - "danny.baron@example1.com Danny Baron CA M TV (UTF-8: 📺) 1 Wed, 4 Jul 2001 \n", - "melanie.white@example2.edu Melanie White NC F 1 2001-07-04T12:08:5\n", - "\"danny.baron@example1.com D.\t(\"\"Tab\"\") Baron CA M Winter jacket 1 2001-07-04 \"\n", - "ben.tyler@example3.org Ben Tyler NV M Flashlight 1 2001/07/04 \n", - "arthur.duff@example4.com Arthur Duff OR M Dining table 1 2001-07 \n", - "danny.baron@example1.com Daniel Baron Bike 1 2001 \n", - "jean.griffith@example5.org Jean Griffith WA F Power drill 1 2000 \n", - "melanie.white@example2.edu Melanie White NC F 'iPad' 1 1999 \n", - "ben.morisson@example6.org Ben Morisson FL M Amplifier 1 1998 \n", - "arthur.duff@example4.com Arthur Duff OR M Night table 1 1997 \n", - "Project 2029778313736 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.txt')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### linesPerRow\n", - "\n", - "should return 6 rows in 2 columns" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1614710460265\n", - "rows: 6\n", - " id: 1614710460265\n", - " url: http://127.0.0.1:3333/project?project=1614710460265\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:08Z\n", - " created: 2019-08-21T23:31:08Z\n", - " rowCount: 6\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.txt', u'storeBlankRows': True, u'encoding': u'', u'ignoreLines': -1, u'projectName': u'duplicates', u'processQuotes': True, u'limit': -1, u'skipDataLines': -1, u'separator': u',', u'trimStrings': False, u'linesPerRow': 2, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False, u'headerLines': 0}]\n", - " column 001: Column 1\n", - " column 002: Column 2\n", - "Column 1\tColumn 2\n", - "email name state gender purchase count date \tdanny.baron@example1.com Danny Baron CA M TV (UTF-8: 📺) 1 Wed, 4 Jul 2001 \n", - "melanie.white@example2.edu Melanie White NC F 1 2001-07-04T12:08:5\t\"danny.baron@example1.com D.\t(\"\"Tab\"\") Baron CA M Winter jacket 1 2001-07-04 \"\n", - "ben.tyler@example3.org Ben Tyler NV M Flashlight 1 2001/07/04 \tarthur.duff@example4.com Arthur Duff OR M Dining table 1 2001-07 \n", - "danny.baron@example1.com Daniel Baron Bike 1 2001 \tjean.griffith@example5.org Jean Griffith WA F Power drill 1 2000 \n", - "melanie.white@example2.edu Melanie White NC F 'iPad' 1 1999 \tben.morisson@example6.org Ben Morisson FL M Amplifier 1 1998 \n", - "arthur.duff@example4.com Arthur Duff OR M Night table 1 1997 \t\n", - "Project 1614710460265 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.txt', linesPerRow=2)\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### fixed-width: columnWidths and headerLines" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 1729341878534\n", - "rows: 10\n", - " id: 1729341878534\n", - " url: http://127.0.0.1:3333/project?project=1729341878534\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:08Z\n", - " created: 2019-08-21T23:31:08Z\n", - " rowCount: 10\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'data/cli/duplicates.txt', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'limit': -1, u'separator': u',', u'trimStrings': False, u'columnWidths': [27, 21, 6, 7, 15, 6, 1000], u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False, u'headerLines': 1}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com \tDanny Baron \tCA \tM \tTV (UTF-8: 📺) \t1 \tWed, 4 Jul 2001 \n", - "melanie.white@example2.edu \tMelanie White \tNC \tF \t \t1 \t2001-07-04T12:08:5\n", - "danny.baron@example1.com \t\" D.\t(\"\"Tab\"\") Baron \"\tCA \tM \tWinter jacket \t1 \t2001-07-04 \n", - "ben.tyler@example3.org \tBen Tyler \tNV \tM \tFlashlight \t1 \t2001/07/04 \n", - "arthur.duff@example4.com \tArthur Duff \tOR \tM \tDining table \t1 \t2001-07 \n", - "danny.baron@example1.com \tDaniel Baron \t \t \tBike \t1 \t2001 \n", - "jean.griffith@example5.org \tJean Griffith \tWA \tF \tPower drill \t1 \t2000 \n", - "melanie.white@example2.edu \tMelanie White \tNC \tF \t'iPad' \t1 \t1999 \n", - "ben.morisson@example6.org \tBen Morisson \tFL \tM \tAmplifier \t1 \t1998 \n", - "arthur.duff@example4.com \tArthur Duff \tOR \tM \tNight table \t1 \t1997 \n", - "Project 1729341878534 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.txt', columnWidths=[27, 21, 6, 7, 15, 6, 1000], headerLines=1)\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ZIP" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default\n", - "\n", - "should contain 16 rows" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2279718038457\n", - "rows: 16\n", - " id: 2279718038457\n", - " url: http://127.0.0.1:3333/project?project=2279718038457\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:08Z\n", - " created: 2019-08-21T23:31:08Z\n", - " rowCount: 16\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'duplicates2.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}, {u'storeEmptyStrings': True, u'fileSource': u'duplicates2.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': False}]\n", - " column 001: email\n", - " column 002: name\n", - " column 003: state\n", - " column 004: gender\n", - " column 005: purchase\n", - " column 006: count\n", - " column 007: date\n", - "email\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "danny.baron@example1.com\t\" D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "danny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "danny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺), Winter jacket, bike\t3\tWed, 4 Jul 2001, 2001-07-04, 2001\n", - "melanie.white@example2.edu\tMelanie White\tNC\tF\t, 'iPad'\t2\t2001-07-04T12:08:56, 1999\n", - "ben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "arthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table, Night table\t2\t2001-07, 1997\n", - "jean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "ben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "Project 2279718038457 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.zip')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### includeFileSources\n", - "\n", - "should contain column File" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: 2100283089198\n", - "rows: 16\n", - " id: 2100283089198\n", - " url: http://127.0.0.1:3333/project?project=2100283089198\n", - " name: duplicates\n", - " modified: 2019-08-21T23:31:08Z\n", - " created: 2019-08-21T23:31:08Z\n", - " rowCount: 16\n", - "importOptionMetadata: [{u'storeEmptyStrings': True, u'fileSource': u'duplicates2.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': True}, {u'storeEmptyStrings': True, u'fileSource': u'duplicates2.csv', u'storeBlankRows': True, u'encoding': u'', u'projectName': u'duplicates', u'processQuotes': True, u'separator': u',', u'trimStrings': False, u'limit': -1, u'storeBlankCellsAsNulls': True, u'guessCellValueTypes': False, u'includeFileSources': True}]\n", - " column 001: File\n", - " column 002: email\n", - " column 003: name\n", - " column 004: state\n", - " column 005: gender\n", - " column 006: purchase\n", - " column 007: count\n", - " column 008: date\n", - "File\temail\tname\tstate\tgender\tpurchase\tcount\tdate\n", - "duplicates.csv\tdanny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺)\t1\tWed, 4 Jul 2001\n", - "duplicates.csv\tmelanie.white@example2.edu\tMelanie White\tNC\tF\t\t1\t2001-07-04T12:08:56\n", - "duplicates.csv\tdanny.baron@example1.com\t\" D.\t(\"\"Tab\"\") Baron\"\tCA\tM\tWinter jacket\t1\t2001-07-04\n", - "duplicates.csv\tben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "duplicates.csv\tarthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table\t1\t2001-07\n", - "duplicates.csv\tdanny.baron@example1.com\tDaniel Baron\t\t\tBike\t1\t2001\n", - "duplicates.csv\tjean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "duplicates.csv\tmelanie.white@example2.edu\tMelanie White\tNC\tF\t'iPad'\t1\t1999\n", - "duplicates.csv\tben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "duplicates.csv\tarthur.duff@example4.com\tArthur Duff\tOR\tM\tNight table\t1\t1997\n", - "duplicates2.csv\tdanny.baron@example1.com\tDanny Baron\tCA\tM\tTV (UTF-8: 📺), Winter jacket, bike\t3\tWed, 4 Jul 2001, 2001-07-04, 2001\n", - "duplicates2.csv\tmelanie.white@example2.edu\tMelanie White\tNC\tF\t, 'iPad'\t2\t2001-07-04T12:08:56, 1999\n", - "duplicates2.csv\tben.tyler@example3.org\tBen Tyler\tNV\tM\tFlashlight\t1\t2001/07/04\n", - "duplicates2.csv\tarthur.duff@example4.com\tArthur Duff\tOR\tM\tDining table, Night table\t2\t2001-07, 1997\n", - "duplicates2.csv\tjean.griffith@example5.org\tJean Griffith\tWA\tF\tPower drill\t1\t2000\n", - "duplicates2.csv\tben.morisson@example6.org\tBen Morisson\tFL\tM\tAmplifier\t1\t1998\n", - "Project 2100283089198 has been successfully deleted\n" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.zip', includeFileSources=True)\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## ODS (broken in OpenRefine >=2.8)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default\n", - "\n", - "many blank columns and rows in OpenRefine <=2.7 (also with manual import via GUI)" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "ename": "Exception", - "evalue": "Project not created", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcli\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data/cli/duplicates.ods'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mcli\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproject_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mcli\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproject_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mcli\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproject_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/home/felix/.local/lib/python2.7/site-packages/google/refine/cli.pyc\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(project_file, project_format, columnWidths, encoding, guessCellValueTypes, headerLines, ignoreLines, includeFileSources, limit, linesPerRow, processQuotes, projectName, projectTags, recordPath, separator, sheets, skipDataLines, storeBlankCellsAsNulls, storeBlankRows, storeEmptyStrings, trimStrings)\u001b[0m\n\u001b[1;32m 125\u001b[0m \u001b[0mstore_blank_cells_as_nulls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstoreBlankCellsAsNulls\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0minclude_file_sources\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mincludeFileSources\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m **kwargs)\n\u001b[0m\u001b[1;32m 128\u001b[0m \u001b[0mrows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mproject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdo_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get-rows'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'total'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrows\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/home/felix/.local/lib/python2.7/site-packages/google/refine/refine.pyc\u001b[0m in \u001b[0;36mnew_project\u001b[0;34m(self, project_file, project_url, project_name, project_format, encoding, separator, ignore_lines, header_lines, skip_data_lines, limit, store_blank_rows, guess_cell_value_types, process_quotes, store_blank_cells_as_nulls, include_file_sources, **opts)\u001b[0m\n\u001b[1;32m 279\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mRefineProject\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproject_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 280\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Project not created'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mException\u001b[0m: Project not created" - ] - } - ], - "source": [ - "p = cli.create('data/cli/duplicates.ods')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### sheets" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "first sheet from file with 2 sheets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates2.ods', sheets=[0])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "both sheets from file with 2 sheets: should contain 16 rows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates2.ods', sheets=[0, 1])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## XLS (broken in OpenRefine >=2.8)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates.xls')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### sheets" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "first sheet from file with 2 sheets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates2.xls', sheets=[0])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "both sheets from file with 2 sheets: should contain 16 rows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates2.xls', sheets=[0, 1])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## XLSX (broken in OpenRefine >=2.8)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates.xlsx')\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### sheets" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "first sheet from file with 2 sheets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates2.xlsx', sheets=[0])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "both sheets from file with 2 sheets: should contain 16 rows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "p = cli.create('data/cli/duplicates2.xlsx', sheets=[0, 1])\n", - "cli.info(p.project_id)\n", - "cli.export(p.project_id)\n", - "cli.delete(p.project_id)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.16" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/create-csv-encoding.sh b/tests/create-csv-encoding.sh new file mode 100644 index 0000000..fb10341 --- /dev/null +++ b/tests/create-csv-encoding.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}-utf8.csv" +a,b,c +1,2,3 +ä,é,ß +$,\,' +DATA +iconv -f UTF-8 -t ISO-8859-1 "tmp/${t}/${t}-utf8.csv" > "tmp/${t}/${t}.csv" + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +ä é ß +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --encoding "ISO-8859-1" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-guessCellValueTypes.sh b/tests/create-csv-guessCellValueTypes.sh new file mode 100644 index 0000000..09fe3ca --- /dev/null +++ b/tests/create-csv-guessCellValueTypes.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +01,02,03 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +1 2 3 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --guessCellValueTypes "true" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-headerLines.sh b/tests/create-csv-headerLines.sh new file mode 100644 index 0000000..5548345 --- /dev/null +++ b/tests/create-csv-headerLines.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Column 1 Column 2 Column 3 +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --headerLines "0" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-ignoreLines.sh b/tests/create-csv-ignoreLines.sh new file mode 100644 index 0000000..fa04da9 --- /dev/null +++ b/tests/create-csv-ignoreLines.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --ignoreLines "1" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-limit.sh b/tests/create-csv-limit.sh new file mode 100644 index 0000000..3141391 --- /dev/null +++ b/tests/create-csv-limit.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +0 0 0 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --limit "2" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-processQuotes.sh b/tests/create-csv-processQuotes.sh new file mode 100644 index 0000000..a6d09de --- /dev/null +++ b/tests/create-csv-processQuotes.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,"2,0",3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c Column 4 +1 2 0 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --processQuotes "false" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-projectTags.sh b/tests/create-csv-projectTags.sh new file mode 100644 index 0000000..cf0f450 --- /dev/null +++ b/tests/create-csv-projectTags.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +if [[ "${version:0:1}" = "2" ]]; then + echo "projectTags were introduced in OpenRefine 3.0" + exit 200 +else + cat << "DATA" > "tmp/${t}/${t}.assert" + tags: [u'beta', u'client1'] +DATA +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --projectTags "beta" --projectTags "client1" +${cmd} --info "${t}" | grep ' tags: ' > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-separator.sh b/tests/create-csv-separator.sh new file mode 100644 index 0000000..175cb45 --- /dev/null +++ b/tests/create-csv-separator.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a;b;c +1;2;3 +0;0;0 +$;\;' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --separator ";" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-skipDataLines.sh b/tests/create-csv-skipDataLines.sh new file mode 100644 index 0000000..fb2650a --- /dev/null +++ b/tests/create-csv-skipDataLines.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --skipDataLines "1" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-storeBlankCellsAsNulls.sh b/tests/create-csv-storeBlankCellsAsNulls.sh new file mode 100644 index 0000000..669b040 --- /dev/null +++ b/tests/create-csv-storeBlankCellsAsNulls.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,,0 +$,\,' +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "b", + "expression": "grel:isNull(value)", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 false 3 +0 false 0 +$ false ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --storeBlankCellsAsNulls "false" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-storeBlankRows.sh b/tests/create-csv-storeBlankRows.sh new file mode 100644 index 0000000..117d020 --- /dev/null +++ b/tests/create-csv-storeBlankRows.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +,, +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --storeBlankRows "false" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv-utf8.sh b/tests/create-csv-utf8.sh new file mode 100644 index 0000000..055a371 --- /dev/null +++ b/tests/create-csv-utf8.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +⌨ code meaning +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc ⛲" +${cmd} --export "${t} biểu tượng cảm xúc ⛲" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-csv.sh b/tests/create-csv.sh new file mode 100644 index 0000000..e617a88 --- /dev/null +++ b/tests/create-csv.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-json-recordPath.sh b/tests/create-json-recordPath.sh new file mode 100644 index 0000000..9ae0cd7 --- /dev/null +++ b/tests/create-json-recordPath.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.json" +{ + "rows":[ + { + "a":1, + "b":2, + "c":3 + }, + { + "a":0, + "b":0, + "c":0 + }, + { + "a":"$", + "b":"\\", + "c":"\"" + } + ] +} +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +_ - a _ - b _ - c +1 2 3 +0 0 0 +$ \ """" +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.json" --recordPath "_" --recordPath "rows" --recordPath "_" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-json-storeEmptyStrings.sh b/tests/create-json-storeEmptyStrings.sh new file mode 100644 index 0000000..d2ebd55 --- /dev/null +++ b/tests/create-json-storeEmptyStrings.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.json" +[ + { + "a": 1, + "b": 2, + "c": 3 + }, + { + "a": "", + "b": "", + "c": "" + }, + { + "a": "$", + "b": "\\", + "c": "\"" + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +_ - a _ - b _ - c +1 2 3 +$ \ """" +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.json" --storeEmptyStrings "false" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-json-trimStrings.sh b/tests/create-json-trimStrings.sh new file mode 100644 index 0000000..20c27b5 --- /dev/null +++ b/tests/create-json-trimStrings.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.json" +[ + { + "a": 1, + "b": 2, + "c": 3 + }, + { + "a": "0", + "b": " 0", + "c": "0 " + }, + { + "a": "$", + "b": "\\", + "c": "\"" + } +] +DATA + +# ================================= ASSERTION ================================ # + +if [[ "${version:0:1}" = "2" || "${version}" = "3.0" || "${version}" = "3.1" || "${version}" = "3.2" || "${version}" = "3.3" ]]; then + echo "trimStrings option does not work in OpenRefine <=3.3" + echo "https://github.com/OpenRefine/OpenRefine/issues/2409" + exit 200 +else + cat << "DATA" > "tmp/${t}/${t}.assert" +_ - a _ - b _ - c +1 2 3 +0 0 0 +$ \ """" +DATA +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.json" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-json-utf8.sh b/tests/create-json-utf8.sh new file mode 100644 index 0000000..c3f1404 --- /dev/null +++ b/tests/create-json-utf8.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.json" +[ + { + "⌨": "⛲", + "code": "1F347", + "meaning": "FOUNTAIN" + }, + { + "⌨": "⛳", + "code": "1F349", + "meaning": "FLAG IN HOLE" + }, + { + "⌨": "⛵", + "code": "1F352", + "meaning": "SAILBOAT" + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +_ - ⌨ _ - code _ - meaning +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.json" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-json.sh b/tests/create-json.sh new file mode 100644 index 0000000..01025da --- /dev/null +++ b/tests/create-json.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.json" +[ + { + "a": 1, + "b": 2, + "c": 3 + }, + { + "a": 0, + "b": 0, + "c": 0 + }, + { + "a": "$", + "b": "\\", + "c": "\"" + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +_ - a _ - b _ - c +1 2 3 +0 0 0 +$ \ """" +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.json" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-ods-sheets-utf8.sh b/tests/create-ods-sheets-utf8.sh new file mode 100644 index 0000000..a9e4252 --- /dev/null +++ b/tests/create-ods-sheets-utf8.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cp "data/example.ods" "tmp/${t}/${t}.ods" + +# ================================= ASSERTION ================================ # + +if [[ "${version}" = "2.7" ]]; then + cat << "DATA" > "tmp/${t}/${t}.assert" +⌨ code meaning Column Column 5 Column 6 Column 7 Column 8 +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA +else + #TODO + echo "https://github.com/opencultureconsulting/openrefine-client/issues/4" + exit 200 +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.ods" --sheets 1 +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-ods.sh b/tests/create-ods.sh new file mode 100644 index 0000000..e13079e --- /dev/null +++ b/tests/create-ods.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cp "data/example.ods" "tmp/${t}/${t}.ods" +#a b c +#1 2 3 +#0 0 0 +#$ \ ' + +# ================================= ASSERTION ================================ # + +if [[ "${version}" = "2.7" ]]; then + cat << "DATA" > "tmp/${t}/${t}.assert" +a b c Column Column 5 Column 6 Column 7 Column 8 +1.0 2.0 3.0 +0.0 0.0 0.0 +$ \ ' +DATA +else + #TODO + echo "https://github.com/opencultureconsulting/openrefine-client/issues/4" + exit 200 +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.ods" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-tsv-utf8.sh b/tests/create-tsv-utf8.sh new file mode 100644 index 0000000..9e918c9 --- /dev/null +++ b/tests/create-tsv-utf8.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.tsv" +⌨ code meaning +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.tsv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.csv" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.csv" diff --git a/tests/create-tsv.sh b/tests/create-tsv.sh new file mode 100644 index 0000000..de71fb1 --- /dev/null +++ b/tests/create-tsv.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.tsv" +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.tsv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.csv" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.csv" diff --git a/tests/create-txt-fixed-width-headerLines.sh b/tests/create-txt-fixed-width-headerLines.sh new file mode 100644 index 0000000..378dd65 --- /dev/null +++ b/tests/create-txt-fixed-width-headerLines.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.txt" +1 2 3 +mon tue wed +$2 $300 $1 +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "1", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + }, + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "2", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + }, + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "3", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +1 2 3 +mon tue wed +$2 $300 $1 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.txt" --columnWidths "6" --columnWidths "6" --columnWidths "6" --headerLines "1" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-txt-fixed-width-utf8.sh b/tests/create-txt-fixed-width-utf8.sh new file mode 100644 index 0000000..c650a4d --- /dev/null +++ b/tests/create-txt-fixed-width-utf8.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.txt" +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "Column 1", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + }, + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "Column 2", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + }, + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "Column 3", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Column 1 Column 2 Column 3 +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.txt" --columnWidths "6" --columnWidths "6" --columnWidths "60" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-txt-fixed-width.sh b/tests/create-txt-fixed-width.sh new file mode 100644 index 0000000..a1679d0 --- /dev/null +++ b/tests/create-txt-fixed-width.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.txt" +1 2 3 +mon tue wed +$2 $300 $1 +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "Column 1", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + }, + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "Column 2", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + }, + { + "op": "core/text-transform", + "engineConfig": { + "facets": [], + "mode": "row-based" + }, + "columnName": "Column 3", + "expression": "grel:value.trim()", + "onError": "keep-original", + "repeat": false, + "repeatCount": 10 + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Column 1 Column 2 Column 3 +1 2 3 +mon tue wed +$2 $300 $1 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.txt" --columnWidths "6" --columnWidths "6" --columnWidths "6" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-txt-linesPerRow.sh b/tests/create-txt-linesPerRow.sh new file mode 100644 index 0000000..d913328 --- /dev/null +++ b/tests/create-txt-linesPerRow.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.txt" +mon tue wed +$2 $300 $1 +thu fri sat +$70 $20 $50 +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Column 1 Column 2 +mon tue wed $2 $300 $1 +thu fri sat $70 $20 $50 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.txt" --linesPerRow "2" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-txt.sh b/tests/create-txt.sh new file mode 100644 index 0000000..b6a8558 --- /dev/null +++ b/tests/create-txt.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.txt" +1 2 3 +mon tue wed +$2 $300 $1 +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Column 1 +1 2 3 +mon tue wed +$2 $300 $1 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.txt" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-xls-sheets-utf8.sh b/tests/create-xls-sheets-utf8.sh new file mode 100644 index 0000000..67ede5f --- /dev/null +++ b/tests/create-xls-sheets-utf8.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cp "data/example.xls" "tmp/${t}/${t}.xls" + +# ================================= ASSERTION ================================ # + +if [[ "${version}" = "2.7" ]]; then + cat << "DATA" > "tmp/${t}/${t}.assert" +⌨ code meaning +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA +else + #TODO + echo "https://github.com/opencultureconsulting/openrefine-client/issues/4" + exit 200 +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.xls" --sheets 1 +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-xls.sh b/tests/create-xls.sh new file mode 100644 index 0000000..711b4cf --- /dev/null +++ b/tests/create-xls.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cp "data/example.xls" "tmp/${t}/${t}.xls" +#a b c +#1 2 3 +#0 0 0 +#$ \ ' + +# ================================= ASSERTION ================================ # + +if [[ "${version}" = "2.7" ]]; then + cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1.0 2.0 3.0 +0.0 0.0 0.0 +$ \ ' +DATA +else + #TODO + echo "https://github.com/opencultureconsulting/openrefine-client/issues/4" + exit 200 +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.xls" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-xlsx-sheets-utf8.sh b/tests/create-xlsx-sheets-utf8.sh new file mode 100644 index 0000000..c90108d --- /dev/null +++ b/tests/create-xlsx-sheets-utf8.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cp "data/example.xlsx" "tmp/${t}/${t}.xlsx" + +# ================================= ASSERTION ================================ # + +if [[ "${version}" = "2.7" ]]; then + cat << "DATA" > "tmp/${t}/${t}.assert" +⌨ code meaning +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA +else + #TODO + echo "https://github.com/opencultureconsulting/openrefine-client/issues/4" + exit 200 +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.xlsx" --sheets 1 +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-xlsx.sh b/tests/create-xlsx.sh new file mode 100644 index 0000000..90fefe8 --- /dev/null +++ b/tests/create-xlsx.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + version="${2}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cp "data/example.xlsx" "tmp/${t}/${t}.xlsx" +#a b c +#1 2 3 +#0 0 0 +#$ \ ' + +# ================================= ASSERTION ================================ # + +if [[ "${version}" = "2.7" ]]; then + cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1.0 2.0 3.0 +0.0 0.0 0.0 +$ \ ' +DATA +else + #TODO + echo "https://github.com/opencultureconsulting/openrefine-client/issues/4" + exit 200 +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.xlsx" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-xml-recordPath.sh b/tests/create-xml-recordPath.sh new file mode 100644 index 0000000..d995255 --- /dev/null +++ b/tests/create-xml-recordPath.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.xml" + + + + 1 + 2 + 3 + + + 0 + 0 + 0 + + + $ + \ + ' + + +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/column-removal", + "columnName": "record" + }, + { + "op": "core/row-removal", + "engineConfig": { + "facets": [ + { + "type": "list", + "name": "Blank Rows", + "expression": "(filter(row.columnNames,cn,isNonBlank(cells[cn].value)).length()==0).toString()", + "columnName": "", + "invert": false, + "omitBlank": false, + "omitError": false, + "selection": [ + { + "v": { + "v": "true", + "l": "true" + } + } + ], + "selectBlank": false, + "selectError": false + } + ], + "mode": "record-based" + } + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +record - a record - b record - c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.xml" --recordPath "root" --recordPath "record" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-xml-utf8.sh b/tests/create-xml-utf8.sh new file mode 100644 index 0000000..b026ab3 --- /dev/null +++ b/tests/create-xml-utf8.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.xml" + + + + ⛲ + 1F347 + FOUNTAIN + + + ⛳ + 1F349 + FLAG IN HOLE + + + ⛵ + 1F352 + SAILBOAT + + +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/column-removal", + "columnName": "root" + }, + { + "op": "core/column-removal", + "columnName": "root - record" + }, + { + "op": "core/row-removal", + "engineConfig": { + "facets": [ + { + "type": "list", + "name": "Blank Rows", + "expression": "(filter(row.columnNames,cn,isNonBlank(cells[cn].value)).length()==0).toString()", + "columnName": "", + "invert": false, + "omitBlank": false, + "omitError": false, + "selection": [ + { + "v": { + "v": "true", + "l": "true" + } + } + ], + "selectBlank": false, + "selectError": false + } + ], + "mode": "record-based" + } + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +root - record - icon root - record - code root - record - meaning +⛲ 1F347 FOUNTAIN +⛳ 1F349 FLAG IN HOLE +⛵ 1F352 SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.xml" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-xml.sh b/tests/create-xml.sh new file mode 100644 index 0000000..c5f1fa9 --- /dev/null +++ b/tests/create-xml.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.xml" + + + + 1 + 2 + 3 + + + 0 + 0 + 0 + + + $ + \ + ' + + +DATA + +cat << "DATA" > "tmp/${t}/${t}.transform" +[ + { + "op": "core/column-removal", + "columnName": "root" + }, + { + "op": "core/column-removal", + "columnName": "root - record" + }, + { + "op": "core/row-removal", + "engineConfig": { + "facets": [ + { + "type": "list", + "name": "Blank Rows", + "expression": "(filter(row.columnNames,cn,isNonBlank(cells[cn].value)).length()==0).toString()", + "columnName": "", + "invert": false, + "omitBlank": false, + "omitError": false, + "selection": [ + { + "v": { + "v": "true", + "l": "true" + } + } + ], + "selectBlank": false, + "selectError": false + } + ], + "mode": "record-based" + } + } +] +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +root - record - a root - record - b root - record - c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.xml" +${cmd} --apply "tmp/${t}/${t}.transform" "${t}" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-zip-includeFileSources.sh b/tests/create-zip-includeFileSources.sh new file mode 100644 index 0000000..a2f1158 --- /dev/null +++ b/tests/create-zip-includeFileSources.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}-1.csv" +a,b,c +1,2,3 +DATA + +cat << "DATA" > "tmp/${t}/${t}-2.csv" +a,b,c +4,5,6 +DATA + +zip "tmp/${t}/${t}.zip" "tmp/${t}/${t}-1.csv" "tmp/${t}/${t}-2.csv" + +# ================================= ASSERTION ================================ # + +cat << DATA > "tmp/${t}/${t}.assert" +File a b c +tmp/${t}/${t}-1.csv 1 2 3 +tmp/${t}/${t}-2.csv 4 5 6 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.zip" --includeFileSources "true" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/create-zip.sh b/tests/create-zip.sh new file mode 100644 index 0000000..99023d1 --- /dev/null +++ b/tests/create-zip.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}-1.csv" +a,b,c +1,2,3 +DATA + +cat << "DATA" > "tmp/${t}/${t}-2.csv" +a,b,c +4,5,6 +DATA + +zip "tmp/${t}/${t}.zip" "tmp/${t}/${t}-1.csv" "tmp/${t}/${t}-2.csv" + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +4 5 6 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.zip" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/data/example.ods b/tests/data/example.ods new file mode 100644 index 0000000..abd5abd Binary files /dev/null and b/tests/data/example.ods differ diff --git a/tests/data/example.xls b/tests/data/example.xls new file mode 100644 index 0000000..cab9cd9 Binary files /dev/null and b/tests/data/example.xls differ diff --git a/tests/data/example.xlsx b/tests/data/example.xlsx new file mode 100644 index 0000000..484eb97 Binary files /dev/null and b/tests/data/example.xlsx differ diff --git a/tests/delete-utf8.sh b/tests/delete-utf8.sh new file mode 100644 index 0000000..cdcdfeb --- /dev/null +++ b/tests/delete-utf8.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +DATA + +# ================================= ASSERTION ================================ # + +cat << DATA > "tmp/${t}/${t}.assert" +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc ⛲" +${cmd} --delete "${t} biểu tượng cảm xúc ⛲" +${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/delete.sh b/tests/delete.sh new file mode 100644 index 0000000..c1c6872 --- /dev/null +++ b/tests/delete.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +DATA + +# ================================= ASSERTION ================================ # + +cat << DATA > "tmp/${t}/${t}.assert" +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --delete "${t}" +${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/download.sh b/tests/download.sh new file mode 100644 index 0000000..57eae8a --- /dev/null +++ b/tests/download.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# ================================== ACTION ================================== # + +${cmd} --download "https://git.io/fj5ju" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "data/duplicates-deletion.json" "tmp/${t}/${t}.output" diff --git a/tests/export-csv-utf8.sh b/tests/export-csv-utf8.sh new file mode 100644 index 0000000..76c5c54 --- /dev/null +++ b/tests/export-csv-utf8.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.tsv" +🔣 code meaning +🍇 1F347 GRAPES +🍉 1F349 WATERMELON +🍒 1F352 CHERRIES +🍓 1F353 STRAWBERRY +🍍 1F34D PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +🔣,code,meaning +🍇,1F347,GRAPES +🍉,1F349,WATERMELON +🍒,1F352,CHERRIES +🍓,1F353,STRAWBERRY +🍍,1F34D,PINEAPPLE +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.tsv" +${cmd} --export "${t}" --output "tmp/${t}/${t} biểu tượng cảm xúc 🍉.csv" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t} biểu tượng cảm xúc 🍉.csv" diff --git a/tests/export-csv.sh b/tests/export-csv.sh new file mode 100644 index 0000000..de71fb1 --- /dev/null +++ b/tests/export-csv.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.tsv" +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.tsv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.csv" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.csv" diff --git a/tests/export-html-utf8.sh b/tests/export-html-utf8.sh new file mode 100644 index 0000000..a883263 --- /dev/null +++ b/tests/export-html-utf8.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ ${2} ]]; then + majorversion="${2%%.*}" +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================= ASSERTION ================================ # + +if [[ "$majorversion" = 2 ]]; then + cat << "DATA" > "tmp/${t}/${t}.assert" + + +export-html-utf8 + + + + + + + + +
⌨codemeaning
⛲1F347FOUNTAIN
⛳1F349FLAG IN HOLE
⛵1F352SAILBOAT
+ + +DATA +else + cat << "DATA" > "tmp/${t}/${t}.assert" + + +export-html-utf8 + + + + + + + + +
⌨codemeaning
⛲1F347FOUNTAIN
⛳1F349FLAG IN HOLE
⛵1F352SAILBOAT
+ + +DATA +fi + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.html" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.html" diff --git a/tests/export-html.sh b/tests/export-html.sh new file mode 100644 index 0000000..f281f25 --- /dev/null +++ b/tests/export-html.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" + + +export-html + + + + + + + + +
abc
123
000
$\'
+ + +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.html" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.html" diff --git a/tests/export-ods-utf8.sh b/tests/export-ods-utf8.sh new file mode 100644 index 0000000..2ba61e3 --- /dev/null +++ b/tests/export-ods-utf8.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,"FLAG IN HOLE" +⛵,1F352,SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.ods" +(cd tmp/"${t}" && + ssconvert -S "${t}.ods" "${t}.csv" && + mv "${t}.csv.1" "${t}.output") + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/export-ods.sh b/tests/export-ods.sh new file mode 100644 index 0000000..41aee04 --- /dev/null +++ b/tests/export-ods.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi +if [[ -z "$(command -v ssconvert 2> /dev/null)" ]] ; then + echo 1>&2 "ERROR: This test requires ssconvert (gnumeric)" + exit 127 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.ods" +(cd tmp/"${t}" && + ssconvert -S "${t}.ods" "${t}.csv" && + mv "${t}.csv.1" "${t}.output") + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/export-tsv-utf8.sh b/tests/export-tsv-utf8.sh new file mode 100644 index 0000000..bbd80c6 --- /dev/null +++ b/tests/export-tsv-utf8.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +🔣,code,meaning +🍇,1F347,GRAPES +🍉,1F349,WATERMELON +🍒,1F352,CHERRIES +🍓,1F353,STRAWBERRY +🍍,1F34D,PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +🔣 code meaning +🍇 1F347 GRAPES +🍉 1F349 WATERMELON +🍒 1F352 CHERRIES +🍓 1F353 STRAWBERRY +🍍 1F34D PINEAPPLE +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.tsv" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.tsv" diff --git a/tests/export-tsv.sh b/tests/export-tsv.sh new file mode 100644 index 0000000..b783d42 --- /dev/null +++ b/tests/export-tsv.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.tsv" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.tsv" diff --git a/tests/export-utf8.sh b/tests/export-utf8.sh new file mode 100644 index 0000000..16a429c --- /dev/null +++ b/tests/export-utf8.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +🔣,code,meaning +🍇,1F347,GRAPES +🍉,1F349,WATERMELON +🍒,1F352,CHERRIES +🍓,1F353,STRAWBERRY +🍍,1F34D,PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +🔣 code meaning +🍇 1F347 GRAPES +🍉 1F349 WATERMELON +🍒 1F352 CHERRIES +🍓 1F353 STRAWBERRY +🍍 1F34D PINEAPPLE +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/export-xls-utf8.sh b/tests/export-xls-utf8.sh new file mode 100644 index 0000000..bf5532a --- /dev/null +++ b/tests/export-xls-utf8.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.xls" +(cd tmp/"${t}" && + ssconvert -S "${t}.xls" "${t}.csv" && + mv "${t}.csv" "${t}.output") + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/export-xls.sh b/tests/export-xls.sh new file mode 100644 index 0000000..3a9894d --- /dev/null +++ b/tests/export-xls.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.xls" +(cd tmp/"${t}" && + ssconvert -S "${t}.xls" "${t}.csv" && + mv "${t}.csv" "${t}.output") + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/export-xlsx-utf8.sh b/tests/export-xlsx-utf8.sh new file mode 100644 index 0000000..17b8a95 --- /dev/null +++ b/tests/export-xlsx-utf8.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +⌨,code,meaning +⛲,1F347,FOUNTAIN +⛳,1F349,FLAG IN HOLE +⛵,1F352,SAILBOAT +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.xlsx" +(cd tmp/"${t}" && + ssconvert -S "${t}.xlsx" "${t}.csv" && + mv "${t}.csv" "${t}.output") + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" + + diff --git a/tests/export-xlsx.sh b/tests/export-xlsx.sh new file mode 100644 index 0000000..dbe77ec --- /dev/null +++ b/tests/export-xlsx.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --output "tmp/${t}/${t}.xlsx" +(cd tmp/"${t}" && + ssconvert -S "${t}.xlsx" "${t}.csv" && + mv "${t}.csv" "${t}.output") + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/export.sh b/tests/export.sh new file mode 100644 index 0000000..a12194c --- /dev/null +++ b/tests/export.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/format-create-separator.sh b/tests/format-create-separator.sh new file mode 100644 index 0000000..8b5b55d --- /dev/null +++ b/tests/format-create-separator.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.txt" +a;b;c +1;2;3 +0;0;0 +$;\;' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a b c +1 2 3 +0 0 0 +$ \ ' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.txt" --format "csv" --separator ";" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/format-create.sh b/tests/format-create.sh new file mode 100644 index 0000000..5a51546 --- /dev/null +++ b/tests/format-create.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Column 1 +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --format "line-based" +${cmd} --export "${t}" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/format-export-output.sh b/tests/format-export-output.sh new file mode 100644 index 0000000..e859b74 --- /dev/null +++ b/tests/format-export-output.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --format "csv" --output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/format-export.sh b/tests/format-export.sh new file mode 100644 index 0000000..eb7fc57 --- /dev/null +++ b/tests/format-export.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +a,b,c +1,2,3 +0,0,0 +$,\,' +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" --format "csv" > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/help.sh b/tests/help.sh new file mode 100644 index 0000000..e7dda25 --- /dev/null +++ b/tests/help.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Script to provide a command line interface to an OpenRefine server. +DATA + +# ================================== ACTION ================================== # + +${cmd} --help | sed '3q;d' > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/info-utf8.sh b/tests/info-utf8.sh new file mode 100644 index 0000000..033bafb --- /dev/null +++ b/tests/info-utf8.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.tsv" +🔣 code meaning +🍇 1F347 GRAPES +🍉 1F349 WATERMELON +🍒 1F352 CHERRIES +🍓 1F353 STRAWBERRY +🍍 1F34D PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << DATA > "tmp/${t}/${t}.assert" + column 001: 🔣 +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.tsv" +${cmd} --info "${t}" | grep 'column 001' > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/info.sh b/tests/info.sh new file mode 100644 index 0000000..5403260 --- /dev/null +++ b/tests/info.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +DATA + +# ================================= ASSERTION ================================ # + +cat << DATA > "tmp/${t}/${t}.assert" + column 002: b +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --info "${t}" | grep 'column 002' > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/list-utf8.sh b/tests/list-utf8.sh new file mode 100644 index 0000000..e3d0359 --- /dev/null +++ b/tests/list-utf8.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +DATA + +# ================================= ASSERTION ================================ # + +cat << DATA > "tmp/${t}/${t}.assert" + ${t} biểu tượng cảm xúc ⛲ +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" --projectName "${t} biểu tượng cảm xúc ⛲" +${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/list.sh b/tests/list.sh new file mode 100644 index 0000000..f592ad1 --- /dev/null +++ b/tests/list.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +a,b,c +1,2,3 +DATA + +# ================================= ASSERTION ================================ # + +cat << DATA > "tmp/${t}/${t}.assert" + ${t} +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --list | grep "${t}" | cut -d ':' -f 2 > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/template-facets.sh b/tests/template-facets.sh new file mode 100644 index 0000000..e6e4c80 --- /dev/null +++ b/tests/template-facets.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +email,name,state,gender,purchase +danny.baron@example1.com,Danny Baron,CA,M,TV +melanie.white@example2.edu,Melanie White,NC,F,iPhone +danny.baron@example1.com,D. Baron,CA,M,Winter jacket +ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight +arthur.duff@example4.com,Arthur Duff,OR,M,Dining table +danny.baron@example1.com,Daniel Baron,CA,M,Bike +jean.griffith@example5.org,Jean Griffith,WA,F,Power drill +melanie.white@example2.edu,Melanie White,NC,F,iPad +ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier +arthur.duff@example4.com,Arthur Duff,OR,M,Night table +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "events" : [ + { "name" : "Melanie White", "purchase" : "iPhone" }, + { "name" : "Jean Griffith", "purchase" : "Power drill" }, + { "name" : "Melanie White", "purchase" : "iPad" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "events" : [ +' \ +--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--facets '{"type":"list","name":"gender","columnName":"gender","expression":"value","omitBlank":false,"omitError":false,"selection":[{"v":{"v":"F","l":"F"}}],"selectBlank":false,"selectError":false,"invert":false}' \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/template-filterQuery-utf8.sh b/tests/template-filterQuery-utf8.sh new file mode 100644 index 0000000..e8dfef9 --- /dev/null +++ b/tests/template-filterQuery-utf8.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +🔣,code,meaning +🍇,1F347,GRAPES +🍉,1F349,WATERMELON +🍒,1F352,CHERRIES +🍓,1F353,STRAWBERRY +🍍,1F34D,PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "emojis" : [ + { "symbol" : "🍇", "meaning" : "GRAPES" }, + { "symbol" : "🍉", "meaning" : "WATERMELON" }, + { "symbol" : "🍍", "meaning" : "PINEAPPLE" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "emojis" : [ +' \ +--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--filterQuery '^1F34' \ +--filterColumn 'code' \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/template-filterQuery.sh b/tests/template-filterQuery.sh new file mode 100644 index 0000000..a2bda73 --- /dev/null +++ b/tests/template-filterQuery.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +email,name,state,gender,purchase +danny.baron@example1.com,Danny Baron,CA,M,TV +melanie.white@example2.edu,Melanie White,NC,F,iPhone +danny.baron@example1.com,D. Baron,CA,M,Winter jacket +ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight +arthur.duff@example4.com,Arthur Duff,OR,M,Dining table +danny.baron@example1.com,Daniel Baron,CA,M,Bike +jean.griffith@example5.org,Jean Griffith,WA,F,Power drill +melanie.white@example2.edu,Melanie White,NC,F,iPad +ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier +arthur.duff@example4.com,Arthur Duff,OR,M,Night table +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "events" : [ + { "name" : "Melanie White", "purchase" : "iPhone" }, + { "name" : "Jean Griffith", "purchase" : "Power drill" }, + { "name" : "Melanie White", "purchase" : "iPad" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "events" : [ +' \ +--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--filterQuery '^F$' \ +--filterColumn 'gender' \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/template-splitToFiles-mode.sh b/tests/template-splitToFiles-mode.sh new file mode 100644 index 0000000..2f8c02e --- /dev/null +++ b/tests/template-splitToFiles-mode.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +email,name,state,gender,purchase +arthur.duff@example4.com,Arthur Duff,OR,M,Dining table +,Arthur Duff,OR,M,Night table +ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier +ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight +danny.baron@example1.com,Daniel Baron,CA,M,Bike +,Danny Baron,CA,M,TV +,D. Baron,CA,M,Winter jacket +jean.griffith@example5.org,Jean Griffith,WA,F,Power drill +melanie.white@example2.edu,Melanie White,NC,F,iPad +,Melanie White,NC,F,iPhone +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "events" : [ + { "name" : "Melanie White", "purchase" : "iPad" } { "name" : "Melanie White", "purchase" : "iPhone" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "events" : [ +' \ +--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--mode "record-based" \ +--splitToFiles true \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +ls "tmp/${t}" +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}_6.output" diff --git a/tests/template-splitToFiles-suffixById-utf8.sh b/tests/template-splitToFiles-suffixById-utf8.sh new file mode 100644 index 0000000..94a2d03 --- /dev/null +++ b/tests/template-splitToFiles-suffixById-utf8.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +🔣,code,meaning +🍇,1F347,GRAPES +🍉,1F349,WATERMELON +🍒,1F352,CHERRIES +🍓,1F353,STRAWBERRY +🍍,1F34D,PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "emojis" : [ + { "symbol" : "🍍", "meaning" : "PINEAPPLE" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "emojis" : [ +' \ +--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--splitToFiles true \ +--suffixById true \ +--output "tmp/${t}/trái cây.json" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/trái cây_🍍.json" diff --git a/tests/template-splitToFiles-suffixById.sh b/tests/template-splitToFiles-suffixById.sh new file mode 100644 index 0000000..4009205 --- /dev/null +++ b/tests/template-splitToFiles-suffixById.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +email,name,state,gender,purchase +danny.baron@example1.com,Danny Baron,CA,M,TV +melanie.white@example2.edu,Melanie White,NC,F,iPhone +danny.baron@example1.com,D. Baron,CA,M,Winter jacket +ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight +arthur.duff@example4.com,Arthur Duff,OR,M,Dining table +danny.baron@example1.com,Daniel Baron,CA,M,Bike +jean.griffith@example5.org,Jean Griffith,WA,F,Power drill +melanie.white@example2.edu,Melanie White,NC,F,iPad +ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier +arthur.duff@example4.com,Arthur Duff,OR,M,Night table +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "events" : [ + { "name" : "Arthur Duff", "purchase" : "Night table" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "events" : [ +' \ +--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--splitToFiles true \ +--suffixById true \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +ls "tmp/${t}" +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}_arthur.duff@example4.com.output" diff --git a/tests/template-splitToFiles-utf8.sh b/tests/template-splitToFiles-utf8.sh new file mode 100644 index 0000000..fa36abf --- /dev/null +++ b/tests/template-splitToFiles-utf8.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +🔣,code,meaning +🍇,1F347,GRAPES +🍉,1F349,WATERMELON +🍒,1F352,CHERRIES +🍓,1F353,STRAWBERRY +🍍,1F34D,PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "emojis" : [ + { "symbol" : "🍍", "meaning" : "PINEAPPLE" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "emojis" : [ +' \ +--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--splitToFiles true \ +--output "tmp/${t}/trái cây.json" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/trái cây_5.json" diff --git a/tests/template-splitToFiles.sh b/tests/template-splitToFiles.sh new file mode 100644 index 0000000..4cb2976 --- /dev/null +++ b/tests/template-splitToFiles.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +email,name,state,gender,purchase +danny.baron@example1.com,Danny Baron,CA,M,TV +melanie.white@example2.edu,Melanie White,NC,F,iPhone +danny.baron@example1.com,D. Baron,CA,M,Winter jacket +ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight +arthur.duff@example4.com,Arthur Duff,OR,M,Dining table +danny.baron@example1.com,Daniel Baron,CA,M,Bike +jean.griffith@example5.org,Jean Griffith,WA,F,Power drill +melanie.white@example2.edu,Melanie White,NC,F,iPad +ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier +arthur.duff@example4.com,Arthur Duff,OR,M,Night table +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "events" : [ + { "name" : "Arthur Duff", "purchase" : "Night table" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "events" : [ +' \ +--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--splitToFiles true \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +ls "tmp/${t}" +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}_10.output" diff --git a/tests/template-utf8.sh b/tests/template-utf8.sh new file mode 100644 index 0000000..851f8e2 --- /dev/null +++ b/tests/template-utf8.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +🔣,code,meaning +🍇,1F347,GRAPES +🍉,1F349,WATERMELON +🍒,1F352,CHERRIES +🍓,1F353,STRAWBERRY +🍍,1F34D,PINEAPPLE +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "emojis" : [ + { "symbol" : "🍇", "meaning" : "GRAPES" }, + { "symbol" : "🍉", "meaning" : "WATERMELON" }, + { "symbol" : "🍒", "meaning" : "CHERRIES" }, + { "symbol" : "🍓", "meaning" : "STRAWBERRY" }, + { "symbol" : "🍍", "meaning" : "PINEAPPLE" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "emojis" : [ +' \ +--template ' { "symbol" : {{jsonize(with(row.columnNames[0],cn,cells[cn].value))}}, "meaning" : {{jsonize(cells["meaning"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/template.sh b/tests/template.sh new file mode 100644 index 0000000..ef31b97 --- /dev/null +++ b/tests/template.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# =================================== DATA =================================== # + +cat << "DATA" > "tmp/${t}/${t}.csv" +email,name,state,gender,purchase +danny.baron@example1.com,Danny Baron,CA,M,TV +melanie.white@example2.edu,Melanie White,NC,F,iPhone +danny.baron@example1.com,D. Baron,CA,M,Winter jacket +ben.tyler@example3.org,Ben Tyler,NV,M,Flashlight +arthur.duff@example4.com,Arthur Duff,OR,M,Dining table +danny.baron@example1.com,Daniel Baron,CA,M,Bike +jean.griffith@example5.org,Jean Griffith,WA,F,Power drill +melanie.white@example2.edu,Melanie White,NC,F,iPad +ben.morisson@example6.org,Ben Morisson,FL,M,Amplifier +arthur.duff@example4.com,Arthur Duff,OR,M,Night table +DATA + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +{ "events" : [ + { "name" : "Danny Baron", "purchase" : "TV" }, + { "name" : "Melanie White", "purchase" : "iPhone" }, + { "name" : "D. Baron", "purchase" : "Winter jacket" }, + { "name" : "Ben Tyler", "purchase" : "Flashlight" }, + { "name" : "Arthur Duff", "purchase" : "Dining table" }, + { "name" : "Daniel Baron", "purchase" : "Bike" }, + { "name" : "Jean Griffith", "purchase" : "Power drill" }, + { "name" : "Melanie White", "purchase" : "iPad" }, + { "name" : "Ben Morisson", "purchase" : "Amplifier" }, + { "name" : "Arthur Duff", "purchase" : "Night table" } +] } +DATA + +# ================================== ACTION ================================== # + +${cmd} --create "tmp/${t}/${t}.csv" +${cmd} --export "${t}" \ +--prefix '{ "events" : [ +' \ +--template ' { "name" : {{jsonize(cells["name"].value)}}, "purchase" : {{jsonize(cells["purchase"].value)}} }' \ +--rowSeparator ', +' \ +--suffix ' +] } +' \ +--output "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output" diff --git a/tests/usage.sh b/tests/usage.sh new file mode 100644 index 0000000..1a7e0d4 --- /dev/null +++ b/tests/usage.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# =============================== ENVIRONMENT ================================ # + +if [[ ${1} ]]; then + cmd="${1}" +else + echo 1>&2 "execute tests-cli.sh to run all tests"; exit 1 +fi + +t="$(basename "${BASH_SOURCE[0]}" .sh)" +cd "${BASH_SOURCE%/*}/" || exit 1 +mkdir -p "tmp/${t}" + +# ================================= ASSERTION ================================ # + +cat << "DATA" > "tmp/${t}/${t}.assert" +Usage: +DATA + +# ================================== ACTION ================================== # + +${cmd} | head -n 1 | cut -c 1-6 > "tmp/${t}/${t}.output" + +# =================================== TEST =================================== # + +diff -u "tmp/${t}/${t}.assert" "tmp/${t}/${t}.output"