opencultureconsulting
/
orcli
mirror of https://github.com/opencultureconsulting/orcli.git
2
0
Fork 0
Code Issues Releases Activity

fix curl quoting with --form-string

This commit is contained in:
felixlohmeier 2023-01-12 10:18:06 +00:00
parent e82b306839
commit cd7d62e254
11 changed files with 90 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
orcli
View File

@ -900,11 +900,10 @@ function get_ids() {
# src/lib/init_import.sh
# common import tasks to support multiple files and URLs
# shellcheck shell=bash
# shellcheck shell=bash disable=SC2154
function init_import() {
local files file
# catch args, convert the space delimited string to an array
files=()
local files=()
eval "files=(${args[file]})"
# create tmp directory
tmpdir="$(mktemp -d)"
@ -915,7 +914,7 @@ function init_import() {
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "download of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# read pipes if name starts with /dev/fd
@ -924,7 +923,7 @@ function init_import() {
if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "reading of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# create a zip archive if there are multiple files
@ -936,25 +935,6 @@ function init_import() {
else
file="${files[0]}"
fi
# basic post data
if [[ ${file} == "-" ]]; then
data+=("project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
data+=("project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
data+=("project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
data+=("project-name=${name}")
fi
}
# src/lib/interactive.sh
@ -994,18 +974,18 @@ function log() {
# shellcheck shell=bash disable=SC2154
function post_export() {
local curloptions
# post
mapfile -t curloptions < <(for d in "$@"; do
echo "--data"
echo "$d"
done)
# support file output
if [[ ${args[--output]} ]]; then
if ! mkdir -p "$(dirname "${args[--output]}")"; then
error "unable to create parent directory for ${args[--output]}"
fi
curloptions+=("--output")
curloptions+=("${args[--output]}")
curloptions+=("--output" "${args[--output]}")
fi
# post
if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then
error "exporting ${args[project]} failed!"
else
@ -1019,15 +999,31 @@ function post_export() {
# post to create-project endpoint and validate
# shellcheck shell=bash disable=SC2154
function post_import() {
local curloptions
local projectid
local projectname
local rows
# post
local curloptions projectid projectname rows
mapfile -t curloptions < <(for d in "$@"; do
echo "--form"
echo "--form-string"
echo "$d"
done)
# basic post data
if [[ ${file} == "-" ]]; then
curloptions+=("--form" "project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
curloptions+=("--form" "project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
curloptions+=("--form-string" "project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
curloptions+=("--form-string" "project-name=${name}")
fi
# post
if ! redirect_url="$(curl -fs --write-out "%{redirect_url}\n" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/create-project-from-upload$(get_csrf)")"; then
error "importing ${args[file]} failed!"
fi
@ -1180,18 +1176,16 @@ orcli_delete_command() {
# :command.function
orcli_import_csv_command() {
# src/import_csv_command.sh
# shellcheck shell=bash
# shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data
init_import
# check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
orcli_import_csv_usage
exit 1
fi
fi
# assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv")
@ -1274,18 +1268,16 @@ orcli_import_csv_command() {
# :command.function
orcli_import_tsv_command() {
# src/import_tsv_command.sh
# shellcheck shell=bash
# shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data
init_import
# check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
orcli_import_tsv_usage
exit 1
fi
fi
# assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv")

6
src/import_csv_command.sh
View File

@ -1,15 +1,13 @@
# shellcheck shell=bash
# shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data
init_import
# check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
orcli_import_csv_usage
exit 1
fi
fi
# assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv")

6
src/import_tsv_command.sh
View File

@ -1,15 +1,13 @@
# shellcheck shell=bash
# shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data
init_import
# check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
if ! read -u 0 -t 0; then
if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
orcli_import_tsv_usage
exit 1
fi
fi
# assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv")

28
src/lib/init_import.sh
View File

@ -1,9 +1,8 @@
# common import tasks to support multiple files and URLs
# shellcheck shell=bash
# shellcheck shell=bash disable=SC2154
function init_import() {
local files file
# catch args, convert the space delimited string to an array
files=()
local files=()
eval "files=(${args[file]})"
# create tmp directory
tmpdir="$(mktemp -d)"
@ -14,7 +13,7 @@ function init_import() {
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "download of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# read pipes if name starts with /dev/fd
@ -23,7 +22,7 @@ function init_import() {
if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "reading of ${files[$i]} failed!"
fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi
done
# create a zip archive if there are multiple files
@ -35,23 +34,4 @@ function init_import() {
else
file="${files[0]}"
fi
# basic post data
if [[ ${file} == "-" ]]; then
data+=("project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
data+=("project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
data+=("project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
data+=("project-name=${name}")
fi
}

6
src/lib/post_export.sh
View File

@ -2,18 +2,18 @@
# shellcheck shell=bash disable=SC2154
function post_export() {
local curloptions
# post
mapfile -t curloptions < <(for d in "$@"; do
echo "--data"
echo "$d"
done)
# support file output
if [[ ${args[--output]} ]]; then
if ! mkdir -p "$(dirname "${args[--output]}")"; then
error "unable to create parent directory for ${args[--output]}"
fi
curloptions+=("--output")
curloptions+=("${args[--output]}")
curloptions+=("--output" "${args[--output]}")
fi
# post
if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then
error "exporting ${args[project]} failed!"
else

28
src/lib/post_import.sh
View File

@ -1,15 +1,31 @@
# post to create-project endpoint and validate
# shellcheck shell=bash disable=SC2154
function post_import() {
local curloptions
local projectid
local projectname
local rows
# post
local curloptions projectid projectname rows
mapfile -t curloptions < <(for d in "$@"; do
echo "--form"
echo "--form-string"
echo "$d"
done)
# basic post data
if [[ ${file} == "-" ]]; then
curloptions+=("--form" "project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
curloptions+=("--form" "project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
curloptions+=("--form-string" "project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
curloptions+=("--form-string" "project-name=${name}")
fi
# post
if ! redirect_url="$(curl -fs --write-out "%{redirect_url}\n" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/create-project-from-upload$(get_csrf)")"; then
error "importing ${args[file]} failed!"
fi

18
tests/delete.sh
View File

@ -15,15 +15,15 @@ DATA
# action
cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv"
orcli list | grep "${t} csv"
orcli delete "${t} csv"
orcli list | grep "${t} csv" > "${t}.output" || exit 0
orcli import csv "${t}.csv"
orcli import csv "${t}.csv"
orcli list | grep "${t} csv"
orcli delete --force "${t} csv"
orcli list | grep "${t} csv" >> "${t}.output" || exit 0
orcli import csv "${t}.csv" --projectName "${t}"
orcli list | grep "${t}"
orcli delete "${t}"
orcli list | grep "${t}" > "${t}.output" || exit 0
orcli import csv "${t}.csv" --projectName "${t}"
orcli import csv "${t}.csv" --projectName "${t}"
orcli list | grep "${t}"
orcli delete --force "${t}"
orcli list | grep "${t}" >> "${t}.output" || exit 0
# test
diff -u "${t}.assert" "${t}.output"

4
tests/export-tsv.sh
View File

@ -14,8 +14,8 @@ cp data/example.tsv "${tmpdir}/${t}.assert"
# action
cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv"
orcli export tsv "${t} csv" --output "${t}.output"
orcli import csv "${t}.csv" --projectName "${t}"
orcli export tsv "${t}" --output "${t}.output"
# test
diff -u "${t}.assert" "${t}.output"

4
tests/import-csv.sh
View File

@ -14,8 +14,8 @@ cp data/example.tsv "${tmpdir}/${t}.assert"
# action
cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv"
orcli export tsv "${t} csv" > "${t}.output"
orcli import csv "${t}.csv" --projectName "${t}"
orcli export tsv "${t}" > "${t}.output"
# test
diff -u "${t}.assert" "${t}.output"

4
tests/info.sh
View File

@ -18,8 +18,8 @@ DATA
# action
cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv"
orcli info "${t} csv" | jq -r .columns[] > "${t}.output"
orcli import csv "${t}.csv" --projectName "${t}"
orcli info "${t}" | jq -r .columns[] > "${t}.output"
# test
diff -u "${t}.assert" "${t}.output"

2
tests/list.sh
View File

@ -11,7 +11,7 @@ cp data/example.csv "${tmpdir}/${t}.csv"
# action
cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv"
orcli import csv "${t}.csv" --projectName "${t}"
orcli list > "${t}.output"
# test