opencultureconsulting
/
orcli
mirror of https://github.com/opencultureconsulting/orcli.git
2
0
Fork 0
Code Issues Releases Activity

fix curl quoting with --form-string

This commit is contained in:
felixlohmeier 2023-01-12 10:18:06 +00:00
parent e82b306839
commit cd7d62e254
11 changed files with 90 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
orcli
View File

@ -900,11 +900,10 @@ function get_ids() {
# src/lib/init_import.sh # src/lib/init_import.sh
# common import tasks to support multiple files and URLs # common import tasks to support multiple files and URLs
# shellcheck shell=bash # shellcheck shell=bash disable=SC2154
function init_import() { function init_import() {
local files file
# catch args, convert the space delimited string to an array # catch args, convert the space delimited string to an array
files=() local files=()
eval "files=(${args[file]})" eval "files=(${args[file]})"
# create tmp directory # create tmp directory
tmpdir="$(mktemp -d)" tmpdir="$(mktemp -d)"
@ -915,7 +914,7 @@ function init_import() {
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "download of ${files[$i]} failed!" error "download of ${files[$i]} failed!"
fi fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi fi
done done
# read pipes if name starts with /dev/fd # read pipes if name starts with /dev/fd
@ -924,7 +923,7 @@ function init_import() {
if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "reading of ${files[$i]} failed!" error "reading of ${files[$i]} failed!"
fi fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi fi
done done
# create a zip archive if there are multiple files # create a zip archive if there are multiple files
@ -936,25 +935,6 @@ function init_import() {
else else
file="${files[0]}" file="${files[0]}"
fi fi
# basic post data
if [[ ${file} == "-" ]]; then
data+=("project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
data+=("project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
data+=("project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
data+=("project-name=${name}")
fi
} }
# src/lib/interactive.sh # src/lib/interactive.sh
@ -994,18 +974,18 @@ function log() {
# shellcheck shell=bash disable=SC2154 # shellcheck shell=bash disable=SC2154
function post_export() { function post_export() {
local curloptions local curloptions
# post
mapfile -t curloptions < <(for d in "$@"; do mapfile -t curloptions < <(for d in "$@"; do
echo "--data" echo "--data"
echo "$d" echo "$d"
done) done)
# support file output
if [[ ${args[--output]} ]]; then if [[ ${args[--output]} ]]; then
if ! mkdir -p "$(dirname "${args[--output]}")"; then if ! mkdir -p "$(dirname "${args[--output]}")"; then
error "unable to create parent directory for ${args[--output]}" error "unable to create parent directory for ${args[--output]}"
fi fi
curloptions+=("--output") curloptions+=("--output" "${args[--output]}")
curloptions+=("${args[--output]}")
fi fi
# post
if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then
error "exporting ${args[project]} failed!" error "exporting ${args[project]} failed!"
else else
@ -1019,15 +999,31 @@ function post_export() {
# post to create-project endpoint and validate # post to create-project endpoint and validate
# shellcheck shell=bash disable=SC2154 # shellcheck shell=bash disable=SC2154
function post_import() { function post_import() {
local curloptions local curloptions projectid projectname rows
local projectid
local projectname
local rows
# post
mapfile -t curloptions < <(for d in "$@"; do mapfile -t curloptions < <(for d in "$@"; do
echo "--form" echo "--form-string"
echo "$d" echo "$d"
done) done)
# basic post data
if [[ ${file} == "-" ]]; then
curloptions+=("--form" "project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
curloptions+=("--form" "project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
curloptions+=("--form-string" "project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
curloptions+=("--form-string" "project-name=${name}")
fi
# post
if ! redirect_url="$(curl -fs --write-out "%{redirect_url}\n" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/create-project-from-upload$(get_csrf)")"; then if ! redirect_url="$(curl -fs --write-out "%{redirect_url}\n" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/create-project-from-upload$(get_csrf)")"; then
error "importing ${args[file]} failed!" error "importing ${args[file]} failed!"
fi fi
@ -1180,18 +1176,16 @@ orcli_delete_command() {
# :command.function # :command.function
orcli_import_csv_command() { orcli_import_csv_command() {
# src/import_csv_command.sh # src/import_csv_command.sh
# shellcheck shell=bash # shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data # call init_import function to eval args and to set basic post data
init_import init_import
# check if stdin is present if selected # check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
if ! read -u 0 -t 0; then
orcli_import_csv_usage orcli_import_csv_usage
exit 1 exit 1
fi fi
fi
# assemble specific post data (some options require json format) # assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv") data+=("format=text/line-based/*sv")
@ -1274,18 +1268,16 @@ orcli_import_csv_command() {
# :command.function # :command.function
orcli_import_tsv_command() { orcli_import_tsv_command() {
# src/import_tsv_command.sh # src/import_tsv_command.sh
# shellcheck shell=bash # shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data # call init_import function to eval args and to set basic post data
init_import init_import
# check if stdin is present if selected # check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
if ! read -u 0 -t 0; then
orcli_import_tsv_usage orcli_import_tsv_usage
exit 1 exit 1
fi fi
fi
# assemble specific post data (some options require json format) # assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv") data+=("format=text/line-based/*sv")

6
src/import_csv_command.sh
View File

@ -1,15 +1,13 @@
# shellcheck shell=bash # shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data # call init_import function to eval args and to set basic post data
init_import init_import
# check if stdin is present if selected # check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
if ! read -u 0 -t 0; then
orcli_import_csv_usage orcli_import_csv_usage
exit 1 exit 1
fi fi
fi
# assemble specific post data (some options require json format) # assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv") data+=("format=text/line-based/*sv")

6
src/import_tsv_command.sh
View File

@ -1,15 +1,13 @@
# shellcheck shell=bash # shellcheck shell=bash disable=SC2154
# call init_import function to eval args and to set basic post data # call init_import function to eval args and to set basic post data
init_import init_import
# check if stdin is present if selected # check if stdin is present if selected
if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then if [[ ${file} == '-' ]] && ! read -u 0 -t 0; then
if ! read -u 0 -t 0; then
orcli_import_tsv_usage orcli_import_tsv_usage
exit 1 exit 1
fi fi
fi
# assemble specific post data (some options require json format) # assemble specific post data (some options require json format)
data+=("format=text/line-based/*sv") data+=("format=text/line-based/*sv")

28
src/lib/init_import.sh
View File

@ -1,9 +1,8 @@
# common import tasks to support multiple files and URLs # common import tasks to support multiple files and URLs
# shellcheck shell=bash # shellcheck shell=bash disable=SC2154
function init_import() { function init_import() {
local files file
# catch args, convert the space delimited string to an array # catch args, convert the space delimited string to an array
files=() local files=()
eval "files=(${args[file]})" eval "files=(${args[file]})"
# create tmp directory # create tmp directory
tmpdir="$(mktemp -d)" tmpdir="$(mktemp -d)"
@ -14,7 +13,7 @@ function init_import() {
if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "download of ${files[$i]} failed!" error "download of ${files[$i]} failed!"
fi fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi fi
done done
# read pipes if name starts with /dev/fd # read pipes if name starts with /dev/fd
@ -23,7 +22,7 @@ function init_import() {
if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
error "reading of ${files[$i]} failed!" error "reading of ${files[$i]} failed!"
fi fi
files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}" files[i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
fi fi
done done
# create a zip archive if there are multiple files # create a zip archive if there are multiple files
@ -35,23 +34,4 @@ function init_import() {
else else
file="${files[0]}" file="${files[0]}"
fi fi
# basic post data
if [[ ${file} == "-" ]]; then
data+=("project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
data+=("project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
data+=("project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
data+=("project-name=${name}")
fi
} }

6
src/lib/post_export.sh
View File

@ -2,18 +2,18 @@
# shellcheck shell=bash disable=SC2154 # shellcheck shell=bash disable=SC2154
function post_export() { function post_export() {
local curloptions local curloptions
# post
mapfile -t curloptions < <(for d in "$@"; do mapfile -t curloptions < <(for d in "$@"; do
echo "--data" echo "--data"
echo "$d" echo "$d"
done) done)
# support file output
if [[ ${args[--output]} ]]; then if [[ ${args[--output]} ]]; then
if ! mkdir -p "$(dirname "${args[--output]}")"; then if ! mkdir -p "$(dirname "${args[--output]}")"; then
error "unable to create parent directory for ${args[--output]}" error "unable to create parent directory for ${args[--output]}"
fi fi
curloptions+=("--output") curloptions+=("--output" "${args[--output]}")
curloptions+=("${args[--output]}")
fi fi
# post
if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then
error "exporting ${args[project]} failed!" error "exporting ${args[project]} failed!"
else else

28
src/lib/post_import.sh
View File

@ -1,15 +1,31 @@
# post to create-project endpoint and validate # post to create-project endpoint and validate
# shellcheck shell=bash disable=SC2154 # shellcheck shell=bash disable=SC2154
function post_import() { function post_import() {
local curloptions local curloptions projectid projectname rows
local projectid
local projectname
local rows
# post
mapfile -t curloptions < <(for d in "$@"; do mapfile -t curloptions < <(for d in "$@"; do
echo "--form" echo "--form-string"
echo "$d" echo "$d"
done) done)
# basic post data
if [[ ${file} == "-" ]]; then
curloptions+=("--form" "project-file=@-")
else
if ! path=$(readlink -e "${file}"); then
error "cannot open ${file} (no such file)!"
fi
curloptions+=("--form" "project-file=@${path}")
fi
if [[ ${args[--projectName]} ]]; then
curloptions+=("--form-string" "project-name=${args[--projectName]}")
else
if [[ ${file} == "-" ]]; then
name="Untitled"
else
name="$(basename "${path}" | tr '.' ' ')"
fi
curloptions+=("--form-string" "project-name=${name}")
fi
# post
if ! redirect_url="$(curl -fs --write-out "%{redirect_url}\n" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/create-project-from-upload$(get_csrf)")"; then if ! redirect_url="$(curl -fs --write-out "%{redirect_url}\n" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/create-project-from-upload$(get_csrf)")"; then
error "importing ${args[file]} failed!" error "importing ${args[file]} failed!"
fi fi

18
tests/delete.sh
View File

@ -15,15 +15,15 @@ DATA
# action # action
cd "${tmpdir}" || exit 1 cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv" orcli import csv "${t}.csv" --projectName "${t}"
orcli list | grep "${t} csv" orcli list | grep "${t}"
orcli delete "${t} csv" orcli delete "${t}"
orcli list | grep "${t} csv" > "${t}.output" || exit 0 orcli list | grep "${t}" > "${t}.output" || exit 0
orcli import csv "${t}.csv" orcli import csv "${t}.csv" --projectName "${t}"
orcli import csv "${t}.csv" orcli import csv "${t}.csv" --projectName "${t}"
orcli list | grep "${t} csv" orcli list | grep "${t}"
orcli delete --force "${t} csv" orcli delete --force "${t}"
orcli list | grep "${t} csv" >> "${t}.output" || exit 0 orcli list | grep "${t}" >> "${t}.output" || exit 0
# test # test
diff -u "${t}.assert" "${t}.output" diff -u "${t}.assert" "${t}.output"

4
tests/export-tsv.sh
View File

@ -14,8 +14,8 @@ cp data/example.tsv "${tmpdir}/${t}.assert"
# action # action
cd "${tmpdir}" || exit 1 cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv" orcli import csv "${t}.csv" --projectName "${t}"
orcli export tsv "${t} csv" --output "${t}.output" orcli export tsv "${t}" --output "${t}.output"
# test # test
diff -u "${t}.assert" "${t}.output" diff -u "${t}.assert" "${t}.output"

4
tests/import-csv.sh
View File

@ -14,8 +14,8 @@ cp data/example.tsv "${tmpdir}/${t}.assert"
# action # action
cd "${tmpdir}" || exit 1 cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv" orcli import csv "${t}.csv" --projectName "${t}"
orcli export tsv "${t} csv" > "${t}.output" orcli export tsv "${t}" > "${t}.output"
# test # test
diff -u "${t}.assert" "${t}.output" diff -u "${t}.assert" "${t}.output"

4
tests/info.sh
View File

@ -18,8 +18,8 @@ DATA
# action # action
cd "${tmpdir}" || exit 1 cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv" orcli import csv "${t}.csv" --projectName "${t}"
orcli info "${t} csv" | jq -r .columns[] > "${t}.output" orcli info "${t}" | jq -r .columns[] > "${t}.output"
# test # test
diff -u "${t}.assert" "${t}.output" diff -u "${t}.assert" "${t}.output"

2
tests/list.sh
View File

@ -11,7 +11,7 @@ cp data/example.csv "${tmpdir}/${t}.csv"
# action # action
cd "${tmpdir}" || exit 1 cd "${tmpdir}" || exit 1
orcli import csv "${t}.csv" orcli import csv "${t}.csv" --projectName "${t}"
orcli list > "${t}.output" orcli list > "${t}.output"
# test # test