opencultureconsulting
/
orcli
mirror of https://github.com/opencultureconsulting/orcli.git
2
0
Fork 0
Code Issues Releases Activity

Merge branch 'main' into fix-csrf_error_export_OR_3.8.4+

This commit is contained in:
Felix Lohmeier 2024-12-08 23:40:59 +01:00 committed by GitHub
parent f27025233c e1d9a76475
commit b03b4f8bde
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 369 additions and 514 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.devcontainer/devcontainer.json
View File

@ -2,7 +2,7 @@
"image": "mcr.microsoft.com/devcontainers/universal:2", "image": "mcr.microsoft.com/devcontainers/universal:2",
"features": { "features": {
}, },
"postCreateCommand": "gem install --silent bashly && wget -q -O openrefine.tar.gz https://github.com/OpenRefine/OpenRefine/releases/download/3.7.6/openrefine-linux-3.7.6.tar.gz && tar --exclude 'licenses' --exclude 'LICENSE.txt' --exclude 'licenses.xml' --exclude 'README.md' -xzf openrefine.tar.gz --strip 1 && rm openrefine.tar.gz && ln -s \"${PWD}/orcli\" ~/.local/bin/", "postCreateCommand": "gem install --silent bashly && wget -q -O openrefine.tar.gz https://github.com/OpenRefine/OpenRefine/releases/download/3.8.7/openrefine-linux-3.8.7.tar.gz && tar --exclude 'licenses' --exclude 'LICENSE.txt' --exclude 'licenses.xml' --exclude 'README.md' -xzf openrefine.tar.gz --strip 1 && rm openrefine.tar.gz && ln -s \"${PWD}/orcli\" ~/.local/bin/",
"forwardPorts": [3333], "forwardPorts": [3333],
"portsAttributes": { "portsAttributes": {
"3333": { "3333": {

2
README.md
View File

@ -6,7 +6,7 @@ Bash script to control OpenRefine via [its HTTP API](https://docs.openrefine.org
## Features ## Features
* works with latest OpenRefine version (currently 3.7) * works with latest OpenRefine version (currently 3.8)
* run batch processes (import, transform, export) * run batch processes (import, transform, export)
* orcli takes care of starting and stopping OpenRefine with temporary workspaces * orcli takes care of starting and stopping OpenRefine with temporary workspaces
* allows execution of arbitrary bash scripts * allows execution of arbitrary bash scripts

677
orcli
View File

File diff suppressed because it is too large Load Diff

2
src/export_jsonl_command.sh
View File

@ -11,7 +11,7 @@ if [[ ${args[--separator]} || ${args[--mode]} == "records" ]]; then
if [[ ${args[--mode]} == "records" ]]; then if [[ ${args[--mode]} == "records" ]]; then
engine='{"facets":[{"type":"list","columnName":"","expression":"grel:filter(row.columnNames,cn,row.record.cells[cn].value.length()>1)","selection":[]}],"mode":"row-based"}' engine='{"facets":[{"type":"list","columnName":"","expression":"grel:filter(row.columnNames,cn,row.record.cells[cn].value.length()>1)","selection":[]}],"mode":"row-based"}'
fi fi
readarray -t columns_mv < <(curl -fs --data project="$projectid" --data "engine=${engine}" "${OPENREFINE_URL}/command/core/compute-facets" | jq -r '.facets[].choices[].v.v') readarray -t columns_mv < <(curl -fs --data project="$projectid" --data "engine=${engine}" "${OPENREFINE_URL}/command/core/compute-facets$(get_csrf)" | jq -r '.facets[].choices[].v.v')
readarray -t columns < <(curl -fs --get --data project="$projectid" "${OPENREFINE_URL}/command/core/get-columns-info" | jq -r '.[].name') readarray -t columns < <(curl -fs --get --data project="$projectid" "${OPENREFINE_URL}/command/core/get-columns-info" | jq -r '.[].name')
readarray -t columns_mix < <(for i in "${columns[@]}"; do readarray -t columns_mix < <(for i in "${columns[@]}"; do
skip= skip=

2
src/lib/post_export.sh
View File

@ -22,7 +22,7 @@ function post_export() {
curloptions+=("--output" "${args[--output]}") curloptions+=("--output" "${args[--output]}")
fi fi
# post # post
if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows"; then if ! curl -fs "${curloptions[@]}" "${OPENREFINE_URL}/command/core/export-rows$(get_csrf)"; then
error "exporting ${args[project]} failed!" error "exporting ${args[project]} failed!"
else else
if [[ ${args[--output]} ]]; then if [[ ${args[--output]} ]]; then

44
src/lib/send_completions.sh
View File

@ -38,11 +38,11 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "rows records")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "rows records")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'export csv\'*\'--mode\')' echo $' \'export tsv\'*\'--mode\')'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "rows records")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "rows records")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'export tsv\'*\'--mode\')' echo $' \'export csv\'*\'--mode\')'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "rows records")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "rows records")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
@ -50,16 +50,16 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --prefix --quiet --separator --suffix -h -q")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --prefix --quiet --separator --suffix -h -q")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'import jsonl\'*)' echo $' \'export jsonl\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --separator -h -q")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'sort columns\'*)' echo $' \'sort columns\'*)'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--first --help -h")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--first --help -h")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'export jsonl\'*)' echo $' \'import jsonl\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --separator -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'completions\'*)' echo $' \'completions\'*)'
@ -70,20 +70,20 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --recordPath --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--guessCellValueTypes --help --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --recordPath --rename --storeEmptyStrings --trimStrings -h -q")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'import csv\'*)' echo $' \'export tsv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--blankCellsAsStrings --columnNames --encoding --guessCellValueTypes --headerLines --help --ignoreLines --ignoreQuoteCharacter --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --quoteCharacter --separator --skipBlankRows --skipDataLines --trimStrings -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --select -h -q")" -- "$cur")'
echo $' ;;'
echo $''
echo $' \'import tsv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--blankCellsAsStrings --columnNames --encoding --guessCellValueTypes --headerLines --help --ignoreLines --ignoreQuoteCharacter --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --quoteCharacter --skipBlankRows --skipDataLines --trimStrings -h -q")" -- "$cur" )'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'export csv\'*)' echo $' \'export csv\'*)'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --select --separator -h -q")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --select --separator -h -q")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'export tsv\'*)' echo $' \'import csv\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --facets --help --mode --output --quiet --select -h -q")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--blankCellsAsStrings --columnNames --encoding --guessCellValueTypes --headerLines --help --ignoreLines --ignoreQuoteCharacter --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --quoteCharacter --separator --skipBlankRows --skipDataLines --trimStrings -h -q")" -- "$cur")'
echo $' ;;'
echo $''
echo $' \'import tsv\'*)'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--blankCellsAsStrings --columnNames --encoding --guessCellValueTypes --headerLines --help --ignoreLines --ignoreQuoteCharacter --includeArchiveFileName --includeFileSources --limit --projectName --projectTags --quiet --quoteCharacter --skipBlankRows --skipDataLines --trimStrings -h -q")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'transform\'*)' echo $' \'transform\'*)'
@ -98,19 +98,15 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h csv json jsonl tsv")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h csv json jsonl tsv")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'search\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --index -h")" -- "$cur" )'
echo $' ;;'
echo $''
echo $' \'export\'*)' echo $' \'export\'*)'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h csv jsonl template tsv")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h csv jsonl template tsv")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'list\'*)' echo $' \'search\'*)'
echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help --index -h")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'info\'*)' echo $' \'test\'*)'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
@ -118,7 +114,11 @@ send_completions() {
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h columns")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h columns")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''
echo $' \'test\'*)' echo $' \'info\'*)'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur")'
echo $' ;;'
echo $''
echo $' \'list\'*)'
echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur")' echo $' while read -r; do COMPREPLY+=("$REPLY"); done < <(compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur")'
echo $' ;;' echo $' ;;'
echo $'' echo $''