Merge pull request #64 from opencultureconsulting:felixlohmeier/batch-use-heredoc-58

support multiple files

.gitpod.yml

@@ -8,7 +8,7 @@ tasks:
       rm openrefine.tar.gz
     command: |
       sudo ln -s "${PWD}/orcli" /usr/local/bin/
-      eval "$(orcli completions)"
+      source <(orcli completions)
       ./refine
 
 ports:

orcli

@@ -28,7 +28,7 @@ orcli_usage() {
   fi
 
   printf "Usage:\n"
-  printf "  orcli COMMAND\n"
+  printf "  orcli [OPTIONS] COMMAND\n"
   printf "  orcli [COMMAND] --help | -h\n"
   printf "  orcli --version | -v\n"
   echo
@@ -54,6 +54,12 @@ orcli_usage() {
     printf "    Show version number\n"
     echo
 
+    # :command.usage_flags
+    # :flag.usage
+    echo "  --quiet, -q"
+    printf "    suppress log output, print errors only\n"
+    echo
+
     # :command.usage_environment_variables
     printf "Environment Variables:\n"
 
@@ -151,11 +157,6 @@ orcli_batch_usage() {
     printf "    Default: 3333\n"
     echo
 
-    # :flag.usage
-    echo "  --quiet, -q"
-    printf "    suppress log output, print errors only\n"
-    echo
-
     # :command.usage_args
     printf "Arguments:\n"
 
@@ -167,9 +168,9 @@ orcli_batch_usage() {
 
     # :command.usage_examples
     printf "Examples:\n"
-    printf "  orcli batch << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n    orcli export tsv \"duplicates\"\n  EOF\n"
+    printf "  orcli batch << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n    orcli info \"duplicates\"\n    orcli export tsv \"duplicates\"\n  EOF\n"
-    printf "  orcli batch --memory \"2000M\" --port \"3334\" << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\"\n    orcli info \"duplicates\"\n    orcli info \"copy\"\n    orcli export tsv \"duplicates\"\n  EOF\n"
+    printf "  orcli batch --memory \"2000M\" --port \"3334\" << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n    wait\n    echo \"finished import\"\n    orcli export csv \"duplicates\" --output duplicates.csv &\n    orcli export tsv \"duplicates\" --output duplicates.tsv &\n    wait\n    wc duplicates*\n  EOF\n"
-    printf "  orcli batch --quiet << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n    wait\n    echo \"finished import\"\n    orcli export csv \"duplicates\" --output duplicates.csv &\n    orcli export tsv \"duplicates\" --output duplicates.tsv &\n    wait\n    wc duplicates*\n    echo \"finished in $SECONDS seconds\"\n  EOF\n"
+    printf "  orcli batch \"file1.sh\" \"file2.sh\" - << EOF\n    echo \"finished in \$SECONDS seconds\"\n  EOF\n"
     echo
 
   fi
@@ -256,11 +257,6 @@ orcli_import_csv_usage() {
     printf "    set a name for the OpenRefine project\n"
     echo
 
-    # :flag.usage
-    echo "  --quiet, -q"
-    printf "    suppress log output, print errors only\n"
-    echo
-
     # :command.usage_args
     printf "Arguments:\n"
 
@@ -295,7 +291,7 @@ orcli_list_usage() {
   fi
 
   printf "Usage:\n"
-  printf "  orcli list [OPTIONS]\n"
+  printf "  orcli list\n"
   printf "  orcli list --help | -h\n"
   echo
 
@@ -308,12 +304,6 @@ orcli_list_usage() {
     printf "    Show this help\n"
     echo
 
-    # :command.usage_flags
-    # :flag.usage
-    echo "  --quiet, -q"
-    printf "    suppress log output, print errors only\n"
-    echo
-
   fi
 }
 
@@ -330,7 +320,7 @@ orcli_info_usage() {
   fi
 
   printf "Usage:\n"
-  printf "  orcli info PROJECT [OPTIONS]\n"
+  printf "  orcli info PROJECT\n"
   printf "  orcli info --help | -h\n"
   echo
 
@@ -343,12 +333,6 @@ orcli_info_usage() {
     printf "    Show this help\n"
     echo
 
-    # :command.usage_flags
-    # :flag.usage
-    echo "  --quiet, -q"
-    printf "    suppress log output, print errors only\n"
-    echo
-
     # :command.usage_args
     printf "Arguments:\n"
 
@@ -437,11 +421,6 @@ orcli_export_tsv_usage() {
     printf "    Default: UTF-8\n"
     echo
 
-    # :flag.usage
-    echo "  --quiet, -q"
-    printf "    suppress log output, print errors only\n"
-    echo
-
     # :command.usage_args
     printf "Arguments:\n"
 
@@ -685,11 +664,11 @@ send_completions() {
   echo $'      ;;'
   echo $''
   echo $'    \'import csv\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --quiet --separator --trimStrings -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --separator --trimStrings -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'export tsv\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'import\'*)'
@@ -701,19 +680,19 @@ send_completions() {
   echo $'      ;;'
   echo $''
   echo $'    \'batch\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'list\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'info\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    *)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v batch completions export import info list")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v batch completions export import info list")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'  esac'
@@ -727,10 +706,7 @@ send_completions() {
 # :command.function
 orcli_completions_command() {
   # src/completions_command.sh
-  # Users can now enable bash completion for this script by running:
+  # shellcheck shell=bash disable=SC2154
-  #
-  #   $ eval "$(orcli completions)"
-  #
   send_completions
 }
 
@@ -745,6 +721,13 @@ orcli_batch_command() {
       exit 1
   fi
 
+  # catch args, convert the space delimited string to an array
+  files=()
+  eval "files=(${args[file]})"
+
+  # update OPENREFINE_URL env
+  OPENREFINE_URL="http://localhost:${args[--port]}"
+
   # locate orcli and OpenRefine
   if command -v orcli &>/dev/null; then
       orcli="orcli"
@@ -763,22 +746,6 @@ orcli_batch_command() {
   tmpdir="$(mktemp -d)"
   trap '{ rm -rf "$tmpdir"; }' 0 2 3 15
 
-  # update OPENREFINE_URL env
-  OPENREFINE_URL="http://localhost:${args[--port]}"
-
-  # catch args, convert the space delimited string to an array
-  files=()
-  eval "files=(${args[file]})"
-  # read pipes if name starts with /dev/fd
-  for i in "${!files[@]}"; do
-      if [[ ${files[$i]} == "/dev/fd"* ]]; then
-          if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
-              error "reading of ${files[$i]} failed!"
-          fi
-          files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
-      fi
-  done
-
   # check if OpenRefine is already running
   if curl -fs "${OPENREFINE_URL}" &>/dev/null; then
       error "OpenRefine is already running on port ${args[--port]}." "Hint: Stop the other process or use another port."
@@ -800,7 +767,8 @@ orcli_batch_command() {
 
   # execute shell script
   export orcli tmpdir OPENREFINE_URL openrefine_pid
-  bash -e "${files[@]}"
+  bash -e <(awk 1 "${files[@]}")
+
 }
 
 # :command.function
@@ -918,6 +886,14 @@ parse_requirements() {
     exit
     ;;
 
+  # :flag.case
+  --quiet | -q )
+
+    # :flag.case_no_arg
+    args[--quiet]=1
+    shift
+    ;;
+
   esac
 
   # :command.environment_variables_filter
@@ -1106,14 +1082,6 @@ orcli_batch_parse_requirements() {
       fi
       ;;
 
-    # :flag.case
-    --quiet | -q )
-
-      # :flag.case_no_arg
-      args[--quiet]=1
-      shift
-      ;;
-
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
@@ -1277,14 +1245,6 @@ orcli_import_csv_parse_requirements() {
       fi
       ;;
 
-    # :flag.case
-    --quiet | -q )
-
-      # :flag.case_no_arg
-      args[--quiet]=1
-      shift
-      ;;
-
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
@@ -1332,13 +1292,6 @@ orcli_list_parse_requirements() {
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
-    # :flag.case
-    --quiet | -q )
-
-      # :flag.case_no_arg
-      args[--quiet]=1
-      shift
-      ;;
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
@@ -1377,13 +1330,6 @@ orcli_info_parse_requirements() {
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
-    # :flag.case
-    --quiet | -q )
-
-      # :flag.case_no_arg
-      args[--quiet]=1
-      shift
-      ;;
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
@@ -1409,7 +1355,7 @@ orcli_info_parse_requirements() {
 
   # :command.required_args_filter
   if [[ -z ${args[project]+x} ]]; then
-    printf "missing required argument: PROJECT\nusage: orcli info PROJECT [OPTIONS]\n" >&2
+    printf "missing required argument: PROJECT\nusage: orcli info PROJECT\n" >&2
     exit 1
   fi
 
@@ -1526,14 +1472,6 @@ orcli_export_tsv_parse_requirements() {
       fi
       ;;
 
-    # :flag.case
-    --quiet | -q )
-
-      # :flag.case_no_arg
-      args[--quiet]=1
-      shift
-      ;;
-
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1

src/bashly.yml

@@ -25,6 +25,11 @@ examples:
       orcli export tsv "duplicates"
     EOF
 
+flags:
+  - long: --quiet
+    short: -q
+    help: suppress log output, print errors only
+
 commands:
   - name: completions
     help: |-
@@ -47,25 +52,15 @@ commands:
         help: PORT on which OpenRefine should listen
         arg: port
         default: "3333"
-      - long: --quiet
-        short: -q
-        help: suppress log output, print errors only
     examples:
       - |-
         orcli batch << EOF
           orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
+          orcli info "duplicates"
           orcli export tsv "duplicates"
         EOF
       - |-
         orcli batch --memory "2000M" --port "3334" << EOF
-          orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
-          orcli import csv "https://git.io/fj5hF" --projectName "copy"
-          orcli info "duplicates"
-          orcli info "copy"
-          orcli export tsv "duplicates"
-        EOF
-      - |-
-        orcli batch --quiet << EOF
           orcli import csv "https://git.io/fj5hF" --projectName "duplicates" &
           orcli import csv "https://git.io/fj5hF" --projectName "copy" &
           wait
@@ -74,7 +69,10 @@ commands:
           orcli export tsv "duplicates" --output duplicates.tsv &
           wait
           wc duplicates*
-          echo "finished in $SECONDS seconds"
+        EOF
+      - |-
+        orcli batch "file1.sh" "file2.sh" - << EOF
+          echo "finished in \$SECONDS seconds"
         EOF
 
   - name: import
@@ -101,9 +99,6 @@ commands:
           - long: --projectName
             arg: projectName
             help: set a name for the OpenRefine project
-          - long: --quiet
-            short: -q
-            help: suppress log output, print errors only
         examples:
           - orcli import csv "file"
           - orcli import csv "file1" "file2"
@@ -118,10 +113,6 @@ commands:
 
   - name: list
     help: list projects on OpenRefine server
-    flags:
-      - long: --quiet
-        short: -q
-        help: suppress log output, print errors only
 
   - name: info
     help: show project metadata
@@ -129,10 +120,6 @@ commands:
       - name: project
         help: project name or id
         required: true
-    flags:
-      - long: --quiet
-        short: -q
-        help: suppress log output, print errors only
     examples:
       - info "duplicates"
       - info 1234567890123
@@ -155,9 +142,6 @@ commands:
             help: set character encoding
             arg: encoding
             default: "UTF-8"
-          - long: --quiet
-            short: -q
-            help: suppress log output, print errors only
         examples:
           - orcli export tsv "duplicates"
           - orcli export tsv "duplicates" --output "duplicates.tsv"

src/batch_command.sh

@@ -6,6 +6,13 @@ if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]] && [ -t 0 ]; then
     exit 1
 fi
 
+# catch args, convert the space delimited string to an array
+files=()
+eval "files=(${args[file]})"
+
+# update OPENREFINE_URL env
+OPENREFINE_URL="http://localhost:${args[--port]}"
+
 # locate orcli and OpenRefine
 if command -v orcli &>/dev/null; then
     orcli="orcli"
@@ -24,22 +31,6 @@ fi
 tmpdir="$(mktemp -d)"
 trap '{ rm -rf "$tmpdir"; }' 0 2 3 15
 
-# update OPENREFINE_URL env
-OPENREFINE_URL="http://localhost:${args[--port]}"
-
-# catch args, convert the space delimited string to an array
-files=()
-eval "files=(${args[file]})"
-# read pipes if name starts with /dev/fd
-for i in "${!files[@]}"; do
-    if [[ ${files[$i]} == "/dev/fd"* ]]; then
-        if ! cat "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
-            error "reading of ${files[$i]} failed!"
-        fi
-        files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
-    fi
-done
-
 # check if OpenRefine is already running
 if curl -fs "${OPENREFINE_URL}" &>/dev/null; then
     error "OpenRefine is already running on port ${args[--port]}." "Hint: Stop the other process or use another port."
@@ -61,4 +52,4 @@ fi
 
 # execute shell script
 export orcli tmpdir OPENREFINE_URL openrefine_pid
-bash -e "${files[@]}"
+bash -e <(awk 1 "${files[@]}")

src/lib/send_completions.sh

@@ -35,11 +35,11 @@ send_completions() {
   echo $'      ;;'
   echo $''
   echo $'    \'import csv\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --quiet --separator --trimStrings -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --projectName --separator --trimStrings -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'export tsv\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'import\'*)'
@@ -51,19 +51,19 @@ send_completions() {
   echo $'      ;;'
   echo $''
   echo $'    \'batch\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --memory --port -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'list\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'info\'*)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet -h -q")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    *)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --version -h -v batch completions export import info list")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v batch completions export import info list")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'  esac'