Merge pull request #73 from opencultureconsulting/felixlohmeier/transform-5

first draft

README.md

@@ -73,7 +73,11 @@ Optional:
   orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
   ```
 
-3. Remove duplicates (coming soon)
+3. Remove duplicates by applying an undo/redo JSON file
+
+  ```sh
+  orcli transform "duplicates" "https://git.io/fj5hF"
+  ```
 
 4. Export data from OpenRefine project to tab-separated-values (TSV) file `duplicates.tsv`
 

orcli

@@ -38,6 +38,7 @@ orcli_usage() {
   echo "  import        commands to create OpenRefine projects from files or URLs"
   echo "  list          list projects on OpenRefine server"
   echo "  info          show OpenRefine project's metadata"
+  echo "  transform     apply undo/redo JSON file(s) to an OpenRefine project"
   echo "  export        commands to export data from OpenRefine projects to files"
   echo "  run           run tmp OpenRefine workspace and execute shell script(s)"
   echo
@@ -74,10 +75,11 @@ orcli_usage() {
     printf "  orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n"
     printf "  orcli list\n"
     printf "  orcli info \"duplicates\"\n"
+    printf "  orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n"
     printf "  orcli export tsv \"duplicates\"\n"
     printf "  orcli export tsv \"duplicates\" --output \"duplicates.tsv\"\n"
     printf "  orcli run --interactive\n"
-    printf "  orcli run << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n    orcli info \"duplicates\"\n    orcli export tsv \"duplicates\"\n  EOF\n"
+    printf "  orcli run << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n    orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n    orcli export tsv \"duplicates\"\n  EOF\n"
     echo
 
     # :command.footer
@@ -286,8 +288,59 @@ orcli_info_usage() {
 
     # :command.usage_examples
     printf "Examples:\n"
-    printf "  info \"duplicates\"\n"
+    printf "  orcli info \"duplicates\"\n"
-    printf "  info 1234567890123\n"
+    printf "  orcli info 1234567890123\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+orcli_transform_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n"
+    echo
+
+  else
+    printf "orcli transform - apply undo/redo JSON file(s) to an OpenRefine project\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  orcli transform PROJECT [FILE...]\n"
+  printf "  orcli transform --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+    # :command.usage_args
+    printf "Arguments:\n"
+
+    # :argument.usage
+    echo "  PROJECT"
+    printf "    project name or id\n"
+    echo
+
+    # :argument.usage
+    echo "  FILE..."
+    printf "    Path to one or more files or URLs containing OpenRefine's undo/redo\n    operation history in JSON format. When FILE is -, read standard input.\n"
+    printf "    Default: -\n"
+    echo
+
+    # :command.usage_examples
+    printf "Examples:\n"
+    printf "  orcli transform \"duplicates\" \"history.json\"\n"
+    printf "  cat \"history.json\" | orcli transform \"duplicates\"\n"
+    printf "  orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n"
+    printf "  orcli transform 1234567890123 \"history.json\"\n"
     echo
 
   fi
@@ -437,7 +490,7 @@ orcli_run_usage() {
     # :command.usage_examples
     printf "Examples:\n"
     printf "  orcli run --interactive\n"
-    printf "  orcli run << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n    orcli info \"duplicates\"\n    orcli export tsv \"duplicates\"\n  EOF\n"
+    printf "  orcli run << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\"\n    orcli transform \"duplicates\" \"https://git.io/fj5ju\"\n    orcli export tsv \"duplicates\"\n  EOF\n"
     printf "  orcli run --memory \"2000M\" --port \"3334\" << EOF\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"duplicates\" &\n    orcli import csv \"https://git.io/fj5hF\" --projectName \"copy\" &\n    wait\n    echo \"finished import\"\n    orcli export csv \"duplicates\" --output duplicates.csv &\n    orcli export tsv \"duplicates\" --output duplicates.tsv &\n    wait\n    wc duplicates*\n  EOF\n"
     printf "  orcli run --interactive \"file1.sh\" \"file2.sh\" - << EOF\n    echo \"finished in \$SECONDS seconds\"\n  EOF\n"
     echo
@@ -693,6 +746,10 @@ send_completions() {
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
+  echo $'    \'transform\'*)'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
+  echo $'      ;;'
+  echo $''
   echo $'    \'import\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )'
   echo $'      ;;'
@@ -714,7 +771,7 @@ send_completions() {
   echo $'      ;;'
   echo $''
   echo $'    *)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'  esac'
@@ -796,6 +853,90 @@ orcli_info_command() {
 
 }
 
+# :command.function
+orcli_transform_command() {
+  # src/transform_command.sh
+  # shellcheck shell=bash disable=SC2154 disable=SC2155
+
+  # check if stdin is present if selected
+  if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
+      if ! read -u 0 -t 0; then
+          orcli_transform_usage
+          exit 1
+      fi
+  fi
+
+  # catch args, convert the space delimited string to an array
+  files=()
+  eval "files=(${args[file]})"
+
+  # create tmp directory
+  tmpdir="$(mktemp -d)"
+  trap 'rm -rf "$tmpdir"' 0 2 3 15
+
+  # download files if name starts with http:// or https://
+  for i in "${!files[@]}"; do
+      if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then
+          if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
+              error "download of ${files[$i]} failed!"
+          fi
+          files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
+      fi
+  done
+
+  # support multiple files
+  for i in "${!files[@]}"; do
+      # read each operation into one line
+      mapfile -t jsonlines < <(jq -c '.[]' "${files[$i]}")
+      for line in "${jsonlines[@]}"; do
+          # parse one line/operation into array
+          declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
+          # map operation names to command endpoints
+          # https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js
+          com="${data[op]#core/}"
+          if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi
+          if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi
+          if [[ $com == "column-addition" ]]; then com="add-column"; fi
+          if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi
+          if [[ $com == "column-removal" ]]; then com="remove-column"; fi
+          if [[ $com == "column-rename" ]]; then com="rename-column"; fi
+          if [[ $com == "column-move" ]]; then com="move-column"; fi
+          if [[ $com == "column-split" ]]; then com="split-column"; fi
+          if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi
+          if [[ $com == "recon" ]]; then com="reconcile"; fi
+          if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi
+          if [[ $com == "row-star" ]]; then com="annotate-rows"; fi
+          if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi
+          if [[ $com == "row-removal" ]]; then com="remove-rows"; fi
+          if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
+          unset "data[op]"
+          # rename engineConfig to engine
+          data[engine]="${data[engineConfig]}"
+          unset "data[engineConfig]"
+          # drop description
+          unset "data[description]"
+          # prepare curl options
+          mapfile -t curloptions < <(for K in "${!data[@]}"; do
+              echo "--data"
+              echo "$K=${data[$K]}"
+          done)
+          # get project id and csrf token; post data to it's individual endpoint
+          if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
+              response_code="$(jq -r '.code' <<<"$response")"
+              if [[ $response_code == "ok" ]]; then
+                  log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")"
+              else
+                  error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")"
+              fi
+          else
+              error "transforming ${args[project]} with ${com} from ${files[$i]} failed!"
+          fi
+          unset data
+      done
+  done
+
+}
+
 # :command.function
 orcli_export_tsv_command() {
   # src/export_tsv_command.sh
@@ -1010,6 +1151,13 @@ parse_requirements() {
     shift $#
     ;;
 
+  transform )
+    action="transform"
+    shift
+    orcli_transform_parse_requirements "$@"
+    shift $#
+    ;;
+
   export )
     action="export"
     shift
@@ -1348,6 +1496,63 @@ orcli_info_parse_requirements() {
 
 }
 
+# :command.parse_requirements
+orcli_transform_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    orcli_transform_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.command_filter
+  action="transform"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+    -?* )
+      printf "invalid option: %s\n" "$key" >&2
+      exit 1
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_repeatable
+      if [[ -z ${args[project]+x} ]]; then
+
+        args[project]=$1
+        shift
+      elif [[ -z ${args[file]+x} ]]; then
+
+        args[file]="\"$1\""
+        shift
+      else
+        args[file]="${args[file]} \"$1\""
+        shift
+      fi
+
+      ;;
+
+    esac
+  done
+
+  # :command.required_args_filter
+  if [[ -z ${args[project]+x} ]]; then
+    printf "missing required argument: PROJECT\nusage: orcli transform PROJECT [FILE...]\n" >&2
+    exit 1
+  fi
+
+  # :command.default_assignments
+  [[ -n ${args[file]:-} ]] || args[file]="-"
+
+}
+
 # :command.parse_requirements
 orcli_export_parse_requirements() {
   # :command.fixed_flags_filter
@@ -1639,6 +1844,14 @@ run() {
       orcli_info_command
     fi
 
+  elif [[ $action == "transform" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      orcli_transform_usage
+    else
+      orcli_transform_command
+    fi
+
   elif [[ $action == "export" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes

src/bashly.yml

@@ -16,13 +16,14 @@ examples:
   - orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
   - orcli list
   - orcli info "duplicates"
+  - orcli transform "duplicates" "https://git.io/fj5ju"
   - orcli export tsv "duplicates"
   - orcli export tsv "duplicates" --output "duplicates.tsv"
   - orcli run --interactive
   - |-
     orcli run << EOF
       orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
-      orcli info "duplicates"
+      orcli transform "duplicates" "https://git.io/fj5ju"
       orcli export tsv "duplicates"
     EOF
 
@@ -83,8 +84,24 @@ commands:
         help: project name or id
         required: true
     examples:
-      - info "duplicates"
+      - orcli info "duplicates"
-      - info 1234567890123
+      - orcli info 1234567890123
+
+  - name: transform
+    help: apply undo/redo JSON file(s) to an OpenRefine project
+    args:
+      - name: project
+        help: project name or id
+        required: true
+      - name: file
+        help: Path to one or more files or URLs containing OpenRefine's undo/redo operation history in JSON format. When FILE is -, read standard input.
+        default: "-"
+        repeatable: true
+    examples:
+      - orcli transform "duplicates" "history.json"
+      - cat "history.json" | orcli transform "duplicates"
+      - orcli transform "duplicates" "https://git.io/fj5ju"
+      - orcli transform 1234567890123 "history.json"
 
   - name: export
     help: commands to export data from OpenRefine projects to files
@@ -131,7 +148,7 @@ commands:
       - |-
         orcli run << EOF
           orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
-          orcli info "duplicates"
+          orcli transform "duplicates" "https://git.io/fj5ju"
           orcli export tsv "duplicates"
         EOF
       - |-

src/lib/send_completions.sh

@@ -42,6 +42,10 @@ send_completions() {
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--encoding --help --output -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
+  echo $'    \'transform\'*)'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h")" -- "$cur" )'
+  echo $'      ;;'
+  echo $''
   echo $'    \'import\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help -h csv")" -- "$cur" )'
   echo $'      ;;'
@@ -63,7 +67,7 @@ send_completions() {
   echo $'      ;;'
   echo $''
   echo $'    *)'
-  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run")" -- "$cur" )'
+  echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_orcli_completions_filter "--help --quiet --version -h -q -v completions export import info list run transform")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'  esac'

src/transform_command.sh

@@ -0,0 +1,78 @@
+# shellcheck shell=bash disable=SC2154 disable=SC2155
+
+# check if stdin is present if selected
+if [[ ${args[file]} == '-' ]] || [[ ${args[file]} == '"-"' ]]; then
+    if ! read -u 0 -t 0; then
+        orcli_transform_usage
+        exit 1
+    fi
+fi
+
+# catch args, convert the space delimited string to an array
+files=()
+eval "files=(${args[file]})"
+
+# create tmp directory
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' 0 2 3 15
+
+# download files if name starts with http:// or https://
+for i in "${!files[@]}"; do
+    if [[ ${files[$i]} == "http://"* ]] || [[ ${files[$i]} == "https://"* ]]; then
+        if ! curl -fs --location "${files[$i]}" >"${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"; then
+            error "download of ${files[$i]} failed!"
+        fi
+        files[$i]="${tmpdir}/${files[$i]//[^A-Za-z0-9._-]/_}"
+    fi
+done
+
+# support multiple files
+for i in "${!files[@]}"; do
+    # read each operation into one line
+    mapfile -t jsonlines < <(jq -c '.[]' "${files[$i]}")
+    for line in "${jsonlines[@]}"; do
+        # parse one line/operation into array
+        declare -A data="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
+        # map operation names to command endpoints
+        # https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js
+        com="${data[op]#core/}"
+        if [[ $com == "multivalued-cell-join" ]]; then com="join-multi-value-cells"; fi
+        if [[ $com == "multivalued-cell-split" ]]; then com="split-multi-value-cells"; fi
+        if [[ $com == "column-addition" ]]; then com="add-column"; fi
+        if [[ $com == "column-addition-by-fetching-urls" ]]; then com="add-column-by-fetching-urls"; fi
+        if [[ $com == "column-removal" ]]; then com="remove-column"; fi
+        if [[ $com == "column-rename" ]]; then com="rename-column"; fi
+        if [[ $com == "column-move" ]]; then com="move-column"; fi
+        if [[ $com == "column-split" ]]; then com="split-column"; fi
+        if [[ $com == "column-reorder" ]]; then com="reorder-columns"; fi
+        if [[ $com == "recon" ]]; then com="reconcile"; fi
+        if [[ $com == "extend-reconciled-data" ]]; then com="extend-data"; fi
+        if [[ $com == "row-star" ]]; then com="annotate-rows"; fi
+        if [[ $com == "row-flag" ]]; then com="annotate-rows"; fi
+        if [[ $com == "row-removal" ]]; then com="remove-rows"; fi
+        if [[ $com == "row-reorder" ]]; then com="reorder-rows"; fi
+        unset "data[op]"
+        # rename engineConfig to engine
+        data[engine]="${data[engineConfig]}"
+        unset "data[engineConfig]"
+        # drop description
+        unset "data[description]"
+        # prepare curl options
+        mapfile -t curloptions < <(for K in "${!data[@]}"; do
+            echo "--data"
+            echo "$K=${data[$K]}"
+        done)
+        # get project id and csrf token; post data to it's individual endpoint
+        if response="$(curl -fs --data "project=$(get_id "${args[project]}")" "${curloptions[@]}" "${OPENREFINE_URL}/command/core/${com}$(get_csrf)")"; then
+            response_code="$(jq -r '.code' <<<"$response")"
+            if [[ $response_code == "ok" ]]; then
+                log "transformed ${args[project]} with ${com}" "Response: $(jq -r '.historyEntry.description' <<<"$response")"
+            else
+                error "transforming ${args[project]} with ${com} from ${files[$i]} failed!" "Response: $(jq -r '.message' <<<"$response")"
+            fi
+        else
+            error "transforming ${args[project]} with ${com} from ${files[$i]} failed!"
+        fi
+        unset data
+    done
+done