diff --git a/README.md b/README.md index a8f86b9..7059664 100644 --- a/README.md +++ b/README.md @@ -144,4 +144,10 @@ orcli uses [bashly](https://github.com/DannyBen/bashly/) for generating the one- ```sh ./orcli test + ``` + +5. Generate help files + + ```sh + ./help.sh ``` \ No newline at end of file diff --git a/help.sh b/help.sh new file mode 100755 index 0000000..df71258 --- /dev/null +++ b/help.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# generate markdown files in subdirectory "help" from cli help screens +name="orcli" +orcli() { + ./orcli "$@" +} + +version="$(${name} --version)" +files=(src/*_command.sh) + +for f in "${files[@]}"; do +command="${f#src/}" +command="${command%_command.sh}" +commands+=( "${command}" ) +done + +mkdir -p help +for command in "${commands[@]}"; do + { echo "# ${name} ${command//_/ }"; } > help/"${command}".md + { echo; echo '```'; } >> help/"${command}".md + ${name} ${command//_/ } --help >> help/"${command}".md + { echo '```'; echo; } >> help/"${command}".md + echo "code: [src/${command}_command.sh](../src/${command}_command.sh)" >> help/"${command}".md +done + +{ echo "# ${name} ${version}"; echo; } > help/README.md +{ echo '## command help screens'; echo; } >> help/README.md +for command in "${commands[@]}"; do + echo "- [${command//_/ }](${command}.md)" >> help/README.md +done +{ echo; echo '## main help screen'; } >> help/README.md +{ echo; echo '```'; } >> help/README.md +${name} --help >> help/README.md +echo '```' >> help/README.md \ No newline at end of file diff --git a/help/README.md b/help/README.md new file mode 100644 index 0000000..6cbf9b7 --- /dev/null +++ b/help/README.md @@ -0,0 +1,67 @@ +# orcli 0.1.0 + +## command help screens + +- [completions](completions.md) +- [delete](delete.md) +- [export template](export_template.md) +- [export tsv](export_tsv.md) +- [import csv](import_csv.md) +- [import tsv](import_tsv.md) +- [info](info.md) +- [list](list.md) +- [run](run.md) +- [test](test.md) +- [transform](transform.md) + +## main help screen + +``` +orcli - OpenRefine command-line interface written in Bash + +Usage: + orcli COMMAND + orcli [COMMAND] --help | -h + orcli --version | -v + +Commands: + completions Generate bash completions + delete delete OpenRefine project + import commands to create OpenRefine projects from files or URLs + list list projects on OpenRefine server + info show OpenRefine project's metadata + test run functional tests on tmp OpenRefine workspace + transform apply undo/redo JSON file(s) to an OpenRefine project + export commands to export data from OpenRefine projects to files + run run tmp OpenRefine workspace and execute shell script(s) + +Options: + --help, -h + Show this help + + --version, -v + Show version number + +Environment Variables: + OPENREFINE_URL + URL to OpenRefine server + Default: http://localhost:3333 + +Examples: + orcli import csv "https://git.io/fj5hF" --projectName "duplicates" + orcli list + orcli info "duplicates" + orcli transform "duplicates" "https://git.io/fj5ju" + orcli export tsv "duplicates" + orcli export tsv "duplicates" --output "duplicates.tsv" + orcli delete "duplicates" + orcli run --interactive + orcli run << EOF + orcli import csv "https://git.io/fj5hF" --projectName "duplicates" + orcli transform "duplicates" "https://git.io/fj5ju" + orcli export tsv "duplicates" + EOF + +https://github.com/opencultureconsulting/orcli + +``` diff --git a/help/completions.md b/help/completions.md new file mode 100644 index 0000000..1c3ab93 --- /dev/null +++ b/help/completions.md @@ -0,0 +1,19 @@ +# orcli completions + +``` +orcli completions + + Generate bash completions + Usage: source <(orcli completions) + +Usage: + orcli completions + orcli completions --help | -h + +Options: + --help, -h + Show this help + +``` + +code: [src/completions_command.sh](../src/completions_command.sh) diff --git a/help/delete.md b/help/delete.md new file mode 100644 index 0000000..e092128 --- /dev/null +++ b/help/delete.md @@ -0,0 +1,32 @@ +# orcli delete + +``` +orcli delete - delete OpenRefine project + +Usage: + orcli delete PROJECT [OPTIONS] + orcli delete --help | -h + +Options: + --force, -f + delete all projects with the same name + + --quiet, -q + suppress log output, print errors only + + --help, -h + Show this help + +Arguments: + PROJECT + project name or id + +Examples: + orcli delete "duplicates" + orcli delete "duplicates" --force + orcli delete 1234567890123 + for p in $(orcli list); do orcli delete ${p:0:13}; done + +``` + +code: [src/delete_command.sh](../src/delete_command.sh) diff --git a/help/export_template.md b/help/export_template.md new file mode 100644 index 0000000..97f14ff --- /dev/null +++ b/help/export_template.md @@ -0,0 +1,69 @@ +# orcli export template + +``` +orcli export template - export to any text format by providing your own GREL template + +Usage: + orcli export template PROJECT [FILE] [OPTIONS] + orcli export template --help | -h + +Options: + --separator SEPARATOR + insert character(s) between rows/records + + --prefix PREFIX + insert character(s) at the beginning of the file + + --suffix SUFFIX + insert character(s) at the end of the file + + --mode MODE + specify if template shall be applied to each row or record + Allowed: rows, records + Default: rows + + --facets FACETS + filter result set by providing an OpenRefine facets config in json + Default: [] + + --output FILE + Write to file instead of stdout + + --encoding ENCODING + set character encoding + Default: UTF-8 + + --quiet, -q + suppress log output, print errors only + + --help, -h + Show this help + +Arguments: + PROJECT + project name or id + + FILE + Path to row/record template file or URL. When FILE is -, read standard + input. + Default: - + +Examples: + orcli export template "duplicates" "template.txt" + cat "template.txt" | orcli export template "duplicates" + orcli export template "duplicates" "https://example.com/template.txt" + orcli export template "duplicates" "template.txt" --output "duplicates.tsv" + orcli export template "duplicates" \ + <<< '{ "name" : {{jsonize(cells["name"].value)}}, "purchase" : + {{jsonize(cells["purchase"].value)}} }' \ + --prefix '{ "events" : [' \ + --separator , \ + --mode records \ + --suffix ]}$'\n' \ + --facets '[ { "type": "text", "name": "foo", "columnName": "name", "mode": + "regex", "caseSensitive": false, "query": "Ben" } ]' \ + | jq + +``` + +code: [src/export_template_command.sh](../src/export_template_command.sh) diff --git a/help/export_tsv.md b/help/export_tsv.md new file mode 100644 index 0000000..7e50010 --- /dev/null +++ b/help/export_tsv.md @@ -0,0 +1,42 @@ +# orcli export tsv + +``` +orcli export tsv - export tab-separated values (TSV) + +Usage: + orcli export tsv PROJECT [OPTIONS] + orcli export tsv --help | -h + +Options: + --facets FACETS + filter result set by providing an OpenRefine facets config in json + Default: [] + + --output FILE + Write to file instead of stdout + + --encoding ENCODING + set character encoding + Default: UTF-8 + + --quiet, -q + suppress log output, print errors only + + --help, -h + Show this help + +Arguments: + PROJECT + project name or id + +Examples: + orcli export tsv "duplicates" + orcli export tsv "duplicates" --output "duplicates.tsv" + orcli export tsv "duplicates" --encoding "ISO-8859-1" + orcli export tsv "duplicates" --facets '[ { "type": "text", "name": "foo", + "columnName": "name", "mode": "regex", "caseSensitive": false, "query": "Ben" + } ]' + +``` + +code: [src/export_tsv_command.sh](../src/export_tsv_command.sh) diff --git a/help/import_csv.md b/help/import_csv.md new file mode 100644 index 0000000..f836ffa --- /dev/null +++ b/help/import_csv.md @@ -0,0 +1,97 @@ +# orcli import csv + +``` +orcli import csv - import character-separated values (CSV) + +Usage: + orcli import csv [FILE...] [OPTIONS] + orcli import csv --help | -h + +Options: + --separator SEPARATOR + character(s) that separates columns + Default: , + + --blankCellsAsStrings + store blank cells as empty strings instead of nulls + + --columnNames COLUMNNAMES + set column names (comma separated) + hint: add --ignoreLines 1 to overwrite existing header row + + --encoding ENCODING + set character encoding + + --guessCellValueTypes + attempt to parse cell text into numbers + + --headerLines HEADERLINES + parse x line(s) as column headers + Default: 1 + + --ignoreLines IGNORELINES + ignore first x line(s) at beginning of file + Default: -1 + + --ignoreQuoteCharacter + do not use any quote character to enclose cells containing column separators + + --includeFileSources + add column with file source + + --includeArchiveFileName + add column with archive file name + + --limit LIMIT + load at most x row(s) of data + Default: -1 + + --quoteCharacter QUOTECHARACTER + quote character to enclose cells containing column separators + Default: " + + --skipBlankRows + do not store blank rows + + --skipDataLines SKIPDATALINES + discard initial x row(s) of data + Default: 0 + + --trimStrings + trim leading & trailing whitespace from strings + + --projectName PROJECTNAME + set a name for the OpenRefine project + + --projectTags PROJECTTAGS + set project tags (comma separated) + + --quiet, -q + suppress log output, print errors only + + --help, -h + Show this help + +Arguments: + FILE... + Path to one or more files or URLs. When FILE is -, read standard input. + Default: - + +Examples: + orcli import csv "file" + orcli import csv "file1" "file2" + head -n 100 "file" | orcli import csv + orcli import csv "https://git.io/fj5hF" + orcli import csv "file" \ + --separator ";" \ + --columnNames "foo,bar,baz" \ + --ignoreLines 1 \ + --encoding "ISO-8859-1" \ + --limit 100 \ + --trimStrings \ + --projectName "duplicates" \ + --projectTags "test,urgent" + +``` + +code: [src/import_csv_command.sh](../src/import_csv_command.sh) diff --git a/help/import_tsv.md b/help/import_tsv.md new file mode 100644 index 0000000..5c39d3a --- /dev/null +++ b/help/import_tsv.md @@ -0,0 +1,93 @@ +# orcli import tsv + +``` +orcli import tsv - import tab-separated values (TSV) + +Usage: + orcli import tsv [FILE...] [OPTIONS] + orcli import tsv --help | -h + +Options: + --blankCellsAsStrings + store blank cells as empty strings instead of nulls + + --columnNames COLUMNNAMES + set column names (comma separated) + hint: add --ignoreLines 1 to overwrite existing header row + + --encoding ENCODING + set character encoding + + --guessCellValueTypes + attempt to parse cell text into numbers + + --headerLines HEADERLINES + parse x line(s) as column headers + Default: 1 + + --ignoreLines IGNORELINES + ignore first x line(s) at beginning of file + Default: -1 + + --ignoreQuoteCharacter + do not use any quote character to enclose cells containing column separators + + --includeFileSources + add column with file source + + --includeArchiveFileName + add column with archive file name + + --limit LIMIT + load at most x row(s) of data + Default: -1 + + --quoteCharacter QUOTECHARACTER + quote character to enclose cells containing column separators + Default: " + + --skipBlankRows + do not store blank rows + + --skipDataLines SKIPDATALINES + discard initial x row(s) of data + Default: 0 + + --trimStrings + trim leading & trailing whitespace from strings + + --projectName PROJECTNAME + set a name for the OpenRefine project + + --projectTags PROJECTTAGS + set project tags (comma separated) + + --quiet, -q + suppress log output, print errors only + + --help, -h + Show this help + +Arguments: + FILE... + Path to one or more files or URLs. When FILE is -, read standard input. + Default: - + +Examples: + orcli import tsv "file" + orcli import tsv "file1" "file2" + head -n 100 "file" | orcli import tsv + orcli import tsv "https://example.com/file.tsv" + orcli import tsv "file" \ + --separator ";" \ + --columnNames "foo,bar,baz" \ + --ignoreLines 1 \ + --encoding "ISO-8859-1" \ + --limit 100 \ + --trimStrings \ + --projectName "duplicates" \ + --projectTags "test,urgent" + +``` + +code: [src/import_tsv_command.sh](../src/import_tsv_command.sh) diff --git a/help/info.md b/help/info.md new file mode 100644 index 0000000..eb97dee --- /dev/null +++ b/help/info.md @@ -0,0 +1,25 @@ +# orcli info + +``` +orcli info - show OpenRefine project's metadata + +Usage: + orcli info PROJECT + orcli info --help | -h + +Options: + --help, -h + Show this help + +Arguments: + PROJECT + project name or id + +Examples: + orcli info "duplicates" + orcli info 1234567890123 + orcli info "duplicates" | jq -r .columns[] + +``` + +code: [src/info_command.sh](../src/info_command.sh) diff --git a/help/list.md b/help/list.md new file mode 100644 index 0000000..4d44b8c --- /dev/null +++ b/help/list.md @@ -0,0 +1,16 @@ +# orcli list + +``` +orcli list - list projects on OpenRefine server + +Usage: + orcli list + orcli list --help | -h + +Options: + --help, -h + Show this help + +``` + +code: [src/list_command.sh](../src/list_command.sh) diff --git a/help/run.md b/help/run.md new file mode 100644 index 0000000..beab649 --- /dev/null +++ b/help/run.md @@ -0,0 +1,56 @@ +# orcli run + +``` +orcli run - run tmp OpenRefine workspace and execute shell script(s) + +Usage: + orcli run [FILE...] [OPTIONS] + orcli run --help | -h + +Options: + --memory RAM + maximum RAM for OpenRefine java heap space + Default: 2048M + + --port PORT + PORT on which OpenRefine should listen + Default: 3333 + + --interactive + do not exit on error and keep bash shell open + + --quiet, -q + suppress log output, print errors only + + --help, -h + Show this help + +Arguments: + FILE... + Path to one or more files or URLs. When FILE is -, read standard input. + Default: - + +Examples: + orcli run --interactive + orcli run << EOF + orcli import csv "https://git.io/fj5hF" --projectName "duplicates" + orcli transform "duplicates" "https://git.io/fj5ju" + orcli export tsv "duplicates" + EOF + orcli run --memory "2000M" --port "3334" << EOF + orcli import csv "https://git.io/fj5hF" --projectName "duplicates" & + orcli import csv "https://git.io/fj5hF" --projectName "copy" & + wait + echo "finished import" + orcli export csv "duplicates" --output duplicates.csv & + orcli export tsv "duplicates" --output duplicates.tsv & + wait + wc duplicates* + EOF + orcli run --interactive "file1.sh" "file2.sh" - << EOF + echo "finished in $SECONDS seconds" + EOF + +``` + +code: [src/run_command.sh](../src/run_command.sh) diff --git a/help/test.md b/help/test.md new file mode 100644 index 0000000..acca7e8 --- /dev/null +++ b/help/test.md @@ -0,0 +1,16 @@ +# orcli test + +``` +orcli test - run functional tests on tmp OpenRefine workspace + +Usage: + orcli test + orcli test --help | -h + +Options: + --help, -h + Show this help + +``` + +code: [src/test_command.sh](../src/test_command.sh) diff --git a/help/transform.md b/help/transform.md new file mode 100644 index 0000000..c4f32d3 --- /dev/null +++ b/help/transform.md @@ -0,0 +1,33 @@ +# orcli transform + +``` +orcli transform - apply undo/redo JSON file(s) to an OpenRefine project + +Usage: + orcli transform PROJECT [FILE...] [OPTIONS] + orcli transform --help | -h + +Options: + --quiet, -q + suppress log output, print errors only + + --help, -h + Show this help + +Arguments: + PROJECT + project name or id + + FILE... + Path to one or more files or URLs. When FILE is -, read standard input. + Default: - + +Examples: + orcli transform "duplicates" "history.json" + cat "history.json" | orcli transform "duplicates" + orcli transform "duplicates" "https://git.io/fj5ju" + orcli transform 1234567890123 "history.json" + +``` + +code: [src/transform_command.sh](../src/transform_command.sh) diff --git a/orcli b/orcli index 4701d39..01c84ba 100755 --- a/orcli +++ b/orcli @@ -92,7 +92,7 @@ orcli_completions_usage() { printf "orcli completions\n" echo - printf " Generate bash completions\n Usage: eval \"\$(orcli completions)\"\n" + printf " Generate bash completions\n Usage: source <(orcli completions)\n" echo else @@ -168,7 +168,7 @@ orcli_delete_usage() { printf " orcli delete \"duplicates\"\n" printf " orcli delete \"duplicates\" --force\n" printf " orcli delete 1234567890123\n" - printf " for p in $(orcli list); do orcli delete ${p:0:13}; done\n" + printf " for p in \$(orcli list); do orcli delete \${p:0:13}; done\n" echo fi @@ -346,7 +346,7 @@ orcli_import_csv_usage() { printf " orcli import csv \"file1\" \"file2\"\n" printf " head -n 100 \"file\" | orcli import csv\n" printf " orcli import csv \"https://git.io/fj5hF\"\n" - printf " orcli import csv \"file\" \\\\\n --separator \";\" \\\\\n --columnNames \"foo,bar,baz\" \\\\\n --ignoreLines 1 \\\\\n --encoding \"ISO-8859-1\" \\\\\n --limit 100 \\\\\n --trimStrings \\\\\n --projectName \"duplicates\"\n --projectTags \"test,urgent\"\n" + printf " orcli import csv \"file\" \\\\\n --separator \";\" \\\\\n --columnNames \"foo,bar,baz\" \\\\\n --ignoreLines 1 \\\\\n --encoding \"ISO-8859-1\" \\\\\n --limit 100 \\\\\n --trimStrings \\\\\n --projectName \"duplicates\" \\\\\n --projectTags \"test,urgent\"\n" echo fi @@ -484,7 +484,7 @@ orcli_import_tsv_usage() { printf " orcli import tsv \"file1\" \"file2\"\n" printf " head -n 100 \"file\" | orcli import tsv\n" printf " orcli import tsv \"https://example.com/file.tsv\"\n" - printf " orcli import tsv \"file\" \\\\\n --separator \";\" \\\\\n --columnNames \"foo,bar,baz\" \\\\\n --ignoreLines 1 \\\\\n --encoding \"ISO-8859-1\" \\\\\n --limit 100 \\\\\n --trimStrings \\\\\n --projectName \"duplicates\"\n --projectTags \"test,urgent\"\n" + printf " orcli import tsv \"file\" \\\\\n --separator \";\" \\\\\n --columnNames \"foo,bar,baz\" \\\\\n --ignoreLines 1 \\\\\n --encoding \"ISO-8859-1\" \\\\\n --limit 100 \\\\\n --trimStrings \\\\\n --projectName \"duplicates\" \\\\\n --projectTags \"test,urgent\"\n" echo fi diff --git a/src/bashly.yml b/src/bashly.yml index 18766de..1f2a283 100644 --- a/src/bashly.yml +++ b/src/bashly.yml @@ -32,7 +32,7 @@ commands: - name: completions help: |- Generate bash completions - Usage: eval "\$(orcli completions)" + Usage: source <(orcli completions) - name: delete help: delete OpenRefine project @@ -53,7 +53,7 @@ commands: - orcli delete "duplicates" - orcli delete "duplicates" --force - orcli delete 1234567890123 - - for p in $(orcli list); do orcli delete ${p:0:13}; done + - for p in \$(orcli list); do orcli delete \${p:0:13}; done - name: import help: commands to create OpenRefine projects from files or URLs @@ -153,7 +153,7 @@ commands: --encoding "ISO-8859-1" \\\\ --limit 100 \\\\ --trimStrings \\\\ - --projectName "duplicates" + --projectName "duplicates" \\\\ --projectTags "test,urgent" - name: tsv @@ -191,7 +191,7 @@ commands: --encoding "ISO-8859-1" \\\\ --limit 100 \\\\ --trimStrings \\\\ - --projectName "duplicates" + --projectName "duplicates" \\\\ --projectTags "test,urgent" - name: list