refactor to allow module execution
This commit is contained in:
parent
04db513453
commit
a0123e9511
32
Makefile
32
Makefile
|
@ -1,32 +0,0 @@
|
||||||
# XXX have a Makefile written by someone that knows Makefiles...
|
|
||||||
|
|
||||||
all: test build install
|
|
||||||
|
|
||||||
readme:
|
|
||||||
# requires docutils, e.g. pip install docutils
|
|
||||||
rst2html.py README.rst > README.html
|
|
||||||
w3m -dump README.html | unix2dos > README.txt
|
|
||||||
|
|
||||||
test:
|
|
||||||
python setup.py test
|
|
||||||
|
|
||||||
# tests that don't require a Refine server running
|
|
||||||
smalltest:
|
|
||||||
python setup.py test --test-suite tests.test_refine_small
|
|
||||||
python setup.py test --test-suite tests.test_facet
|
|
||||||
python setup.py test --test-suite tests.test_history
|
|
||||||
|
|
||||||
build:
|
|
||||||
python setup.py build
|
|
||||||
|
|
||||||
install:
|
|
||||||
sudo python setup.py install
|
|
||||||
|
|
||||||
clean:
|
|
||||||
find . -name '*.pyc' | xargs rm -f
|
|
||||||
# XXX is there some way of having setup.py clean up its junk?
|
|
||||||
rm -rf README.{html,txt} build dist refine_client.egg-info distribute-*
|
|
||||||
|
|
||||||
upload: clean
|
|
||||||
python setup.py sdist upload
|
|
||||||
|
|
26
README.md
26
README.md
|
@ -1,6 +1,7 @@
|
||||||
# OpenRefine Python Client with extended command line interface
|
# OpenRefine Python Client with extended command line interface
|
||||||
|
|
||||||
The [OpenRefine Python Client Library from PaulMakepeace](https://github.com/PaulMakepeace/refine-client-py) provides an interface to communicating with an [OpenRefine](http://openrefine.org) server. This fork extends the command line interface (CLI) and supports communication between docker containers.
|
The [OpenRefine Python Client Library from PaulMakepeace](https://github.com/PaulMakepeace/refine-client-py) provides an interface to communicating with an [OpenRefine](http://openrefine.org) server.
|
||||||
|
This fork extends the command line interface (CLI) and supports communication between docker containers.
|
||||||
|
|
||||||
## Download
|
## Download
|
||||||
|
|
||||||
|
@ -31,7 +32,7 @@ Command line interface:
|
||||||
- delete project: `--delete [PROJECTID/PROJECTNAME]`
|
- delete project: `--delete [PROJECTID/PROJECTNAME]`
|
||||||
- check `--help` for further options...
|
- check `--help` for further options...
|
||||||
|
|
||||||
If you are familiar with python you may try all functions interactively (`python -i refine.py`) or use this library in your own python scripts. Some Examples:
|
If you are familiar with python you may try all functions interactively (`python -i refine/refine.py`) or use this library in your own python scripts. Some Examples:
|
||||||
|
|
||||||
* show version of OpenRefine server: `refine.RefineServer().get_version()`
|
* show version of OpenRefine server: `refine.RefineServer().get_version()`
|
||||||
* show total rows of project 2151545447855: `refine.RefineProject(refine.RefineServer(),'2151545447855').do_json('get-rows')['total']`
|
* show total rows of project 2151545447855: `refine.RefineProject(refine.RefineServer(),'2151545447855').do_json('get-rows')['total']`
|
||||||
|
@ -45,25 +46,16 @@ The environment variables `OPENREFINE_HOST` and `OPENREFINE_PORT` enable overrid
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
Install dependencies, which currently is `urllib2_file`:
|
```pip install openrefine-client```
|
||||||
|
|
||||||
```
|
(requires Python 2.x, depends on urllib2_file>=0.2.1)
|
||||||
sudo pip install -r requirements.txt
|
|
||||||
```
|
|
||||||
|
|
||||||
Ensure you have a Refine server running somewhere and, if necessary, set the environment vars as above.
|
## Tests
|
||||||
|
|
||||||
Run tests, build, and install:
|
Ensure you have an OpenRefine server running somewhere and, if necessary, set the environment vars as above.
|
||||||
|
|
||||||
```
|
* all tests: ```python setup.py test```
|
||||||
python setup.py test # to do a subset, e.g., --test-suite tests.test_facet
|
* a subset: ```python setup.py test --test-suite tests.test_facet```
|
||||||
|
|
||||||
python setup.py build
|
|
||||||
|
|
||||||
python setup.py install
|
|
||||||
```
|
|
||||||
|
|
||||||
There is a Makefile that will do this too, and more.
|
|
||||||
|
|
||||||
## Credits
|
## Credits
|
||||||
|
|
||||||
|
|
|
@ -3,26 +3,26 @@ LABEL maintainer="felixlohmeier@opencultureconsulting.com"
|
||||||
# The OpenRefine Python Client Library from PaulMakepeace provides an interface to communicating with an OpenRefine server. This fork extends the command line interface (CLI) and supports communication between docker containers.
|
# The OpenRefine Python Client Library from PaulMakepeace provides an interface to communicating with an OpenRefine server. This fork extends the command line interface (CLI) and supports communication between docker containers.
|
||||||
# Source: https://github.com/opencultureconsulting/openrefine-client
|
# Source: https://github.com/opencultureconsulting/openrefine-client
|
||||||
|
|
||||||
# Install python and pip
|
# Install python, pip, unzip, curl and grep
|
||||||
# ... and curl for https://github.com/opencultureconsulting/openrefine-batch
|
|
||||||
RUN apk add --no-cache \
|
RUN apk add --no-cache \
|
||||||
python \
|
python \
|
||||||
py-pip \
|
py-pip \
|
||||||
curl
|
unzip \
|
||||||
|
curl \
|
||||||
|
grep
|
||||||
|
|
||||||
# Install dependency urllib2_file
|
# Install dependency urllib2_file
|
||||||
RUN pip install urllib2_file==0.2.1
|
RUN pip install urllib2_file==0.2.1
|
||||||
|
|
||||||
# Download and build openrefine-client-master
|
# Copy python scripts
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY google google
|
COPY refine/* ./
|
||||||
COPY refine.py .
|
|
||||||
|
|
||||||
# Change docker WORKDIR (shall be mounted by user)
|
# Change docker WORKDIR (shall be mounted by user)
|
||||||
WORKDIR /data
|
WORKDIR /data
|
||||||
|
|
||||||
# Execute refine.py
|
# Execute main python script
|
||||||
ENTRYPOINT ["/app/refine.py"]
|
ENTRYPOINT ["/app/__main__.py"]
|
||||||
|
|
||||||
# Default command: print help
|
# Default command: print help
|
||||||
CMD ["-h"]
|
CMD ["-h"]
|
||||||
|
|
|
@ -25,7 +25,7 @@ import sys
|
||||||
import time
|
import time
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from google.refine import refine
|
import refine
|
||||||
|
|
||||||
reload(sys)
|
reload(sys)
|
||||||
sys.setdefaultencoding('utf-8')
|
sys.setdefaultencoding('utf-8')
|
||||||
|
@ -37,7 +37,7 @@ class myParser(optparse.OptionParser):
|
||||||
|
|
||||||
PARSER = \
|
PARSER = \
|
||||||
myParser(description='Script to provide a command line interface to an OpenRefine server.',
|
myParser(description='Script to provide a command line interface to an OpenRefine server.',
|
||||||
usage='usage: %prog [--help | OPTIONS]',
|
usage='usage: openrefine-client [--help | OPTIONS]',
|
||||||
epilog="""
|
epilog="""
|
||||||
Examples:
|
Examples:
|
||||||
--list # show list of projects (id: name)
|
--list # show list of projects (id: name)
|
|
@ -30,8 +30,8 @@ import urllib2_file
|
||||||
import urllib2
|
import urllib2
|
||||||
import urlparse
|
import urlparse
|
||||||
|
|
||||||
from google.refine import facet
|
import facet
|
||||||
from google.refine import history
|
import history
|
||||||
|
|
||||||
REFINE_HOST = os.environ.get('OPENREFINE_HOST', os.environ.get('GOOGLE_REFINE_HOST', '127.0.0.1'))
|
REFINE_HOST = os.environ.get('OPENREFINE_HOST', os.environ.get('GOOGLE_REFINE_HOST', '127.0.0.1'))
|
||||||
REFINE_PORT = os.environ.get('OPENREFINE_PORT', os.environ.get('GOOGLE_REFINE_PORT', '3333'))
|
REFINE_PORT = os.environ.get('OPENREFINE_PORT', os.environ.get('GOOGLE_REFINE_PORT', '3333'))
|
6
setup.py
6
setup.py
|
@ -25,7 +25,7 @@ def read(filename):
|
||||||
return open(os.path.join(os.path.dirname(__file__), filename)).read()
|
return open(os.path.join(os.path.dirname(__file__), filename)).read()
|
||||||
|
|
||||||
setup(name='openrefine-client',
|
setup(name='openrefine-client',
|
||||||
version='0.3.4',
|
version='0.3.5',
|
||||||
description=('The OpenRefine Python Client Library provides an '
|
description=('The OpenRefine Python Client Library provides an '
|
||||||
'interface to communicating with an OpenRefine server. '
|
'interface to communicating with an OpenRefine server. '
|
||||||
'This fork extends the command line interface (CLI).'),
|
'This fork extends the command line interface (CLI).'),
|
||||||
|
@ -34,7 +34,11 @@ setup(name='openrefine-client',
|
||||||
author_email='felix.lohmeier@opencultureconsulting.com',
|
author_email='felix.lohmeier@opencultureconsulting.com',
|
||||||
url='https://github.com/opencultureconsulting/openrefine-client',
|
url='https://github.com/opencultureconsulting/openrefine-client',
|
||||||
packages=find_packages(exclude=['tests']),
|
packages=find_packages(exclude=['tests']),
|
||||||
|
python_requires='>2.6, !=3.*',
|
||||||
install_requires=['urllib2_file'],
|
install_requires=['urllib2_file'],
|
||||||
|
entry_points={
|
||||||
|
'console_scripts': [ 'openrefine-client = refine.__main__:main' ]
|
||||||
|
},
|
||||||
platforms=['Any'],
|
platforms=['Any'],
|
||||||
classifiers = [
|
classifiers = [
|
||||||
'Development Status :: 3 - Alpha',
|
'Development Status :: 3 - Alpha',
|
||||||
|
|
|
@ -15,7 +15,7 @@ and REFINE_PORT.
|
||||||
import os
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from google.refine import refine
|
from refine import refine
|
||||||
|
|
||||||
PATH_TO_TEST_DATA = os.path.join(os.path.dirname(__file__), 'data')
|
PATH_TO_TEST_DATA = os.path.join(os.path.dirname(__file__), 'data')
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,7 @@ test_facet.py
|
||||||
import json
|
import json
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from google.refine.facet import *
|
from refine.facet import *
|
||||||
|
|
||||||
|
|
||||||
class CamelTest(unittest.TestCase):
|
class CamelTest(unittest.TestCase):
|
||||||
|
|
|
@ -7,7 +7,7 @@ test_history.py
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from google.refine.history import *
|
from refine.history import *
|
||||||
|
|
||||||
|
|
||||||
class HistoryTest(unittest.TestCase):
|
class HistoryTest(unittest.TestCase):
|
||||||
|
|
|
@ -12,7 +12,7 @@ OPENREFINE_HOST and OPENREFINE_PORT.
|
||||||
import csv
|
import csv
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from google.refine import refine
|
from refine import refine
|
||||||
from tests import refinetest
|
from tests import refinetest
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,7 @@ test_refine_small.py
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from google.refine import refine
|
from refine import refine
|
||||||
|
|
||||||
|
|
||||||
class RefineRowsTest(unittest.TestCase):
|
class RefineRowsTest(unittest.TestCase):
|
||||||
|
|
|
@ -16,7 +16,7 @@ OPENREFINE_HOST and OPENREFINE_PORT.
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from google.refine import facet
|
from refine import facet
|
||||||
from tests import refinetest
|
from tests import refinetest
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue