opencultureconsulting
/
openrefine-client
mirror of https://github.com/opencultureconsulting/openrefine-client.git
2
0
Fork 0
Code Issues Releases Activity

Add NumericFacet & camelcase tests. Rename text_facet to compute_facets. Make FacetResponse more generic, with support for bins & baseBins.

This commit is contained in:
Paul Makepeace 2011-04-24 18:36:19 -04:00
parent 263c991804
commit da85e698dd
3 changed files with 78 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
google/refine.py
View File

@ -19,10 +19,17 @@ REFINE_HOST = os.environ.get('GOOGLE_REFINE_HOST', '127.0.0.1')
REFINE_PORT = os.environ.get('GOOGLE_REFINE_PORT', '3333') REFINE_PORT = os.environ.get('GOOGLE_REFINE_PORT', '3333')
def to_camel(attr): def to_camel(attr):
"""convert this_attr_name to thisAttrName.""" """convert this_attr_name to thisAttrName."""
return re.sub(r'_(.)', lambda x: x.group(1).upper(), attr) # Do lower case first letter
return (attr[0].lower() +
re.sub(r'_(.)', lambda x: x.group(1).upper(), attr[1:]))
def from_camel(attr):
"""convert thisAttrName to this_attr_name."""
# Don't add an underscore for capitalized first letter
return re.sub(r'(?<=.)([A-Z])', lambda x: '_' + x.group(1), attr).lower()
class Facet(object): class Facet(object):
def __init__(self, column, type, expression='value', **options): def __init__(self, column, type, expression='value', **options):
@ -35,7 +42,8 @@ class Facet(object):
setattr(self, k, v) setattr(self, k, v)
def as_dict(self): def as_dict(self):
return dict([(to_camel(k), v) for k, v in self.__dict__.items()]) return dict([(to_camel(k), v) for k, v in self.__dict__.items()
if v is not None])
def include(self, selection): def include(self, selection):
for s in self.selection: for s in self.selection:
@ -62,9 +70,9 @@ class TextFacet(Facet):
invert=invert, invert=invert,
**options) **options)
# Capitalize 'From' to get around python's reserved word.
class NumericFacet(Facet): class NumericFacet(Facet):
def __init__(self, column, select_blank=True, select_error=True, select_non_numeric=True, select_numeric=True, **options): def __init__(self, column, From=None, to=None, select_blank=True, select_error=True, select_non_numeric=True, select_numeric=True, **options):
super(NumericFacet, self).__init__( super(NumericFacet, self).__init__(
column, column,
type='range', type='range',
@ -72,27 +80,32 @@ class NumericFacet(Facet):
select_error=select_error, select_error=select_error,
select_non_numeric=select_non_numeric, select_non_numeric=select_non_numeric,
select_numeric=select_numeric, select_numeric=select_numeric,
From=From,
to=to,
**options) **options)
class FacetResponse(object): class FacetResponse(object):
def __init__(self, facet): def __init__(self, facet):
self.name = facet['name'] for k, v in facet.items():
self.column = self.name if isinstance(k, bool) or isinstance(k, basestring):
self.expression = facet['expression'] setattr(self, from_camel(k), v)
self.invert = facet['invert']
self.choices = {} self.choices = {}
class FacetChoice(object): class FacetChoice(object):
def __init__(self, c): def __init__(self, c):
self.count = c['c'] self.count = c['c']
self.selected = c['s'] self.selected = c['s']
for choice in facet['choices']: if 'choices' in facet:
self.choices[choice['v']['v']] = FacetChoice(choice) for choice in facet['choices']:
if 'blankChoice' in facet: self.choices[choice['v']['v']] = FacetChoice(choice)
self.blank_choice = FacetChoice(facet['blankChoice']) if 'blankChoice' in facet:
else: self.blank_choice = FacetChoice(facet['blankChoice'])
self.blank_choice = None else:
self.blank_choice = None
if 'bins' in facet:
self.bins = facet['bins']
self.base_bins = facet['baseBins']
class FacetsResponse(object): class FacetsResponse(object):
@ -138,6 +151,7 @@ class RefineServer(object):
if data is None: if data is None:
data = {} data = {}
if project_id: if project_id:
# XXX haven't figured out pattern on qs v body
if 'delete' in command: if 'delete' in command:
data['project'] = project_id data['project'] = project_id
else: else:
@ -362,7 +376,7 @@ class RefineProject:
response_json = self.do_json('delete-project') response_json = self.do_json('delete-project')
return 'code' in response_json and response_json['code'] == 'ok' return 'code' in response_json and response_json['code'] == 'ok'
def text_facet(self, facets=None): def compute_facets(self, facets=None):
if facets: if facets:
self.engine = Engine(facets) self.engine = Engine(facets)
response = self.do_json('compute-facets', response = self.do_json('compute-facets',

7
google/test/test_engine.py
View File

@ -21,6 +21,9 @@ class FacetTest(unittest.TestCase):
engine = Engine(facet) engine = Engine(facet)
self.assertEqual(facet.selection, []) self.assertEqual(facet.selection, [])
self.assertTrue(str(engine)) self.assertTrue(str(engine))
facet = NumericFacet('column name', From=1, to=5)
self.assertEqual(facet.to, 5)
self.assertEqual(facet.From, 1)
def test_serialize(self): def test_serialize(self):
engine = Engine() engine = Engine()
@ -28,8 +31,8 @@ class FacetTest(unittest.TestCase):
self.assertEqual(engine_json, '{"facets": [], "mode": "row-based"}') self.assertEqual(engine_json, '{"facets": [], "mode": "row-based"}')
facet = TextFacet(column='column') facet = TextFacet(column='column')
self.assertEqual(facet.as_dict(), {'selectError': False, 'name': 'column', 'selection': [], 'expression': 'value', 'invert': False, 'columnName': 'column', 'selectBlank': False, 'omitBlank': False, 'type': 'list', 'omitError': False}) self.assertEqual(facet.as_dict(), {'selectError': False, 'name': 'column', 'selection': [], 'expression': 'value', 'invert': False, 'columnName': 'column', 'selectBlank': False, 'omitBlank': False, 'type': 'list', 'omitError': False})
facet = NumericFacet(column='column') facet = NumericFacet(column='column', From=1, to=5)
self.assertEqual(facet.as_dict(), {'selectBlank': True, 'name': 'column', 'selectError': True, 'expression': 'value', 'selection': [], 'selectNumeric': True, 'columnName': 'column', 'selectNonNumeric': True, 'type': 'range'}) self.assertEqual(facet.as_dict(), {'from': 1, 'to': 5, 'selectBlank': True, 'name': 'column', 'selectError': True, 'expression': 'value', 'selection': [], 'selectNumeric': True, 'columnName': 'column', 'selectNonNumeric': True, 'type': 'range'})
def test_add_facet(self): def test_add_facet(self):
facet = TextFacet(column='Party Code') facet = TextFacet(column='Party Code')

49
google/test/test_refine.py
View File

@ -11,11 +11,35 @@ import sys
import os import os
import unittest import unittest
from google.refine import REFINE_HOST, REFINE_PORT from google.refine import REFINE_HOST, REFINE_PORT
from google.refine import TextFacet, Engine from google.refine import NumericFacet, TextFacet, Engine
from google.refine import RefineServer, Refine, RefineProject from google.refine import RefineServer, Refine, RefineProject
from google.refine import to_camel, from_camel
PATH_TO_TEST_DATA = os.path.join('google', 'test', 'data') PATH_TO_TEST_DATA = os.path.join('google', 'test', 'data')
class CamelTest(unittest.TestCase):
def test_to_camel(self):
pairs = (
('this', 'this'),
('this_attr', 'thisAttr'),
('From', 'from'),
)
for attr, camel_attr in pairs:
self.assertEqual(to_camel(attr), camel_attr)
def test_from_camel(self):
pairs = (
('this', 'this'),
('This', 'this'),
('thisAttr', 'this_attr'),
('ThisAttr', 'this_attr'),
('From', 'from'),
)
for camel_attr, attr in pairs:
self.assertEqual(from_camel(camel_attr), attr)
class RefineTestCase(unittest.TestCase): class RefineTestCase(unittest.TestCase):
project_file = None project_file = None
project = None project = None
@ -80,7 +104,7 @@ class TutorialTestFacets(RefineTestCase):
def test_basic_facet(self): def test_basic_facet(self):
# {4} # {4}
party_code_facet = TextFacet(column='Party Code') party_code_facet = TextFacet(column='Party Code')
response = self.project.text_facet(party_code_facet) response = self.project.compute_facets(party_code_facet)
pc = response.facets[0] pc = response.facets[0]
self.assertEqual(pc.name, 'Party Code') self.assertEqual(pc.name, 'Party Code')
self.assertEqual(pc.choices['D'].count, 3700) self.assertEqual(pc.choices['D'].count, 3700)
@ -91,7 +115,7 @@ class TutorialTestFacets(RefineTestCase):
ethnicity_facet = TextFacet(column='Ethnicity') ethnicity_facet = TextFacet(column='Ethnicity')
engine.add_facet(ethnicity_facet) engine.add_facet(ethnicity_facet)
self.project.engine = engine self.project.engine = engine
response = self.project.text_facet() response = self.project.compute_facets()
e = response.facets[1] e = response.facets[1]
self.assertEqual(e.choices['B'].count, 1255) self.assertEqual(e.choices['B'].count, 1255)
self.assertEqual(e.choices['W'].count, 4469) self.assertEqual(e.choices['W'].count, 4469)
@ -102,7 +126,7 @@ class TutorialTestFacets(RefineTestCase):
indexes = [r.index for r in response.rows] indexes = [r.index for r in response.rows]
self.assertEqual(indexes, [1, 2, 3, 4, 6, 12, 18, 26, 28, 32]) self.assertEqual(indexes, [1, 2, 3, 4, 6, 12, 18, 26, 28, 32])
# {8} # {8}
response = self.project.text_facet() response = self.project.compute_facets()
pc = response.facets[0] pc = response.facets[0]
self.assertEqual(pc.name, 'Party Code') self.assertEqual(pc.name, 'Party Code')
self.assertEqual(pc.choices['D'].count, 1179) self.assertEqual(pc.choices['D'].count, 1179)
@ -110,7 +134,7 @@ class TutorialTestFacets(RefineTestCase):
self.assertEqual(pc.blank_choice.count, 46) self.assertEqual(pc.blank_choice.count, 46)
# {9} # {9}
party_code_facet.include('R') party_code_facet.include('R')
response = self.project.text_facet() response = self.project.compute_facets()
e = response.facets[1] e = response.facets[1]
self.assertEqual(e.choices['B'].count, 11) self.assertEqual(e.choices['B'].count, 11)
# {10} # {10}
@ -121,8 +145,21 @@ class TutorialTestFacets(RefineTestCase):
# {11} # {11}
office_title_facet = TextFacet('Office Title') office_title_facet = TextFacet('Office Title')
self.project.engine.add_facet(office_title_facet) self.project.engine.add_facet(office_title_facet)
response = self.project.text_facet() response = self.project.compute_facets()
self.assertEqual(len(response.facets[2].choices), 76) self.assertEqual(len(response.facets[2].choices), 76)
# {12} - XXX not sure how to interpret bins & baseBins yet
office_level_facet = NumericFacet('Office Level')
self.project.engine.add_facet(office_level_facet)
# {13}
office_level_facet.From = 300 # from reserved word
office_level_facet.to = 320
response = self.project.get_rows()
self.assertEqual(response.filtered, 1907)
response = self.project.compute_facets()
ot = response.facets[2] # Office Title
self.assertEqual(len(ot.choices), 21)
self.assertEqual(ot.choices['Chief of Police'].count, 2)
self.assertEqual(ot.choices['Chief of Police '].count, 211)
if __name__ == '__main__': if __name__ == '__main__':