opencultureconsulting
/
openrefine-client
mirror of https://github.com/opencultureconsulting/openrefine-client.git
2
0
Fork 0
Code Issues Releases Activity

attempt to fix tests

This commit is contained in:
Wolf Vollprecht 2020-01-13 11:19:34 +01:00
parent 7826fa1037
commit cca067a87c
6 changed files with 55 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
google/refine/__main__.py
View File

@ -213,7 +213,7 @@ def main():
projects = list(refine.Refine(refine.RefineServer()).list_projects().items())
idlist = []
for project_id, project_info in projects:
if args[0].decode('UTF-8') == project_info['name']:
if args[0] == project_info['name']:
idlist.append(str(project_id))
if len(idlist) > 1:
print(('Error: Found %s projects with name %s.\n'

25
google/refine/cli.py
View File

@ -24,6 +24,7 @@ import os
import ssl
import sys
import time
import requests
import urllib.request, urllib.parse, urllib.error
from xml.etree import ElementTree
@ -156,9 +157,10 @@ def download(url, output_file=None):
'Delete existing file or try command --output '
'to specify a different filename.' % output_file))
return
# Workaround for SSL verification problems in one-file-executables
context = ssl._create_unverified_context()
urllib.request.urlretrieve(url, output_file, context=context)
myfile = requests.get(url)
with open(output_file, 'wb') as fo:
fo.write(myfile.content)
print(('Download to file %s complete' % output_file))
@ -171,7 +173,7 @@ def export(project_id, encoding=None, output_file=None, export_format=None):
if export_format in ['csv', 'tsv', 'txt']:
encoding = 'UTF-8'
sys.stdout.write(project.export(
export_format=export_format, encoding=encoding).read())
export_format=export_format, encoding=encoding).text)
else:
ext = os.path.splitext(output_file)[1][1:]
if ext:
@ -180,10 +182,9 @@ def export(project_id, encoding=None, output_file=None, export_format=None):
encoding = 'UTF-8'
with open(output_file, 'wb') as f:
f.write(project.export(
export_format=export_format, encoding=encoding).read())
export_format=export_format, encoding=encoding).content)
print(('Export to file %s complete' % output_file))
def info(project_id):
"""Show project metadata"""
projects = refine.Refine(refine.RefineServer()).list_projects()
@ -267,10 +268,10 @@ def templating(project_id,
# normal output
if not output_file:
sys.stdout.write(project.export_templating(
**templateconfig).read())
**templateconfig).text)
else:
with open(output_file, 'wb') as f:
f.write(project.export_templating(**templateconfig).read())
f.write(project.export_templating(**templateconfig).content)
print(('Export to file %s complete' % output_file))
else:
# splitToFiles functionality
@ -294,7 +295,7 @@ def templating(project_id,
'rowSeparator': '\n',
'encoding': encoding}
ids = [line.rstrip('\n') for line in project.export_templating(
**ids_templateconfig) if line.rstrip('\n')]
**ids_templateconfig).text if line.rstrip('\n')]
# generate common config
if mode == 'record-based':
# record-based: split-character into template
@ -316,12 +317,12 @@ def templating(project_id,
'rowSeparator': ''})
# execute
records = project.export_templating(
**templateconfig).read().split(split)
**templateconfig).text.split(split)
del records[0] # skip first blank entry
if suffixById:
for index, record in enumerate(records):
output_file = base + '_' + ids[index] + '.' + ext
with open(output_file, 'wb') as f:
with open(output_file, 'w') as f:
f.writelines([prefix, record, suffix])
print(('Export to files complete. Last file: %s' % output_file))
else:
@ -329,6 +330,6 @@ def templating(project_id,
for index, record in enumerate(records):
output_file = base + '_' + \
str(index + 1).zfill(zeros) + '.' + ext
with open(output_file, 'wb') as f:
with open(output_file, 'w') as f:
f.writelines([prefix, record, suffix])
print(('Export to files complete. Last file: %s' % output_file))

5
google/refine/facet.py
View File

@ -208,7 +208,10 @@ class FacetsResponse(object):
return self.facets[index]
self.facets = FacetResponseContainer(facets['facets'])
self.mode = facets['mode']
if facets.get('mode'):
self.mode = facets['mode']
else:
self.mode = facets['engine-mode']
class Engine(object):

23
google/refine/refine.py
View File

@ -55,7 +55,7 @@ class RefineServer(object):
self.server = server[:-1] if server.endswith('/') else server
self.__version = None # see version @property below
def urlopen(self, command, data=None, params=None, project_id=None):
def urlopen(self, command, data=None, params=None, project_id=None, files=None):
"""Open a Refine URL and with optional query params and POST data.
data: POST data dict
@ -83,12 +83,13 @@ class RefineServer(object):
response = requests.get(url)
else:
response = requests.post(url, data=data, files=files)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
raise Exception('HTTP %d "%s" for %s\n\t%s' % (e.code, e.msg, e.geturl(), data))
raise Exception('HTTP Error: %s' % (e))
except requests.exceptions.URLRequired as e:
raise requests.exceptions.URLRequired(
'%s for %s. No Refine server reachable/running; ENV set?' %
(e.reason, self.server))
(e, self.server))
if response.encoding == 'gzip':
# Need a seekable filestream for gzip
@ -102,8 +103,16 @@ class RefineServer(object):
"""Open a Refine URL, optionally POST data, and return parsed JSON."""
response = self.urlopen(*args, **kwargs).json()
if 'code' in response and response['code'] not in ('ok', 'pending'):
error_message = ('server ' + response['code'] + ': ' +
response.get('message', response.get('stack', response)))
error_hint = ""
if response.get('message') and response['message'] is not None:
error_hint += response['message']
if response.get('stack') and response['stack'] is not None:
error_hint += response['stack']
if not error_hint:
error_hint += str(response)
error_message = 'server ' + response['code'] + ':\n' + error_hint
raise Exception(error_message)
return response
@ -274,9 +283,11 @@ class Refine:
response = self.server.urlopen(
'create-project-from-upload', options, params, files=files
)
if project_file:
files['project-file'].close()
# expecting a redirect to the new project containing the id in the url
url_params = urllib.parse.parse_qs(
urllib.parse.urlparse(response.geturl()).query)
urllib.parse.urlparse(response.url).query)
if 'project' in url_params:
project_id = url_params['project'][0]
return RefineProject(self.server, project_id)

22
tests/test_refine.py
View File

@ -13,7 +13,9 @@ import csv
import unittest
from google.refine import refine
from tests import refinetest
import refinetest
from io import StringIO
class RefineServerTest(refinetest.RefineTestCase):
@ -37,7 +39,7 @@ class RefineServerTest(refinetest.RefineTestCase):
self.assertTrue(item in version_info)
def test_version(self):
self.assertTrue(self.server.version in ('2.0', '2.1', '2.5'))
self.assertTrue(self.server.version in ('3.2'))
class RefineTest(refinetest.RefineTestCase):
@ -59,21 +61,19 @@ class RefineTest(refinetest.RefineTestCase):
self.assertTrue(self.project.delete())
def test_open_export(self):
fp = refine.RefineProject(self.project.project_url()).export()
line = fp.next()
self.assertTrue('email' in line)
for line in fp:
response = refine.RefineProject(self.project.project_url()).export()
lines = response.text.splitlines()
self.assertTrue('email' in lines[0])
for line in lines[1:]:
self.assertTrue('M' in line or 'F' in line)
fp.close()
def test_open_export_csv(self):
fp = refine.RefineProject(self.project.project_url()).export()
csv_fp = csv.reader(fp, dialect='excel-tab')
row = csv_fp.next()
response = refine.RefineProject(self.project.project_url()).export()
csv_fp = csv.reader(StringIO(response.text), dialect='excel-tab')
row = csv_fp.__next__()
self.assertTrue(row[0] == 'email')
for row in csv_fp:
self.assertTrue(row[3] == 'F' or row[3] == 'M')
fp.close()
if __name__ == '__main__':

18
tests/test_tutorial.py
View File

@ -17,7 +17,7 @@ OPENREFINE_HOST and OPENREFINE_PORT.
import unittest
from google.refine import facet
from tests import refinetest
import refinetest
class TutorialTestFacets(refinetest.RefineTestCase):
@ -138,7 +138,7 @@ class TutorialTestEditing(refinetest.RefineTestCase):
# {2}
self.project.text_transform(column='Zip Code 2',
expression='value.toString()[0, 5]')
self.assertInResponse('transform on 6067 cells in column Zip Code 2')
self.assertInResponse('transform on 6958 cells in column Zip Code 2')
# {3} - XXX history
# {4}
office_title_facet = facet.TextFacet('Office Title')
@ -156,14 +156,14 @@ class TutorialTestEditing(refinetest.RefineTestCase):
self.assertEqual(len(response.facets[office_title_facet].choices), 66)
# {6}
response = self.project.compute_clusters('Office Title')
self.assertTrue(not response)
self.assertTrue(response)
# {7}
clusters = self.project.compute_clusters('Office Title', 'knn')
self.assertEqual(len(clusters), 7)
first_cluster = clusters[0]
self.assertEqual(len(first_cluster), 2)
self.assertEqual(first_cluster[0]['value'], 'RSCC Member')
self.assertEqual(first_cluster[0]['count'], 233)
self.assertEqual(first_cluster[0]['value'], 'DPEC Member at Large')
self.assertEqual(first_cluster[0]['count'], 6)
# Not strictly necessary to repeat 'Council Member' but a test
# of mass_edit, and it's also what the front end sends.
self.project.mass_edit('Office Title', [{
@ -194,9 +194,9 @@ class TutorialTestEditing(refinetest.RefineTestCase):
# {5}, {6}, {7}
response = self.project.compute_facets(facet.StarredFacet(True))
self.assertEqual(len(response.facets[0].choices), 2) # true & false
self.assertEqual(response.facets[0].choices[True].count, 3)
self.assertEqual(response.facets[0].choices[True].count, 2)
self.project.remove_rows()
self.assertInResponse('3 rows')
self.assertInResponse('2 rows')
class TutorialTestDuplicateDetection(refinetest.RefineTestCase):
@ -214,7 +214,7 @@ class TutorialTestDuplicateDetection(refinetest.RefineTestCase):
self.assertInResponse('Reorder rows')
response = self.project.get_rows()
indexes = [row.index for row in response.rows]
self.assertEqual(indexes, range(10))
self.assertEqual(indexes, list(range(10)))
# {10}
self.project.add_column(
'email', 'count', 'facetCount(value, "value", "email")')
@ -393,7 +393,7 @@ class TutorialTestTransposeVariableNumberOfRowsIntoColumns(
'Column', 'row.record.cells["Column"].value[1, -1].join("|")')
self.assertInResponse('18 cells')
# {26}
self.project.engine.mode = 'row-based'
self.project.engine.mode = 'fd'
# {27}
blank_facet = facet.BlankFacet('First Line', selection=True)
self.project.remove_rows(blank_facet)