attempt to fix tests

This commit is contained in:
Wolf Vollprecht 2020-01-13 11:19:34 +01:00
parent 7826fa1037
commit cca067a87c
6 changed files with 55 additions and 40 deletions

View File

@ -213,7 +213,7 @@ def main():
projects = list(refine.Refine(refine.RefineServer()).list_projects().items()) projects = list(refine.Refine(refine.RefineServer()).list_projects().items())
idlist = [] idlist = []
for project_id, project_info in projects: for project_id, project_info in projects:
if args[0].decode('UTF-8') == project_info['name']: if args[0] == project_info['name']:
idlist.append(str(project_id)) idlist.append(str(project_id))
if len(idlist) > 1: if len(idlist) > 1:
print(('Error: Found %s projects with name %s.\n' print(('Error: Found %s projects with name %s.\n'

View File

@ -24,6 +24,7 @@ import os
import ssl import ssl
import sys import sys
import time import time
import requests
import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.parse, urllib.error
from xml.etree import ElementTree from xml.etree import ElementTree
@ -156,9 +157,10 @@ def download(url, output_file=None):
'Delete existing file or try command --output ' 'Delete existing file or try command --output '
'to specify a different filename.' % output_file)) 'to specify a different filename.' % output_file))
return return
# Workaround for SSL verification problems in one-file-executables
context = ssl._create_unverified_context() myfile = requests.get(url)
urllib.request.urlretrieve(url, output_file, context=context) with open(output_file, 'wb') as fo:
fo.write(myfile.content)
print(('Download to file %s complete' % output_file)) print(('Download to file %s complete' % output_file))
@ -171,7 +173,7 @@ def export(project_id, encoding=None, output_file=None, export_format=None):
if export_format in ['csv', 'tsv', 'txt']: if export_format in ['csv', 'tsv', 'txt']:
encoding = 'UTF-8' encoding = 'UTF-8'
sys.stdout.write(project.export( sys.stdout.write(project.export(
export_format=export_format, encoding=encoding).read()) export_format=export_format, encoding=encoding).text)
else: else:
ext = os.path.splitext(output_file)[1][1:] ext = os.path.splitext(output_file)[1][1:]
if ext: if ext:
@ -180,10 +182,9 @@ def export(project_id, encoding=None, output_file=None, export_format=None):
encoding = 'UTF-8' encoding = 'UTF-8'
with open(output_file, 'wb') as f: with open(output_file, 'wb') as f:
f.write(project.export( f.write(project.export(
export_format=export_format, encoding=encoding).read()) export_format=export_format, encoding=encoding).content)
print(('Export to file %s complete' % output_file)) print(('Export to file %s complete' % output_file))
def info(project_id): def info(project_id):
"""Show project metadata""" """Show project metadata"""
projects = refine.Refine(refine.RefineServer()).list_projects() projects = refine.Refine(refine.RefineServer()).list_projects()
@ -267,10 +268,10 @@ def templating(project_id,
# normal output # normal output
if not output_file: if not output_file:
sys.stdout.write(project.export_templating( sys.stdout.write(project.export_templating(
**templateconfig).read()) **templateconfig).text)
else: else:
with open(output_file, 'wb') as f: with open(output_file, 'wb') as f:
f.write(project.export_templating(**templateconfig).read()) f.write(project.export_templating(**templateconfig).content)
print(('Export to file %s complete' % output_file)) print(('Export to file %s complete' % output_file))
else: else:
# splitToFiles functionality # splitToFiles functionality
@ -294,7 +295,7 @@ def templating(project_id,
'rowSeparator': '\n', 'rowSeparator': '\n',
'encoding': encoding} 'encoding': encoding}
ids = [line.rstrip('\n') for line in project.export_templating( ids = [line.rstrip('\n') for line in project.export_templating(
**ids_templateconfig) if line.rstrip('\n')] **ids_templateconfig).text if line.rstrip('\n')]
# generate common config # generate common config
if mode == 'record-based': if mode == 'record-based':
# record-based: split-character into template # record-based: split-character into template
@ -316,12 +317,12 @@ def templating(project_id,
'rowSeparator': ''}) 'rowSeparator': ''})
# execute # execute
records = project.export_templating( records = project.export_templating(
**templateconfig).read().split(split) **templateconfig).text.split(split)
del records[0] # skip first blank entry del records[0] # skip first blank entry
if suffixById: if suffixById:
for index, record in enumerate(records): for index, record in enumerate(records):
output_file = base + '_' + ids[index] + '.' + ext output_file = base + '_' + ids[index] + '.' + ext
with open(output_file, 'wb') as f: with open(output_file, 'w') as f:
f.writelines([prefix, record, suffix]) f.writelines([prefix, record, suffix])
print(('Export to files complete. Last file: %s' % output_file)) print(('Export to files complete. Last file: %s' % output_file))
else: else:
@ -329,6 +330,6 @@ def templating(project_id,
for index, record in enumerate(records): for index, record in enumerate(records):
output_file = base + '_' + \ output_file = base + '_' + \
str(index + 1).zfill(zeros) + '.' + ext str(index + 1).zfill(zeros) + '.' + ext
with open(output_file, 'wb') as f: with open(output_file, 'w') as f:
f.writelines([prefix, record, suffix]) f.writelines([prefix, record, suffix])
print(('Export to files complete. Last file: %s' % output_file)) print(('Export to files complete. Last file: %s' % output_file))

View File

@ -208,7 +208,10 @@ class FacetsResponse(object):
return self.facets[index] return self.facets[index]
self.facets = FacetResponseContainer(facets['facets']) self.facets = FacetResponseContainer(facets['facets'])
self.mode = facets['mode'] if facets.get('mode'):
self.mode = facets['mode']
else:
self.mode = facets['engine-mode']
class Engine(object): class Engine(object):

View File

@ -55,7 +55,7 @@ class RefineServer(object):
self.server = server[:-1] if server.endswith('/') else server self.server = server[:-1] if server.endswith('/') else server
self.__version = None # see version @property below self.__version = None # see version @property below
def urlopen(self, command, data=None, params=None, project_id=None): def urlopen(self, command, data=None, params=None, project_id=None, files=None):
"""Open a Refine URL and with optional query params and POST data. """Open a Refine URL and with optional query params and POST data.
data: POST data dict data: POST data dict
@ -83,12 +83,13 @@ class RefineServer(object):
response = requests.get(url) response = requests.get(url)
else: else:
response = requests.post(url, data=data, files=files) response = requests.post(url, data=data, files=files)
response.raise_for_status()
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
raise Exception('HTTP %d "%s" for %s\n\t%s' % (e.code, e.msg, e.geturl(), data)) raise Exception('HTTP Error: %s' % (e))
except requests.exceptions.URLRequired as e: except requests.exceptions.URLRequired as e:
raise requests.exceptions.URLRequired( raise requests.exceptions.URLRequired(
'%s for %s. No Refine server reachable/running; ENV set?' % '%s for %s. No Refine server reachable/running; ENV set?' %
(e.reason, self.server)) (e, self.server))
if response.encoding == 'gzip': if response.encoding == 'gzip':
# Need a seekable filestream for gzip # Need a seekable filestream for gzip
@ -102,8 +103,16 @@ class RefineServer(object):
"""Open a Refine URL, optionally POST data, and return parsed JSON.""" """Open a Refine URL, optionally POST data, and return parsed JSON."""
response = self.urlopen(*args, **kwargs).json() response = self.urlopen(*args, **kwargs).json()
if 'code' in response and response['code'] not in ('ok', 'pending'): if 'code' in response and response['code'] not in ('ok', 'pending'):
error_message = ('server ' + response['code'] + ': ' + error_hint = ""
response.get('message', response.get('stack', response))) if response.get('message') and response['message'] is not None:
error_hint += response['message']
if response.get('stack') and response['stack'] is not None:
error_hint += response['stack']
if not error_hint:
error_hint += str(response)
error_message = 'server ' + response['code'] + ':\n' + error_hint
raise Exception(error_message) raise Exception(error_message)
return response return response
@ -274,9 +283,11 @@ class Refine:
response = self.server.urlopen( response = self.server.urlopen(
'create-project-from-upload', options, params, files=files 'create-project-from-upload', options, params, files=files
) )
if project_file:
files['project-file'].close()
# expecting a redirect to the new project containing the id in the url # expecting a redirect to the new project containing the id in the url
url_params = urllib.parse.parse_qs( url_params = urllib.parse.parse_qs(
urllib.parse.urlparse(response.geturl()).query) urllib.parse.urlparse(response.url).query)
if 'project' in url_params: if 'project' in url_params:
project_id = url_params['project'][0] project_id = url_params['project'][0]
return RefineProject(self.server, project_id) return RefineProject(self.server, project_id)

View File

@ -13,7 +13,9 @@ import csv
import unittest import unittest
from google.refine import refine from google.refine import refine
from tests import refinetest import refinetest
from io import StringIO
class RefineServerTest(refinetest.RefineTestCase): class RefineServerTest(refinetest.RefineTestCase):
@ -37,7 +39,7 @@ class RefineServerTest(refinetest.RefineTestCase):
self.assertTrue(item in version_info) self.assertTrue(item in version_info)
def test_version(self): def test_version(self):
self.assertTrue(self.server.version in ('2.0', '2.1', '2.5')) self.assertTrue(self.server.version in ('3.2'))
class RefineTest(refinetest.RefineTestCase): class RefineTest(refinetest.RefineTestCase):
@ -59,21 +61,19 @@ class RefineTest(refinetest.RefineTestCase):
self.assertTrue(self.project.delete()) self.assertTrue(self.project.delete())
def test_open_export(self): def test_open_export(self):
fp = refine.RefineProject(self.project.project_url()).export() response = refine.RefineProject(self.project.project_url()).export()
line = fp.next() lines = response.text.splitlines()
self.assertTrue('email' in line) self.assertTrue('email' in lines[0])
for line in fp: for line in lines[1:]:
self.assertTrue('M' in line or 'F' in line) self.assertTrue('M' in line or 'F' in line)
fp.close()
def test_open_export_csv(self): def test_open_export_csv(self):
fp = refine.RefineProject(self.project.project_url()).export() response = refine.RefineProject(self.project.project_url()).export()
csv_fp = csv.reader(fp, dialect='excel-tab') csv_fp = csv.reader(StringIO(response.text), dialect='excel-tab')
row = csv_fp.next() row = csv_fp.__next__()
self.assertTrue(row[0] == 'email') self.assertTrue(row[0] == 'email')
for row in csv_fp: for row in csv_fp:
self.assertTrue(row[3] == 'F' or row[3] == 'M') self.assertTrue(row[3] == 'F' or row[3] == 'M')
fp.close()
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -17,7 +17,7 @@ OPENREFINE_HOST and OPENREFINE_PORT.
import unittest import unittest
from google.refine import facet from google.refine import facet
from tests import refinetest import refinetest
class TutorialTestFacets(refinetest.RefineTestCase): class TutorialTestFacets(refinetest.RefineTestCase):
@ -138,7 +138,7 @@ class TutorialTestEditing(refinetest.RefineTestCase):
# {2} # {2}
self.project.text_transform(column='Zip Code 2', self.project.text_transform(column='Zip Code 2',
expression='value.toString()[0, 5]') expression='value.toString()[0, 5]')
self.assertInResponse('transform on 6067 cells in column Zip Code 2') self.assertInResponse('transform on 6958 cells in column Zip Code 2')
# {3} - XXX history # {3} - XXX history
# {4} # {4}
office_title_facet = facet.TextFacet('Office Title') office_title_facet = facet.TextFacet('Office Title')
@ -156,14 +156,14 @@ class TutorialTestEditing(refinetest.RefineTestCase):
self.assertEqual(len(response.facets[office_title_facet].choices), 66) self.assertEqual(len(response.facets[office_title_facet].choices), 66)
# {6} # {6}
response = self.project.compute_clusters('Office Title') response = self.project.compute_clusters('Office Title')
self.assertTrue(not response) self.assertTrue(response)
# {7} # {7}
clusters = self.project.compute_clusters('Office Title', 'knn') clusters = self.project.compute_clusters('Office Title', 'knn')
self.assertEqual(len(clusters), 7) self.assertEqual(len(clusters), 7)
first_cluster = clusters[0] first_cluster = clusters[0]
self.assertEqual(len(first_cluster), 2) self.assertEqual(len(first_cluster), 2)
self.assertEqual(first_cluster[0]['value'], 'RSCC Member') self.assertEqual(first_cluster[0]['value'], 'DPEC Member at Large')
self.assertEqual(first_cluster[0]['count'], 233) self.assertEqual(first_cluster[0]['count'], 6)
# Not strictly necessary to repeat 'Council Member' but a test # Not strictly necessary to repeat 'Council Member' but a test
# of mass_edit, and it's also what the front end sends. # of mass_edit, and it's also what the front end sends.
self.project.mass_edit('Office Title', [{ self.project.mass_edit('Office Title', [{
@ -194,9 +194,9 @@ class TutorialTestEditing(refinetest.RefineTestCase):
# {5}, {6}, {7} # {5}, {6}, {7}
response = self.project.compute_facets(facet.StarredFacet(True)) response = self.project.compute_facets(facet.StarredFacet(True))
self.assertEqual(len(response.facets[0].choices), 2) # true & false self.assertEqual(len(response.facets[0].choices), 2) # true & false
self.assertEqual(response.facets[0].choices[True].count, 3) self.assertEqual(response.facets[0].choices[True].count, 2)
self.project.remove_rows() self.project.remove_rows()
self.assertInResponse('3 rows') self.assertInResponse('2 rows')
class TutorialTestDuplicateDetection(refinetest.RefineTestCase): class TutorialTestDuplicateDetection(refinetest.RefineTestCase):
@ -214,7 +214,7 @@ class TutorialTestDuplicateDetection(refinetest.RefineTestCase):
self.assertInResponse('Reorder rows') self.assertInResponse('Reorder rows')
response = self.project.get_rows() response = self.project.get_rows()
indexes = [row.index for row in response.rows] indexes = [row.index for row in response.rows]
self.assertEqual(indexes, range(10)) self.assertEqual(indexes, list(range(10)))
# {10} # {10}
self.project.add_column( self.project.add_column(
'email', 'count', 'facetCount(value, "value", "email")') 'email', 'count', 'facetCount(value, "value", "email")')
@ -393,7 +393,7 @@ class TutorialTestTransposeVariableNumberOfRowsIntoColumns(
'Column', 'row.record.cells["Column"].value[1, -1].join("|")') 'Column', 'row.record.cells["Column"].value[1, -1].join("|")')
self.assertInResponse('18 cells') self.assertInResponse('18 cells')
# {26} # {26}
self.project.engine.mode = 'row-based' self.project.engine.mode = 'fd'
# {27} # {27}
blank_facet = facet.BlankFacet('First Line', selection=True) blank_facet = facet.BlankFacet('First Line', selection=True)
self.project.remove_rows(blank_facet) self.project.remove_rows(blank_facet)