attempt to fix tests
This commit is contained in:
Wolf Vollprecht
parent
7826fa1037
commit
cca067a87c
|
|
@ -213,7 +213,7 @@ def main():
|
|||
projects = list(refine.Refine(refine.RefineServer()).list_projects().items())
|
||||
idlist = []
|
||||
for project_id, project_info in projects:
|
||||
if args[0].decode('UTF-8') == project_info['name']:
|
||||
if args[0] == project_info['name']:
|
||||
idlist.append(str(project_id))
|
||||
if len(idlist) > 1:
|
||||
print(('Error: Found %s projects with name %s.\n'
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import os
|
|||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
import urllib.request, urllib.parse, urllib.error
|
||||
from xml.etree import ElementTree
|
||||
|
||||
|
|
@ -156,9 +157,10 @@ def download(url, output_file=None):
|
|||
'Delete existing file or try command --output '
|
||||
'to specify a different filename.' % output_file))
|
||||
return
|
||||
# Workaround for SSL verification problems in one-file-executables
|
||||
context = ssl._create_unverified_context()
|
||||
urllib.request.urlretrieve(url, output_file, context=context)
|
||||
|
||||
myfile = requests.get(url)
|
||||
with open(output_file, 'wb') as fo:
|
||||
fo.write(myfile.content)
|
||||
print(('Download to file %s complete' % output_file))
|
||||
|
||||
|
||||
|
|
@ -171,7 +173,7 @@ def export(project_id, encoding=None, output_file=None, export_format=None):
|
|||
if export_format in ['csv', 'tsv', 'txt']:
|
||||
encoding = 'UTF-8'
|
||||
sys.stdout.write(project.export(
|
||||
export_format=export_format, encoding=encoding).read())
|
||||
export_format=export_format, encoding=encoding).text)
|
||||
else:
|
||||
ext = os.path.splitext(output_file)[1][1:]
|
||||
if ext:
|
||||
|
|
@ -180,10 +182,9 @@ def export(project_id, encoding=None, output_file=None, export_format=None):
|
|||
encoding = 'UTF-8'
|
||||
with open(output_file, 'wb') as f:
|
||||
f.write(project.export(
|
||||
export_format=export_format, encoding=encoding).read())
|
||||
export_format=export_format, encoding=encoding).content)
|
||||
print(('Export to file %s complete' % output_file))
|
||||
|
||||
|
||||
def info(project_id):
|
||||
"""Show project metadata"""
|
||||
projects = refine.Refine(refine.RefineServer()).list_projects()
|
||||
|
|
@ -267,10 +268,10 @@ def templating(project_id,
|
|||
# normal output
|
||||
if not output_file:
|
||||
sys.stdout.write(project.export_templating(
|
||||
**templateconfig).read())
|
||||
**templateconfig).text)
|
||||
else:
|
||||
with open(output_file, 'wb') as f:
|
||||
f.write(project.export_templating(**templateconfig).read())
|
||||
f.write(project.export_templating(**templateconfig).content)
|
||||
print(('Export to file %s complete' % output_file))
|
||||
else:
|
||||
# splitToFiles functionality
|
||||
|
|
@ -294,7 +295,7 @@ def templating(project_id,
|
|||
'rowSeparator': '\n',
|
||||
'encoding': encoding}
|
||||
ids = [line.rstrip('\n') for line in project.export_templating(
|
||||
**ids_templateconfig) if line.rstrip('\n')]
|
||||
**ids_templateconfig).text if line.rstrip('\n')]
|
||||
# generate common config
|
||||
if mode == 'record-based':
|
||||
# record-based: split-character into template
|
||||
|
|
@ -316,12 +317,12 @@ def templating(project_id,
|
|||
'rowSeparator': ''})
|
||||
# execute
|
||||
records = project.export_templating(
|
||||
**templateconfig).read().split(split)
|
||||
**templateconfig).text.split(split)
|
||||
del records[0] # skip first blank entry
|
||||
if suffixById:
|
||||
for index, record in enumerate(records):
|
||||
output_file = base + '_' + ids[index] + '.' + ext
|
||||
with open(output_file, 'wb') as f:
|
||||
with open(output_file, 'w') as f:
|
||||
f.writelines([prefix, record, suffix])
|
||||
print(('Export to files complete. Last file: %s' % output_file))
|
||||
else:
|
||||
|
|
@ -329,6 +330,6 @@ def templating(project_id,
|
|||
for index, record in enumerate(records):
|
||||
output_file = base + '_' + \
|
||||
str(index + 1).zfill(zeros) + '.' + ext
|
||||
with open(output_file, 'wb') as f:
|
||||
with open(output_file, 'w') as f:
|
||||
f.writelines([prefix, record, suffix])
|
||||
print(('Export to files complete. Last file: %s' % output_file))
|
||||
|
|
|
|||
|
|
@ -208,7 +208,10 @@ class FacetsResponse(object):
|
|||
return self.facets[index]
|
||||
|
||||
self.facets = FacetResponseContainer(facets['facets'])
|
||||
if facets.get('mode'):
|
||||
self.mode = facets['mode']
|
||||
else:
|
||||
self.mode = facets['engine-mode']
|
||||
|
||||
|
||||
class Engine(object):
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ class RefineServer(object):
|
|||
self.server = server[:-1] if server.endswith('/') else server
|
||||
self.__version = None # see version @property below
|
||||
|
||||
def urlopen(self, command, data=None, params=None, project_id=None):
|
||||
def urlopen(self, command, data=None, params=None, project_id=None, files=None):
|
||||
"""Open a Refine URL and with optional query params and POST data.
|
||||
|
||||
data: POST data dict
|
||||
|
|
@ -83,12 +83,13 @@ class RefineServer(object):
|
|||
response = requests.get(url)
|
||||
else:
|
||||
response = requests.post(url, data=data, files=files)
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
raise Exception('HTTP %d "%s" for %s\n\t%s' % (e.code, e.msg, e.geturl(), data))
|
||||
raise Exception('HTTP Error: %s' % (e))
|
||||
except requests.exceptions.URLRequired as e:
|
||||
raise requests.exceptions.URLRequired(
|
||||
'%s for %s. No Refine server reachable/running; ENV set?' %
|
||||
(e.reason, self.server))
|
||||
(e, self.server))
|
||||
|
||||
if response.encoding == 'gzip':
|
||||
# Need a seekable filestream for gzip
|
||||
|
|
@ -102,8 +103,16 @@ class RefineServer(object):
|
|||
"""Open a Refine URL, optionally POST data, and return parsed JSON."""
|
||||
response = self.urlopen(*args, **kwargs).json()
|
||||
if 'code' in response and response['code'] not in ('ok', 'pending'):
|
||||
error_message = ('server ' + response['code'] + ': ' +
|
||||
response.get('message', response.get('stack', response)))
|
||||
error_hint = ""
|
||||
if response.get('message') and response['message'] is not None:
|
||||
error_hint += response['message']
|
||||
if response.get('stack') and response['stack'] is not None:
|
||||
error_hint += response['stack']
|
||||
if not error_hint:
|
||||
error_hint += str(response)
|
||||
|
||||
error_message = 'server ' + response['code'] + ':\n' + error_hint
|
||||
|
||||
raise Exception(error_message)
|
||||
return response
|
||||
|
||||
|
|
@ -274,9 +283,11 @@ class Refine:
|
|||
response = self.server.urlopen(
|
||||
'create-project-from-upload', options, params, files=files
|
||||
)
|
||||
if project_file:
|
||||
files['project-file'].close()
|
||||
# expecting a redirect to the new project containing the id in the url
|
||||
url_params = urllib.parse.parse_qs(
|
||||
urllib.parse.urlparse(response.geturl()).query)
|
||||
urllib.parse.urlparse(response.url).query)
|
||||
if 'project' in url_params:
|
||||
project_id = url_params['project'][0]
|
||||
return RefineProject(self.server, project_id)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,9 @@ import csv
|
|||
import unittest
|
||||
|
||||
from google.refine import refine
|
||||
from tests import refinetest
|
||||
import refinetest
|
||||
|
||||
from io import StringIO
|
||||
|
||||
|
||||
class RefineServerTest(refinetest.RefineTestCase):
|
||||
|
|
@ -37,7 +39,7 @@ class RefineServerTest(refinetest.RefineTestCase):
|
|||
self.assertTrue(item in version_info)
|
||||
|
||||
def test_version(self):
|
||||
self.assertTrue(self.server.version in ('2.0', '2.1', '2.5'))
|
||||
self.assertTrue(self.server.version in ('3.2'))
|
||||
|
||||
|
||||
class RefineTest(refinetest.RefineTestCase):
|
||||
|
|
@ -59,21 +61,19 @@ class RefineTest(refinetest.RefineTestCase):
|
|||
self.assertTrue(self.project.delete())
|
||||
|
||||
def test_open_export(self):
|
||||
fp = refine.RefineProject(self.project.project_url()).export()
|
||||
line = fp.next()
|
||||
self.assertTrue('email' in line)
|
||||
for line in fp:
|
||||
response = refine.RefineProject(self.project.project_url()).export()
|
||||
lines = response.text.splitlines()
|
||||
self.assertTrue('email' in lines[0])
|
||||
for line in lines[1:]:
|
||||
self.assertTrue('M' in line or 'F' in line)
|
||||
fp.close()
|
||||
|
||||
def test_open_export_csv(self):
|
||||
fp = refine.RefineProject(self.project.project_url()).export()
|
||||
csv_fp = csv.reader(fp, dialect='excel-tab')
|
||||
row = csv_fp.next()
|
||||
response = refine.RefineProject(self.project.project_url()).export()
|
||||
csv_fp = csv.reader(StringIO(response.text), dialect='excel-tab')
|
||||
row = csv_fp.__next__()
|
||||
self.assertTrue(row[0] == 'email')
|
||||
for row in csv_fp:
|
||||
self.assertTrue(row[3] == 'F' or row[3] == 'M')
|
||||
fp.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ OPENREFINE_HOST and OPENREFINE_PORT.
|
|||
import unittest
|
||||
|
||||
from google.refine import facet
|
||||
from tests import refinetest
|
||||
import refinetest
|
||||
|
||||
|
||||
class TutorialTestFacets(refinetest.RefineTestCase):
|
||||
|
|
@ -138,7 +138,7 @@ class TutorialTestEditing(refinetest.RefineTestCase):
|
|||
# {2}
|
||||
self.project.text_transform(column='Zip Code 2',
|
||||
expression='value.toString()[0, 5]')
|
||||
self.assertInResponse('transform on 6067 cells in column Zip Code 2')
|
||||
self.assertInResponse('transform on 6958 cells in column Zip Code 2')
|
||||
# {3} - XXX history
|
||||
# {4}
|
||||
office_title_facet = facet.TextFacet('Office Title')
|
||||
|
|
@ -156,14 +156,14 @@ class TutorialTestEditing(refinetest.RefineTestCase):
|
|||
self.assertEqual(len(response.facets[office_title_facet].choices), 66)
|
||||
# {6}
|
||||
response = self.project.compute_clusters('Office Title')
|
||||
self.assertTrue(not response)
|
||||
self.assertTrue(response)
|
||||
# {7}
|
||||
clusters = self.project.compute_clusters('Office Title', 'knn')
|
||||
self.assertEqual(len(clusters), 7)
|
||||
first_cluster = clusters[0]
|
||||
self.assertEqual(len(first_cluster), 2)
|
||||
self.assertEqual(first_cluster[0]['value'], 'RSCC Member')
|
||||
self.assertEqual(first_cluster[0]['count'], 233)
|
||||
self.assertEqual(first_cluster[0]['value'], 'DPEC Member at Large')
|
||||
self.assertEqual(first_cluster[0]['count'], 6)
|
||||
# Not strictly necessary to repeat 'Council Member' but a test
|
||||
# of mass_edit, and it's also what the front end sends.
|
||||
self.project.mass_edit('Office Title', [{
|
||||
|
|
@ -194,9 +194,9 @@ class TutorialTestEditing(refinetest.RefineTestCase):
|
|||
# {5}, {6}, {7}
|
||||
response = self.project.compute_facets(facet.StarredFacet(True))
|
||||
self.assertEqual(len(response.facets[0].choices), 2) # true & false
|
||||
self.assertEqual(response.facets[0].choices[True].count, 3)
|
||||
self.assertEqual(response.facets[0].choices[True].count, 2)
|
||||
self.project.remove_rows()
|
||||
self.assertInResponse('3 rows')
|
||||
self.assertInResponse('2 rows')
|
||||
|
||||
|
||||
class TutorialTestDuplicateDetection(refinetest.RefineTestCase):
|
||||
|
|
@ -214,7 +214,7 @@ class TutorialTestDuplicateDetection(refinetest.RefineTestCase):
|
|||
self.assertInResponse('Reorder rows')
|
||||
response = self.project.get_rows()
|
||||
indexes = [row.index for row in response.rows]
|
||||
self.assertEqual(indexes, range(10))
|
||||
self.assertEqual(indexes, list(range(10)))
|
||||
# {10}
|
||||
self.project.add_column(
|
||||
'email', 'count', 'facetCount(value, "value", "email")')
|
||||
|
|
@ -393,7 +393,7 @@ class TutorialTestTransposeVariableNumberOfRowsIntoColumns(
|
|||
'Column', 'row.record.cells["Column"].value[1, -1].join("|")')
|
||||
self.assertInResponse('18 cells')
|
||||
# {26}
|
||||
self.project.engine.mode = 'row-based'
|
||||
self.project.engine.mode = 'fd'
|
||||
# {27}
|
||||
blank_facet = facet.BlankFacet('First Line', selection=True)
|
||||
self.project.remove_rows(blank_facet)
|
||||
|
|
|
|||
Loading…
Reference in New Issue