opencultureconsulting
/
openrefine-client
mirror of https://github.com/opencultureconsulting/openrefine-client
2
0
Fork 0
Code Issues Releases 13 Activity
This fork extends the command line interface (CLI) and is distributed as a convenient one-file-executable (Windows, Linux, Mac). It is also available via Docker Hub, PyPI and Binder.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
258 Commits
1 Branch
2.8 MiB
 
 
 
Branch: master
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'master'
${ noResults }
openrefine-client/tests/test_tutorial.py

491 lines
21 KiB
Raw Permalink Blame History 

  1. #!/usr/bin/env python

  2. """

  3. test_tutorial.py

  4. 

  5. The tests here are based on David Huynh's Refine tutorial at

  6. http://davidhuynh.net/spaces/nicar2011/tutorial.pdf The tests perform all the

  7. Refine actions given in the tutorial (except the web scraping) and verify the

  8. changes expected to be observed explained in the tutorial.

  9. 

  10. These tests require a connection to a Refine server either at

  11. http://127.0.0.1:3333/ or by specifying environment variables

  12. OPENREFINE_HOST and OPENREFINE_PORT.

  13. """

  14. 

  15. # Copyright (c) 2011 Paul Makepeace, Real Programmers. All rights reserved.

  16. 

  17. import unittest

  18. 

  19. from google.refine import facet

  20. from tests import refinetest

  21. 

  22. 

  23. class TutorialTestFacets(refinetest.RefineTestCase):

  24.     project_file = 'louisiana-elected-officials.csv'

  25.     project_options = {'guess_cell_value_types': True}

  26. 

  27.     def test_get_rows(self):

  28.         # Section "2. Exploration using Facets": {3}

  29.         response = self.project.get_rows(limit=10)

  30.         self.assertEqual(len(response.rows), 10)

  31.         self.assertEqual(response.limit, 10)

  32.         self.assertEqual(response.total, 6958)

  33.         self.assertEqual(response.filtered, 6958)

  34.         for row in response.rows:

  35.             self.assertFalse(row.flagged)

  36.             self.assertFalse(row.starred)

  37. 

  38.     def test_facet(self):

  39.         # Section "2. Exploration using Facets": {4}

  40.         party_code_facet = facet.TextFacet(column='Party Code')

  41.         response = self.project.compute_facets(party_code_facet)

  42.         pc = response.facets[0]

  43.         # test look by index same as look up by facet object

  44.         self.assertEqual(pc, response.facets[party_code_facet])

  45.         self.assertEqual(pc.name, 'Party Code')

  46.         self.assertEqual(pc.choices['D'].count, 3700)

  47.         self.assertEqual(pc.choices['N'].count, 15)

  48.         self.assertEqual(pc.blank_choice.count, 1446)

  49.         # {5}, {6}

  50.         engine = facet.Engine(party_code_facet)

  51.         ethnicity_facet = facet.TextFacet(column='Ethnicity')

  52.         engine.add_facet(ethnicity_facet)

  53.         self.project.engine = engine

  54.         response = self.project.compute_facets()

  55.         e = response.facets[ethnicity_facet]

  56.         self.assertEqual(e.choices['B'].count, 1255)

  57.         self.assertEqual(e.choices['W'].count, 4469)

  58.         # {7}

  59.         ethnicity_facet.include('B')

  60.         response = self.project.get_rows()

  61.         self.assertEqual(response.filtered, 1255)

  62.         indexes = [row.index for row in response.rows]

  63.         self.assertEqual(indexes, [1, 2, 3, 4, 6, 12, 18, 26, 28, 32])

  64.         # {8}

  65.         response = self.project.compute_facets()

  66.         pc = response.facets[party_code_facet]

  67.         self.assertEqual(pc.name, 'Party Code')

  68.         self.assertEqual(pc.choices['D'].count, 1179)

  69.         self.assertEqual(pc.choices['R'].count, 11)

  70.         self.assertEqual(pc.blank_choice.count, 46)

  71.         # {9}

  72.         party_code_facet.include('R')

  73.         response = self.project.compute_facets()

  74.         e = response.facets[ethnicity_facet]

  75.         self.assertEqual(e.choices['B'].count, 11)

  76.         # {10}

  77.         party_code_facet.reset()

  78.         ethnicity_facet.reset()

  79.         response = self.project.get_rows()

  80.         self.assertEqual(response.filtered, 6958)

  81.         # {11}

  82.         office_title_facet = facet.TextFacet('Office Title')

  83.         self.project.engine.add_facet(office_title_facet)

  84.         response = self.project.compute_facets()

  85.         self.assertEqual(len(response.facets[2].choices), 76)

  86.         # {12} - XXX not sure how to interpret bins & baseBins yet

  87.         office_level_facet = facet.NumericFacet('Office Level')

  88.         self.project.engine.add_facet(office_level_facet)

  89.         # {13}

  90.         office_level_facet.From = 300   # from reserved word

  91.         office_level_facet.to = 320

  92.         response = self.project.get_rows()

  93.         self.assertEqual(response.filtered, 1907)

  94.         response = self.project.compute_facets()

  95.         ot = response.facets[office_title_facet]

  96.         self.assertEqual(len(ot.choices), 21)

  97.         self.assertEqual(ot.choices['Chief of Police'].count, 2)

  98.         self.assertEqual(ot.choices['Chief of Police          '].count, 211)

  99.         # {14}

  100.         self.project.engine.remove_all()

  101.         response = self.project.get_rows()

  102.         self.assertEqual(response.filtered, 6958)

  103.         # {15}

  104.         phone_facet = facet.TextFacet('Phone', expression='value[0, 3]')

  105.         self.project.engine.add_facet(phone_facet)

  106.         response = self.project.compute_facets()

  107.         p = response.facets[phone_facet]

  108.         self.assertEqual(p.expression, 'value[0, 3]')

  109.         self.assertEqual(p.choices['318'].count, 2331)

  110.         # {16}

  111.         commissioned_date_facet = facet.NumericFacet(

  112.             'Commissioned Date',

  113.             expression='value.toDate().datePart("year")')

  114.         self.project.engine.add_facet(commissioned_date_facet)

  115.         response = self.project.compute_facets()

  116.         cd = response.facets[commissioned_date_facet]

  117.         self.assertEqual(cd.error_count, 959)

  118.         self.assertEqual(cd.numeric_count, 5999)

  119.         # {17}

  120.         office_description_facet = facet.NumericFacet(

  121.             'Office Description',

  122.             expression=r'value.match(/\D*(\d+)\w\w Rep.*/)[0].toNumber()')

  123.         self.project.engine.add_facet(office_description_facet)

  124.         response = self.project.compute_facets()

  125.         od = response.facets[office_description_facet]

  126.         self.assertEqual(od.min, 0)

  127.         self.assertEqual(od.max, 110)

  128.         self.assertEqual(od.numeric_count, 548)

  129. 

  130. 

  131. class TutorialTestEditing(refinetest.RefineTestCase):

  132.     project_file = 'louisiana-elected-officials.csv'

  133.     project_options = {'guess_cell_value_types': True}

  134. 

  135.     def test_editing(self):

  136.         # Section "3. Cell Editing": {1}

  137.         self.project.engine.remove_all()    # redundant due to setUp

  138.         # {2}

  139.         self.project.text_transform(column='Zip Code 2',

  140.                                     expression='value.toString()[0, 5]')

  141.         self.assertInResponse('transform on 6067 cells in column Zip Code 2')

  142.         # {3} - XXX history

  143.         # {4}

  144.         office_title_facet = facet.TextFacet('Office Title')

  145.         self.project.engine.add_facet(office_title_facet)

  146.         response = self.project.compute_facets()

  147.         self.assertEqual(len(response.facets[office_title_facet].choices), 76)

  148.         self.project.text_transform('Office Title', 'value.trim()')

  149.         self.assertInResponse('6895')

  150.         response = self.project.compute_facets()

  151.         self.assertEqual(len(response.facets[office_title_facet].choices), 67)

  152.         # {5}

  153.         self.project.edit('Office Title', 'Councilmen', 'Councilman')

  154.         self.assertInResponse('13')

  155.         response = self.project.compute_facets()

  156.         self.assertEqual(len(response.facets[office_title_facet].choices), 66)

  157.         # {6}

  158.         response = self.project.compute_clusters('Office Title')

  159.         self.assertTrue(not response)

  160.         # {7}

  161.         clusters = self.project.compute_clusters('Office Title', 'knn')

  162.         self.assertEqual(len(clusters), 7)

  163.         first_cluster = clusters[0]

  164.         self.assertEqual(len(first_cluster), 2)

  165.         self.assertEqual(first_cluster[0]['value'], 'RSCC Member')

  166.         self.assertEqual(first_cluster[0]['count'], 233)

  167.         # Not strictly necessary to repeat 'Council Member' but a test

  168.         # of mass_edit, and it's also what the front end sends.

  169.         self.project.mass_edit('Office Title', [{

  170.             'from': ['Council Member', 'Councilmember'],

  171.             'to': 'Council Member'

  172.         }])

  173.         self.assertInResponse('372')

  174.         response = self.project.compute_facets()

  175.         self.assertEqual(len(response.facets[office_title_facet].choices), 65)

  176. 

  177.         # Section "4. Row and Column Editing, Batched Row Deletion"

  178.         # Test doesn't strictly follow the tutorial as the "Browse this

  179.         # cluster" performs a text facet which the server can't complete

  180.         # as it busts its max facet count. The useful work is done with

  181.         # get_rows(). Also, we can facet & select in one; the UI can't.

  182.         # {1}, {2}, {3}, {4}

  183.         clusters = self.project.compute_clusters('Candidate Name')

  184.         for cluster in clusters[0:3]:   # just do a few

  185.             for match in cluster:

  186.                 # {2}

  187.                 if match['value'].endswith(', '):

  188.                     response = self.project.get_rows(

  189.                         facet.TextFacet('Candidate Name', match['value']))

  190.                     self.assertEqual(len(response.rows), 1)

  191.                     for row in response.rows:

  192.                         self.project.star_row(row)

  193.                         self.assertInResponse(str(row.index + 1))

  194.         # {5}, {6}, {7}

  195.         response = self.project.compute_facets(facet.StarredFacet(True))

  196.         self.assertEqual(len(response.facets[0].choices), 2)    # true & false

  197.         self.assertEqual(response.facets[0].choices[True].count, 3)

  198.         self.project.remove_rows()

  199.         self.assertInResponse('3 rows')

  200. 

  201. 

  202. class TutorialTestDuplicateDetection(refinetest.RefineTestCase):

  203.     project_file = 'duplicates.csv'

  204. 

  205.     def test_duplicate_detection(self):

  206.         # Section "4. Row and Column Editing,

  207.         #             Duplicate Row Detection and Deletion"

  208.         # {7}, {8}

  209.         response = self.project.get_rows(sort_by='email')

  210.         indexes = [row.index for row in response.rows]

  211.         self.assertEqual(indexes, [4, 9, 8, 3, 0, 2, 5, 6, 1, 7])

  212.         # {9}

  213.         self.project.reorder_rows()

  214.         self.assertInResponse('Reorder rows')

  215.         response = self.project.get_rows()

  216.         indexes = [row.index for row in response.rows]

  217.         self.assertEqual(indexes, range(10))

  218.         # {10}

  219.         self.project.add_column(

  220.             'email', 'count', 'facetCount(value, "value", "email")')

  221.         self.assertInResponse('column email by filling 10 rows')

  222.         response = self.project.get_rows()

  223.         self.assertEqual(self.project.column_order['email'], 0)  # i.e. 1st

  224.         self.assertEqual(self.project.column_order['count'], 1)  # i.e. 2nd

  225.         counts = [row['count'] for row in response.rows]

  226.         self.assertEqual(counts, [2, 2, 1, 1, 3, 3, 3, 1, 2, 2])

  227.         # {11}

  228.         self.assertFalse(self.project.has_records)

  229.         self.project.blank_down('email')

  230.         self.assertInResponse('Blank down 4 cells')

  231.         self.assertTrue(self.project.has_records)

  232.         response = self.project.get_rows()

  233.         emails = [1 if row['email'] else 0 for row in response.rows]

  234.         self.assertEqual(emails, [1, 0, 1, 1, 1, 0, 0, 1, 1, 0])

  235.         # {12}

  236.         blank_facet = facet.BlankFacet('email', selection=True)

  237.         # {13}

  238.         self.project.remove_rows(blank_facet)

  239.         self.assertInResponse('Remove 4 rows')

  240.         self.project.engine.remove_all()

  241.         response = self.project.get_rows()

  242.         email_counts = [(row['email'], row['count']) for row in response.rows]

  243.         self.assertEqual(email_counts, [

  244.             (u'arthur.duff@example4.com', 2),

  245.             (u'ben.morisson@example6.org', 1),

  246.             (u'ben.tyler@example3.org', 1),

  247.             (u'danny.baron@example1.com', 3),

  248.             (u'jean.griffith@example5.org', 1),

  249.             (u'melanie.white@example2.edu', 2)

  250.         ])

  251. 

  252. 

  253. class TutorialTestTransposeColumnsIntoRows(refinetest.RefineTestCase):

  254.     project_file = 'us_economic_assistance.csv'

  255. 

  256.     def test_transpose_columns_into_rows(self):

  257.         # Section "5. Structural Editing, Transpose Columns into Rows"

  258.         # {1}, {2}, {3}

  259.         self.project.transpose_columns_into_rows('FY1946', 64, 'pair')

  260.         self.assertInResponse('64 column(s) starting with FY1946')

  261.         # {4}

  262.         self.project.add_column('pair', 'year', 'value[2,6].toNumber()')

  263.         self.assertInResponse('filling 26185 rows')

  264.         # {5}

  265.         self.project.text_transform(

  266.             column='pair', expression='value.substring(7).toNumber()')

  267.         self.assertInResponse('transform on 26185 cells')

  268.         # {6}

  269.         self.project.rename_column('pair', 'amount')

  270.         self.assertInResponse('Rename column pair to amount')

  271.         # {7}

  272.         self.project.fill_down('country_name')

  273.         self.assertInResponse('Fill down 23805 cells')

  274.         self.project.fill_down('program_name')

  275.         self.assertInResponse('Fill down 23805 cells')

  276.         # spot check of last row for transforms and fill down

  277.         response = self.project.get_rows()

  278.         row10 = response.rows[9]

  279.         self.assertEqual(row10['country_name'], 'Afghanistan')

  280.         self.assertEqual(row10['program_name'],

  281.                          'Department of Defense Security Assistance')

  282.         self.assertEqual(row10['amount'], 113777303)

  283. 

  284. 

  285. class TutorialTestTransposeFixedNumberOfRowsIntoColumns(

  286.         refinetest.RefineTestCase):

  287.     project_file = 'fixed-rows.csv'

  288.     project_format = 'text/line-based'

  289.     project_options = {'header_lines': 0}

  290. 

  291.     def test_transpose_fixed_number_of_rows_into_columns(self):

  292.         if self.server.version not in ('2.0', '2.1'):

  293.             self.project.rename_column('Column 1', 'Column')

  294.         # Section "5. Structural Editing,

  295.         #             Transpose Fixed Number of Rows into Columns"

  296.         # {1}

  297.         self.assertTrue('Column' in self.project.column_order)

  298.         # {8}

  299.         self.project.transpose_rows_into_columns('Column', 4)

  300.         self.assertInResponse('Transpose every 4 cells in column Column')

  301.         # {9} - renaming column triggers a bug in Refine <= 2.1

  302.         if self.server.version not in ('2.0', '2.1'):

  303.             self.project.rename_column('Column 2', 'Address')

  304.             self.project.rename_column('Column 3', 'Address 2')

  305.             self.project.rename_column('Column 4', 'Status')

  306.         # {10}

  307.         self.project.add_column(

  308.             'Column 1', 'Transaction',

  309.             'if(value.contains(" sent "), "send", "receive")')

  310.         self.assertInResponse('Column 1 by filling 4 rows')

  311.         # {11}

  312.         transaction_facet = facet.TextFacet(column='Transaction',

  313.                                             selection='send')

  314.         self.project.engine.add_facet(transaction_facet)

  315.         self.project.compute_facets()

  316.         # {12}, {13}, {14}

  317.         self.project.add_column(

  318.             'Column 1', 'Sender',

  319.             'value.partition(" sent ")[0]')

  320.         # XXX resetting the facet shows data in rows with Transaction=receive

  321.         #     which shouldn't have been possible with the facet.

  322.         self.project.add_column(

  323.             'Column 1', 'Recipient',

  324.             'value.partition(" to ")[2].partition(" on ")[0]')

  325.         self.project.add_column(

  326.             'Column 1', 'Amount',

  327.             'value.partition(" sent ")[2].partition(" to ")[0]')

  328.         # {15}

  329.         transaction_facet.reset().include('receive')

  330.         self.project.get_rows()

  331.         # XXX there seems to be some kind of bug where the model doesn't

  332.         #     match get_rows() output - cellIndex being returned that are

  333.         #     out of range.

  334.         #self.assertTrue(a_row['Sender'] is None)

  335.         #self.assertTrue(a_row['Recipient'] is None)

  336.         #self.assertTrue(a_row['Amount'] is None)

  337.         # {16}

  338.         for column, expression in (

  339.             ('Sender',

  340.              'cells["Column 1"].value.partition(" from ")[2].partition(" on ")[0]'),

  341.             ('Recipient',

  342.              'cells["Column 1"].value.partition(" received ")[0]'),

  343.             ('Amount',

  344.              'cells["Column 1"].value.partition(" received ")[2].partition(" from ")[0]')

  345.         ):

  346.             self.project.text_transform(column, expression)

  347.             self.assertInResponse('2 cells')

  348.         # {17}

  349.         transaction_facet.reset()

  350.         # {18}

  351.         self.project.text_transform('Column 1', 'value.partition(" on ")[2]')

  352.         self.assertInResponse('4 cells')

  353.         # {19}

  354.         self.project.reorder_columns(['Transaction', 'Amount', 'Sender',

  355.                                       'Recipient'])

  356.         self.assertInResponse('Reorder columns')

  357. 

  358. 

  359. class TutorialTestTransposeVariableNumberOfRowsIntoColumns(

  360.         refinetest.RefineTestCase):

  361.     project_file = 'variable-rows.csv'

  362.     project_format = 'text/line-based'

  363.     project_options = {'header_lines': 0}

  364. 

  365.     def test_transpose_variable_number_of_rows_into_columns(self):

  366.         # {20}, {21}

  367.         if self.server.version not in ('2.0', '2.1') :

  368.             self.project.rename_column('Column 1', 'Column')

  369.         self.project.add_column(

  370.             'Column', 'First Line', 'if(value.contains(" on "), value, null)')

  371.         self.assertInResponse('Column by filling 4 rows')

  372.         response = self.project.get_rows()

  373.         first_names = [row['First Line'][0:10] if row['First Line'] else None

  374.                        for row in response.rows]

  375.         self.assertEqual(first_names, [

  376.             'Tom Dalton', None, None, None,

  377.             'Morgan Law', None, None, None, None, 'Eric Batem'])

  378.         # {22}

  379.         self.project.move_column('First Line', 0)

  380.         self.assertInResponse('Move column First Line to position 0')

  381.         self.assertEqual(self.project.column_order['First Line'], 0)

  382.         # {23}

  383.         self.project.engine.mode = 'record-based'

  384.         response = self.project.get_rows()

  385.         self.assertEqual(response.mode, 'record-based')

  386.         self.assertEqual(response.filtered, 4)

  387.         # {24}

  388.         self.project.add_column(

  389.             'Column', 'Status', 'row.record.cells["Column"].value[-1]')

  390.         self.assertInResponse('filling 18 rows')

  391.         # {25}

  392.         self.project.text_transform(

  393.             'Column', 'row.record.cells["Column"].value[1, -1].join("|")')

  394.         self.assertInResponse('18 cells')

  395.         # {26}

  396.         self.project.engine.mode = 'row-based'

  397.         # {27}

  398.         blank_facet = facet.BlankFacet('First Line', selection=True)

  399.         self.project.remove_rows(blank_facet)

  400.         self.assertInResponse('Remove 14 rows')

  401.         self.project.engine.remove_all()

  402.         # {28}

  403.         self.project.split_column('Column', separator='|')

  404.         self.assertInResponse('Split 4 cell(s) in column Column')

  405. 

  406. 

  407. class TutorialTestWebScraping(refinetest.RefineTestCase):

  408.     project_file = 'eli-lilly.csv'

  409. 

  410.     filter_expr_1 = """

  411.         forEach(

  412.             value[2,-2].replace("&#160;", " ").split("), ("),

  413.             v,

  414.             v[0,-1].partition(", '", true).join(":")

  415.         ).join("|")

  416.     """

  417.     filter_expr_2 = """

  418.         filter(

  419.             value.split("|"), p, p.partition(":")[0].toNumber() == %d

  420.         )[0].partition(":")[2]

  421.     """

  422. 

  423.     def test_web_scraping(self):

  424.         # Section "6. Web Scraping"

  425.         # {1}, {2}

  426.         self.project.split_column('key', separator=':')

  427.         self.assertInResponse('Split 5409 cell(s) in column key')

  428.         self.project.rename_column('key 1', 'page')

  429.         self.assertInResponse('Rename column key 1 to page')

  430.         self.project.rename_column('key 2', 'top')

  431.         self.assertInResponse('Rename column key 2 to top')

  432.         self.project.move_column('line', 'end')

  433.         self.assertInResponse('Move column line to position 2')

  434.         # {3}

  435.         self.project.sorting = facet.Sorting([

  436.             {'column': 'page', 'valueType': 'number'},

  437.             {'column': 'top',  'valueType': 'number'},

  438.         ])

  439.         self.project.reorder_rows()

  440.         self.assertInResponse('Reorder rows')

  441.         first_row = self.project.get_rows(limit=1).rows[0]

  442.         self.assertEqual(first_row['page'], 1)

  443.         self.assertEqual(first_row['top'], 24)

  444.         # {4}

  445.         filter_facet = facet.TextFilterFacet('line', 'ahman')

  446.         rows = self.project.get_rows(filter_facet).rows

  447.         self.assertEqual(len(rows), 1)

  448.         self.assertEqual(rows[0]['top'], 106)

  449.         filter_facet.query = 'alvarez'

  450.         rows = self.project.get_rows().rows

  451.         self.assertEqual(len(rows), 2)

  452.         self.assertEqual(rows[-1]['top'], 567)

  453.         self.project.engine.remove_all()

  454.         # {5} - tutorial says 'line'; it means 'top'

  455.         line_facet = facet.NumericFacet('top')

  456.         line_facet.to = 100

  457.         self.project.remove_rows(line_facet)

  458.         self.assertInResponse('Remove 775 rows')

  459.         line_facet.From = 570

  460.         line_facet.to = 600

  461.         self.project.remove_rows(line_facet)

  462.         self.assertInResponse('Remove 71 rows')

  463.         line_facet.reset()

  464.         response = self.project.get_rows()

  465.         self.assertEqual(response.filtered, 4563)

  466.         # {6}

  467.         page_facet = facet.TextFacet('page', 1)   # 1 not '1'

  468.         self.project.engine.add_facet(page_facet)

  469.         # {7}

  470.         rows = self.project.get_rows().rows

  471.         # Look for a row with a name in it by skipping HTML

  472.         name_row = [row for row in rows if '<b>' not in row['line']][0]

  473.         self.assertTrue('WELLNESS' in name_row['line'])

  474.         self.assertEqual(name_row['top'], 161)

  475.         line_facet.From = 20

  476.         line_facet.to = 160

  477.         self.project.remove_rows()

  478.         self.assertInResponse('Remove 9 rows')

  479.         self.project.engine.remove_all()

  480.         # {8}

  481.         self.project.text_transform('line', expression=self.filter_expr_1)

  482.         self.assertInResponse('Text transform on 4554 cells in column line')

  483.         # {9} - XXX following is generating Java exceptions

  484.         #filter_expr = self.filter_expr_2 % 16

  485.         #self.project.add_column('line', 'Name', expression=filter_expr)

  486.         # {10} to the final {19} - nothing new in terms of exercising the API.

  487. 

  488. 

  489. if __name__ == '__main__':

  490.     unittest.main()