TST: Test iterator parsers as iterators

Author: toobaz

pandas/io/tests/test_common.py

@@ -9,6 +9,8 @@
 
 from pandas.io import common
 
+from pandas import read_csv, concat
+
 try:
     from pathlib import Path
 except ImportError:
@@ -21,6 +23,14 @@
 
 
 class TestCommonIOCapabilities(tm.TestCase):
+    data1 = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
 
     def test_expand_user(self):
         filename = '~/sometest'
@@ -64,3 +74,16 @@ def test_get_filepath_or_buffer_with_buffer(self):
         input_buffer = StringIO()
         filepath_or_buffer, _, _ = common.get_filepath_or_buffer(input_buffer)
         self.assertEqual(filepath_or_buffer, input_buffer)
+
+    def test_iterator(self):
+        reader = read_csv(StringIO(self.data1), chunksize=1)
+        result = concat(reader, ignore_index=True)
+        expected = read_csv(StringIO(self.data1))
+        tm.assert_frame_equal(result, expected)
+
+        # GH12153
+        it = read_csv(StringIO(self.data1), chunksize=1)
+        first = next(it)
+        tm.assert_frame_equal(first, expected.iloc[[0]])
+        expected.index = [0 for i in range(len(expected))]
+        tm.assert_frame_equal(concat(it), expected.iloc[1:])

pandas/io/tests/test_sas.py

@@ -88,6 +88,11 @@ def test1_incremental(self):
 
         tm.assert_frame_equal(data, data_csv, check_index_type=False)
 
+        reader = XportReader(self.file01, index="SEQN", chunksize=1000)
+        data = pd.concat(reader, axis=0)
+
+        tm.assert_frame_equal(data, data_csv, check_index_type=False)
+
     def test2(self):
         # Test with SSHSV1_A.XPT
 

pandas/io/tests/test_stata.py

@@ -1033,6 +1033,10 @@ def test_iterator(self):
         chunk = itr.get_chunk()
         tm.assert_frame_equal(parsed.iloc[0:5, :], chunk)
 
+        # GH12153
+        from_chunks = pd.concat(read_stata(fname, chunksize=4))
+        tm.assert_frame_equal(parsed, from_chunks)
+
     def test_read_chunks_115(self):
         files_115 = [self.dta2_115, self.dta3_115, self.dta4_115,
                      self.dta14_115, self.dta15_115, self.dta16_115,