ENH: Implemented lazy iteration.

Fixes GH20783.

Author: mitar

doc/source/whatsnew/v0.23.1.txt

@@ -16,6 +16,8 @@ New features
 ~~~~~~~~~~~~
 
 - :meth:`Index.droplevel` is now implemented also for flat indexes, for compatibility with MultiIndex (:issue:`21115`)
+- Iterating over a :class:`Series` and using :meth:`DataFrame.itertuples` now create iterators without internally
+  allocating lists of all elements (:issue:`20783`)
 
 
 .. _whatsnew_0231.deprecations:

pandas/core/base.py

@@ -23,7 +23,7 @@
 import pandas.core.nanops as nanops
 import pandas._libs.lib as lib
 from pandas.compat.numpy import function as nv
-from pandas.compat import PYPY
+from pandas.compat import PYPY, map, range
 from pandas.util._decorators import (Appender, cache_readonly,
                                      deprecate_kwarg, Substitution)
 
@@ -917,7 +917,13 @@ def __iter__(self):
         (for str, int, float) or a pandas scalar
         (for Timestamp/Timedelta/Interval/Period)
         """
-        return iter(self.tolist())
+        # We are explicity making element iterators.
+        if is_datetimelike(self._values):
+            return map(com._maybe_box_datetimelike, self._values)
+        elif is_extension_array_dtype(self._values):
+            return iter(self._values)
+        else:
+            return map(self._values.item, range(self._values.size))
 
     @cache_readonly
     def hasnans(self):

pandas/core/frame.py

@@ -815,14 +815,14 @@ def itertuples(self, index=True, name="Pandas"):
         Pandas(Index='b', col1=2, col2=0.20000000000000001)
 
         """
-        arrays = []
+        iterators = []
         fields = []
         if index:
-            arrays.append(self.index)
+            iterators.append(self.index)
             fields.append("Index")
 
         # use integer indexing because of possible duplicate column names
-        arrays.extend(self.iloc[:, k] for k in range(len(self.columns)))
+        iterators.extend(self.iloc[:, k] for k in range(len(self.columns)))
 
         # Python 3 supports at most 255 arguments to constructor, and
         # things get slow with this many fields in Python 2
@@ -832,12 +832,12 @@ def itertuples(self, index=True, name="Pandas"):
                 itertuple = collections.namedtuple(name,
                                                    fields + list(self.columns),
                                                    rename=True)
-                return map(itertuple._make, zip(*arrays))
+                return map(itertuple._make, zip(*iterators))
             except Exception:
                 pass
 
         # fallback to regular tuples
-        return zip(*arrays)
+        return zip(*iterators)
 
     items = iteritems