lookup_linke_turbidity speed improvement, mat to h5 (#442)

lookup_linke_turbidity speed improvement, mat to h5

Author: cedricleroy

appveyor.yml

@@ -28,7 +28,7 @@ install:
     - cmd: conda info -a
 
     # install depenencies
-    - cmd: conda create -n test_env --yes --quiet python=%PYTHON_VERSION% pip numpy scipy pandas nose pytest pytz ephem numba siphon -c conda-forge
+    - cmd: conda create -n test_env --yes --quiet python=%PYTHON_VERSION% pip numpy scipy pytables pandas nose pytest pytz ephem numba siphon -c conda-forge
     - cmd: activate test_env
     - cmd: python --version
     - cmd: conda list

ci/requirements-py27.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=2.7
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     - ephem

ci/requirements-py34.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=3.4
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     - ephem

ci/requirements-py35.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=3.5
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     - ephem

ci/requirements-py36.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=3.6
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     #- ephem

docs/environment.yml

@@ -7,6 +7,7 @@ dependencies:
     - mock  # needed for local python 2.7 builds
     - numpy=1.11.2
     - scipy
+    - tables
     - pandas=0.19.1
     - pytz
     - ephem

docs/sphinx/source/clearsky.rst

@@ -136,24 +136,24 @@ the year. You could run it in a loop to create plots for all months.
 
     In [1]: import os
 
-    In [1]: import scipy.io
+    In [1]: import tables
 
     In [1]: pvlib_path = os.path.dirname(os.path.abspath(pvlib.clearsky.__file__))
 
-    In [1]: filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.mat')
-
-    In [1]: mat = scipy.io.loadmat(filepath)
+    In [1]: filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.h5')
 
     # data is in units of 20 x turbidity
-    In [1]: linke_turbidity_table = mat['LinkeTurbidity']  # / 20.   # crashes on rtd
+    In [1]: lt_h5_file = tables.open_file(filepath)
 
     In [1]: def plot_turbidity_map(month, vmin=1, vmax=100):
        ...:     plt.figure();
-       ...:     plt.imshow(linke_turbidity_table[:, :, month-1], vmin=vmin, vmax=vmax);
+       ...:     plt.imshow(lt_h5_file.root.LinkeTurbidity[:, :, month-1], vmin=vmin, vmax=vmax);
        ...:     plt.title('Linke turbidity x 20, ' + calendar.month_name[month]);
        ...:     plt.colorbar(shrink=0.5);
        ...:     plt.tight_layout();
 
+    In [1]: lt_h5_file.close()
+
     @savefig turbidity-1.png width=10in
     In [1]: plot_turbidity_map(1)
 

docs/sphinx/source/whatsnew/v0.5.2.rst

@@ -9,7 +9,8 @@ API Changes
 
 Enhancements
 ~~~~~~~~~~~~
-*
+* Improve clearsky.lookup_linke_turbidity speed, changing .mat source
+  file to .h5 (:issue:`437`)
 
 Bug fixes
 ~~~~~~~~~
@@ -42,5 +43,4 @@ Contributors
 * Will Holmgren
 * KonstantinTr
 * Anton Driesse
-
-
+* Cedric Leroy

pvlib/clearsky.py

@@ -153,7 +153,7 @@ def ineichen(apparent_zenith, airmass_absolute, linke_turbidity,
 def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
                            interp_turbidity=True):
     """
-    Look up the Linke Turibidity from the ``LinkeTurbidities.mat``
+    Look up the Linke Turibidity from the ``LinkeTurbidities.h5``
     data file supplied with pvlib.
 
     Parameters
@@ -165,18 +165,18 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
     longitude : float
 
     filepath : None or string, default None
-        The path to the ``.mat`` file.
+        The path to the ``.h5`` file.
 
     interp_turbidity : bool, default True
         If ``True``, interpolates the monthly Linke turbidity values
-        found in ``LinkeTurbidities.mat`` to daily values.
+        found in ``LinkeTurbidities.h5`` to daily values.
 
     Returns
     -------
     turbidity : Series
     """
 
-    # The .mat file 'LinkeTurbidities.mat' contains a single 2160 x 4320 x 12
+    # The .h5 file 'LinkeTurbidities.h5' contains a single 2160 x 4320 x 12
     # matrix of type uint8 called 'LinkeTurbidity'. The rows represent global
     # latitudes from 90 to -90 degrees; the columns represent global longitudes
     # from -180 to 180; and the depth (third dimension) represents months of
@@ -194,18 +194,15 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
     # 1st column: 179.9583 W, 2nd column: 179.875 W
 
     try:
-        import scipy.io
+        import tables
     except ImportError:
-        raise ImportError('The Linke turbidity lookup table requires scipy. '
+        raise ImportError('The Linke turbidity lookup table requires tables. '
                           'You can still use clearsky.ineichen if you '
                           'supply your own turbidities.')
 
     if filepath is None:
         pvlib_path = os.path.dirname(os.path.abspath(__file__))
-        filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.mat')
-
-    mat = scipy.io.loadmat(filepath)
-    linke_turbidity_table = mat['LinkeTurbidity']
+        filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.h5')
 
     latitude_index = (
         np.around(_linearly_scale(latitude, 90, -90, 0, 2160))
@@ -214,7 +211,14 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
         np.around(_linearly_scale(longitude, -180, 180, 0, 4320))
         .astype(np.int64))
 
-    lts = linke_turbidity_table[latitude_index][longitude_index]
+    lt_h5_file = tables.open_file(filepath)
+    try:
+        lts = lt_h5_file.root.LinkeTurbidity[latitude_index, longitude_index, :]
+    except IndexError:
+        raise IndexError('Latitude should be between 90 and -90, '
+                         'longitude between -180 and 180.')
+    finally:
+        lt_h5_file.close()
 
     if interp_turbidity:
         linke_turbidity = _interpolate_turbidity(lts, time)

pvlib/data/LinkeTurbidities.mat renamed to pvlib/data/LinkeTurbidities.h5

@@ -19,6 +19,15 @@
 requires_scipy = pytest.mark.skipif(not has_scipy, reason='requires scipy')
 
 
+try:
+    import tables
+    has_tables = True
+except ImportError:
+    has_tables = False
+
+requires_tables = pytest.mark.skipif(not has_tables, reason='requires tables')
+
+
 try:
     import ephem
     has_ephem = True

pvlib/test/conftest.py

@@ -16,7 +16,7 @@
 from pvlib import atmosphere
 from pvlib import irradiance
 
-from conftest import requires_scipy
+from conftest import requires_scipy, requires_tables
 
 
 def test_ineichen_series():
@@ -158,7 +158,7 @@ def test_ineichen_altitude():
     assert_frame_equal(expected, out)
 
 
-@requires_scipy
+@requires_tables
 def test_lookup_linke_turbidity():
     times = pd.date_range(start='2014-06-24', end='2014-06-25',
                           freq='12h', tz='America/Phoenix')
@@ -171,7 +171,7 @@ def test_lookup_linke_turbidity():
     assert_series_equal(expected, out)
 
 
-@requires_scipy
+@requires_tables
 def test_lookup_linke_turbidity_leapyear():
     times = pd.date_range(start='2016-06-24', end='2016-06-25',
                           freq='12h', tz='America/Phoenix')
@@ -184,7 +184,7 @@ def test_lookup_linke_turbidity_leapyear():
     assert_series_equal(expected, out)
 
 
-@requires_scipy
+@requires_tables
 def test_lookup_linke_turbidity_nointerp():
     times = pd.date_range(start='2014-06-24', end='2014-06-25',
                           freq='12h', tz='America/Phoenix')
@@ -195,7 +195,7 @@ def test_lookup_linke_turbidity_nointerp():
     assert_series_equal(expected, out)
 
 
-@requires_scipy
+@requires_tables
 def test_lookup_linke_turbidity_months():
     times = pd.date_range(start='2014-04-01', end='2014-07-01',
                           freq='1M', tz='America/Phoenix')
@@ -206,7 +206,7 @@ def test_lookup_linke_turbidity_months():
     assert_series_equal(expected, out)
 
 
-@requires_scipy
+@requires_tables
 def test_lookup_linke_turbidity_months_leapyear():
     times = pd.date_range(start='2016-04-01', end='2016-07-01',
                           freq='1M', tz='America/Phoenix')
@@ -217,7 +217,7 @@ def test_lookup_linke_turbidity_months_leapyear():
     assert_series_equal(expected, out)
 
 
-@requires_scipy
+@requires_tables
 def test_lookup_linke_turbidity_nointerp_months():
     times = pd.date_range(start='2014-04-10', end='2014-07-10',
                           freq='1M', tz='America/Phoenix')
@@ -448,7 +448,7 @@ def test_simplified_solis_nans_series():
     assert_frame_equal(expected, out)
 
 
-@requires_scipy
+@requires_tables
 def test_linke_turbidity_corners():
     """Test Linke turbidity corners out of bounds."""
     months = pd.DatetimeIndex('%d/1/2016' % (m + 1) for m in range(12))