Surfrad data reader (#595)

* surfrad reader

* minor updates

* coding style fixes

* DatetimeIndex updates for compatibility with pandas 0.14.0

* documentation updates

* Doc updates, addition to what's new and api

* bump min pandas version to 0.15.0, update tests and docs, add map_variables option

* style fix

* typo in whatsnew/doc update

* rebase, small fix to pvsystems.py

* remove whitespace

* fix duplicate whatsnew entry after rebase

Author: lboeman

.travis.yml

@@ -69,7 +69,7 @@ install:
         pip uninstall numpy --yes;
         pip uninstall pandas --yes;
         pip install --no-cache-dir numpy==1.10.1;
-        pip install --no-cache-dir pandas==0.14.0;
+        pip install --no-cache-dir pandas==0.15.0;
       fi
     - conda list
     - echo $PATH

docs/sphinx/source/api.rst

@@ -311,6 +311,7 @@ relevant to solar energy modeling.
    iotools.read_tmy3
    iotools.read_srml
    iotools.read_srml_month_from_solardat
+   iotools.read_surfrad
 
 A :py:class:`~pvlib.location.Location` object may be created from metadata
 in some files.

docs/sphinx/source/whatsnew/v0.6.1.rst

@@ -9,6 +9,8 @@ release.
 **Python 2.7 support will end on June 1, 2019**. Releases made after this
 date will require Python 3. (:issue:`501`)
 
+**Minimum pandas requirement bumped 0.14.0=>0.15.0**
+
 
 API Changes
 ~~~~~~~~~~~
@@ -30,7 +32,7 @@ Enhancements
 * Created :py:func:`pvlib.iotools.read_srml` and
   :py:func:`pvlib.iotools.read_srml_month_from_solardat` to read University of
   Oregon Solar Radiation Monitoring Laboratory data. (:issue:`589`)
-
+* Created :py:func:`pvlib.iotools.read_surfrad` to read NOAA SURFRAD data. (:issue:`590`)
 
 Bug fixes
 ~~~~~~~~~

pvlib/data/surfrad-slv16001.dat

@@ -2,3 +2,4 @@
 from pvlib.iotools.tmy import read_tmy3  # noqa: F401
 from pvlib.iotools.srml import read_srml  # noqa: F401
 from pvlib.iotools.srml import read_srml_month_from_solardat  # noqa: F401
+from pvlib.iotools.surfrad import read_surfrad  # noqa: F401

pvlib/iotools/__init__.py

@@ -0,0 +1,184 @@
+"""
+Import functions for NOAA SURFRAD Data.
+"""
+import io
+
+try:
+    # python 2 compatibility
+    from urllib2 import urlopen, Request
+except ImportError:
+    from urllib.request import urlopen, Request
+
+import pandas as pd
+import numpy as np
+
+SURFRAD_COLUMNS = [
+    'year', 'jday', 'month', 'day', 'hour', 'minute', 'dt', 'zen',
+    'dw_solar', 'dw_solar_flag', 'uw_solar', 'uw_solar_flag', 'direct_n',
+    'direct_n_flag', 'diffuse', 'diffuse_flag', 'dw_ir', 'dw_ir_flag',
+    'dw_casetemp', 'dw_casetemp_flag', 'dw_dometemp', 'dw_dometemp_flag',
+    'uw_ir', 'uw_ir_flag', 'uw_casetemp', 'uw_casetemp_flag', 'uw_dometemp',
+    'uw_dometemp_flag', 'uvb', 'uvb_flag', 'par', 'par_flag', 'netsolar',
+    'netsolar_flag', 'netir', 'netir_flag', 'totalnet', 'totalnet_flag',
+    'temp', 'temp_flag', 'rh', 'rh_flag', 'windspd', 'windspd_flag',
+    'winddir', 'winddir_flag', 'pressure', 'pressure_flag']
+
+# Dictionary mapping surfrad variables to pvlib names
+VARIABLE_MAP = {
+    'zen': 'solar_zenith',
+    'dw_solar': 'ghi',
+    'dw_solar_flag': 'ghi_flag',
+    'direct_n': 'dni',
+    'direct_n_flag': 'dni_flag',
+    'diffuse': 'dhi',
+    'diffuse_flag': 'dhi_flag',
+    'temp': 'temp_air',
+    'temp_flag': 'temp_air_flag',
+    'windspd': 'wind_speed',
+    'windspd_flag': 'wind_speed_flag',
+    'winddir': 'wind_direction',
+    'winddir_flag': 'wind_direction_flag',
+    'rh': 'relative_humidity',
+    'rh_flag': 'relative_humidity_flag'
+}
+
+
+def read_surfrad(filename, map_variables=True):
+    """Read in a daily NOAA SURFRAD[1] file.
+
+    Parameters
+    ----------
+    filename: str
+        Filepath or url.
+    map_variables: bool
+        When true, renames columns of the Dataframe to pvlib variable names
+        where applicable. See variable SURFRAD_COLUMNS.
+
+    Returns
+    -------
+    Tuple of the form (data, metadata).
+
+    data: Dataframe
+        Dataframe with the fields found below.
+    metadata: dict
+        Site metadata included in the file.
+
+    Notes
+    -----
+    Metadata dictionary includes the following fields:
+
+    ===============  ======  ===============
+    Key              Format  Description
+    ===============  ======  ===============
+    station          String  site name
+    latitude         Float   site latitude
+    longitude        Float   site longitude
+    elevation        Int     site elevation
+    surfrad_version  Int     surfrad version
+    tz               String  Timezone (UTC)
+    ===============  ======  ===============
+
+    Dataframe includes the following fields:
+
+    =======================  ======  ==========================================
+    raw, mapped              Format  Description
+    =======================  ======  ==========================================
+    **Mapped field names are returned when the map_variables argument is True**
+    ---------------------------------------------------------------------------
+    year                     int     year as 4 digit int
+    jday                     int     day of year 1-365(or 366)
+    month                    int     month (1-12)
+    day                      int     day of month(1-31)
+    hour                     int     hour (0-23)
+    minute                   int     minute (0-59)
+    dt                       float   decimal time i.e. 23.5 = 2330
+    zen, solar_zenith        float   solar zenith angle (deg)
+    **Fields below have associated qc flags labeled <field>_flag.**
+    ---------------------------------------------------------------------------
+    dw_solar, ghi            float   downwelling global solar(W/m^2)
+    uw_solar                 float   updownwelling global solar(W/m^2)
+    direct_n, dni            float   direct normal solar (W/m^2)
+    diffuse, dhi             float   downwelling diffuse solar (W/m^2)
+    dw_ir                    float   downwelling thermal infrared (W/m^2)
+    dw_casetemp              float   downwelling IR case temp (K)
+    dw_dometemp              float   downwelling IR dome temp (K)
+    uw_ir                    float   upwelling thermal infrared (W/m^2)
+    uw_casetemp              float   upwelling IR case temp (K)
+    uw_dometemp              float   upwelling IR case temp (K)
+    uvb                      float   global uvb (miliWatts/m^2)
+    par                      float   photosynthetically active radiation(W/m^2)
+    netsolar                 float   net solar (dw_solar - uw_solar) (W/m^2)
+    netir                    float   net infrared (dw_ir - uw_ir) (W/m^2)
+    totalnet                 float   net radiation (netsolar+netir) (W/m^2)
+    temp, temp_air           float   10-meter air temperature (?C)
+    rh, relative_humidity    float   relative humidity (%)
+    windspd, wind_speed      float   wind speed (m/s)
+    winddir, wind_direction  float   wind direction (deg, clockwise from north)
+    pressure                 float   station pressure (mb)
+    =======================  ======  ==========================================
+
+    See README files located in the station directories in the SURFRAD
+    data archives[2] for details on SURFRAD daily data files.
+
+    References
+    ----------
+    [1] NOAA Earth System Research Laboratory Surface Radiation Budget Network
+        `SURFRAD Homepage <https://www.esrl.noaa.gov/gmd/grad/surfrad/>`_
+    [2] NOAA SURFRAD Data Archive
+        `SURFRAD Archive <ftp://aftp.cmdl.noaa.gov/data/radiation/surfrad/>`_
+    """
+    if filename.startswith('ftp'):
+        req = Request(filename)
+        response = urlopen(req)
+        file_buffer = io.StringIO(response.read().decode(errors='ignore'))
+    else:
+        file_buffer = open(filename, 'r')
+
+    # Read and parse the first two lines to build the metadata dict.
+    station = file_buffer.readline()
+    file_metadata = file_buffer.readline()
+
+    metadata_list = file_metadata.split()
+    metadata = {}
+    metadata['name'] = station.strip()
+    metadata['latitude'] = float(metadata_list[0])
+    metadata['longitude'] = float(metadata_list[1])
+    metadata['elevation'] = float(metadata_list[2])
+    metadata['surfrad_version'] = int(metadata_list[-1])
+    metadata['tz'] = 'UTC'
+
+    data = pd.read_csv(file_buffer, delim_whitespace=True,
+                       header=None, names=SURFRAD_COLUMNS)
+    file_buffer.close()
+
+    data = format_index(data)
+    missing = data == -9999.9
+    data = data.where(~missing, np.NaN)
+
+    if map_variables:
+        data.rename(columns=VARIABLE_MAP, inplace=True)
+    return data, metadata
+
+
+def format_index(data):
+    """Create UTC localized DatetimeIndex for the dataframe.
+
+    Parameters
+    ----------
+    data: Dataframe
+        Must contain columns 'year', 'jday', 'hour' and
+        'minute'.
+
+    Return
+    ------
+    data: Dataframe
+        Dataframe with a DatetimeIndex localized to UTC.
+    """
+    year = data.year.apply(str)
+    jday = data.jday.apply(lambda x: '{:03d}'.format(x))
+    hours = data.hour.apply(lambda x: '{:02d}'.format(x))
+    minutes = data.minute.apply(lambda x: '{:02d}'.format(x))
+    index = pd.to_datetime(year + jday + hours + minutes, format="%Y%j%H%M")
+    data.index = index
+    data = data.tz_localize('UTC')
+    return data

pvlib/iotools/surfrad.py

@@ -2648,8 +2648,8 @@ def adrinverter(v_dc, p_dc, inverter, vtol=0.10):
     mppt_hi = inverter['MPPTHi']
     mppt_low = inverter['MPPTLow']
 
-    v_lim_upper = np.nanmax([v_max, vdc_max, mppt_hi]) * (1 + vtol)
-    v_lim_lower = np.nanmax([v_min, mppt_low]) * (1 - vtol)
+    v_lim_upper = float(np.nanmax([v_max, vdc_max, mppt_hi]) * (1 + vtol))
+    v_lim_lower = float(np.nanmax([v_min, mppt_low]) * (1 - vtol))
 
     pdc = p_dc / p_nom
     vdc = v_dc / v_nom

pvlib/pvsystem.py

@@ -0,0 +1,65 @@
+import inspect
+import os
+
+from pandas import Timestamp, DatetimeIndex
+from pandas.util.testing import network
+
+from pvlib.iotools import surfrad
+
+test_dir = os.path.dirname(
+    os.path.abspath(inspect.getfile(inspect.currentframe())))
+testfile = os.path.join(test_dir, '../data/surfrad-slv16001.dat')
+network_testfile = ('ftp://aftp.cmdl.noaa.gov/data/radiation/surfrad/'
+                    'Alamosa_CO/2016/slv16001.dat')
+
+
+@network
+def test_read_surfrad_network():
+    # If this test begins failing, SURFRAD's data structure or data
+    # archive may have changed.
+    local_data, _ = surfrad.read_surfrad(testfile)
+    network_data, _ = surfrad.read_surfrad(network_testfile)
+    assert local_data.equals(network_data)
+
+
+def test_read_surfrad_columns_no_map():
+    data, _ = surfrad.read_surfrad(testfile, map_variables=False)
+    assert 'zen' in data.columns
+    assert 'temp' in data.columns
+    assert 'par' in data.columns
+    assert 'pressure' in data.columns
+
+
+def test_read_surfrad_columns_map():
+    data, _ = surfrad.read_surfrad(testfile)
+    assert 'solar_zenith' in data.columns
+    assert 'ghi' in data.columns
+    assert 'ghi_flag' in data.columns
+    assert 'dni' in data.columns
+    assert 'dni_flag' in data.columns
+    assert 'dhi' in data.columns
+    assert 'dhi_flag' in data.columns
+    assert 'wind_direction' in data.columns
+    assert 'wind_direction_flag' in data.columns
+    assert 'wind_speed' in data.columns
+    assert 'wind_speed_flag' in data.columns
+    assert 'temp_air' in data.columns
+    assert 'temp_air_flag' in data.columns
+
+
+def test_format_index():
+    start = Timestamp('20160101 00:00')
+    expected = DatetimeIndex(start=start, periods=1440, freq='1min', tz='UTC')
+    actual, _ = surfrad.read_surfrad(testfile)
+    assert actual.index.equals(expected)
+
+
+def test_read_surfrad_metadata():
+    expected = {'name': 'Alamosa',
+                'latitude': 37.70,
+                'longitude': 105.92,
+                'elevation': 2317,
+                'surfrad_version': 1,
+                'tz': 'UTC'}
+    _, metadata = surfrad.read_surfrad(testfile)
+    assert metadata == expected

pvlib/test/test_surfrad.py

@@ -38,7 +38,7 @@
 URL = 'https://github.com/pvlib/pvlib-python'
 
 INSTALL_REQUIRES = ['numpy >= 1.10.1',
-                    'pandas >= 0.14.0',
+                    'pandas >= 0.15.0',
                     'pytz',
                     'six',
                     ]