Create pvlib.iotools.get_solrad (#1967)

* create get_solrad function

* Add init.py and iotools.rst entry

* Apply suggestions from code review

Co-authored-by: Adam R. Jensen <[email protected]>

* See Also cross-links with read_solrad

* move Examples section below References

* don't use "inclusive" with pd.date_range

it is only available from pandas v1.4 onwards, so can't use it in pvlib yet

* station.lower()

* add test

* whatsnew

* lint

* add another test for coverage

* remove stray empty line

* fix broken test

* fix problem with read_solrad for 404 URLs

---------

Co-authored-by: Adam R. Jensen <[email protected]>

Author: kandersolar

docs/sphinx/source/reference/iotools.rst

@@ -26,6 +26,7 @@ of sources and file formats relevant to solar energy modeling.
    iotools.read_midc_raw_data_from_nrel
    iotools.read_crn
    iotools.read_solrad
+   iotools.get_solrad
    iotools.get_psm3
    iotools.read_psm3
    iotools.parse_psm3

docs/sphinx/source/whatsnew/v0.10.4.rst

@@ -8,6 +8,8 @@ v0.10.4 (Anticipated March, 2024)
 Enhancements
 ~~~~~~~~~~~~
 * Added the Huld PV model used by PVGIS (:pull:`1940`)
+* Added :py:func:`~pvlib.iotools.get_solrad` for fetching irradiance data from
+  the SOLRAD ground station network. (:pull:`1967`)
 * Added metadata parsing to :py:func:`~pvlib.iotools.read_solrad` to follow the standard iotools
   convention of returning a tuple of (data, meta). Previously the function only returned a dataframe. (:pull:`1968`)
 
@@ -52,4 +54,5 @@ Contributors
 * Cliff Hansen (:ghuser:`cwhanse`)
 * :ghuser:`matsuobasho`
 * Adam R. Jensen (:ghuser:`AdamRJensen`)
+* Kevin Anderson (:ghuser:`kandersolar`)
 * Peter Dudfield (:ghuser:`peterdudfield`)

pvlib/iotools/__init__.py

@@ -8,6 +8,7 @@
 from pvlib.iotools.midc import read_midc_raw_data_from_nrel  # noqa: F401
 from pvlib.iotools.crn import read_crn  # noqa: F401
 from pvlib.iotools.solrad import read_solrad  # noqa: F401
+from pvlib.iotools.solrad import get_solrad  # noqa: F401
 from pvlib.iotools.psm3 import get_psm3  # noqa: F401
 from pvlib.iotools.psm3 import read_psm3  # noqa: F401
 from pvlib.iotools.psm3 import parse_psm3  # noqa: F401

pvlib/iotools/solrad.py

@@ -1,6 +1,7 @@
 """Functions to read data from the NOAA SOLRAD network."""
 
 import pandas as pd
+import warnings
 import requests
 import io
 
@@ -72,6 +73,10 @@ def read_solrad(filename):
     metadata : dict
         Metadata.
 
+    See Also
+    --------
+    get_solrad
+
     Notes
     -----
     SOLRAD data resolution is described by the README_SOLRAD.txt:
@@ -104,6 +109,7 @@ def read_solrad(filename):
 
     if str(filename).startswith('ftp') or str(filename).startswith('http'):
         response = requests.get(filename)
+        response.raise_for_status()
         file_buffer = io.StringIO(response.content.decode())
     else:
         with open(str(filename), 'r') as file_buffer:
@@ -135,3 +141,81 @@ def read_solrad(filename):
     data = data.set_index(dtindex)
 
     return data, meta
+
+
+def get_solrad(station, start, end,
+               url="https://gml.noaa.gov/aftp/data/radiation/solrad/"):
+    """Request data from NOAA SOLRAD and read it into a Dataframe.
+
+    A list of stations and their descriptions can be found in [1]_,
+    The data files are described in [2]_.
+
+    Data is returned for complete days, including ``start`` and ``end``.
+
+    Parameters
+    ----------
+    station : str
+        Three letter station abbreviation.
+    start : datetime-like
+        First day of the requested period
+    end : datetime-like
+        Last day of the requested period
+    url : str, default: 'https://gml.noaa.gov/aftp/data/radiation/solrad/'
+        API endpoint URL
+
+    Returns
+    -------
+    data : pd.DataFrame
+        Dataframe with data from SOLRAD.
+    meta : dict
+        Metadata.
+
+    See Also
+    --------
+    read_solrad
+
+    Notes
+    -----
+    Recent SOLRAD data is 1-minute averages.  Prior to 2015-01-01, it was
+    3-minute averages.
+
+    References
+    ----------
+    .. [1] https://gml.noaa.gov/grad/solrad/index.html
+    .. [2] https://gml.noaa.gov/aftp/data/radiation/solrad/README_SOLRAD.txt
+
+    Examples
+    --------
+    >>> # Retrieve one month of irradiance data from the ABQ SOLRAD station
+    >>> data, metadata = pvlib.iotools.get_solrad(
+    >>>     station='abq', start="2020-01-01", end="2020-01-31")
+    """
+    # Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
+    start = pd.to_datetime(start)
+    end = pd.to_datetime(end)
+
+    # Generate list of filenames
+    dates = pd.date_range(start.floor('d'), end, freq='d')
+    station = station.lower()
+    filenames = [
+        f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat"
+        for d in dates
+    ]
+
+    dfs = []  # Initialize list of monthly dataframes
+    for f in filenames:
+        try:
+            dfi, file_metadata = read_solrad(url + f)
+            dfs.append(dfi)
+        except requests.exceptions.HTTPError:
+            warnings.warn(f"The following file was not found: {f}")
+
+    data = pd.concat(dfs, axis='rows')
+
+    meta = {'station': station,
+            'filenames': filenames,
+            # all file should have the same metadata, so just merge in the
+            # metadata from the last file
+            **file_metadata}
+
+    return data, meta

pvlib/tests/iotools/test_solrad.py

@@ -117,3 +117,36 @@ def test_read_solrad_https():
     remote_data, _ = solrad.read_solrad(https_testfile)
     # local file only contains four rows to save space
     assert_frame_equal(local_data, remote_data.iloc[:4])
+
+
+@pytest.mark.remote_data
+@pytest.mark.parametrize('testfile, station', [
+    (testfile, 'abq'),
+    (testfile_mad, 'msn'),
+])
+def test_get_solrad(testfile, station):
+    df, meta = solrad.get_solrad(station, "2019-02-25", "2019-02-25")
+
+    assert meta['station'] == station
+    assert isinstance(meta['filenames'], list)
+
+    assert len(df) == 1440
+    assert df.index[0] == pd.to_datetime('2019-02-25 00:00+00:00')
+    assert df.index[-1] == pd.to_datetime('2019-02-25 23:59+00:00')
+
+    expected, _ = solrad.read_solrad(testfile)
+    actual = df.reindex(expected.index)
+    # ABQ test file has an unexplained NaN in row 4; just verify first 3 rows
+    assert_frame_equal(actual.iloc[:3], expected.iloc[:3])
+
+
+@pytest.mark.remote_data
+def test_get_solrad_missing_day():
+    # data availability begins for ABQ on 2002-02-01 (DOY 32), so requesting
+    # data before that will raise a warning
+    message = 'The following file was not found: abq/2002/abq02031.dat'
+    with pytest.warns(UserWarning, match=message):
+        df, meta = solrad.get_solrad('abq', '2002-01-31', '2002-02-01')
+
+    # but the data for 2022-02-01 is still returned
+    assert not df.empty