Skip to content

Commit 923e025

Browse files
Create pvlib.iotools.get_solrad (#1967)
* create get_solrad function * Add init.py and iotools.rst entry * Apply suggestions from code review Co-authored-by: Adam R. Jensen <[email protected]> * See Also cross-links with read_solrad * move Examples section below References * don't use "inclusive" with pd.date_range it is only available from pandas v1.4 onwards, so can't use it in pvlib yet * station.lower() * add test * whatsnew * lint * add another test for coverage * remove stray empty line * fix broken test * fix problem with read_solrad for 404 URLs --------- Co-authored-by: Adam R. Jensen <[email protected]>
1 parent 33045d2 commit 923e025

File tree

5 files changed

+122
-0
lines changed

5 files changed

+122
-0
lines changed

docs/sphinx/source/reference/iotools.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ of sources and file formats relevant to solar energy modeling.
2626
iotools.read_midc_raw_data_from_nrel
2727
iotools.read_crn
2828
iotools.read_solrad
29+
iotools.get_solrad
2930
iotools.get_psm3
3031
iotools.read_psm3
3132
iotools.parse_psm3

docs/sphinx/source/whatsnew/v0.10.4.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ v0.10.4 (Anticipated March, 2024)
88
Enhancements
99
~~~~~~~~~~~~
1010
* Added the Huld PV model used by PVGIS (:pull:`1940`)
11+
* Added :py:func:`~pvlib.iotools.get_solrad` for fetching irradiance data from
12+
the SOLRAD ground station network. (:pull:`1967`)
1113
* Added metadata parsing to :py:func:`~pvlib.iotools.read_solrad` to follow the standard iotools
1214
convention of returning a tuple of (data, meta). Previously the function only returned a dataframe. (:pull:`1968`)
1315

@@ -52,4 +54,5 @@ Contributors
5254
* Cliff Hansen (:ghuser:`cwhanse`)
5355
* :ghuser:`matsuobasho`
5456
* Adam R. Jensen (:ghuser:`AdamRJensen`)
57+
* Kevin Anderson (:ghuser:`kandersolar`)
5558
* Peter Dudfield (:ghuser:`peterdudfield`)

pvlib/iotools/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from pvlib.iotools.midc import read_midc_raw_data_from_nrel # noqa: F401
99
from pvlib.iotools.crn import read_crn # noqa: F401
1010
from pvlib.iotools.solrad import read_solrad # noqa: F401
11+
from pvlib.iotools.solrad import get_solrad # noqa: F401
1112
from pvlib.iotools.psm3 import get_psm3 # noqa: F401
1213
from pvlib.iotools.psm3 import read_psm3 # noqa: F401
1314
from pvlib.iotools.psm3 import parse_psm3 # noqa: F401

pvlib/iotools/solrad.py

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Functions to read data from the NOAA SOLRAD network."""
22

33
import pandas as pd
4+
import warnings
45
import requests
56
import io
67

@@ -72,6 +73,10 @@ def read_solrad(filename):
7273
metadata : dict
7374
Metadata.
7475
76+
See Also
77+
--------
78+
get_solrad
79+
7580
Notes
7681
-----
7782
SOLRAD data resolution is described by the README_SOLRAD.txt:
@@ -104,6 +109,7 @@ def read_solrad(filename):
104109

105110
if str(filename).startswith('ftp') or str(filename).startswith('http'):
106111
response = requests.get(filename)
112+
response.raise_for_status()
107113
file_buffer = io.StringIO(response.content.decode())
108114
else:
109115
with open(str(filename), 'r') as file_buffer:
@@ -135,3 +141,81 @@ def read_solrad(filename):
135141
data = data.set_index(dtindex)
136142

137143
return data, meta
144+
145+
146+
def get_solrad(station, start, end,
147+
url="https://gml.noaa.gov/aftp/data/radiation/solrad/"):
148+
"""Request data from NOAA SOLRAD and read it into a Dataframe.
149+
150+
A list of stations and their descriptions can be found in [1]_,
151+
The data files are described in [2]_.
152+
153+
Data is returned for complete days, including ``start`` and ``end``.
154+
155+
Parameters
156+
----------
157+
station : str
158+
Three letter station abbreviation.
159+
start : datetime-like
160+
First day of the requested period
161+
end : datetime-like
162+
Last day of the requested period
163+
url : str, default: 'https://gml.noaa.gov/aftp/data/radiation/solrad/'
164+
API endpoint URL
165+
166+
Returns
167+
-------
168+
data : pd.DataFrame
169+
Dataframe with data from SOLRAD.
170+
meta : dict
171+
Metadata.
172+
173+
See Also
174+
--------
175+
read_solrad
176+
177+
Notes
178+
-----
179+
Recent SOLRAD data is 1-minute averages. Prior to 2015-01-01, it was
180+
3-minute averages.
181+
182+
References
183+
----------
184+
.. [1] https://gml.noaa.gov/grad/solrad/index.html
185+
.. [2] https://gml.noaa.gov/aftp/data/radiation/solrad/README_SOLRAD.txt
186+
187+
Examples
188+
--------
189+
>>> # Retrieve one month of irradiance data from the ABQ SOLRAD station
190+
>>> data, metadata = pvlib.iotools.get_solrad(
191+
>>> station='abq', start="2020-01-01", end="2020-01-31")
192+
"""
193+
# Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
194+
start = pd.to_datetime(start)
195+
end = pd.to_datetime(end)
196+
197+
# Generate list of filenames
198+
dates = pd.date_range(start.floor('d'), end, freq='d')
199+
station = station.lower()
200+
filenames = [
201+
f"{station}/{d.year}/{station}{d.strftime('%y')}{d.dayofyear:03}.dat"
202+
for d in dates
203+
]
204+
205+
dfs = [] # Initialize list of monthly dataframes
206+
for f in filenames:
207+
try:
208+
dfi, file_metadata = read_solrad(url + f)
209+
dfs.append(dfi)
210+
except requests.exceptions.HTTPError:
211+
warnings.warn(f"The following file was not found: {f}")
212+
213+
data = pd.concat(dfs, axis='rows')
214+
215+
meta = {'station': station,
216+
'filenames': filenames,
217+
# all file should have the same metadata, so just merge in the
218+
# metadata from the last file
219+
**file_metadata}
220+
221+
return data, meta

pvlib/tests/iotools/test_solrad.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,3 +117,36 @@ def test_read_solrad_https():
117117
remote_data, _ = solrad.read_solrad(https_testfile)
118118
# local file only contains four rows to save space
119119
assert_frame_equal(local_data, remote_data.iloc[:4])
120+
121+
122+
@pytest.mark.remote_data
123+
@pytest.mark.parametrize('testfile, station', [
124+
(testfile, 'abq'),
125+
(testfile_mad, 'msn'),
126+
])
127+
def test_get_solrad(testfile, station):
128+
df, meta = solrad.get_solrad(station, "2019-02-25", "2019-02-25")
129+
130+
assert meta['station'] == station
131+
assert isinstance(meta['filenames'], list)
132+
133+
assert len(df) == 1440
134+
assert df.index[0] == pd.to_datetime('2019-02-25 00:00+00:00')
135+
assert df.index[-1] == pd.to_datetime('2019-02-25 23:59+00:00')
136+
137+
expected, _ = solrad.read_solrad(testfile)
138+
actual = df.reindex(expected.index)
139+
# ABQ test file has an unexplained NaN in row 4; just verify first 3 rows
140+
assert_frame_equal(actual.iloc[:3], expected.iloc[:3])
141+
142+
143+
@pytest.mark.remote_data
144+
def test_get_solrad_missing_day():
145+
# data availability begins for ABQ on 2002-02-01 (DOY 32), so requesting
146+
# data before that will raise a warning
147+
message = 'The following file was not found: abq/2002/abq02031.dat'
148+
with pytest.warns(UserWarning, match=message):
149+
df, meta = solrad.get_solrad('abq', '2002-01-31', '2002-02-01')
150+
151+
# but the data for 2022-02-01 is still returned
152+
assert not df.empty

0 commit comments

Comments
 (0)