Skip to content

Commit 32e5350

Browse files
xylarxylar
committed
Switch analysis scripts to open_multifile_datset
1 parent b64255a commit 32e5350

File tree

5 files changed

+139
-137
lines changed

5 files changed

+139
-137
lines changed

‎mpas_analysis/ocean/ocean_modelvsobs.py

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,20 @@
1111
import matplotlib.pyplot as plt
1212
import matplotlib.colors as cols
1313

14-
import numpy as np
1514
import xarray as xr
15+
import numpy as np
1616
import datetime
17-
from netCDF4 import Dataset as netcdf_dataset
17+
import netCDF4
1818

19-
from ..shared.mpas_xarray.mpas_xarray import preprocess_mpas, \
20-
remove_repeated_time_index
2119
from ..shared.plot.plotting import plot_global_comparison
2220
from ..shared.interpolation.interpolate import interp_fields, init_tree
2321
from ..shared.constants import constants
2422

2523
from ..shared.io import NameList, StreamsFile
2624
from ..shared.io.utility import buildConfigFullPath
2725

26+
from ..shared.dataset.utility import open_multifile_dataset
27+
2828

2929
def ocn_modelvsobs(config, field, streamMap=None, variableMap=None):
3030

@@ -85,7 +85,7 @@ def ocn_modelvsobs(config, field, streamMap=None, variableMap=None):
8585
sectionName = 'regridded{}'.format(field.upper())
8686
outputTimes = config.getExpression(sectionName, 'comparisonTimes')
8787

88-
ncFile = netcdf_dataset(restartFile, mode='r')
88+
ncFile = netCDF4.Dataset(restartFile, mode='r')
8989
lonCell = ncFile.variables["lonCell"][:]
9090
latCell = ncFile.variables["latCell"][:]
9191
ncFile.close()
@@ -94,7 +94,7 @@ def ocn_modelvsobs(config, field, streamMap=None, variableMap=None):
9494

9595
if field == 'mld':
9696

97-
selvals = None
97+
iselvals = None
9898

9999
# Load MLD observational data
100100
obsFileName = "{}/holtetalley_mld_climatology.nc".format(
@@ -122,7 +122,7 @@ def ocn_modelvsobs(config, field, streamMap=None, variableMap=None):
122122

123123
elif field == 'sst':
124124

125-
selvals = {'nVertLevels': 0}
125+
iselvals = {'nVertLevels': 0}
126126

127127
obsFileName = \
128128
"{}/MODEL.SST.HAD187001-198110.OI198111-201203.nc".format(
@@ -154,7 +154,7 @@ def ocn_modelvsobs(config, field, streamMap=None, variableMap=None):
154154

155155
elif field == 'sss':
156156

157-
selvals = {'nVertLevels': 0}
157+
iselvals = {'nVertLevels': 0}
158158

159159
obsFileName = "{}/Aquarius_V3_SSS_Monthly.nc".format(
160160
observationsDirectory)
@@ -183,14 +183,15 @@ def ocn_modelvsobs(config, field, streamMap=None, variableMap=None):
183183
outFileLabel = 'sssAquarius'
184184
unitsLabel = 'PSU'
185185

186-
ds = xr.open_mfdataset(
187-
inputFiles,
188-
preprocess=lambda x: preprocess_mpas(x, yearoffset=yearOffset,
189-
timestr='Time',
190-
onlyvars=varList,
191-
selvals=selvals,
192-
varmap=variableMap))
193-
ds = remove_repeated_time_index(ds)
186+
ds = open_multifile_dataset(file_names=inputFiles,
187+
calendar=calendar,
188+
time_variable_name='Time',
189+
variable_list=varList,
190+
isel_values=iselvals,
191+
variable_map=variableMap,
192+
start_date=startDate,
193+
end_date=endDate,
194+
year_offset=yearOffset)
194195

195196
timeStart = datetime.datetime(yearOffset+startYear, 1, 1)
196197
timeEnd = datetime.datetime(yearOffset+endYear, 12, 31)

‎mpas_analysis/ocean/ohc_timeseries.py

Lines changed: 43 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,15 @@
11
import numpy as np
22
import netCDF4
3-
from netCDF4 import Dataset as netcdf_dataset
4-
import xarray as xr
53
import pandas as pd
64
import datetime
75

8-
from ..shared.mpas_xarray.mpas_xarray import preprocess_mpas, \
9-
remove_repeated_time_index
10-
116
from ..shared.plot.plotting import timeseries_analysis_plot
127

138
from ..shared.io import NameList, StreamsFile
149
from ..shared.io.utility import buildConfigFullPath
1510

11+
from ..shared.dataset.utility import open_multifile_dataset
12+
1613
from ..shared.timekeeping.utility import stringToDatetime, \
1714
clampToNumpyDatetime64
1815

@@ -33,7 +30,7 @@ def ohc_timeseries(config, streamMap=None, variableMap=None):
3330
to their mpas_analysis counterparts.
3431
3532
Author: Xylar Asay-Davis, Milena Veneziani
36-
Last Modified: 02/02/2017
33+
Last Modified: 02/08/2017
3734
"""
3835

3936
inDirectory = config.get('input', 'baseDirectory')
@@ -82,13 +79,13 @@ def ohc_timeseries(config, streamMap=None, variableMap=None):
8279
startDate = config.get('timeSeries', 'startDate')
8380
endDate = config.get('timeSeries', 'endDate')
8481
streamName = streams.find_stream(streamMap['timeSeriesStats'])
85-
inFiles = streams.readpath(streamName, startDate=startDate,
86-
endDate=endDate, calendar=calendar)
87-
print 'Reading files {} through {}'.format(inFiles[0], inFiles[-1])
82+
file_names = streams.readpath(streamName, startDate=startDate,
83+
endDate=endDate, calendar=calendar)
84+
print 'Reading files {} through {}'.format(file_names[0], file_names[-1])
8885

8986
# Define/read in general variables
9087
print ' Read in depth and compute specific depth indexes...'
91-
ncFile = netcdf_dataset(restartFile, mode='r')
88+
ncFile = netCDF4.Dataset(restartFile, mode='r')
9289
# reference depth [m]
9390
depth = ncFile.variables['refBottomDepth'][:]
9491
# simulation start time
@@ -109,48 +106,51 @@ def ohc_timeseries(config, streamMap=None, variableMap=None):
109106

110107
# Load data
111108
print ' Load ocean data...'
112-
variableList = ['avgLayerTemperature',
113-
'sumLayerMaskValue',
114-
'avgLayerArea',
115-
'avgLayerThickness']
116-
ds = xr.open_mfdataset(
117-
inFiles,
118-
preprocess=lambda x: preprocess_mpas(x,
119-
yearoffset=yearOffset,
120-
timestr='Time',
121-
onlyvars=variableList,
122-
varmap=variableMap))
123-
124-
ds = remove_repeated_time_index(ds)
109+
variable_list = ['avgLayerTemperature',
110+
'sumLayerMaskValue',
111+
'avgLayerArea',
112+
'avgLayerThickness']
113+
ds = open_multifile_dataset(file_names=file_names,
114+
calendar=calendar,
115+
time_variable_name='Time',
116+
variable_list=variable_list,
117+
variable_map=variableMap,
118+
start_date=startDate,
119+
end_date=endDate,
120+
year_offset=yearOffset)
125121

126122
timeStart = clampToNumpyDatetime64(stringToDatetime(startDate), yearOffset)
127123
timeEnd = clampToNumpyDatetime64(stringToDatetime(endDate), yearOffset)
128-
# select only the data in the specified range of years
129-
ds = ds.sel(Time=slice(timeStart, timeEnd))
130124

131125
# Select year-1 data and average it (for later computing anomalies)
132126
timeStartFirstYear = clampToNumpyDatetime64(
133127
stringToDatetime(simulationStartTime), yearOffset)
134128
if timeStartFirstYear < timeStart:
135129
startDateFirstYear = simulationStartTime
136-
endDateFirstYear = '{}-12-31_23:59:59'.format(startDateFirstYear[0:4])
130+
firstYear = int(startDateFirstYear[0:4])
131+
endDateFirstYear = '{:04d}-12-31_23:59:59'.format(firstYear)
137132
filesFirstYear = streams.readpath(streamName,
138133
startDate=startDateFirstYear,
139134
endDate=endDateFirstYear,
140135
calendar=calendar)
141-
dsFirstYear = xr.open_mfdataset(
142-
filesFirstYear,
143-
preprocess=lambda x: preprocess_mpas(x,
144-
yearoffset=yearOffset,
145-
timestr='Time',
146-
onlyvars=variableList,
147-
varmap=variableMap))
148-
149-
dsFirstYear = remove_repeated_time_index(dsFirstYear)
136+
dsFirstYear = open_multifile_dataset(file_names=filesFirstYear,
137+
calendar=calendar,
138+
time_variable_name='Time',
139+
variable_list=variable_list,
140+
variable_map=variableMap,
141+
start_date=startDateFirstYear,
142+
end_date=endDateFirstYear,
143+
year_offset=yearOffset)
144+
firstYear += yearOffset
150145
else:
151-
timeStart = datetime.datetime(timeStart.year, 1, 1)
152-
timeEnd = datetime.datetime(timeStart.year, 12, 31)
153-
dsFirstYear = ds.sel(Time=slice(timeStart, timeEnd))
146+
dsFirstYear = ds
147+
firstYear = timeStart.year
148+
149+
timeStartFirstYear = datetime.datetime(firstYear, 1, 1)
150+
timeEndFirstYear = datetime.datetime(firstYear, 12, 31)
151+
dsFirstYear = dsFirstYear.sel(Time=slice(timeStartFirstYear,
152+
timeEndFirstYear))
153+
154154
meanFirstYear = dsFirstYear.mean('Time')
155155

156156
print ' Compute temperature anomalies...'
@@ -167,12 +167,12 @@ def ohc_timeseries(config, streamMap=None, variableMap=None):
167167

168168
if preprocessedReferenceRunName != 'None':
169169
print ' Load in OHC from preprocessed reference run...'
170-
inFilesPreprocesses = '{}/OHC.{}.year*.nc'.format(
170+
inFilesPreprocessed = '{}/OHC.{}.year*.nc'.format(
171171
preprocessedInputDirectory, preprocessedReferenceRunName)
172-
dsPreprocessed = xr.open_mfdataset(
173-
inFilesPreprocesses,
174-
preprocess=lambda x: preprocess_mpas(x, yearoffset=yearOffset))
175-
dsPreprocessed = remove_repeated_time_index(dsPreprocessed)
172+
dsPreprocessed = open_multifile_dataset(file_names=inFilesPreprocessed,
173+
calendar=calendar,
174+
time_variable_name='xtime',
175+
year_offset=yearOffset)
176176
yearEndPreprocessed = \
177177
(pd.to_datetime(dsPreprocessed.Time.max().values)).year
178178
if yearStart <= yearEndPreprocessed:

‎mpas_analysis/ocean/sst_timeseries.py

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
1-
import xarray as xr
21
import pandas as pd
32
import datetime
43

5-
from ..shared.mpas_xarray.mpas_xarray import preprocess_mpas, \
6-
remove_repeated_time_index
7-
84
from ..shared.plot.plotting import timeseries_analysis_plot
95

106
from ..shared.io import NameList, StreamsFile
117
from ..shared.io.utility import buildConfigFullPath
128

9+
from ..shared.dataset.utility import open_multifile_dataset
10+
1311
from ..shared.timekeeping.utility import stringToDatetime, \
1412
clampToNumpyDatetime64
1513

@@ -50,9 +48,9 @@ def sst_timeseries(config, streamMap=None, variableMap=None):
5048
startDate = config.get('timeSeries', 'startDate')
5149
endDate = config.get('timeSeries', 'endDate')
5250
streamName = streams.find_stream(streamMap['timeSeriesStats'])
53-
inFiles = streams.readpath(streamName, startDate=startDate,
54-
endDate=endDate, calendar=calendar)
55-
print 'Reading files {} through {}'.format(inFiles[0], inFiles[-1])
51+
file_names = streams.readpath(streamName, startDate=startDate,
52+
endDate=endDate, calendar=calendar)
53+
print 'Reading files {} through {}'.format(file_names[0], file_names[-1])
5654

5755
mainRunName = config.get('runs', 'mainRunName')
5856
preprocessedReferenceRunName = config.get('runs',
@@ -73,13 +71,14 @@ def sst_timeseries(config, streamMap=None, variableMap=None):
7371

7472
# Load data:
7573
varList = ['avgSurfaceTemperature']
76-
ds = xr.open_mfdataset(
77-
inFiles,
78-
preprocess=lambda x: preprocess_mpas(x, yearoffset=yearOffset,
79-
timestr='Time',
80-
onlyvars=varList,
81-
varmap=variableMap))
82-
ds = remove_repeated_time_index(ds)
74+
ds = open_multifile_dataset(file_names=file_names,
75+
calendar=calendar,
76+
time_variable_name='Time',
77+
variable_list=varList,
78+
variable_map=variableMap,
79+
start_date=startDate,
80+
end_date=endDate,
81+
year_offset=yearOffset)
8382

8483
timeStart = clampToNumpyDatetime64(stringToDatetime(startDate), yearOffset)
8584
timeEnd = clampToNumpyDatetime64(stringToDatetime(endDate), yearOffset)
@@ -97,10 +96,10 @@ def sst_timeseries(config, streamMap=None, variableMap=None):
9796
print ' Load in SST for a preprocesses reference run...'
9897
inFilesPreprocessed = '{}/SST.{}.year*.nc'.format(
9998
preprocessedInputDirectory, preprocessedReferenceRunName)
100-
dsPreprocessed = xr.open_mfdataset(
101-
inFilesPreprocessed,
102-
preprocess=lambda x: preprocess_mpas(x, yearoffset=yearOffset))
103-
dsPreprocessed = remove_repeated_time_index(dsPreprocessed)
99+
dsPreprocessed = open_multifile_dataset(file_names=inFilesPreprocessed,
100+
calendar=calendar,
101+
time_variable_name='xtime',
102+
year_offset=yearOffset)
104103
yearEndPreprocessed = \
105104
(pd.to_datetime(dsPreprocessed.Time.max().values)).year
106105
if yearStart <= yearEndPreprocessed:

‎mpas_analysis/sea_ice/modelvsobs.py

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,17 @@
66

77
import numpy as np
88
import numpy.ma as ma
9-
import xarray as xr
109
import datetime
1110

12-
from netCDF4 import Dataset as netcdf_dataset
11+
import netCDF4
1312

14-
from ..shared.mpas_xarray.mpas_xarray import preprocess_mpas, \
15-
remove_repeated_time_index
1613
from ..shared.plot.plotting import plot_polar_comparison
1714

1815
from ..shared.io import NameList, StreamsFile
1916
from ..shared.io.utility import buildConfigFullPath
2017

18+
from ..shared.dataset.utility import open_multifile_dataset
19+
2120

2221
def seaice_modelvsobs(config, streamMap=None, variableMap=None):
2322
"""
@@ -53,9 +52,9 @@ def seaice_modelvsobs(config, streamMap=None, variableMap=None):
5352
startDate = config.get('climatology', 'startDate')
5453
endDate = config.get('climatology', 'endDate')
5554
streamName = streams.find_stream(streamMap['timeSeriesStats'])
56-
infiles = streams.readpath(streamName, startDate=startDate,
57-
endDate=endDate, calendar=calendar)
58-
print 'Reading files {} through {}'.format(infiles[0], infiles[-1])
55+
file_names = streams.readpath(streamName, startDate=startDate,
56+
endDate=endDate, calendar=calendar)
57+
print 'Reading files {} through {}'.format(file_names[0], file_names[-1])
5958

6059
plotsDirectory = buildConfigFullPath(config, 'output', 'plotsSubdirectory')
6160
obsDirectory = config.get('seaIceObservations', 'baseDirectory')
@@ -154,14 +153,15 @@ def seaice_modelvsobs(config, streamMap=None, variableMap=None):
154153

155154
# Load data
156155
print " Load sea-ice data..."
157-
ds = xr.open_mfdataset(
158-
infiles,
159-
preprocess=lambda x: preprocess_mpas(x, yearoffset=yearOffset,
160-
timestr='Time',
161-
onlyvars=['iceAreaCell',
162-
'iceVolumeCell'],
163-
varmap=variableMap))
164-
ds = remove_repeated_time_index(ds)
156+
ds = open_multifile_dataset(file_names=file_names,
157+
calendar=calendar,
158+
time_variable_name='Time',
159+
variable_list=['iceAreaCell',
160+
'iceVolumeCell'],
161+
variable_map=variableMap,
162+
start_date=startDate,
163+
end_date=endDate,
164+
year_offset=yearOffset)
165165

166166
# Compute climatologies (first motnhly and then seasonally)
167167
print " Compute seasonal climatologies..."
@@ -250,7 +250,7 @@ def seaice_modelvsobs(config, streamMap=None, variableMap=None):
250250
# ice concentrations
251251
fileName = "{}/{}".format(climatologyRegriddedDirectory,
252252
climatologyFiles[climName])
253-
ncFile = netcdf_dataset(fileName, mode='r')
253+
ncFile = netCDF4.Dataset(fileName, mode='r')
254254
iceConcentration = ncFile.variables["iceAreaCell"][:]
255255
if(first):
256256
lons = ncFile.variables["lon"][:]
@@ -267,7 +267,7 @@ def seaice_modelvsobs(config, streamMap=None, variableMap=None):
267267

268268
fileName = obsIceConcFileNames[
269269
'{}_{}'.format(climName, obsName)]
270-
ncFile = netcdf_dataset(fileName, mode='r')
270+
ncFile = netCDF4.Dataset(fileName, mode='r')
271271
obsIceConcentration = ncFile.variables["AICE"][:]
272272
ncFile.close()
273273

@@ -315,7 +315,7 @@ def seaice_modelvsobs(config, streamMap=None, variableMap=None):
315315
# ice concentrations
316316
fileName = "{}/{}".format(climatologyRegriddedDirectory,
317317
climatologyFiles[climName])
318-
ncFile = netcdf_dataset(fileName, mode='r')
318+
ncFile = netCDF4.Dataset(fileName, mode='r')
319319
iceThickness = ncFile.variables["iceVolumeCell"][:]
320320
ncFile.close()
321321

@@ -358,7 +358,7 @@ def seaice_modelvsobs(config, streamMap=None, variableMap=None):
358358

359359
fileName = obsIceThickFileNames['{}{}'.format(climName,
360360
hemisphere)]
361-
ncFile = netcdf_dataset(fileName, mode='r')
361+
ncFile = netCDF4.Dataset(fileName, mode='r')
362362
obsIceThickness = ncFile.variables["HI"][:]
363363
ncFile.close()
364364
# Mask thickness fields

0 commit comments

Comments
 (0)