From 30f3c72a8e2fe736dd23a30ddf32864a0e007de9 Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Mon, 16 Apr 2018 15:38:49 -0600 Subject: [PATCH 01/11] Adds xml-based functionality to keep track of data --- docs/Makefile | 7 +++- docs/environment.yml | 4 ++- docs/index.rst | 5 +++ docs/observations.rst | 9 ++++++ docs/parse_table.py | 48 +++++++++++++++++++++++++++ docs/tier1a.xml | 75 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 docs/observations.rst create mode 100755 docs/parse_table.py create mode 100644 docs/tier1a.xml diff --git a/docs/Makefile b/docs/Makefile index 1489bead0..a229e0ef3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -12,9 +12,14 @@ BUILDDIR = _build help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +clean: + rm -rf tier1a.rst + @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + python parse_table.py -x tier1a.xml -r tier1a.rst + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/environment.yml b/docs/environment.yml index 509b389e9..76d97501f 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -21,4 +21,6 @@ dependencies: - sphinx - sphinx_rtd_theme - numpydoc - - recommonmark + - tabulate + - pip: + - recommonmark diff --git a/docs/index.rst b/docs/index.rst index eea59046d..e5d0002d8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,11 @@ Analysis for simulations produced with Model for Prediction Across Scales (MPAS) components and the Accelerated Climate Model for Energy (ACME), which used those components. +Observations +============ +.. toctree:: + + observations Documentation ============= diff --git a/docs/observations.rst b/docs/observations.rst new file mode 100644 index 000000000..86a3af9f8 --- /dev/null +++ b/docs/observations.rst @@ -0,0 +1,9 @@ +Observations +========== + +A variety of observational datasets are used within MPAS-Analysis: + +Tier1a metrics +-------------- + +.. include:: tier1a.rst diff --git a/docs/parse_table.py b/docs/parse_table.py new file mode 100755 index 000000000..690dac9a7 --- /dev/null +++ b/docs/parse_table.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +""" +This script is used to convert an xml-based table into an rst table for +python documentation and pythonic parsing. + +Phillip J. Wolfram +04/16/2018 +""" + +import xml.etree.ElementTree as ET +import tabulate +import argparse + +def build_rst_table_from_xml(xmlfile, rstfile): + + # open xml file for reading + xml = ET.parse(xmlfile) + + # open rst file for writing + rst = open(rstfile, 'w') + + # get headers and build list to write entries + headers = xml.getroot().attrib['headers'].replace(' ','').split(',') + headernames = [aname.strip() for aname in xml.getroot().attrib['headernames'].split(',')] + data = [] + for entry in xml.findall('aobs'): + line = [] + for aheader in headers: + line.append(entry.findall(aheader)[0].text.strip().replace('\n', ' ')) + data.append(line) + + rst.write(tabulate.tabulate(data, headernames, tablefmt='rst')) + + rst.close() + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-x", "--xml_table", dest="xml_table", + help="Path to file containing xml description of table", + metavar="FILE", required=True) + parser.add_argument("-r", "--rst_table", dest="rst_table", + help="Path to file containing rst description of table for output", + metavar="FILE", required=True) + + args = parser.parse_args() + + build_rst_table_from_xml(args.xml_table, args.rst_table) diff --git a/docs/tier1a.xml b/docs/tier1a.xml new file mode 100644 index 000000000..8094bc6e2 --- /dev/null +++ b/docs/tier1a.xml @@ -0,0 +1,75 @@ + + + + + sst Hadley-NOAA + + + ocean + + + SST global annual climatology and trend + + + Global latitude/longitude plot on ocean native grid. Annual climatologies + computed over 5-10 year periods at regular intervals throughout the run. + Time series of area-weighted (AW) global mean (1-year running average). + + + Merged Hadley Center-NOAA/OI data set from Hurrell et al. 2008 + (https://climatedataguide.ucar.edu/climate-data/merged-hadley-noaaoi-sea-surface-temperature-sea-ice-concentration-hurrell-et-al-2008) + + Update 4/11/18: emailed UCAR to ask about their release policy. + + + + 1. CESM Atm diagnostic package, 2. Gent et al. (2011, Fig.1) + + + + + + + + + + SSS NASA Aquarius + + + ocean + + + SSS global annual climatology + + + Global latitude/longitude plot on ocean native grid . Annual climatologies + computed over 5-10 year periods at regular intervals throughout the run. + + + NASA Aquarius satellite data ~4 years until Jun 2015 + (ftp://podaac-ftp.jpl.nasa.gov/allData/aquarius/L3/mapped/V4/7day_running/SCI/) + + This statement is from the NASA daac (Distributed + Active Archive Center) website: "NASA data are not copyrighted; however, + when you publish our data or results derived therefrom, we request that + you include an acknowledgment within the text of the publication and + reference list." This should apply to all NASA data sets. + + + + PHC (http://psc.apl.washington.edu/nonwp_projects/PHC/Climatology.html) + or WOA13 climatology (https://www.nodc.noaa.gov/OC5/woa13/) + + + 1. Large and Danabasoglu (2006, Fig.1), 2. Griffies et al. (2009, Fig. 8) + + + + + + + + + From eb28e36e8320cc105890583af40beb334bbcd27d Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Mon, 16 Apr 2018 15:39:09 -0600 Subject: [PATCH 02/11] Fixes author name and whitespace typos --- docs/authors.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/authors.rst b/docs/authors.rst index 3b67178e5..5f5711aa6 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -2,7 +2,7 @@ Main Authors ============ * Xylar Asay-Davis * Milena Veniziani -* Philip Wolfram +* Phillip J. Wolfram Contributors ============ @@ -16,4 +16,4 @@ Contributors * Jeremy Fyke For a list of all the contributions: -https://github.com/MPAS-Dev/MPAS-Analysis/graphs/contributors \ No newline at end of file +https://github.com/MPAS-Dev/MPAS-Analysis/graphs/contributors From eec997249dec941bf7438a46e342042b8efd729f Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 08:51:05 -0600 Subject: [PATCH 03/11] Supports Markdown urls in xml --- docs/Makefile | 4 ++-- docs/{tier1a.xml => observations.xml} | 10 ++++------ docs/parse_table.py | 24 +++++++++++++++++++++++- 3 files changed, 29 insertions(+), 9 deletions(-) rename docs/{tier1a.xml => observations.xml} (83%) diff --git a/docs/Makefile b/docs/Makefile index a229e0ef3..eef84e754 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -13,7 +13,7 @@ help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) clean: - rm -rf tier1a.rst + rm -rf observations.rst @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile @@ -21,5 +21,5 @@ clean: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - python parse_table.py -x tier1a.xml -r tier1a.rst + python parse_table.py -x observations.xml -r observations.rst @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/tier1a.xml b/docs/observations.xml similarity index 83% rename from docs/tier1a.xml rename to docs/observations.xml index 8094bc6e2..6021b50d7 100644 --- a/docs/tier1a.xml +++ b/docs/observations.xml @@ -18,8 +18,7 @@ Time series of area-weighted (AW) global mean (1-year running average). - Merged Hadley Center-NOAA/OI data set from Hurrell et al. 2008 - (https://climatedataguide.ucar.edu/climate-data/merged-hadley-noaaoi-sea-surface-temperature-sea-ice-concentration-hurrell-et-al-2008) + [Merged Hadley Center-NOAA/OI data set from Hurrell et al. 2008](https://climatedataguide.ucar.edu/climate-data/merged-hadley-noaaoi-sea-surface-temperature-sea-ice-concentration-hurrell-et-al-2008) Update 4/11/18: emailed UCAR to ask about their release policy. @@ -48,8 +47,7 @@ computed over 5-10 year periods at regular intervals throughout the run. - NASA Aquarius satellite data ~4 years until Jun 2015 - (ftp://podaac-ftp.jpl.nasa.gov/allData/aquarius/L3/mapped/V4/7day_running/SCI/) + [NASA Aquarius satellite data ~4 years until Jun 2015](ftp://podaac-ftp.jpl.nasa.gov/allData/aquarius/L3/mapped/V4/7day_running/SCI/) This statement is from the NASA daac (Distributed Active Archive Center) website: "NASA data are not copyrighted; however, @@ -59,8 +57,8 @@ - PHC (http://psc.apl.washington.edu/nonwp_projects/PHC/Climatology.html) - or WOA13 climatology (https://www.nodc.noaa.gov/OC5/woa13/) + [PHC](http://psc.apl.washington.edu/nonwp_projects/PHC/Climatology.html) + or [WOA13 climatology](https://www.nodc.noaa.gov/OC5/woa13/). 1. Large and Danabasoglu (2006, Fig.1), 2. Griffies et al. (2009, Fig. 8) diff --git a/docs/parse_table.py b/docs/parse_table.py index 690dac9a7..817f13658 100755 --- a/docs/parse_table.py +++ b/docs/parse_table.py @@ -10,6 +10,24 @@ import xml.etree.ElementTree as ET import tabulate import argparse +import re + +def markdown_links(data, footer): + urlscmd = re.findall(r"\[.*?\]\(.*?\)", data) + urls = re.findall(r"\[.*?\]\((.*?)\)", data) + linknames = re.findall(r"\[(.*?)\]\(.*?\)", data) + + for alinkname, aurl, aurlscmd in zip(linknames, urls, urlscmd): + data = data.replace(aurlscmd, '`' + alinkname +'`_') + footer += ".. _`" + alinkname + "`: " + aurl + "\n" + + return data, footer + +def cleanup(linedata, footer): + cleanups = [markdown_links] + for acleanup in cleanups: + cleandata, footer = acleanup(linedata, footer) + return cleandata, footer def build_rst_table_from_xml(xmlfile, rstfile): @@ -23,13 +41,17 @@ def build_rst_table_from_xml(xmlfile, rstfile): headers = xml.getroot().attrib['headers'].replace(' ','').split(',') headernames = [aname.strip() for aname in xml.getroot().attrib['headernames'].split(',')] data = [] + footer = '\n' for entry in xml.findall('aobs'): line = [] for aheader in headers: - line.append(entry.findall(aheader)[0].text.strip().replace('\n', ' ')) + linedata = entry.findall(aheader)[0].text.strip().replace('\n', ' ') + linedata, footer = cleanup(linedata, footer) + line.append(linedata) data.append(line) rst.write(tabulate.tabulate(data, headernames, tablefmt='rst')) + rst.write(footer) rst.close() From d83f13ef0161cf6d264618c1194d03b429afa64c Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 10:41:37 -0600 Subject: [PATCH 04/11] Adds list capability and component filtering --- docs/Makefile | 6 +- docs/environment.yml | 2 +- docs/observations.rst | 17 ++++- docs/observations.xml | 73 ------------------ docs/observationstable.xml | 153 +++++++++++++++++++++++++++++++++++++ docs/parse_table.py | 24 ++++-- 6 files changed, 189 insertions(+), 86 deletions(-) delete mode 100644 docs/observations.xml create mode 100644 docs/observationstable.xml diff --git a/docs/Makefile b/docs/Makefile index eef84e754..299240787 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -13,7 +13,7 @@ help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) clean: - rm -rf observations.rst + rm -rf *observationstable.rst @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile @@ -21,5 +21,7 @@ clean: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - python parse_table.py -x observations.xml -r observations.rst + python parse_table.py -x observationstable.xml -r landicobservationstable.rst -c landice + python parse_table.py -x observationstable.xml -r oceanobservationstable.rst -c ocean + python parse_table.py -x observationstable.xml -r seaicobservationstable.rst -c seaice @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/environment.yml b/docs/environment.yml index 76d97501f..d67f332fb 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -21,6 +21,6 @@ dependencies: - sphinx - sphinx_rtd_theme - numpydoc - - tabulate + - tabulate >= 0.8.2 - pip: - recommonmark diff --git a/docs/observations.rst b/docs/observations.rst index 86a3af9f8..487135fbc 100644 --- a/docs/observations.rst +++ b/docs/observations.rst @@ -1,9 +1,18 @@ Observations -========== +============ A variety of observational datasets are used within MPAS-Analysis: -Tier1a metrics --------------- +Land Ice metrics +---------------- +.. include:: landicobservationstable.rst + +Ocean metrics +------------- + +.. include:: oceanobservationstable.rst + +Sea Ice metrics +--------------- +.. include:: seaicobservationstable.rst -.. include:: tier1a.rst diff --git a/docs/observations.xml b/docs/observations.xml deleted file mode 100644 index 6021b50d7..000000000 --- a/docs/observations.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - - sst Hadley-NOAA - - - ocean - - - SST global annual climatology and trend - - - Global latitude/longitude plot on ocean native grid. Annual climatologies - computed over 5-10 year periods at regular intervals throughout the run. - Time series of area-weighted (AW) global mean (1-year running average). - - - [Merged Hadley Center-NOAA/OI data set from Hurrell et al. 2008](https://climatedataguide.ucar.edu/climate-data/merged-hadley-noaaoi-sea-surface-temperature-sea-ice-concentration-hurrell-et-al-2008) - - Update 4/11/18: emailed UCAR to ask about their release policy. - - - - 1. CESM Atm diagnostic package, 2. Gent et al. (2011, Fig.1) - - - - - - - - - - SSS NASA Aquarius - - - ocean - - - SSS global annual climatology - - - Global latitude/longitude plot on ocean native grid . Annual climatologies - computed over 5-10 year periods at regular intervals throughout the run. - - - [NASA Aquarius satellite data ~4 years until Jun 2015](ftp://podaac-ftp.jpl.nasa.gov/allData/aquarius/L3/mapped/V4/7day_running/SCI/) - - This statement is from the NASA daac (Distributed - Active Archive Center) website: "NASA data are not copyrighted; however, - when you publish our data or results derived therefrom, we request that - you include an acknowledgment within the text of the publication and - reference list." This should apply to all NASA data sets. - - - - [PHC](http://psc.apl.washington.edu/nonwp_projects/PHC/Climatology.html) - or [WOA13 climatology](https://www.nodc.noaa.gov/OC5/woa13/). - - - 1. Large and Danabasoglu (2006, Fig.1), 2. Griffies et al. (2009, Fig. 8) - - - - - - - - - diff --git a/docs/observationstable.xml b/docs/observationstable.xml new file mode 100644 index 000000000..f222a7153 --- /dev/null +++ b/docs/observationstable.xml @@ -0,0 +1,153 @@ + + + + + + sst Hadley-NOAA + + + ocean + + + SST global annual climatology and trend + + + - Global latitude/longitude plot on ocean native grid. Annual climatologies computed over 5-10 year periods at regular intervals throughout the run. + - Time series of area-weighted (AW) global mean (1-year running average). + + + [Merged Hadley Center-NOAA/OI data set from Hurrell et al. 2008](https://climatedataguide.ucar.edu/climate-data/merged-hadley-noaaoi-sea-surface-temperature-sea-ice-concentration-hurrell-et-al-2008) + + Update 4/11/18: emailed UCAR to ask about their release policy. + + + + - CESM Atm diagnostic package + - Gent et al. (2011, Fig.1) + + + + + + + + + + SSS NASA Aquarius + + + ocean + + + SSS global annual climatology + + + - Global latitude/longitude plot on ocean native grid . Annual climatologies computed over 5-10 year periods at regular intervals throughout the run. + + + [NASA Aquarius satellite data ~4 years until Jun 2015](ftp://podaac-ftp.jpl.nasa.gov/allData/aquarius/L3/mapped/V4/7day_running/SCI/) + + This statement is from the NASA daac (Distributed + Active Archive Center) website: "NASA data are not copyrighted; however, + when you publish our data or results derived therefrom, we request that + you include an acknowledgment within the text of the publication and + reference list." This should apply to all NASA data sets. + + + + [PHC](http://psc.apl.washington.edu/nonwp_projects/PHC/Climatology.html) + or [WOA13 climatology](https://www.nodc.noaa.gov/OC5/woa13/). + + + - Large and Danabasoglu (2006, Fig.1) + - Griffies et al. (2009, Fig. 8) + + + + + + + + + + + Ice area/extent annual climatology, and annual and seasonal trends + + + seaice + + + Sea ice concentration + + + - Arctic and Antarctic seasonal and annual climatologies (maps) of ice area and extent. Annual climatologies computed over 5-10 year periods at regular intervals throughout the run. + - Daily, monthly and 1-year average time series of area-weighted ice area and extent. + + + - [Ice concentration: SSM/I, NASATeam algorithm(both NH and SH)](http://nsidc.org/data/NSIDC-0051) + - [Ice concentration: SSM/I, Bootstrap algorithm (both NH and SH)](http://nsidc.org/data/NSIDC-0079) + - [Ice area time series: SSM/I derived (both NH and SH)](http://neptune.gsfc.nasa.gov/csb/index.php?section=59) + + Same as all NASA data. + + + + - [Antarctic ship-based data](http://aspect.antarctica.gov.au/data) + - [Ice type, and drift](http://marine.copernicus.eu/services-portfolio/access-to-products/?option=com_csw&view=details&product_id=SEAICE_GLO_SEAICE_L4_NRT_OBSERVATIONS_011_001) + - [MANY data sets available for both ocean and sea ice, both global and regional](high-reshttp://marine.copernicus.edu) + + + - CESM CICE diagnostics + - Ivanova et al. (2012) + - Worby et al. (2008) + + + See CESM sea ice diagnostic package + + + + + + + + + Present-day ice sheet area and volume (scalar) + + + landice + + + Land ice area / volume + + + - Difference between modeled and observed values. + + + Greenland ice thickness at bed topography at 1 km posting + - [Bedmap2 paper](http://www.the-cryosphere.net/7/375/2013/), [Bedmap2 data](https://www.bas.ac.uk/project/bedmap-2/) + - [R-Topo2 paper](http://www.earth-syst-sci-data-discuss.net/essd-2016-3/), [R-Topo2 data](https://www.pangaea.de/PHP/hs.php?s=Maps&d=RTopo-2.0&ID=856844) + Antarctic ice thickness and bed topography at 1 km posting + + Same as all NASA data. + + + + - [Antarctic ship-based data](http://aspect.antarctica.gov.au/data) + - [Ice type, and drift](http://marine.copernicus.eu/services-portfolio/access-to-products/?option=com_csw&view=details&product_id=SEAICE_GLO_SEAICE_L4_NRT_OBSERVATIONS_011_001) + - [MANY data sets available for both ocean and sea ice, both global and regional](high-reshttp://marine.copernicus.edu) + + + - Bamber et al. (2013) + - Fretwell et al. (2013) + + + validate initial conditions + + + + + + + diff --git a/docs/parse_table.py b/docs/parse_table.py index 817f13658..e6a7988ee 100755 --- a/docs/parse_table.py +++ b/docs/parse_table.py @@ -23,13 +23,20 @@ def markdown_links(data, footer): return data, footer +def spurious_newline_whitespace(data, _): + whitespace = re.findall('\n\s*', data) + if len(whitespace) > 0: + astr = min(whitespace) + data = data.replace(astr, "\n") + return data, _ + def cleanup(linedata, footer): - cleanups = [markdown_links] + cleanups = [spurious_newline_whitespace, markdown_links] for acleanup in cleanups: - cleandata, footer = acleanup(linedata, footer) - return cleandata, footer + linedata, footer = acleanup(linedata, footer) + return linedata, footer -def build_rst_table_from_xml(xmlfile, rstfile): +def build_rst_table_from_xml(xmlfile, rstfile, component): # open xml file for reading xml = ET.parse(xmlfile) @@ -43,9 +50,11 @@ def build_rst_table_from_xml(xmlfile, rstfile): data = [] footer = '\n' for entry in xml.findall('aobs'): + if component != 'all' and entry.findall('component')[0].text.strip() != component: + continue line = [] for aheader in headers: - linedata = entry.findall(aheader)[0].text.strip().replace('\n', ' ') + linedata = entry.findall(aheader)[0].text.strip() linedata, footer = cleanup(linedata, footer) line.append(linedata) data.append(line) @@ -64,7 +73,10 @@ def build_rst_table_from_xml(xmlfile, rstfile): parser.add_argument("-r", "--rst_table", dest="rst_table", help="Path to file containing rst description of table for output", metavar="FILE", required=True) + parser.add_argument("-c", "--component", dest="component", + help="Component for parsing of table, 'landice', 'ocean', " + "'seaice', or 'all'", metavar="STRING", default="all") args = parser.parse_args() - build_rst_table_from_xml(args.xml_table, args.rst_table) + build_rst_table_from_xml(args.xml_table, args.rst_table, args.component) From c1aed90d665fd136a6af863d4078a0ff95415480 Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 10:44:26 -0600 Subject: [PATCH 05/11] Moves observations xml to python code location --- docs/Makefile | 6 +++--- {docs => mpas_analysis/obs}/observationstable.xml | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename {docs => mpas_analysis/obs}/observationstable.xml (100%) diff --git a/docs/Makefile b/docs/Makefile index 299240787..b417471f9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -21,7 +21,7 @@ clean: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - python parse_table.py -x observationstable.xml -r landicobservationstable.rst -c landice - python parse_table.py -x observationstable.xml -r oceanobservationstable.rst -c ocean - python parse_table.py -x observationstable.xml -r seaicobservationstable.rst -c seaice + python parse_table.py -x ../mpas_analysis/obs/observationstable.xml -r landicobservationstable.rst -c landice + python parse_table.py -x ../mpas_analysis/obs/observationstable.xml -r oceanobservationstable.rst -c ocean + python parse_table.py -x ../mpas_analysis/obs/observationstable.xml -r seaicobservationstable.rst -c seaice @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/observationstable.xml b/mpas_analysis/obs/observationstable.xml similarity index 100% rename from docs/observationstable.xml rename to mpas_analysis/obs/observationstable.xml From 83ffba6317a8f7bc37f06a903c8f79d38efa474b Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 10:48:49 -0600 Subject: [PATCH 06/11] Fixes aobs to observations --- docs/parse_table.py | 2 +- mpas_analysis/obs/observationstable.xml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/parse_table.py b/docs/parse_table.py index e6a7988ee..a609ff3c0 100755 --- a/docs/parse_table.py +++ b/docs/parse_table.py @@ -49,7 +49,7 @@ def build_rst_table_from_xml(xmlfile, rstfile, component): headernames = [aname.strip() for aname in xml.getroot().attrib['headernames'].split(',')] data = [] footer = '\n' - for entry in xml.findall('aobs'): + for entry in xml.findall('observation'): if component != 'all' and entry.findall('component')[0].text.strip() != component: continue line = [] diff --git a/mpas_analysis/obs/observationstable.xml b/mpas_analysis/obs/observationstable.xml index f222a7153..bd202df5d 100644 --- a/mpas_analysis/obs/observationstable.xml +++ b/mpas_analysis/obs/observationstable.xml @@ -3,7 +3,7 @@ headers="shortdesc, obsDataSet, references" headernames="Dataset, Observational Dataset, References"> - + sst Hadley-NOAA @@ -32,8 +32,8 @@ - - + + SSS NASA Aquarius @@ -69,9 +69,9 @@ - + - + Ice area/extent annual climatology, and annual and seasonal trends @@ -109,9 +109,9 @@ - + - + Present-day ice sheet area and volume (scalar) @@ -148,6 +148,6 @@ - + From 8e593dee3280c73a132f0f0354d6038ca89e3e82 Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 10:51:07 -0600 Subject: [PATCH 07/11] Adds support for MD url tags on sep lines [title](http://example.com) and [title] (http://example.com) both now work --- docs/parse_table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/parse_table.py b/docs/parse_table.py index a609ff3c0..3eff90416 100755 --- a/docs/parse_table.py +++ b/docs/parse_table.py @@ -13,9 +13,9 @@ import re def markdown_links(data, footer): - urlscmd = re.findall(r"\[.*?\]\(.*?\)", data) - urls = re.findall(r"\[.*?\]\((.*?)\)", data) - linknames = re.findall(r"\[(.*?)\]\(.*?\)", data) + urlscmd = re.findall(r"\[.*?\]\n*\(.*?\)", data) + urls = re.findall(r"\[.*?\]\n*\((.*?)\)", data) + linknames = re.findall(r"\[(.*?)\]\n*\(.*?\)", data) for alinkname, aurl, aurlscmd in zip(linknames, urls, urlscmd): data = data.replace(aurlscmd, '`' + alinkname +'`_') From 55650c770bfc8adf3f8e9ba7379f34de0903aa84 Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 10:58:15 -0600 Subject: [PATCH 08/11] PEP8 cleanup --- docs/parse_table.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/parse_table.py b/docs/parse_table.py index 3eff90416..105edb91a 100755 --- a/docs/parse_table.py +++ b/docs/parse_table.py @@ -12,17 +12,19 @@ import argparse import re + def markdown_links(data, footer): urlscmd = re.findall(r"\[.*?\]\n*\(.*?\)", data) urls = re.findall(r"\[.*?\]\n*\((.*?)\)", data) linknames = re.findall(r"\[(.*?)\]\n*\(.*?\)", data) for alinkname, aurl, aurlscmd in zip(linknames, urls, urlscmd): - data = data.replace(aurlscmd, '`' + alinkname +'`_') + data = data.replace(aurlscmd, '`' + alinkname + '`_') footer += ".. _`" + alinkname + "`: " + aurl + "\n" return data, footer + def spurious_newline_whitespace(data, _): whitespace = re.findall('\n\s*', data) if len(whitespace) > 0: @@ -30,12 +32,14 @@ def spurious_newline_whitespace(data, _): data = data.replace(astr, "\n") return data, _ + def cleanup(linedata, footer): cleanups = [spurious_newline_whitespace, markdown_links] for acleanup in cleanups: linedata, footer = acleanup(linedata, footer) return linedata, footer + def build_rst_table_from_xml(xmlfile, rstfile, component): # open xml file for reading @@ -45,12 +49,14 @@ def build_rst_table_from_xml(xmlfile, rstfile, component): rst = open(rstfile, 'w') # get headers and build list to write entries - headers = xml.getroot().attrib['headers'].replace(' ','').split(',') - headernames = [aname.strip() for aname in xml.getroot().attrib['headernames'].split(',')] + headers = xml.getroot().attrib['headers'].replace(' ', '').split(',') + headernames = [aname.strip() for aname in + xml.getroot().attrib['headernames'].split(',')] data = [] footer = '\n' for entry in xml.findall('observation'): - if component != 'all' and entry.findall('component')[0].text.strip() != component: + if (component != 'all' and + entry.findall('component')[0].text.strip() != component): continue line = [] for aheader in headers: @@ -64,18 +70,22 @@ def build_rst_table_from_xml(xmlfile, rstfile, component): rst.close() + if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-x", "--xml_table", dest="xml_table", - help="Path to file containing xml description of table", + help="Path to file containing " + "xml description of table", metavar="FILE", required=True) parser.add_argument("-r", "--rst_table", dest="rst_table", - help="Path to file containing rst description of table for output", + help="Path to file containing rst description " + "of table for output", metavar="FILE", required=True) parser.add_argument("-c", "--component", dest="component", - help="Component for parsing of table, 'landice', 'ocean', " - "'seaice', or 'all'", metavar="STRING", default="all") + help="Component for parsing of table: " + "'landice', 'ocean', 'seaice', or 'all'", + metavar="STRING", default="all") args = parser.parse_args() From 7c3bb1d2e31e806c84ada80b44006348277e4cee Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 11:00:10 -0600 Subject: [PATCH 09/11] Cleans up order of observations in docs --- docs/index.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index e5d0002d8..1df535021 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,12 +14,6 @@ Analysis for simulations produced with Model for Prediction Across Scales (MPAS) components and the Accelerated Climate Model for Energy (ACME), which used those components. -Observations -============ -.. toctree:: - - observations - Documentation ============= .. toctree:: @@ -38,6 +32,11 @@ MPAS Components and E3SM mpasseaice e3sm +Observations +============ +.. toctree:: + + observations Indices and tables ================== From 31dfbe1478efc681e088ae2c786029d44509fa04 Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 11:18:15 -0600 Subject: [PATCH 10/11] Fixes line issue warning building doc --- docs/parse_table.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/parse_table.py b/docs/parse_table.py index 105edb91a..c44ab3c30 100755 --- a/docs/parse_table.py +++ b/docs/parse_table.py @@ -53,7 +53,7 @@ def build_rst_table_from_xml(xmlfile, rstfile, component): headernames = [aname.strip() for aname in xml.getroot().attrib['headernames'].split(',')] data = [] - footer = '\n' + footer = '' for entry in xml.findall('observation'): if (component != 'all' and entry.findall('component')[0].text.strip() != component): @@ -65,8 +65,10 @@ def build_rst_table_from_xml(xmlfile, rstfile, component): line.append(linedata) data.append(line) - rst.write(tabulate.tabulate(data, headernames, tablefmt='rst')) + rst.writelines(tabulate.tabulate(data, headernames, tablefmt='rst') + '\n') + rst.write('\n') rst.write(footer) + rst.write('\n') rst.close() From d46209302a7f20d30d37f0b1c77814bfa651ff6e Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Tue, 17 Apr 2018 11:49:38 -0600 Subject: [PATCH 11/11] White space fix --- docs/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/environment.yml b/docs/environment.yml index d67f332fb..8d5b05f2c 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -21,6 +21,6 @@ dependencies: - sphinx - sphinx_rtd_theme - numpydoc - - tabulate >= 0.8.2 + - tabulate >= 0.8.2 - pip: - recommonmark