Euclid: upgrade the TAP method upload_table to upload tables in fits format

CHANGES.rst

@@ -26,12 +26,14 @@ heasarc
 - Heasarc.locate_data returns empty rows with an error in the error_message column if there are
   no data associated with that row rather than filtering it out. [#3275]
 
-
 utils.tap
 ^^^^^^^^^
 
 - Get the cookie associated to the keys JSESSIONID or SESSION due to the tap library release at ESAC. [#3289]
 
+- The method ``upload_table`` accepts file formats accepted by astropy's
+  ``Table.read()``. [#3295]
+
 
 Infrastructure, Utility and Other Changes and Additions
 -------------------------------------------------------

astroquery/esa/euclid/__init__.py

@@ -15,12 +15,6 @@ class Conf(_config.ConfigNamespace):
     Configuration parameters for `astroquery.esa.euclid`.
     """
 
-    URL_BASE = _config.ConfigItem('https://eas.esac.esa.int/', 'Euclid base URL')
-
-    EUCLID_TAP_SERVER = _config.ConfigItem('https://easidr.esac.esa.int/tap-server/tap', 'Euclid TAP Server')
-    EUCLID_DATALINK_SERVER = _config.ConfigItem("https://easidr.esac.esa.int/sas-dd/data?", "Euclid DataLink Server")
-    EUCLID_CUTOUT_SERVER = _config.ConfigItem("https://easidr.esac.esa.int/sas-cutout/cutout?", "Euclid Cutout Server")
-
     ROW_LIMIT = _config.ConfigItem(50,
                                    "Number of rows to return from database query (set to -1 for unlimited).")
 

astroquery/utils/tap/conn/tests/DummyConnHandler.py

@@ -14,10 +14,11 @@
 
 
 """
-from astroquery.utils.tap import taputils
-
 import requests
 
+from astroquery.utils.tap import taputils
+from astroquery.utils.tap.conn.tapconn import TapConn
+
 
 class DummyConnHandler:
 
@@ -158,3 +159,12 @@ def execute_secure(self, subcontext=None, data=None, verbose=False):
 
     def get_host_url(self):
         return "my fake object"
+
+    def encode_multipart(self, fields, files):
+        tap = TapConn(ishttps=False, host='host')
+        return tap.encode_multipart(fields, files)
+
+    def execute_upload(self, data,
+                       content_type="application/x-www-form-urlencoded", *,
+                       verbose=False):
+        return self.defaultResponse

astroquery/utils/tap/core.py

@@ -15,11 +15,12 @@
 """
 import getpass
 import os
-import requests
 import tempfile
-from astropy.table.table import Table
 from urllib.parse import urlencode
 
+import requests
+from astropy.table.table import Table
+
 from astroquery import log
 from astroquery.utils.tap import taputils
 from astroquery.utils.tap.conn.tapconn import TapConn
@@ -1341,8 +1342,9 @@ def upload_table(self, *, upload_resource=None, table_name=None, table_descripti
             resource temporary table name associated to the uploaded resource
         table_description : str, optional, default None
             table description
-        format : str, optional, default 'VOTable'
-            resource format
+        format : str, optional, default 'votable'
+            resource format.  Only formats described in
+            https://docs.astropy.org/en/stable/io/unified.html#built-in-table-readers-writers are accepted.
         verbose : bool, optional, default 'False'
             flag to display information about the process
         """
@@ -1381,9 +1383,7 @@ def upload_table(self, *, upload_resource=None, table_name=None, table_descripti
             log.info(f"Uploaded table '{table_name}'.")
             return None
 
-    def __uploadTableMultipart(self, resource, *, table_name=None,
-                               table_description=None,
-                               resource_format="VOTable",
+    def __uploadTableMultipart(self, resource, *, table_name=None, table_description=None, resource_format="votable",
                                verbose=False):
         connHandler = self.__getconnhandler()
         if isinstance(resource, Table):
@@ -1397,24 +1397,38 @@ def __uploadTableMultipart(self, resource, *, table_name=None,
             fh = tempfile.NamedTemporaryFile(delete=False)
             resource.write(fh, format='votable')
             fh.close()
-            f = open(fh.name, "r")
-            chunk = f.read()
-            f.close()
+
+            with open(fh.name, "r") as f:
+                chunk = f.read()
+
             os.unlink(fh.name)
             files = [['FILE', 'pytable', chunk]]
-            contentType, body = connHandler.encode_multipart(args, files)
+            content_type, body = connHandler.encode_multipart(args, files)
         else:
             if not (str(resource).startswith("http")):  # upload from file
                 args = {
                     "TASKID": str(-1),
                     "TABLE_NAME": str(table_name),
                     "TABLE_DESC": str(table_description),
-                    "FORMAT": str(resource_format)}
+                    "FORMAT": 'votable'}
                 log.info(f"Sending file: {resource}")
-                with open(resource, "r") as f:
-                    chunk = f.read()
-                files = [['FILE', os.path.basename(resource), chunk]]
-                contentType, body = connHandler.encode_multipart(args, files)
+                if resource_format.lower() == 'votable':
+                    with open(resource, "r") as f:
+                        chunk = f.read()
+                    files = [['FILE', os.path.basename(resource), chunk]]
+                else:
+                    table = Table.read(str(resource), format=resource_format)
+                    fh = tempfile.NamedTemporaryFile(delete=False)
+                    table.write(fh, format='votable')
+                    fh.close()
+
+                    with open(fh.name, "r") as f:
+                        chunk = f.read()
+
+                    os.unlink(fh.name)
+                    files = [['FILE', 'pytable', chunk]]
+
+                content_type, body = connHandler.encode_multipart(args, files)
             else:  # upload from URL
                 args = {
                     "TASKID": str(-1),
@@ -1423,8 +1437,8 @@ def __uploadTableMultipart(self, resource, *, table_name=None,
                     "FORMAT": str(resource_format),
                     "URL": str(resource)}
                 files = [['FILE', "", ""]]
-                contentType, body = connHandler.encode_multipart(args, files)
-        response = connHandler.execute_upload(body, contentType)
+                content_type, body = connHandler.encode_multipart(args, files)
+        response = connHandler.execute_upload(body, content_type)
         if verbose:
             print(response.status, response.reason)
             print(response.getheaders())

astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.csv

@@ -0,0 +1,3 @@
+source_id,ra,dec
+3834447128563320320,149.8678677871318,1.12773018116361
+3834447162923057280,149.8953864417848,1.1345712682426434

astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.ecsv

@@ -0,0 +1,41 @@
+# %ECSV 1.0
+# ---
+# delimiter: ','
+# datatype:
+# -
+#   name: source_id
+#   datatype: int64
+#   description: Unique source identifier (unique within a particular Data Release)
+#   meta:
+#     ucd: meta.id
+# -
+#   name: ra
+#   datatype: float64
+#   unit: deg
+#   description: Right ascension
+#   meta:
+#     ucd: pos.eq.ra;meta.main
+#     utype: stc:AstroCoords.Position3D.Value3.C1
+#     CoosysSystem: ICRS
+#     CoosysEpoch: J2016.0
+# -
+#   name: dec
+#   datatype: float64
+#   unit: deg
+#   description: Declination
+#   meta:
+#     ucd: pos.eq.dec;meta.main
+#     utype: stc:AstroCoords.Position3D.Value3.C2
+#     CoosysSystem: ICRS
+#     CoosysEpoch: J2016.0
+# meta:
+#   name: votable
+#   QUERY_STATUS: OK
+#   QUERY: 'SELECT TOP 2 source_id, ra, dec FROM gaiadr3.gaia_source '
+#   CAPTION: 'How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html'
+#   CITATION: 'How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html'
+#   JOBID: 1744351221317O
+#   RELEASE: Gaia DR3
+source_id,ra,dec
+3834447128563320320,149.8678677871318,1.12773018116361
+3834447162923057280,149.8953864417848,1.1345712682426434

astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.json

@@ -0,0 +1,95 @@
+{
+	"VOTABLE": {
+		"RESOURCE": {
+			"INFO": [
+				{
+					"_name": "QUERY_STATUS",
+					"_value": "OK"
+				},
+				{
+					"_name": "QUERY",
+					"_value": "SELECT TOP 20 * FROM user_jferna01.my_votable_fits ",
+					"__cdata": "SELECT TOP 20 *\nFROM user_jferna01.my_votable_fits "
+				},
+				{
+					"_name": "CAPTION",
+					"_value": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html",
+					"__cdata": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"
+				},
+				{
+					"_name": "CITATION",
+					"_value": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html",
+					"_ucd": "meta.bib",
+					"__cdata": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"
+				},
+				{
+					"_name": "PAGE",
+					"_value": ""
+				},
+				{
+					"_name": "PAGE_SIZE",
+					"_value": ""
+				},
+				{
+					"_name": "JOBID",
+					"_value": "1744360074103O",
+					"__cdata": "1744360074103O"
+				},
+				{
+					"_name": "JOBNAME",
+					"_value": ""
+				}
+			],
+			"TABLE": {
+				"FIELD": [
+					{
+						"DESCRIPTION": "Object Identifier",
+						"_datatype": "int",
+						"_name": "my_votable_fits_oid"
+					},
+					{
+						"_datatype": "long",
+						"_name": "source_id"
+					},
+					{
+						"_datatype": "double",
+						"_name": "ra",
+						"_unit": "deg"
+					},
+					{
+						"_datatype": "double",
+						"_name": "dec",
+						"_unit": "deg"
+					}
+				],
+				"DATA": {
+					"TABLEDATA": {
+						"TR": [
+							{
+								"TD": [
+									"1",
+									"3834447128563320320",
+									"149.8678677871318",
+									"1.12773018116361"
+								]
+							},
+							{
+								"TD": [
+									"2",
+									"3834447162923057280",
+									"149.8953864417848",
+									"1.1345712682426434"
+								]
+							}
+						]
+					}
+				}
+			},
+			"_type": "results"
+		},
+		"_version": "1.4",
+		"_xmlns": "http://www.ivoa.net/xml/VOTable/v1.3",
+		"_xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
+		"_xsi:schemaLocation": "http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/votable-1.4.xsd"
+	}
+}

astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.vot

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/votable-1.4.xsd">
+<RESOURCE type="results">
+<INFO name="QUERY_STATUS" value="OK"/>
+
+<INFO name="QUERY" value="SELECT TOP 20 *
+FROM user_jferna01.my_votable_fits "><![CDATA[SELECT TOP 20 *
+FROM user_jferna01.my_votable_fits ]]></INFO>
+<INFO name="CAPTION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
+<INFO name="CITATION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html" ucd="meta.bib"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
+<INFO name="PAGE" value=""/>
+<INFO name="PAGE_SIZE" value=""/>
+<INFO name="JOBID" value="1744360074103O"><![CDATA[1744360074103O]]></INFO>
+<INFO name="JOBNAME" value=""></INFO>
+
+<TABLE>
+<FIELD datatype="int" name="my_votable_fits_oid">
+<DESCRIPTION>Object Identifier</DESCRIPTION>
+</FIELD>
+<FIELD datatype="long" name="source_id"/>
+<FIELD datatype="double" name="ra" unit="deg"/>
+<FIELD datatype="double" name="dec" unit="deg"/>
+<DATA>
+<BINARY2>
+<STREAM encoding='base64'>
+AAAAAAE1NrFZAAiGAEBiu8WSql90P/ILLs1s9TAAAAAAAjU2sWEACICAQGK8pwF3
+l+w/8ic0M8FVmA==
+</STREAM>
+</BINARY2>
+</DATA>
+</TABLE>
+</RESOURCE>
+</VOTABLE>

astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result_plain.vot

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/votable-1.4.xsd">
+<RESOURCE type="results">
+<INFO name="QUERY_STATUS" value="OK"/>
+
+<INFO name="QUERY" value="SELECT TOP 20 * FROM user_jferna01.my_votable_fits "><![CDATA[SELECT TOP 20 *
+FROM user_jferna01.my_votable_fits ]]></INFO>
+<INFO name="CAPTION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
+<INFO name="CITATION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html" ucd="meta.bib"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
+<INFO name="PAGE" value=""/>
+<INFO name="PAGE_SIZE" value=""/>
+<INFO name="JOBID" value="1744360074103O"><![CDATA[1744360074103O]]></INFO>
+<INFO name="JOBNAME" value=""/>
+
+<TABLE>
+<FIELD datatype="int" name="my_votable_fits_oid">
+<DESCRIPTION>Object Identifier</DESCRIPTION>
+</FIELD>
+<FIELD datatype="long" name="source_id"/>
+<FIELD datatype="double" name="ra" unit="deg"/>
+<FIELD datatype="double" name="dec" unit="deg"/>
+<DATA>
+<TABLEDATA>
+<TR><TD>1</TD><TD>3834447128563320320</TD><TD>149.8678677871318</TD><TD>1.12773018116361</TD></TR>
+<TR><TD>2</TD><TD>3834447162923057280</TD><TD>149.8953864417848</TD><TD>1.1345712682426434</TD></TR>
+</TABLEDATA>
+</DATA>
+
+</TABLE>
+</RESOURCE>
+</VOTABLE>

astroquery/utils/tap/tests/setup_package.py

@@ -15,7 +15,6 @@
 
 """
 
-
 import os
 
 
@@ -24,7 +23,17 @@
 def get_package_data():
     paths = [os.path.join('data', '*.vot'),
              os.path.join('data', '*.xml'),
+             os.path.join('data', '*.csv'),
+             os.path.join('data', '*.ecsv'),
+             os.path.join('data', '*.json'),
+             os.path.join('data', '*.fits'),
              os.path.join('data', '*.fits.gz'),
+             os.path.join('data/test_upload_file', '*.vot'),
+             os.path.join('data/test_upload_file', '*.xml'),
+             os.path.join('data/test_upload_file', '*.csv'),
+             os.path.join('data/test_upload_file', '*.ecsv'),
+             os.path.join('data/test_upload_file', '*.json'),
+             os.path.join('data/test_upload_file', '*.fits'),
              ]  # etc, add other extensions
     # you can also enlist files individually by names
     # finally construct and return a dict for the sub module