Pinot BIG_DECIMAL columns are not accessible via SQLAlchemy #64
Description
When retrieving Pinot table metadata with sqlalchemy the Pinot BIG_DECIMAL type is not currently supported. This prevents Apache SuperSet from creating datasets for any Pinot tables that contain BIG_DECIMAL columns. The following code triggers this:
from sqlalchemy import *
from sqlalchemy.engine import create_engine
engine = create_engine("pinot+http://<broker_host>:<broker_port>/query/sql?controller=http://<controller_host>:<controller_port>/")
conn = engine.connect()
metadata = MetaData()
t = Table("mytable", metadata, autoload_with=conn)This results in the following error:
Traceback (most recent call last):
File "/home/keri/testbed/test.py", line 23, in <module>
t = Table("arm_commission", metadata, autoload_with=conn)
File "<string>", line 2, in __new__
File "/path/to/sqlalchemy/util/deprecations.py", line 277, in warned
return fn(*args, **kwargs) # type: ignore[no-any-return]
File "/path/to/sqlalchemy/sql/schema.py", line 432, in __new__
return cls._new(*args, **kw)
File "/path/to/sqlalchemy/sql/schema.py", line 486, in _new
with util.safe_reraise():
File "/path/to/sqlalchemy/util/langhelpers.py", line 147, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/path/to/sqlalchemy/sql/schema.py", line 482, in _new
table.__init__(name, metadata, *args, _no_init=False, **kw)
File "/path/to/sqlalchemy/sql/schema.py", line 858, in __init__
self._autoload(
File "/path/to/sqlalchemy/sql/schema.py", line 890, in _autoload
conn_insp.reflect_table(
File "/path/to/sqlalchemy/engine/reflection.py", line 1535, in reflect_table
_reflect_info = self._get_reflection_info(
File "/path/to/sqlalchemy/engine/reflection.py", line 2014, in _get_reflection_info
columns=run(
File "/path/to/sqlalchemy/engine/reflection.py", line 2000, in run
res = meth(filter_names=_fn, **kw)
File "/path/to/sqlalchemy/engine/reflection.py", line 928, in get_multi_columns
table_col_defs = dict(
File "/path/to/sqlalchemy/engine/default.py", line 917, in _default_multi_reflect
single_tbl_method(
File "/path/to/pinotdb/sqlalchemy.py", line 252, in get_columns
columns = [
File "/path/to/pinotdb/sqlalchemy.py", line 255, in <listcomp>
"type": get_type(spec["dataType"], spec.get("fieldSize")),
File "/path/to/pinotdb/sqlalchemy.py", line 329, in get_type
return type_map[data_type.lower()]
KeyError: 'big_decimal'
Metadata
Metadata
Assignees
Labels
No labels