Skip to content

Annotate to_json #26449

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 8 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions pandas/io/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,23 @@
import lzma
import mmap
import os
import pathlib
from urllib.error import URLError # noqa
from urllib.parse import ( # noqa
urlencode, urljoin, urlparse as parse_url, uses_netloc, uses_params,
uses_relative)
from urllib.request import pathname2url, urlopen
import zipfile
from typing import AnyStr, IO, Union

from pandas.errors import ( # noqa
AbstractMethodError, DtypeWarning, EmptyDataError, ParserError,
ParserWarning)

from pandas.core.dtypes.common import is_file_like

from pandas._typing import FilePathOrBuffer

# gh-12665: Alias for now and remove later.
CParserError = ParserError

Expand Down Expand Up @@ -67,7 +71,8 @@ def _is_url(url):
return False


def _expand_user(filepath_or_buffer):
def _expand_user(
filepath_or_buffer: FilePathOrBuffer) -> FilePathOrBuffer:
"""Return the argument with an initial component of ~ or ~user
replaced by that user's home directory.

Expand All @@ -82,6 +87,8 @@ def _expand_user(filepath_or_buffer):
"""
if isinstance(filepath_or_buffer, str):
return os.path.expanduser(filepath_or_buffer)
elif isinstance(filepath_or_buffer, pathlib.Path):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We weren't expanding Path objects with existing code so this should handle that as well. Reviewing where to add test for this

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might split this off into a separate issue/PR if I can find a bug using Path objects not expanding

return filepath_or_buffer.expanduser()
return filepath_or_buffer


Expand All @@ -93,7 +100,8 @@ def _validate_header_arg(header):
"the row(s) making up the column names")


def _stringify_path(filepath_or_buffer):
def _stringify_path(
filepath_or_buffer: FilePathOrBuffer) -> Union[str, IO[AnyStr]]:
"""Attempt to convert a path-like object to a string.

Parameters
Expand All @@ -115,25 +123,24 @@ def _stringify_path(filepath_or_buffer):
Any other object is passed through unchanged, which includes bytes,
strings, buffers, or anything else that's not even path-like.
"""
try:
import pathlib
_PATHLIB_INSTALLED = True
except ImportError:
_PATHLIB_INSTALLED = False

try:
from py.path import local as LocalPath
_PY_PATH_INSTALLED = True
except ImportError:
_PY_PATH_INSTALLED = False

if hasattr(filepath_or_buffer, '__fspath__'):
return filepath_or_buffer.__fspath__()
if _PATHLIB_INSTALLED and isinstance(filepath_or_buffer, pathlib.Path):
return str(filepath_or_buffer)
# mypy lacks comprehensive support for hasattr; see mypy#1424
# TODO (PY36): refactor to use os.PathLike
return filepath_or_buffer.__fspath__() # type: ignore
if _PY_PATH_INSTALLED and isinstance(filepath_or_buffer, LocalPath):
return filepath_or_buffer.strpath
return _expand_user(filepath_or_buffer)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had to refactor this because Mypy wasn't narrowing types at this point and still guessed that Path objects could be returned here based off of the signature of _expand_user even though the preceding isinstance check wouldn't allow that.

I commented on python/mypy#6847 (comment) so will see if that gets any attention, but the refactor works as well


expanded = _expand_user(filepath_or_buffer)
if isinstance(expanded, pathlib.Path):
return str(expanded)

return expanded


def is_s3_url(url):
Expand Down
33 changes: 23 additions & 10 deletions pandas/io/json/json.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from io import StringIO
from itertools import islice
import os
from typing import Callable, Optional, Type, Union

import numpy as np

Expand All @@ -12,6 +13,7 @@
from pandas.core.dtypes.common import is_period_dtype

from pandas import DataFrame, MultiIndex, Series, isna, to_datetime
from pandas._typing import FilePathOrBuffer
from pandas.core.reshape.concat import concat

from pandas.io.common import (
Expand All @@ -30,24 +32,32 @@


# interface to/from
def to_json(path_or_buf, obj, orient=None, date_format='epoch',
double_precision=10, force_ascii=True, date_unit='ms',
default_handler=None, lines=False, compression='infer',
index=True):
def to_json(
path_or_buf: FilePathOrBuffer,
obj: Union[Series, DataFrame],
orient: str = None,
date_format: str = 'epoch',
double_precision: int = 10,
force_ascii: bool = True,
date_unit: str = 'ms',
default_handler: Callable = None,
lines: bool = False,
compression: str = 'infer',
index: bool = True) -> Optional[str]:

if not index and orient not in ['split', 'table']:
raise ValueError("'index=False' is only valid when 'orient' is "
"'split' or 'table'")

path_or_buf = _stringify_path(path_or_buf)
str_or_io = _stringify_path(path_or_buf)
if lines and orient != 'records':
raise ValueError(
"'lines' keyword only valid when 'orient' is records")

if orient == 'table' and isinstance(obj, Series):
obj = obj.to_frame(name=obj.name or 'values')
if orient == 'table' and isinstance(obj, DataFrame):
writer = JSONTableWriter
writer = JSONTableWriter # type: Type[Writer]
elif isinstance(obj, Series):
writer = SeriesWriter
elif isinstance(obj, DataFrame):
Expand All @@ -64,16 +74,19 @@ def to_json(path_or_buf, obj, orient=None, date_format='epoch',
if lines:
s = _convert_to_line_delimits(s)

if isinstance(path_or_buf, str):
fh, handles = _get_handle(path_or_buf, 'w', compression=compression)
if isinstance(str_or_io, str):
fh, handles = _get_handle(
str_or_io, 'w', compression=compression)
try:
fh.write(s)
finally:
fh.close()
elif path_or_buf is None:
elif str_or_io is None:
return s
else:
path_or_buf.write(s)
str_or_io.write(s)

return None


class Writer:
Expand Down