saad
/
ai-content-maker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
ai-content-maker/.venv/Lib/site-packages/pandas/io/json/_json.py

1415 lines
43 KiB
Python
Raw Normal View History

first commit
2024-05-03 04:18:51 +03:00
from __future__ import annotations
from abc import (
ABC,
abstractmethod,
)
from collections import abc
import functools
from io import StringIO
from itertools import islice
from typing import (
TYPE_CHECKING,
Any,
Callable,
Generic,
Literal,
Mapping,
TypeVar,
overload,
)
import numpy as np
import pandas._libs.json as json
from pandas._libs.tslibs import iNaT
from pandas._typing import (
CompressionOptions,
DtypeArg,
FilePath,
IndexLabel,
JSONSerializable,
ReadBuffer,
StorageOptions,
WriteBuffer,
)
from pandas.errors import AbstractMethodError
from pandas.util._decorators import (
deprecate_kwarg,
deprecate_nonkeyword_arguments,
doc,
)
from pandas.core.dtypes.common import (
ensure_str,
is_period_dtype,
)
from pandas import (
DataFrame,
MultiIndex,
Series,
isna,
notna,
to_datetime,
)
from pandas.core.construction import create_series_with_explicit_dtype
from pandas.core.reshape.concat import concat
from pandas.core.shared_docs import _shared_docs
from pandas.io.common import (
IOHandles,
_extension_to_compression,
file_exists,
get_handle,
is_fsspec_url,
is_url,
stringify_path,
)
from pandas.io.json._normalize import convert_to_line_delimits
from pandas.io.json._table_schema import (
build_table_schema,
parse_table_schema,
)
from pandas.io.parsers.readers import validate_integer
if TYPE_CHECKING:
from pandas.core.generic import NDFrame
FrameSeriesStrT = TypeVar("FrameSeriesStrT", bound=Literal["frame", "series"])
loads = json.loads
dumps = json.dumps
# interface to/from
@overload
def to_json(
path_or_buf: FilePath | WriteBuffer[str] | WriteBuffer[bytes],
obj: NDFrame,
orient: str | None = ...,
date_format: str = ...,
double_precision: int = ...,
force_ascii: bool = ...,
date_unit: str = ...,
default_handler: Callable[[Any], JSONSerializable] | None = ...,
lines: bool = ...,
compression: CompressionOptions = ...,
index: bool = ...,
indent: int = ...,
storage_options: StorageOptions = ...,
) -> None:
...
@overload
def to_json(
path_or_buf: None,
obj: NDFrame,
orient: str | None = ...,
date_format: str = ...,
double_precision: int = ...,
force_ascii: bool = ...,
date_unit: str = ...,
default_handler: Callable[[Any], JSONSerializable] | None = ...,
lines: bool = ...,
compression: CompressionOptions = ...,
index: bool = ...,
indent: int = ...,
storage_options: StorageOptions = ...,
) -> str:
...
def to_json(
path_or_buf: FilePath | WriteBuffer[str] | WriteBuffer[bytes] | None,
obj: NDFrame,
orient: str | None = None,
date_format: str = "epoch",
double_precision: int = 10,
force_ascii: bool = True,
date_unit: str = "ms",
default_handler: Callable[[Any], JSONSerializable] | None = None,
lines: bool = False,
compression: CompressionOptions = "infer",
index: bool = True,
indent: int = 0,
storage_options: StorageOptions = None,
) -> str | None:
if not index and orient not in ["split", "table"]:
raise ValueError(
"'index=False' is only valid when 'orient' is 'split' or 'table'"
)
if lines and orient != "records":
raise ValueError("'lines' keyword only valid when 'orient' is records")
if orient == "table" and isinstance(obj, Series):
obj = obj.to_frame(name=obj.name or "values")
writer: type[Writer]
if orient == "table" and isinstance(obj, DataFrame):
writer = JSONTableWriter
elif isinstance(obj, Series):
writer = SeriesWriter
elif isinstance(obj, DataFrame):
writer = FrameWriter
else:
raise NotImplementedError("'obj' should be a Series or a DataFrame")
s = writer(
obj,
orient=orient,
date_format=date_format,
double_precision=double_precision,
ensure_ascii=force_ascii,
date_unit=date_unit,
default_handler=default_handler,
index=index,
indent=indent,
).write()
if lines:
s = convert_to_line_delimits(s)
if path_or_buf is not None:
# apply compression and byte/text conversion
with get_handle(
path_or_buf, "w", compression=compression, storage_options=storage_options
) as handles:
handles.handle.write(s)
else:
return s
return None
class Writer(ABC):
_default_orient: str
def __init__(
self,
obj,
orient: str | None,
date_format: str,
double_precision: int,
ensure_ascii: bool,
date_unit: str,
index: bool,
default_handler: Callable[[Any], JSONSerializable] | None = None,
indent: int = 0,
) -> None:
self.obj = obj
if orient is None:
orient = self._default_orient
self.orient = orient
self.date_format = date_format
self.double_precision = double_precision
self.ensure_ascii = ensure_ascii
self.date_unit = date_unit
self.default_handler = default_handler
self.index = index
self.indent = indent
self.is_copy = None
self._format_axes()
def _format_axes(self):
raise AbstractMethodError(self)
def write(self) -> str:
iso_dates = self.date_format == "iso"
return dumps(
self.obj_to_write,
orient=self.orient,
double_precision=self.double_precision,
ensure_ascii=self.ensure_ascii,
date_unit=self.date_unit,
iso_dates=iso_dates,
default_handler=self.default_handler,
indent=self.indent,
)
@property
@abstractmethod
def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
"""Object to write in JSON format."""
pass
class SeriesWriter(Writer):
_default_orient = "index"
@property
def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
if not self.index and self.orient == "split":
return {"name": self.obj.name, "data": self.obj.values}
else:
return self.obj
def _format_axes(self):
if not self.obj.index.is_unique and self.orient == "index":
raise ValueError(f"Series index must be unique for orient='{self.orient}'")
class FrameWriter(Writer):
_default_orient = "columns"
@property
def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
if not self.index and self.orient == "split":
obj_to_write = self.obj.to_dict(orient="split")
del obj_to_write["index"]
else:
obj_to_write = self.obj
return obj_to_write
def _format_axes(self):
"""
Try to format axes if they are datelike.
"""
if not self.obj.index.is_unique and self.orient in ("index", "columns"):
raise ValueError(
f"DataFrame index must be unique for orient='{self.orient}'."
)
if not self.obj.columns.is_unique and self.orient in (
"index",
"columns",
"records",
):
raise ValueError(
f"DataFrame columns must be unique for orient='{self.orient}'."
)
class JSONTableWriter(FrameWriter):
_default_orient = "records"
def __init__(
self,
obj,
orient: str | None,
date_format: str,
double_precision: int,
ensure_ascii: bool,
date_unit: str,
index: bool,
default_handler: Callable[[Any], JSONSerializable] | None = None,
indent: int = 0,
) -> None:
"""
Adds a `schema` attribute with the Table Schema, resets
the index (can't do in caller, because the schema inference needs
to know what the index is, forces orient to records, and forces
date_format to 'iso'.
"""
super().__init__(
obj,
orient,
date_format,
double_precision,
ensure_ascii,
date_unit,
index,
default_handler=default_handler,
indent=indent,
)
if date_format != "iso":
msg = (
"Trying to write with `orient='table'` and "
f"`date_format='{date_format}'`. Table Schema requires dates "
"to be formatted with `date_format='iso'`"
)
raise ValueError(msg)
self.schema = build_table_schema(obj, index=self.index)
# NotImplemented on a column MultiIndex
if obj.ndim == 2 and isinstance(obj.columns, MultiIndex):
raise NotImplementedError(
"orient='table' is not supported for MultiIndex columns"
)
# TODO: Do this timedelta properly in objToJSON.c See GH #15137
if (
(obj.ndim == 1)
and (obj.name in set(obj.index.names))
or len(obj.columns.intersection(obj.index.names))
):
msg = "Overlapping names between the index and columns"
raise ValueError(msg)
obj = obj.copy()
timedeltas = obj.select_dtypes(include=["timedelta"]).columns
if len(timedeltas):
obj[timedeltas] = obj[timedeltas].applymap(lambda x: x.isoformat())
# Convert PeriodIndex to datetimes before serializing
if is_period_dtype(obj.index.dtype):
obj.index = obj.index.to_timestamp()
# exclude index from obj if index=False
if not self.index:
self.obj = obj.reset_index(drop=True)
else:
self.obj = obj.reset_index(drop=False)
self.date_format = "iso"
self.orient = "records"
self.index = index
@property
def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
return {"schema": self.schema, "data": self.obj}
@overload
def read_json(
path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
*,
orient: str | None = ...,
typ: Literal["frame"] = ...,
dtype: DtypeArg | None = ...,
convert_axes=...,
convert_dates: bool | list[str] = ...,
keep_default_dates: bool = ...,
numpy: bool = ...,
precise_float: bool = ...,
date_unit: str | None = ...,
encoding: str | None = ...,
encoding_errors: str | None = ...,
lines: bool = ...,
chunksize: int,
compression: CompressionOptions = ...,
nrows: int | None = ...,
storage_options: StorageOptions = ...,
) -> JsonReader[Literal["frame"]]:
...
@overload
def read_json(
path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
*,
orient: str | None = ...,
typ: Literal["series"],
dtype: DtypeArg | None = ...,
convert_axes=...,
convert_dates: bool | list[str] = ...,
keep_default_dates: bool = ...,
numpy: bool = ...,
precise_float: bool = ...,
date_unit: str | None = ...,
encoding: str | None = ...,
encoding_errors: str | None = ...,
lines: bool = ...,
chunksize: int,
compression: CompressionOptions = ...,
nrows: int | None = ...,
storage_options: StorageOptions = ...,
) -> JsonReader[Literal["series"]]:
...
@overload
def read_json(
path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
*,
orient: str | None = ...,
typ: Literal["series"],
dtype: DtypeArg | None = ...,
convert_axes=...,
convert_dates: bool | list[str] = ...,
keep_default_dates: bool = ...,
numpy: bool = ...,
precise_float: bool = ...,
date_unit: str | None = ...,
encoding: str | None = ...,
encoding_errors: str | None = ...,
lines: bool = ...,
chunksize: None = ...,
compression: CompressionOptions = ...,
nrows: int | None = ...,
storage_options: StorageOptions = ...,
) -> Series:
...
@overload
def read_json(
path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
orient: str | None = ...,
typ: Literal["frame"] = ...,
dtype: DtypeArg | None = ...,
convert_axes=...,
convert_dates: bool | list[str] = ...,
keep_default_dates: bool = ...,
numpy: bool = ...,
precise_float: bool = ...,
date_unit: str | None = ...,
encoding: str | None = ...,
encoding_errors: str | None = ...,
lines: bool = ...,
chunksize: None = ...,
compression: CompressionOptions = ...,
nrows: int | None = ...,
storage_options: StorageOptions = ...,
) -> DataFrame:
...
@doc(
storage_options=_shared_docs["storage_options"],
decompression_options=_shared_docs["decompression_options"] % "path_or_buf",
)
@deprecate_kwarg(old_arg_name="numpy", new_arg_name=None)
@deprecate_nonkeyword_arguments(version="2.0", allowed_args=["path_or_buf"])
def read_json(
path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
orient: str | None = None,
typ: Literal["frame", "series"] = "frame",
dtype: DtypeArg | None = None,
convert_axes=None,
convert_dates: bool | list[str] = True,
keep_default_dates: bool = True,
numpy: bool = False,
precise_float: bool = False,
date_unit: str | None = None,
encoding: str | None = None,
encoding_errors: str | None = "strict",
lines: bool = False,
chunksize: int | None = None,
compression: CompressionOptions = "infer",
nrows: int | None = None,
storage_options: StorageOptions = None,
) -> DataFrame | Series | JsonReader:
"""
Convert a JSON string to pandas object.
Parameters
----------
path_or_buf : a valid JSON str, path object or file-like object
Any valid string path is acceptable. The string could be a URL. Valid
URL schemes include http, ftp, s3, and file. For file URLs, a host is
expected. A local file could be:
``file://localhost/path/to/table.json``.
If you want to pass in a path object, pandas accepts any
``os.PathLike``.
By file-like object, we refer to objects with a ``read()`` method,
such as a file handle (e.g. via builtin ``open`` function)
or ``StringIO``.
orient : str
Indication of expected JSON string format.
Compatible JSON strings can be produced by ``to_json()`` with a
corresponding orient value.
The set of possible orients is:
- ``'split'`` : dict like
``{{index -> [index], columns -> [columns], data -> [values]}}``
- ``'records'`` : list like
``[{{column -> value}}, ... , {{column -> value}}]``
- ``'index'`` : dict like ``{{index -> {{column -> value}}}}``
- ``'columns'`` : dict like ``{{column -> {{index -> value}}}}``
- ``'values'`` : just the values array
The allowed and default values depend on the value
of the `typ` parameter.
* when ``typ == 'series'``,
- allowed orients are ``{{'split','records','index'}}``
- default is ``'index'``
- The Series index must be unique for orient ``'index'``.
* when ``typ == 'frame'``,
- allowed orients are ``{{'split','records','index',
'columns','values', 'table'}}``
- default is ``'columns'``
- The DataFrame index must be unique for orients ``'index'`` and
``'columns'``.
- The DataFrame columns must be unique for orients ``'index'``,
``'columns'``, and ``'records'``.
typ : {{'frame', 'series'}}, default 'frame'
The type of object to recover.
dtype : bool or dict, default None
If True, infer dtypes; if a dict of column to dtype, then use those;
if False, then don't infer dtypes at all, applies only to the data.
For all ``orient`` values except ``'table'``, default is True.
.. versionchanged:: 0.25.0
Not applicable for ``orient='table'``.
convert_axes : bool, default None
Try to convert the axes to the proper dtypes.
For all ``orient`` values except ``'table'``, default is True.
.. versionchanged:: 0.25.0
Not applicable for ``orient='table'``.
convert_dates : bool or list of str, default True
If True then default datelike columns may be converted (depending on
keep_default_dates).
If False, no dates will be converted.
If a list of column names, then those columns will be converted and
default datelike columns may also be converted (depending on
keep_default_dates).
keep_default_dates : bool, default True
If parsing dates (convert_dates is not False), then try to parse the
default datelike columns.
A column label is datelike if
* it ends with ``'_at'``,
* it ends with ``'_time'``,
* it begins with ``'timestamp'``,
* it is ``'modified'``, or
* it is ``'date'``.
numpy : bool, default False
Direct decoding to numpy arrays. Supports numeric data only, but
non-numeric column and index labels are supported. Note also that the
JSON ordering MUST be the same for each term if numpy=True.
.. deprecated:: 1.0.0
precise_float : bool, default False
Set to enable usage of higher precision (strtod) function when
decoding string to double values. Default (False) is to use fast but
less precise builtin functionality.
date_unit : str, default None
The timestamp unit to detect if converting dates. The default behaviour
is to try and detect the correct precision, but if this is not desired
then pass one of 's', 'ms', 'us' or 'ns' to force parsing only seconds,
milliseconds, microseconds or nanoseconds respectively.
encoding : str, default is 'utf-8'
The encoding to use to decode py3 bytes.
encoding_errors : str, optional, default "strict"
How encoding errors are treated. `List of possible values
<https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
.. versionadded:: 1.3.0
lines : bool, default False
Read the file as a json object per line.
chunksize : int, optional
Return JsonReader object for iteration.
See the `line-delimited json docs
<https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#line-delimited-json>`_
for more information on ``chunksize``.
This can only be passed if `lines=True`.
If this is None, the file will be read into memory all at once.
.. versionchanged:: 1.2
``JsonReader`` is a context manager.
{decompression_options}
.. versionchanged:: 1.4.0 Zstandard support.
nrows : int, optional
The number of lines from the line-delimited jsonfile that has to be read.
This can only be passed if `lines=True`.
If this is None, all the rows will be returned.
.. versionadded:: 1.1
{storage_options}
.. versionadded:: 1.2.0
Returns
-------
Series or DataFrame
The type returned depends on the value of `typ`.
See Also
--------
DataFrame.to_json : Convert a DataFrame to a JSON string.
Series.to_json : Convert a Series to a JSON string.
json_normalize : Normalize semi-structured JSON data into a flat table.
Notes
-----
Specific to ``orient='table'``, if a :class:`DataFrame` with a literal
:class:`Index` name of `index` gets written with :func:`to_json`, the
subsequent read operation will incorrectly set the :class:`Index` name to
``None``. This is because `index` is also used by :func:`DataFrame.to_json`
to denote a missing :class:`Index` name, and the subsequent
:func:`read_json` operation cannot distinguish between the two. The same
limitation is encountered with a :class:`MultiIndex` and any names
beginning with ``'level_'``.
Examples
--------
>>> df = pd.DataFrame([['a', 'b'], ['c', 'd']],
... index=['row 1', 'row 2'],
... columns=['col 1', 'col 2'])
Encoding/decoding a Dataframe using ``'split'`` formatted JSON:
>>> df.to_json(orient='split')
'\
{{\
"columns":["col 1","col 2"],\
"index":["row 1","row 2"],\
"data":[["a","b"],["c","d"]]\
}}\
'
>>> pd.read_json(_, orient='split')
col 1 col 2
row 1 a b
row 2 c d
Encoding/decoding a Dataframe using ``'index'`` formatted JSON:
>>> df.to_json(orient='index')
'{{"row 1":{{"col 1":"a","col 2":"b"}},"row 2":{{"col 1":"c","col 2":"d"}}}}'
>>> pd.read_json(_, orient='index')
col 1 col 2
row 1 a b
row 2 c d
Encoding/decoding a Dataframe using ``'records'`` formatted JSON.
Note that index labels are not preserved with this encoding.
>>> df.to_json(orient='records')
'[{{"col 1":"a","col 2":"b"}},{{"col 1":"c","col 2":"d"}}]'
>>> pd.read_json(_, orient='records')
col 1 col 2
0 a b
1 c d
Encoding with Table Schema
>>> df.to_json(orient='table')
'\
{{"schema":{{"fields":[\
{{"name":"index","type":"string"}},\
{{"name":"col 1","type":"string"}},\
{{"name":"col 2","type":"string"}}],\
"primaryKey":["index"],\
"pandas_version":"1.4.0"}},\
"data":[\
{{"index":"row 1","col 1":"a","col 2":"b"}},\
{{"index":"row 2","col 1":"c","col 2":"d"}}]\
}}\
'
"""
if orient == "table" and dtype:
raise ValueError("cannot pass both dtype and orient='table'")
if orient == "table" and convert_axes:
raise ValueError("cannot pass both convert_axes and orient='table'")
if dtype is None and orient != "table":
# error: Incompatible types in assignment (expression has type "bool", variable
# has type "Union[ExtensionDtype, str, dtype[Any], Type[str], Type[float],
# Type[int], Type[complex], Type[bool], Type[object], Dict[Hashable,
# Union[ExtensionDtype, Union[str, dtype[Any]], Type[str], Type[float],
# Type[int], Type[complex], Type[bool], Type[object]]], None]")
dtype = True # type: ignore[assignment]
if convert_axes is None and orient != "table":
convert_axes = True
json_reader = JsonReader(
path_or_buf,
orient=orient,
typ=typ,
dtype=dtype,
convert_axes=convert_axes,
convert_dates=convert_dates,
keep_default_dates=keep_default_dates,
numpy=numpy,
precise_float=precise_float,
date_unit=date_unit,
encoding=encoding,
lines=lines,
chunksize=chunksize,
compression=compression,
nrows=nrows,
storage_options=storage_options,
encoding_errors=encoding_errors,
)
if chunksize:
return json_reader
with json_reader:
return json_reader.read()
class JsonReader(abc.Iterator, Generic[FrameSeriesStrT]):
"""
JsonReader provides an interface for reading in a JSON file.
If initialized with ``lines=True`` and ``chunksize``, can be iterated over
``chunksize`` lines at a time. Otherwise, calling ``read`` reads in the
whole document.
"""
def __init__(
self,
filepath_or_buffer,
orient,
typ: FrameSeriesStrT,
dtype,
convert_axes,
convert_dates,
keep_default_dates: bool,
numpy: bool,
precise_float: bool,
date_unit,
encoding,
lines: bool,
chunksize: int | None,
compression: CompressionOptions,
nrows: int | None,
storage_options: StorageOptions = None,
encoding_errors: str | None = "strict",
) -> None:
self.orient = orient
self.typ = typ
self.dtype = dtype
self.convert_axes = convert_axes
self.convert_dates = convert_dates
self.keep_default_dates = keep_default_dates
self.numpy = numpy
self.precise_float = precise_float
self.date_unit = date_unit
self.encoding = encoding
self.compression = compression
self.storage_options = storage_options
self.lines = lines
self.chunksize = chunksize
self.nrows_seen = 0
self.nrows = nrows
self.encoding_errors = encoding_errors
self.handles: IOHandles[str] | None = None
if self.chunksize is not None:
self.chunksize = validate_integer("chunksize", self.chunksize, 1)
if not self.lines:
raise ValueError("chunksize can only be passed if lines=True")
if self.nrows is not None:
self.nrows = validate_integer("nrows", self.nrows, 0)
if not self.lines:
raise ValueError("nrows can only be passed if lines=True")
data = self._get_data_from_filepath(filepath_or_buffer)
self.data = self._preprocess_data(data)
def _preprocess_data(self, data):
"""
At this point, the data either has a `read` attribute (e.g. a file
object or a StringIO) or is a string that is a JSON document.
If self.chunksize, we prepare the data for the `__next__` method.
Otherwise, we read it into memory for the `read` method.
"""
if hasattr(data, "read") and not (self.chunksize or self.nrows):
with self:
data = data.read()
if not hasattr(data, "read") and (self.chunksize or self.nrows):
data = StringIO(data)
return data
def _get_data_from_filepath(self, filepath_or_buffer):
"""
The function read_json accepts three input types:
1. filepath (string-like)
2. file-like object (e.g. open file object, StringIO)
3. JSON string
This method turns (1) into (2) to simplify the rest of the processing.
It returns input types (2) and (3) unchanged.
It raises FileNotFoundError if the input is a string ending in
one of .json, .json.gz, .json.bz2, etc. but no such file exists.
"""
# if it is a string but the file does not exist, it might be a JSON string
filepath_or_buffer = stringify_path(filepath_or_buffer)
if (
not isinstance(filepath_or_buffer, str)
or is_url(filepath_or_buffer)
or is_fsspec_url(filepath_or_buffer)
or file_exists(filepath_or_buffer)
):
self.handles = get_handle(
filepath_or_buffer,
"r",
encoding=self.encoding,
compression=self.compression,
storage_options=self.storage_options,
errors=self.encoding_errors,
)
filepath_or_buffer = self.handles.handle
elif (
isinstance(filepath_or_buffer, str)
and filepath_or_buffer.lower().endswith(
(".json",) + tuple(f".json{c}" for c in _extension_to_compression)
)
and not file_exists(filepath_or_buffer)
):
raise FileNotFoundError(f"File {filepath_or_buffer} does not exist")
return filepath_or_buffer
def _combine_lines(self, lines) -> str:
"""
Combines a list of JSON objects into one JSON object.
"""
return (
f'[{",".join([line for line in (line.strip() for line in lines) if line])}]'
)
@overload
def read(self: JsonReader[Literal["frame"]]) -> DataFrame:
...
@overload
def read(self: JsonReader[Literal["series"]]) -> Series:
...
@overload
def read(self: JsonReader[Literal["frame", "series"]]) -> DataFrame | Series:
...
def read(self) -> DataFrame | Series:
"""
Read the whole JSON input into a pandas object.
"""
obj: DataFrame | Series
if self.lines:
if self.chunksize:
obj = concat(self)
elif self.nrows:
lines = list(islice(self.data, self.nrows))
lines_json = self._combine_lines(lines)
obj = self._get_object_parser(lines_json)
else:
data = ensure_str(self.data)
data_lines = data.split("\n")
obj = self._get_object_parser(self._combine_lines(data_lines))
else:
obj = self._get_object_parser(self.data)
self.close()
return obj
def _get_object_parser(self, json) -> DataFrame | Series:
"""
Parses a json document into a pandas object.
"""
typ = self.typ
dtype = self.dtype
kwargs = {
"orient": self.orient,
"dtype": self.dtype,
"convert_axes": self.convert_axes,
"convert_dates": self.convert_dates,
"keep_default_dates": self.keep_default_dates,
"numpy": self.numpy,
"precise_float": self.precise_float,
"date_unit": self.date_unit,
}
obj = None
if typ == "frame":
obj = FrameParser(json, **kwargs).parse()
if typ == "series" or obj is None:
if not isinstance(dtype, bool):
kwargs["dtype"] = dtype
obj = SeriesParser(json, **kwargs).parse()
return obj
def close(self) -> None:
"""
If we opened a stream earlier, in _get_data_from_filepath, we should
close it.
If an open stream or file was passed, we leave it open.
"""
if self.handles is not None:
self.handles.close()
def __iter__(self: JsonReader[FrameSeriesStrT]) -> JsonReader[FrameSeriesStrT]:
return self
@overload
def __next__(self: JsonReader[Literal["frame"]]) -> DataFrame:
...
@overload
def __next__(self: JsonReader[Literal["series"]]) -> Series:
...
@overload
def __next__(self: JsonReader[Literal["frame", "series"]]) -> DataFrame | Series:
...
def __next__(self) -> DataFrame | Series:
if self.nrows:
if self.nrows_seen >= self.nrows:
self.close()
raise StopIteration
lines = list(islice(self.data, self.chunksize))
if lines:
lines_json = self._combine_lines(lines)
obj = self._get_object_parser(lines_json)
# Make sure that the returned objects have the right index.
obj.index = range(self.nrows_seen, self.nrows_seen + len(obj))
self.nrows_seen += len(obj)
return obj
self.close()
raise StopIteration
def __enter__(self) -> JsonReader[FrameSeriesStrT]:
return self
def __exit__(self, exc_type, exc_value, traceback) -> None:
self.close()
class Parser:
_split_keys: tuple[str, ...]
_default_orient: str
_STAMP_UNITS = ("s", "ms", "us", "ns")
_MIN_STAMPS = {
"s": 31536000,
"ms": 31536000000,
"us": 31536000000000,
"ns": 31536000000000000,
}
def __init__(
self,
json,
orient,
dtype: DtypeArg | None = None,
convert_axes: bool = True,
convert_dates: bool | list[str] = True,
keep_default_dates: bool = False,
numpy: bool = False,
precise_float: bool = False,
date_unit=None,
) -> None:
self.json = json
if orient is None:
orient = self._default_orient
self.orient = orient
self.dtype = dtype
if orient == "split":
numpy = False
if date_unit is not None:
date_unit = date_unit.lower()
if date_unit not in self._STAMP_UNITS:
raise ValueError(f"date_unit must be one of {self._STAMP_UNITS}")
self.min_stamp = self._MIN_STAMPS[date_unit]
else:
self.min_stamp = self._MIN_STAMPS["s"]
self.numpy = numpy
self.precise_float = precise_float
self.convert_axes = convert_axes
self.convert_dates = convert_dates
self.date_unit = date_unit
self.keep_default_dates = keep_default_dates
self.obj: DataFrame | Series | None = None
def check_keys_split(self, decoded) -> None:
"""
Checks that dict has only the appropriate keys for orient='split'.
"""
bad_keys = set(decoded.keys()).difference(set(self._split_keys))
if bad_keys:
bad_keys_joined = ", ".join(bad_keys)
raise ValueError(f"JSON data had unexpected key(s): {bad_keys_joined}")
def parse(self):
if self.numpy:
self._parse_numpy()
else:
self._parse_no_numpy()
if self.obj is None:
return None
if self.convert_axes:
self._convert_axes()
self._try_convert_types()
return self.obj
def _parse_numpy(self):
raise AbstractMethodError(self)
def _parse_no_numpy(self):
raise AbstractMethodError(self)
def _convert_axes(self):
"""
Try to convert axes.
"""
obj = self.obj
assert obj is not None # for mypy
for axis_name in obj._AXIS_ORDERS:
new_axis, result = self._try_convert_data(
name=axis_name,
data=obj._get_axis(axis_name),
use_dtypes=False,
convert_dates=True,
)
if result:
setattr(self.obj, axis_name, new_axis)
def _try_convert_types(self):
raise AbstractMethodError(self)
def _try_convert_data(
self,
name,
data,
use_dtypes: bool = True,
convert_dates: bool | list[str] = True,
):
"""
Try to parse a ndarray like into a column by inferring dtype.
"""
# don't try to coerce, unless a force conversion
if use_dtypes:
if not self.dtype:
if all(notna(data)):
return data, False
return data.fillna(np.nan), True
# error: Non-overlapping identity check (left operand type:
# "Union[ExtensionDtype, str, dtype[Any], Type[object],
# Dict[Hashable, Union[ExtensionDtype, Union[str, dtype[Any]],
# Type[str], Type[float], Type[int], Type[complex], Type[bool],
# Type[object]]]]", right operand type: "Literal[True]")
elif self.dtype is True: # type: ignore[comparison-overlap]
pass
else:
# dtype to force
dtype = (
self.dtype.get(name) if isinstance(self.dtype, dict) else self.dtype
)
if dtype is not None:
try:
return data.astype(dtype), True
except (TypeError, ValueError):
return data, False
if convert_dates:
new_data, result = self._try_convert_to_date(data)
if result:
return new_data, True
if data.dtype == "object":
# try float
try:
data = data.astype("float64")
except (TypeError, ValueError):
pass
if data.dtype.kind == "f":
if data.dtype != "float64":
# coerce floats to 64
try:
data = data.astype("float64")
except (TypeError, ValueError):
pass
# don't coerce 0-len data
if len(data) and (data.dtype == "float" or data.dtype == "object"):
# coerce ints if we can
try:
new_data = data.astype("int64")
if (new_data == data).all():
data = new_data
except (TypeError, ValueError, OverflowError):
pass
# coerce ints to 64
if data.dtype == "int":
# coerce floats to 64
try:
data = data.astype("int64")
except (TypeError, ValueError):
pass
# if we have an index, we want to preserve dtypes
if name == "index" and len(data):
if self.orient == "split":
return data, False
return data, True
def _try_convert_to_date(self, data):
"""
Try to parse a ndarray like into a date column.
Try to coerce object in epoch/iso formats and integer/float in epoch
formats. Return a boolean if parsing was successful.
"""
# no conversion on empty
if not len(data):
return data, False
new_data = data
if new_data.dtype == "object":
try:
new_data = data.astype("int64")
except (TypeError, ValueError, OverflowError):
pass
# ignore numbers that are out of range
if issubclass(new_data.dtype.type, np.number):
in_range = (
isna(new_data._values)
| (new_data > self.min_stamp)
| (new_data._values == iNaT)
)
if not in_range.all():
return data, False
date_units = (self.date_unit,) if self.date_unit else self._STAMP_UNITS
for date_unit in date_units:
try:
new_data = to_datetime(new_data, errors="raise", unit=date_unit)
except (ValueError, OverflowError, TypeError):
continue
return new_data, True
return data, False
def _try_convert_dates(self):
raise AbstractMethodError(self)
class SeriesParser(Parser):
_default_orient = "index"
_split_keys = ("name", "index", "data")
def _parse_no_numpy(self):
data = loads(self.json, precise_float=self.precise_float)
if self.orient == "split":
decoded = {str(k): v for k, v in data.items()}
self.check_keys_split(decoded)
self.obj = create_series_with_explicit_dtype(**decoded)
else:
self.obj = create_series_with_explicit_dtype(data, dtype_if_empty=object)
def _parse_numpy(self):
load_kwargs = {
"dtype": None,
"numpy": True,
"precise_float": self.precise_float,
}
if self.orient in ["columns", "index"]:
load_kwargs["labelled"] = True
loads_ = functools.partial(loads, **load_kwargs)
data = loads_(self.json)
if self.orient == "split":
decoded = {str(k): v for k, v in data.items()}
self.check_keys_split(decoded)
self.obj = create_series_with_explicit_dtype(**decoded)
elif self.orient in ["columns", "index"]:
# error: "create_series_with_explicit_dtype"
# gets multiple values for keyword argument "dtype_if_empty
self.obj = create_series_with_explicit_dtype(
*data, dtype_if_empty=object
) # type: ignore[misc]
else:
self.obj = create_series_with_explicit_dtype(data, dtype_if_empty=object)
def _try_convert_types(self):
if self.obj is None:
return
obj, result = self._try_convert_data(
"data", self.obj, convert_dates=self.convert_dates
)
if result:
self.obj = obj
class FrameParser(Parser):
_default_orient = "columns"
_split_keys = ("columns", "index", "data")
def _parse_numpy(self):
json = self.json
orient = self.orient
if orient == "columns":
args = loads(
json,
dtype=None,
numpy=True,
labelled=True,
precise_float=self.precise_float,
)
if len(args):
args = (args[0].T, args[2], args[1])
self.obj = DataFrame(*args)
elif orient == "split":
decoded = loads(
json, dtype=None, numpy=True, precise_float=self.precise_float
)
decoded = {str(k): v for k, v in decoded.items()}
self.check_keys_split(decoded)
self.obj = DataFrame(**decoded)
elif orient == "values":
self.obj = DataFrame(
loads(json, dtype=None, numpy=True, precise_float=self.precise_float)
)
else:
self.obj = DataFrame(
*loads(
json,
dtype=None,
numpy=True,
labelled=True,
precise_float=self.precise_float,
)
)
def _parse_no_numpy(self):
json = self.json
orient = self.orient
if orient == "columns":
self.obj = DataFrame(
loads(json, precise_float=self.precise_float), dtype=None
)
elif orient == "split":
decoded = {
str(k): v
for k, v in loads(json, precise_float=self.precise_float).items()
}
self.check_keys_split(decoded)
self.obj = DataFrame(dtype=None, **decoded)
elif orient == "index":
self.obj = DataFrame.from_dict(
loads(json, precise_float=self.precise_float),
dtype=None,
orient="index",
)
elif orient == "table":
self.obj = parse_table_schema(json, precise_float=self.precise_float)
else:
self.obj = DataFrame(
loads(json, precise_float=self.precise_float), dtype=None
)
def _process_converter(self, f, filt=None):
"""
Take a conversion function and possibly recreate the frame.
"""
if filt is None:
filt = lambda col, c: True
obj = self.obj
assert obj is not None # for mypy
needs_new_obj = False
new_obj = {}
for i, (col, c) in enumerate(obj.items()):
if filt(col, c):
new_data, result = f(col, c)
if result:
c = new_data
needs_new_obj = True
new_obj[i] = c
if needs_new_obj:
# possibly handle dup columns
new_frame = DataFrame(new_obj, index=obj.index)
new_frame.columns = obj.columns
self.obj = new_frame
def _try_convert_types(self):
if self.obj is None:
return
if self.convert_dates:
self._try_convert_dates()
self._process_converter(
lambda col, c: self._try_convert_data(col, c, convert_dates=False)
)
def _try_convert_dates(self):
if self.obj is None:
return
# our columns to parse
convert_dates_list_bool = self.convert_dates
if isinstance(convert_dates_list_bool, bool):
convert_dates_list_bool = []
convert_dates = set(convert_dates_list_bool)
def is_ok(col) -> bool:
"""
Return if this col is ok to try for a date parse.
"""
if not isinstance(col, str):
return False
col_lower = col.lower()
if (
col_lower.endswith("_at")
or col_lower.endswith("_time")
or col_lower == "modified"
or col_lower == "date"
or col_lower == "datetime"
or col_lower.startswith("timestamp")
):
return True
return False
self._process_converter(
lambda col, c: self._try_convert_to_date(c),
lambda col, c: (
(self.keep_default_dates and is_ok(col)) or col in convert_dates
),
)