From 748a7e80e4696546dae0eb46eae3bb356f4f563e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 9 Jul 2020 23:30:37 +0300 Subject: [PATCH 1/4] gh-86457: Fix signature for code.replace(). --- Objects/clinic/codeobject.c.h | 42 +++++++++++++------------ Objects/codeobject.c | 58 +++++++++++++++++------------------ Tools/clinic/clinic.py | 2 +- 3 files changed, 52 insertions(+), 50 deletions(-) diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index da33f4a6a20c1b..cfd0f025095ca3 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -163,12 +163,15 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(code_replace__doc__, -"replace($self, /, *, co_argcount=-1, co_posonlyargcount=-1,\n" -" co_kwonlyargcount=-1, co_nlocals=-1, co_stacksize=-1,\n" -" co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n" -" co_names=None, co_varnames=None, co_freevars=None,\n" -" co_cellvars=None, co_filename=None, co_name=None,\n" -" co_qualname=None, co_linetable=None, co_exceptiontable=None)\n" +"replace($self, /, *, co_argcount=unchanged,\n" +" co_posonlyargcount=unchanged, co_kwonlyargcount=unchanged,\n" +" co_nlocals=unchanged, co_stacksize=unchanged,\n" +" co_flags=unchanged, co_firstlineno=unchanged,\n" +" co_code=unchanged, co_consts=unchanged, co_names=unchanged,\n" +" co_varnames=unchanged, co_freevars=unchanged,\n" +" co_cellvars=unchanged, co_filename=unchanged,\n" +" co_name=unchanged, co_qualname=unchanged,\n" +" co_linetable=unchanged, co_exceptiontable=unchanged)\n" "--\n" "\n" "Return a copy of the code object with new values for the specified fields."); @@ -180,13 +183,12 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable); + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable); static PyObject * code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -226,7 +228,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje int co_stacksize = self->co_stacksize; int co_flags = self->co_flags; int co_firstlineno = self->co_firstlineno; - PyBytesObject *co_code = NULL; + PyObject *co_code = NULL; PyObject *co_consts = self->co_consts; PyObject *co_names = self->co_names; PyObject *co_varnames = NULL; @@ -235,8 +237,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *co_filename = self->co_filename; PyObject *co_name = self->co_name; PyObject *co_qualname = self->co_qualname; - PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable; - PyBytesObject *co_exceptiontable = (PyBytesObject *)self->co_exceptiontable; + PyObject *co_linetable = self->co_linetable; + PyObject *co_exceptiontable = self->co_exceptiontable; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); if (!args) { @@ -313,7 +315,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_code'", "bytes", args[7]); goto exit; } - co_code = (PyBytesObject *)args[7]; + co_code = args[7]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -412,7 +414,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_linetable'", "bytes", args[16]); goto exit; } - co_linetable = (PyBytesObject *)args[16]; + co_linetable = args[16]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -421,7 +423,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[17]); goto exit; } - co_exceptiontable = (PyBytesObject *)args[17]; + co_exceptiontable = args[17]; skip_optional_kwonly: return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_linetable, co_exceptiontable); @@ -488,4 +490,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=b6c98f17c60ace53 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6247f27cfdf9d1a6 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0c197d767b0a23..7449ebdc6e9242 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1949,24 +1949,24 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) code.replace * - co_argcount: int(c_default="self->co_argcount") = -1 - co_posonlyargcount: int(c_default="self->co_posonlyargcount") = -1 - co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = -1 - co_nlocals: int(c_default="self->co_nlocals") = -1 - co_stacksize: int(c_default="self->co_stacksize") = -1 - co_flags: int(c_default="self->co_flags") = -1 - co_firstlineno: int(c_default="self->co_firstlineno") = -1 - co_code: PyBytesObject(c_default="NULL") = None - co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None - co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None - co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_filename: unicode(c_default="self->co_filename") = None - co_name: unicode(c_default="self->co_name") = None - co_qualname: unicode(c_default="self->co_qualname") = None - co_linetable: PyBytesObject(c_default="(PyBytesObject *)self->co_linetable") = None - co_exceptiontable: PyBytesObject(c_default="(PyBytesObject *)self->co_exceptiontable") = None + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = unchanged + co_nlocals: int(c_default="self->co_nlocals") = unchanged + co_stacksize: int(c_default="self->co_stacksize") = unchanged + co_flags: int(c_default="self->co_flags") = unchanged + co_firstlineno: int(c_default="self->co_firstlineno") = unchanged + co_code: object(subclass_of="&PyBytes_Type", c_default="NULL") = unchanged + co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = unchanged + co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = unchanged + co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_filename: unicode(c_default="self->co_filename") = unchanged + co_name: unicode(c_default="self->co_name") = unchanged + co_qualname: unicode(c_default="self->co_qualname") = unchanged + co_linetable: object(subclass_of="&PyBytes_Type", c_default="self->co_linetable") = unchanged + co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="self->co_exceptiontable") = unchanged Return a copy of the code object with new values for the specified fields. [clinic start generated code]*/ @@ -1975,14 +1975,13 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable) -/*[clinic end generated code: output=b6cd9988391d5711 input=f6f68e03571f8d7c]*/ + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable) +/*[clinic end generated code: output=e75c48a15def18b9 input=520204253f64c415]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -2007,13 +2006,14 @@ code_replace_impl(PyCodeObject *self, int co_argcount, if (code == NULL) { return NULL; } - co_code = (PyBytesObject *)code; + co_code = code; } if (PySys_Audit("code.__new__", "OOOiiiiii", co_code, co_filename, co_name, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags) < 0) { + Py_XDECREF(code); return NULL; } @@ -2045,10 +2045,10 @@ code_replace_impl(PyCodeObject *self, int co_argcount, co = PyCode_NewWithPosOnlyArgs( co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, - co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names, + co_stacksize, co_flags, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_firstlineno, - (PyObject*)co_linetable, (PyObject*)co_exceptiontable); + co_linetable, co_exceptiontable); error: Py_XDECREF(code); diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0ece814e8f1883..8690a49595071f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2652,7 +2652,7 @@ def __init__(self, name, py_name, function, default=unspecified, *, c_default=No self.name = ensure_legal_c_identifier(name) self.py_name = py_name - if default is not unspecified: + if default is not unspecified and (default is not NULL or c_default == 'NULL'): if self.default_type and not isinstance(default, (self.default_type, Unknown)): if isinstance(self.default_type, type): types_str = self.default_type.__name__ From 65a165b1387347caf50a307bb8c8665512680706 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 29 Jun 2023 16:23:54 +0300 Subject: [PATCH 2/4] Completely override signature. --- Objects/clinic/codeobject.c.h | 12 +- Objects/codeobject.c | 3 +- Tools/clinic/clinic.py | 257 ++++++++++++++++++---------------- 3 files changed, 137 insertions(+), 135 deletions(-) diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index cc63f0862c6fc7..511a8e4aaffea4 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -154,15 +154,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(code_replace__doc__, -"replace($self, /, *, co_argcount=unchanged,\n" -" co_posonlyargcount=unchanged, co_kwonlyargcount=unchanged,\n" -" co_nlocals=unchanged, co_stacksize=unchanged,\n" -" co_flags=unchanged, co_firstlineno=unchanged,\n" -" co_code=unchanged, co_consts=unchanged, co_names=unchanged,\n" -" co_varnames=unchanged, co_freevars=unchanged,\n" -" co_cellvars=unchanged, co_filename=unchanged,\n" -" co_name=unchanged, co_qualname=unchanged,\n" -" co_linetable=unchanged, co_exceptiontable=unchanged)\n" +"replace($self, /, **changes)\n" "--\n" "\n" "Return a copy of the code object with new values for the specified fields."); @@ -472,4 +464,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=ac5dc77c07bafee5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=16c95266bbc4bc03 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d232d59f86dae9..d39dec7e045d55 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1967,6 +1967,7 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) } /*[clinic input] +@text_signature "($self, /, **changes)" code.replace * @@ -2002,7 +2003,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, PyObject *co_filename, PyObject *co_name, PyObject *co_qualname, PyObject *co_linetable, PyObject *co_exceptiontable) -/*[clinic end generated code: output=e75c48a15def18b9 input=520204253f64c415]*/ +/*[clinic end generated code: output=e75c48a15def18b9 input=18e280e07846c122]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c01d1679f8b34a..e359d2d28a1938 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2751,7 +2751,7 @@ def __init__(self, self.py_name = py_name self.unused = unused - if default is not unspecified and (default is not NULL or c_default == 'NULL'): + if default is not unspecified: if (self.default_type and default is not unknown and not isinstance(default, self.default_type) @@ -4314,6 +4314,7 @@ def reset(self) -> None: self.indent = IndentStack() self.kind = CALLABLE self.coexist = False + self.forced_text_signature: str | None = None self.parameter_continuation = '' self.preserve_output = False @@ -4448,6 +4449,11 @@ def at_coexist(self) -> None: fail("Called @coexist twice!") self.coexist = True + def at_text_signature(self, text_signature) -> None: + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + def parse(self, block: Block) -> None: self.reset() self.block = block @@ -5151,142 +5157,145 @@ def format_docstring(self): add(f.cls.name) else: add(f.name) - add('(') + if self.forced_text_signature: + add(self.forced_text_signature) + else: + add('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + parameters[0].right_bracket_count = 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired): + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - parameters[0].right_bracket_count = 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired): - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - need_slash = False - added_slash = False - need_a_trailing_slash = False + added_star = False - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(text)) + indent = " " * line_length + def add_parameter(text): + nonlocal line_length + nonlocal first_parameter + if first_parameter: + s = text + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + add('\n') + add(indent) + line_length = len(indent) + s = text + line_length += len(s) + add(s) + + for p in parameters: if not p.converter.show_in_signature: continue - if p.is_positional_only(): - need_a_trailing_slash = True - break + assert p.name + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue - added_star = False - - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(text)) - indent = " " * line_length - def add_parameter(text): - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - add('\n') - add(indent) - line_length = len(indent) - s = text - line_length += len(s) - add(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name - - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue - - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_add('$') + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_add, p_output = text_accumulator() + p_add(fix_right_bracket_count(p.right_bracket_count)) + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_add('$') - if p.is_vararg(): - p_add("*") + if p.is_vararg(): + p_add("*") - name = p.converter.signature_name or p.name - p_add(name) + name = p.converter.signature_name or p.name + p_add(name) - if not p.is_vararg() and p.converter.is_optional(): - p_add('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_add(value) + if not p.is_vararg() and p.converter.is_optional(): + p_add('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_add(value) - if (p != last_p) or need_a_trailing_slash: - p_add(',') + if (p != last_p) or need_a_trailing_slash: + p_add(',') - add_parameter(p_output()) + add_parameter(p_output()) - add(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - add(')') + add(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + add(')') # PEP 8 says: # From f58f9260718864785d75080ba998ee8e90f51ea0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 16:28:15 +0300 Subject: [PATCH 3/4] Fix MyPy complain. --- Tools/clinic/clinic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a3062277391490..478beb34ae69be 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4710,7 +4710,7 @@ def at_coexist(self) -> None: fail("Called @coexist twice!") self.coexist = True - def at_text_signature(self, text_signature) -> None: + def at_text_signature(self, text_signature: str) -> None: if self.forced_text_signature: fail("Called @text_signature twice!") self.forced_text_signature = text_signature From 7b8791f6febc1d0e44fe8b2856b2ac1f450474ec Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Aug 2023 16:30:56 +0300 Subject: [PATCH 4/4] Add a NEWS entry. --- .../Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst new file mode 100644 index 00000000000000..41cfd72cb36760 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -0,0 +1,2 @@ +Argument Clinic now supports overriding automatically generated signature by +using directive `@text_signature`.