Skip to content
This repository was archived by the owner on Jul 5, 2023. It is now read-only.

Commit f29a0fb

Browse files
ddfisherddfisher
authored
[2.7, 3.5, conversions] Support the per-argument function comment syntax (#5)
Python 2.7: Adds a type_comments list of strings to the arguments object containing the per-argument type comments of a function, if any. Python 3.5: Puts per-argument type comments into the annotation slot of the argument as a string expression. Conversions: Converts the new type_comments list to argument annotations. This change contains some autogenerated files: - `ast*/Python/graminit.c` is autogenerated from `ast*/Grammar/Grammar`. - `ast*/Include/Python-ast.h` and `ast*/Python/Python-ast.c` are autogenerated from `ast*/Parser/Python.asdl`.
1 parent 651fbe5 commit f29a0fb

File tree

12 files changed

+310
-101
lines changed

12 files changed

+310
-101
lines changed

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ compatible with Python 3.3 - 3.5.
66

77
### Current Caveats for Use
88
- Type comments in invalid locations produce syntax errors.
9+
- When using per-argument function comment annotations, the type comments must
10+
come after the argument-separating comma.
911

1012
## Development Notes
1113
### General Notes

‎ast27/Grammar/Grammar

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ decorators: decorator+
2525
decorated: decorators (classdef | funcdef)
2626
funcdef: 'def' NAME parameters ':' [TYPE_COMMENT] suite
2727
parameters: '(' [varargslist] ')'
28-
varargslist: ((fpdef ['=' test] ',')*
29-
('*' NAME [',' '**' NAME] | '**' NAME) |
30-
fpdef ['=' test] (',' fpdef ['=' test])* [','])
28+
varargslist: ((fpdef ['=' test] ',' [TYPE_COMMENT])*
29+
('*' NAME [',' [TYPE_COMMENT] '**' NAME] [TYPE_COMMENT] | '**' NAME [TYPE_COMMENT]) |
30+
fpdef ['=' test] (',' [TYPE_COMMENT] fpdef ['=' test])* [','] [TYPE_COMMENT])
3131
fpdef: NAME | '(' fplist ')'
3232
fplist: fpdef (',' fpdef)* [',']
3333

‎ast27/Include/Python-ast.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,7 @@ struct _arguments {
367367
identifier vararg;
368368
identifier kwarg;
369369
asdl_seq *defaults;
370+
asdl_seq *type_comments;
370371
};
371372

372373
struct _keyword {
@@ -531,9 +532,9 @@ comprehension_ty _Ta27_comprehension(expr_ty target, expr_ty iter, asdl_seq * if
531532
#define ExceptHandler(a0, a1, a2, a3, a4, a5) _Ta27_ExceptHandler(a0, a1, a2, a3, a4, a5)
532533
excepthandler_ty _Ta27_ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int
533534
col_offset, PyArena *arena);
534-
#define arguments(a0, a1, a2, a3, a4) _Ta27_arguments(a0, a1, a2, a3, a4)
535+
#define arguments(a0, a1, a2, a3, a4, a5) _Ta27_arguments(a0, a1, a2, a3, a4, a5)
535536
arguments_ty _Ta27_arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq *
536-
defaults, PyArena *arena);
537+
defaults, asdl_seq * type_comments, PyArena *arena);
537538
#define keyword(a0, a1, a2) _Ta27_keyword(a0, a1, a2)
538539
keyword_ty _Ta27_keyword(identifier arg, expr_ty value, PyArena *arena);
539540
#define alias(a0, a1, a2) _Ta27_alias(a0, a1, a2)

‎ast27/Parser/Python.asdl

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,12 @@ module Python version "$Revision$"
105105
excepthandler = ExceptHandler(expr? type, expr? name, stmt* body)
106106
attributes (int lineno, int col_offset)
107107

108+
-- type_comments is used to support the per-argument type comment syntax.
109+
-- It is either an empty list or a list with length equal to the number of
110+
-- args (including varargs and kwargs, if present) and with members set to the
111+
-- string of each arg's type comment, if present, or None otherwise.
108112
arguments = (expr* args, identifier? vararg,
109-
identifier? kwarg, expr* defaults)
113+
identifier? kwarg, expr* defaults, string* type_comments)
110114

111115
-- keyword arguments supplied to call
112116
keyword = (identifier arg, expr value)

‎ast27/Python/Python-ast.c

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ static char *arguments_fields[]={
376376
"vararg",
377377
"kwarg",
378378
"defaults",
379+
"type_comments",
379380
};
380381
static PyTypeObject *keyword_type;
381382
static PyObject* ast2obj_keyword(void*);
@@ -963,7 +964,7 @@ static int init_types(void)
963964
ExceptHandler_type = make_type("ExceptHandler", excepthandler_type, ExceptHandler_fields,
964965
3);
965966
if (!ExceptHandler_type) return 0;
966-
arguments_type = make_type("arguments", &AST_type, arguments_fields, 4);
967+
arguments_type = make_type("arguments", &AST_type, arguments_fields, 5);
967968
if (!arguments_type) return 0;
968969
keyword_type = make_type("keyword", &AST_type, keyword_fields, 2);
969970
if (!keyword_type) return 0;
@@ -2090,7 +2091,8 @@ ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body, int lineno, int col_o
20902091
}
20912092

20922093
arguments_ty
2093-
arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defaults, PyArena *arena)
2094+
arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defaults, asdl_seq *
2095+
type_comments, PyArena *arena)
20942096
{
20952097
arguments_ty p;
20962098
p = (arguments_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -2100,6 +2102,7 @@ arguments(asdl_seq * args, identifier vararg, identifier kwarg, asdl_seq * defau
21002102
p->vararg = vararg;
21012103
p->kwarg = kwarg;
21022104
p->defaults = defaults;
2105+
p->type_comments = type_comments;
21032106
return p;
21042107
}
21052108

@@ -3304,6 +3307,11 @@ ast2obj_arguments(void* _o)
33043307
if (PyObject_SetAttrString(result, "defaults", value) == -1)
33053308
goto failed;
33063309
Py_DECREF(value);
3310+
value = ast2obj_list(o->type_comments, ast2obj_string);
3311+
if (!value) goto failed;
3312+
if (PyObject_SetAttrString(result, "type_comments", value) == -1)
3313+
goto failed;
3314+
Py_DECREF(value);
33073315
return result;
33083316
failed:
33093317
Py_XDECREF(value);
@@ -6617,6 +6625,7 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
66176625
identifier vararg;
66186626
identifier kwarg;
66196627
asdl_seq* defaults;
6628+
asdl_seq* type_comments;
66206629

66216630
if (PyObject_HasAttrString(obj, "args")) {
66226631
int res;
@@ -6690,7 +6699,32 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
66906699
PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments");
66916700
return 1;
66926701
}
6693-
*out = arguments(args, vararg, kwarg, defaults, arena);
6702+
if (PyObject_HasAttrString(obj, "type_comments")) {
6703+
int res;
6704+
Py_ssize_t len;
6705+
Py_ssize_t i;
6706+
tmp = PyObject_GetAttrString(obj, "type_comments");
6707+
if (tmp == NULL) goto failed;
6708+
if (!PyList_Check(tmp)) {
6709+
PyErr_Format(PyExc_TypeError, "arguments field \"type_comments\" must be a list, not a %.200s", tmp->ob_type->tp_name);
6710+
goto failed;
6711+
}
6712+
len = PyList_GET_SIZE(tmp);
6713+
type_comments = asdl_seq_new(len, arena);
6714+
if (type_comments == NULL) goto failed;
6715+
for (i = 0; i < len; i++) {
6716+
string value;
6717+
res = obj2ast_string(PyList_GET_ITEM(tmp, i), &value, arena);
6718+
if (res != 0) goto failed;
6719+
asdl_seq_SET(type_comments, i, value);
6720+
}
6721+
Py_XDECREF(tmp);
6722+
tmp = NULL;
6723+
} else {
6724+
PyErr_SetString(PyExc_TypeError, "required field \"type_comments\" missing from arguments");
6725+
return 1;
6726+
}
6727+
*out = arguments(args, vararg, kwarg, defaults, type_comments, arena);
66946728
return 0;
66956729
failed:
66966730
Py_XDECREF(tmp);

‎ast27/Python/ast.c

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -732,17 +732,18 @@ static arguments_ty
732732
ast_for_arguments(struct compiling *c, const node *n)
733733
{
734734
/* parameters: '(' [varargslist] ')'
735-
varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME]
736-
| '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']
735+
varargslist: ((fpdef ['=' test] ',' [TYPE_COMMENT])*
736+
('*' NAME [',' [TYPE_COMMENT] '**' NAME] [TYPE_COMMENT] | '**' NAME [TYPE_COMMENT]) |
737+
fpdef ['=' test] (',' [TYPE_COMMENT] fpdef ['=' test])* [','] [TYPE_COMMENT])
737738
*/
738-
int i, j, k, n_args = 0, n_defaults = 0, found_default = 0;
739-
asdl_seq *args, *defaults;
739+
int i, j, k, l, n_args = 0, n_all_args = 0, n_defaults = 0, found_default = 0;
740+
asdl_seq *args, *defaults, *type_comments = NULL;
740741
identifier vararg = NULL, kwarg = NULL;
741742
node *ch;
742743

743744
if (TYPE(n) == parameters) {
744745
if (NCH(n) == 2) /* () as argument list */
745-
return arguments(NULL, NULL, NULL, NULL, c->c_arena);
746+
return arguments(NULL, NULL, NULL, NULL, NULL, c->c_arena);
746747
n = CHILD(n, 1);
747748
}
748749
REQ(n, varargslist);
@@ -754,20 +755,30 @@ ast_for_arguments(struct compiling *c, const node *n)
754755
n_args++;
755756
if (TYPE(ch) == EQUAL)
756757
n_defaults++;
758+
if (TYPE(ch) == STAR || TYPE(ch) == DOUBLESTAR)
759+
n_all_args++;
757760
}
761+
n_all_args += n_args;
758762
args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL);
759763
if (!args && n_args)
760764
return NULL;
761765
defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL);
762766
if (!defaults && n_defaults)
763767
return NULL;
768+
/* type_comments will be lazily initialized if needed. If there are no
769+
per-argument type comments, it will remain NULL. Otherwise, it will be
770+
an asdl_seq with length equal to the number of args (including varargs
771+
and kwargs, if present) and with members set to the string of each arg's
772+
type comment, if present, or NULL otherwise.
773+
*/
764774

765775
/* fpdef: NAME | '(' fplist ')'
766776
fplist: fpdef (',' fpdef)* [',']
767777
*/
768778
i = 0;
769779
j = 0; /* index for defaults */
770780
k = 0; /* index for args */
781+
l = 0; /* index for type comments */
771782
while (i < NCH(n)) {
772783
ch = CHILD(n, i);
773784
switch (TYPE(ch)) {
@@ -834,7 +845,9 @@ ast_for_arguments(struct compiling *c, const node *n)
834845
asdl_seq_SET(args, k++, name);
835846

836847
}
837-
i += 2; /* the name and the comma */
848+
i += 1; /* the name */
849+
if (TYPE(CHILD(n, i)) == COMMA)
850+
i += 1; /* the comma, if present */
838851
if (parenthesized && Py_Py3kWarningFlag &&
839852
!ast_warn(c, ch, "parenthesized argument names "
840853
"are invalid in 3.x"))
@@ -848,15 +861,36 @@ ast_for_arguments(struct compiling *c, const node *n)
848861
vararg = NEW_IDENTIFIER(CHILD(n, i+1));
849862
if (!vararg)
850863
return NULL;
851-
i += 3;
864+
i += 2; /* the star and the name */
865+
if (TYPE(CHILD(n, i)) == COMMA)
866+
i += 1; /* the comma, if present */
852867
break;
853868
case DOUBLESTAR:
854869
if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))
855870
return NULL;
856871
kwarg = NEW_IDENTIFIER(CHILD(n, i+1));
857872
if (!kwarg)
858873
return NULL;
859-
i += 3;
874+
i += 2; /* the double star and the name */
875+
if (TYPE(CHILD(n, i)) == COMMA)
876+
i += 1; /* the comma, if present */
877+
break;
878+
case TYPE_COMMENT:
879+
assert(l < k + !!vararg + !!kwarg);
880+
881+
if (!type_comments) {
882+
/* lazily allocate the type_comments seq for perf reasons */
883+
type_comments = asdl_seq_new(n_all_args, c->c_arena);
884+
if (!type_comments)
885+
return NULL;
886+
}
887+
888+
while (l < k + !!vararg + !!kwarg - 1) {
889+
asdl_seq_SET(type_comments, l++, NULL);
890+
}
891+
892+
asdl_seq_SET(type_comments, l++, NEW_TYPE_COMMENT(ch));
893+
i += 1;
860894
break;
861895
default:
862896
PyErr_Format(PyExc_SystemError,
@@ -866,7 +900,13 @@ ast_for_arguments(struct compiling *c, const node *n)
866900
}
867901
}
868902

869-
return arguments(args, vararg, kwarg, defaults, c->c_arena);
903+
if (type_comments) {
904+
while (l < n_all_args) {
905+
asdl_seq_SET(type_comments, l++, NULL);
906+
}
907+
}
908+
909+
return arguments(args, vararg, kwarg, defaults, type_comments, c->c_arena);
870910
}
871911

872912
static expr_ty
@@ -1038,7 +1078,7 @@ ast_for_lambdef(struct compiling *c, const node *n)
10381078
expr_ty expression;
10391079

10401080
if (NCH(n) == 3) {
1041-
args = arguments(NULL, NULL, NULL, NULL, c->c_arena);
1081+
args = arguments(NULL, NULL, NULL, NULL, NULL, c->c_arena);
10421082
if (!args)
10431083
return NULL;
10441084
expression = ast_for_expr(c, CHILD(n, 2));

‎ast27/Python/graminit.c

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -159,51 +159,72 @@ static arc arcs_8_0[3] = {
159159
{31, 2},
160160
{32, 3},
161161
};
162-
static arc arcs_8_1[3] = {
162+
static arc arcs_8_1[4] = {
163163
{28, 4},
164164
{30, 5},
165+
{24, 6},
165166
{0, 1},
166167
};
167168
static arc arcs_8_2[1] = {
168-
{21, 6},
169+
{21, 7},
169170
};
170171
static arc arcs_8_3[1] = {
171-
{21, 7},
172+
{21, 8},
172173
};
173174
static arc arcs_8_4[1] = {
174-
{29, 8},
175+
{29, 9},
175176
};
176-
static arc arcs_8_5[4] = {
177+
static arc arcs_8_5[5] = {
177178
{27, 1},
179+
{24, 10},
178180
{31, 2},
179181
{32, 3},
180182
{0, 5},
181183
};
182-
static arc arcs_8_6[2] = {
183-
{30, 9},
184+
static arc arcs_8_6[1] = {
184185
{0, 6},
185186
};
186-
static arc arcs_8_7[1] = {
187+
static arc arcs_8_7[3] = {
188+
{30, 11},
189+
{24, 6},
187190
{0, 7},
188191
};
189192
static arc arcs_8_8[2] = {
190-
{30, 5},
193+
{24, 6},
191194
{0, 8},
192195
};
193-
static arc arcs_8_9[1] = {
196+
static arc arcs_8_9[3] = {
197+
{30, 5},
198+
{24, 6},
199+
{0, 9},
200+
};
201+
static arc arcs_8_10[4] = {
202+
{27, 1},
203+
{31, 2},
204+
{32, 3},
205+
{0, 10},
206+
};
207+
static arc arcs_8_11[2] = {
208+
{24, 12},
209+
{32, 3},
210+
};
211+
static arc arcs_8_12[1] = {
194212
{32, 3},
195213
};
196-
static state states_8[10] = {
214+
static state states_8[13] = {
197215
{3, arcs_8_0},
198-
{3, arcs_8_1},
216+
{4, arcs_8_1},
199217
{1, arcs_8_2},
200218
{1, arcs_8_3},
201219
{1, arcs_8_4},
202-
{4, arcs_8_5},
203-
{2, arcs_8_6},
204-
{1, arcs_8_7},
220+
{5, arcs_8_5},
221+
{1, arcs_8_6},
222+
{3, arcs_8_7},
205223
{2, arcs_8_8},
206-
{1, arcs_8_9},
224+
{3, arcs_8_9},
225+
{4, arcs_8_10},
226+
{2, arcs_8_11},
227+
{1, arcs_8_12},
207228
};
208229
static arc arcs_9_0[2] = {
209230
{21, 1},
@@ -1971,7 +1992,7 @@ static dfa dfas[88] = {
19711992
"\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
19721993
{263, "parameters", 0, 4, states_7,
19731994
"\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
1974-
{264, "varargslist", 0, 10, states_8,
1995+
{264, "varargslist", 0, 13, states_8,
19751996
"\000\040\040\200\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
19761997
{265, "fpdef", 0, 4, states_9,
19771998
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},

‎ast35/Grammar/Grammar

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ async_funcdef: ASYNC funcdef
3030
funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] suite
3131

3232
parameters: '(' [typedargslist] ')'
33-
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
34-
['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
35-
| '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
33+
typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT]
34+
['*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT] '**' tfpdef] | '**' tfpdef]] [TYPE_COMMENT]
35+
| '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* [',' [TYPE_COMMENT] '**' tfpdef] [TYPE_COMMENT]
36+
| '**' tfpdef [TYPE_COMMENT])
3637
tfpdef: NAME [':' test]
3738
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
3839
['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]

0 commit comments

Comments
 (0)