Implement PEP 526 Variable Annotations Syntax (#16)

An implementation of the PEP 526 syntax with minimal changes in typed_ast API. The only externally visible change is:
```
Assign(expr* targets, expr? value, string? type_comment, expr? annotation)
```

Author: ilevkivskyi

ast35/Grammar/Grammar

@@ -44,8 +44,9 @@ stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
-expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist_star_expr))* [TYPE_COMMENT])
+annassign: ':' test ['=' test]
 testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
 augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')

ast35/Include/Python-ast.h

@@ -119,6 +119,7 @@ struct _stmt {
             asdl_seq *targets;
             expr_ty value;
             string type_comment;
+            expr_ty annotation;
         } Assign;
 
         struct {
@@ -468,9 +469,10 @@ stmt_ty _Ta35_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Delete(a0, a1, a2, a3) _Ta35_Delete(a0, a1, a2, a3)
 stmt_ty _Ta35_Delete(asdl_seq * targets, int lineno, int col_offset, PyArena
                      *arena);
-#define Assign(a0, a1, a2, a3, a4, a5) _Ta35_Assign(a0, a1, a2, a3, a4, a5)
+#define Assign(a0, a1, a2, a3, a4, a5, a6) _Ta35_Assign(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Ta35_Assign(asdl_seq * targets, expr_ty value, string type_comment,
-                     int lineno, int col_offset, PyArena *arena);
+                     expr_ty annotation, int lineno, int col_offset, PyArena
+                     *arena);
 #define AugAssign(a0, a1, a2, a3, a4, a5) _Ta35_AugAssign(a0, a1, a2, a3, a4, a5)
 stmt_ty _Ta35_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
                         lineno, int col_offset, PyArena *arena);

ast35/Include/compile.h

@@ -6,6 +6,6 @@
 #define Py_single_input 256
 #define Py_file_input 257
 #define Py_eval_input 258
-#define Py_func_type_input 341
+#define Py_func_type_input 342
 
 #endif /* !Ta35_COMPILE_H */

ast35/Include/graminit.h

@@ -17,74 +17,75 @@
 #define simple_stmt 270
 #define small_stmt 271
 #define expr_stmt 272
-#define testlist_star_expr 273
-#define augassign 274
-#define del_stmt 275
-#define pass_stmt 276
-#define flow_stmt 277
-#define break_stmt 278
-#define continue_stmt 279
-#define return_stmt 280
-#define yield_stmt 281
-#define raise_stmt 282
-#define import_stmt 283
-#define import_name 284
-#define import_from 285
-#define import_as_name 286
-#define dotted_as_name 287
-#define import_as_names 288
-#define dotted_as_names 289
-#define dotted_name 290
-#define global_stmt 291
-#define nonlocal_stmt 292
-#define assert_stmt 293
-#define compound_stmt 294
-#define async_stmt 295
-#define if_stmt 296
-#define while_stmt 297
-#define for_stmt 298
-#define try_stmt 299
-#define with_stmt 300
-#define with_item 301
-#define except_clause 302
-#define suite 303
-#define test 304
-#define test_nocond 305
-#define lambdef 306
-#define lambdef_nocond 307
-#define or_test 308
-#define and_test 309
-#define not_test 310
-#define comparison 311
-#define comp_op 312
-#define star_expr 313
-#define expr 314
-#define xor_expr 315
-#define and_expr 316
-#define shift_expr 317
-#define arith_expr 318
-#define term 319
-#define factor 320
-#define power 321
-#define atom_expr 322
-#define atom 323
-#define testlist_comp 324
-#define trailer 325
-#define subscriptlist 326
-#define subscript 327
-#define sliceop 328
-#define exprlist 329
-#define testlist 330
-#define dictorsetmaker 331
-#define classdef 332
-#define arglist 333
-#define argument 334
-#define comp_iter 335
-#define comp_for 336
-#define comp_if 337
-#define encoding_decl 338
-#define yield_expr 339
-#define yield_arg 340
-#define func_type_input 341
-#define func_type 342
-#define typelist 343
+#define annassign 273
+#define testlist_star_expr 274
+#define augassign 275
+#define del_stmt 276
+#define pass_stmt 277
+#define flow_stmt 278
+#define break_stmt 279
+#define continue_stmt 280
+#define return_stmt 281
+#define yield_stmt 282
+#define raise_stmt 283
+#define import_stmt 284
+#define import_name 285
+#define import_from 286
+#define import_as_name 287
+#define dotted_as_name 288
+#define import_as_names 289
+#define dotted_as_names 290
+#define dotted_name 291
+#define global_stmt 292
+#define nonlocal_stmt 293
+#define assert_stmt 294
+#define compound_stmt 295
+#define async_stmt 296
+#define if_stmt 297
+#define while_stmt 298
+#define for_stmt 299
+#define try_stmt 300
+#define with_stmt 301
+#define with_item 302
+#define except_clause 303
+#define suite 304
+#define test 305
+#define test_nocond 306
+#define lambdef 307
+#define lambdef_nocond 308
+#define or_test 309
+#define and_test 310
+#define not_test 311
+#define comparison 312
+#define comp_op 313
+#define star_expr 314
+#define expr 315
+#define xor_expr 316
+#define and_expr 317
+#define shift_expr 318
+#define arith_expr 319
+#define term 320
+#define factor 321
+#define power 322
+#define atom_expr 323
+#define atom 324
+#define testlist_comp 325
+#define trailer 326
+#define subscriptlist 327
+#define subscript 328
+#define sliceop 329
+#define exprlist 330
+#define testlist 331
+#define dictorsetmaker 332
+#define classdef 333
+#define arglist 334
+#define argument 335
+#define comp_iter 336
+#define comp_for 337
+#define comp_if 338
+#define encoding_decl 339
+#define yield_expr 340
+#define yield_arg 341
+#define func_type_input 342
+#define func_type 343
+#define typelist 344

ast35/Parser/Python.asdl

@@ -23,7 +23,7 @@ module Python
           | Return(expr? value)
 
           | Delete(expr* targets)
-          | Assign(expr* targets, expr value, string? type_comment)
+          | Assign(expr* targets, expr? value, string? type_comment, expr? annotation)
           | AugAssign(expr target, operator op, expr value)
 
           -- use 'orelse' because else is a keyword in target languages

ast35/Python/Python-ast.c

@@ -85,10 +85,12 @@ static char *Delete_fields[]={
     "targets",
 };
 static PyTypeObject *Assign_type;
+_Py_IDENTIFIER(annotation);
 static char *Assign_fields[]={
     "targets",
     "value",
     "type_comment",
+    "annotation",
 };
 static PyTypeObject *AugAssign_type;
 _Py_IDENTIFIER(target);
@@ -464,7 +466,6 @@ static char *arg_attributes[] = {
     "col_offset",
 };
 _Py_IDENTIFIER(arg);
-_Py_IDENTIFIER(annotation);
 static char *arg_fields[]={
     "arg",
     "annotation",
@@ -862,7 +863,7 @@ static int init_types(void)
     if (!Return_type) return 0;
     Delete_type = make_type("Delete", stmt_type, Delete_fields, 1);
     if (!Delete_type) return 0;
-    Assign_type = make_type("Assign", stmt_type, Assign_fields, 3);
+    Assign_type = make_type("Assign", stmt_type, Assign_fields, 4);
     if (!Assign_type) return 0;
     AugAssign_type = make_type("AugAssign", stmt_type, AugAssign_fields, 3);
     if (!AugAssign_type) return 0;
@@ -1369,22 +1370,18 @@ Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
 }
 
 stmt_ty
-Assign(asdl_seq * targets, expr_ty value, string type_comment, int lineno, int
-       col_offset, PyArena *arena)
+Assign(asdl_seq * targets, expr_ty value, string type_comment, expr_ty
+       annotation, int lineno, int col_offset, PyArena *arena)
 {
     stmt_ty p;
-    if (!value) {
-        PyErr_SetString(PyExc_ValueError,
-                        "field value is required for Assign");
-        return NULL;
-    }
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
     if (!p)
         return NULL;
     p->kind = Assign_kind;
     p->v.Assign.targets = targets;
     p->v.Assign.value = value;
     p->v.Assign.type_comment = type_comment;
+    p->v.Assign.annotation = annotation;
     p->lineno = lineno;
     p->col_offset = col_offset;
     return p;
@@ -2729,6 +2726,11 @@ ast2obj_stmt(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_expr(o->v.Assign.annotation);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_annotation, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case AugAssign_kind:
         result = PyType_GenericNew(AugAssign_type, NULL, NULL);
@@ -4554,6 +4556,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         asdl_seq* targets;
         expr_ty value;
         string type_comment;
+        expr_ty annotation;
 
         if (_PyObject_HasAttrId(obj, &PyId_targets)) {
             int res;
@@ -4579,16 +4582,15 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             PyErr_SetString(PyExc_TypeError, "required field \"targets\" missing from Assign");
             return 1;
         }
-        if (_PyObject_HasAttrId(obj, &PyId_value)) {
+        if (exists_not_none(obj, &PyId_value)) {
             int res;
             tmp = _PyObject_GetAttrId(obj, &PyId_value);
             if (tmp == NULL) goto failed;
             res = obj2ast_expr(tmp, &value, arena);
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         } else {
-            PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Assign");
-            return 1;
+            value = NULL;
         }
         if (exists_not_none(obj, &PyId_type_comment)) {
             int res;
@@ -4600,7 +4602,18 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         } else {
             type_comment = NULL;
         }
-        *out = Assign(targets, value, type_comment, lineno, col_offset, arena);
+        if (exists_not_none(obj, &PyId_annotation)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_annotation);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_expr(tmp, &annotation, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            annotation = NULL;
+        }
+        *out = Assign(targets, value, type_comment, annotation, lineno,
+                      col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }