diff --git a/ast27/Include/Python-ast.h b/ast27/Include/Python-ast.h index 3b39f5cd..2d100e9b 100644 --- a/ast27/Include/Python-ast.h +++ b/ast27/Include/Python-ast.h @@ -286,7 +286,7 @@ struct _expr { struct { string s; - int has_b; + string kind; } Str; struct { @@ -505,7 +505,7 @@ expr_ty _Ta27_Repr(expr_ty value, int lineno, int col_offset, PyArena *arena); #define Num(a0, a1, a2, a3) _Ta27_Num(a0, a1, a2, a3) expr_ty _Ta27_Num(object n, int lineno, int col_offset, PyArena *arena); #define Str(a0, a1, a2, a3, a4) _Ta27_Str(a0, a1, a2, a3, a4) -expr_ty _Ta27_Str(string s, int has_b, int lineno, int col_offset, PyArena *arena); +expr_ty _Ta27_Str(string s, string kind, int lineno, int col_offset, PyArena *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Ta27_Attribute(a0, a1, a2, a3, a4, a5) expr_ty _Ta27_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena); diff --git a/ast27/Parser/Python.asdl b/ast27/Parser/Python.asdl index 5a69a1c6..00bbf2af 100644 --- a/ast27/Parser/Python.asdl +++ b/ast27/Parser/Python.asdl @@ -71,7 +71,7 @@ module Python version "$Revision$" expr? starargs, expr? kwargs) | Repr(expr value) | Num(object n) -- a number as a PyObject. - | Str(string s, int? has_b) -- need to specify raw, unicode, etc? + | Str(string s, string kind) -- other literals? bools? -- the following expression can appear in assignment context diff --git a/ast27/Python/Python-ast.c b/ast27/Python/Python-ast.c index 0d1212ae..7f5e39ac 100644 --- a/ast27/Python/Python-ast.c +++ b/ast27/Python/Python-ast.c @@ -252,7 +252,7 @@ static char *Num_fields[]={ static PyTypeObject *Str_type; static char *Str_fields[]={ "s", - "has_b", + "kind", }; static PyTypeObject *Attribute_type; static char *Attribute_fields[]={ @@ -1850,7 +1850,7 @@ Num(object n, int lineno, int col_offset, PyArena *arena) } expr_ty -Str(string s, int has_b, int lineno, int col_offset, PyArena *arena) +Str(string s, string kind, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!s) { @@ -1858,12 +1858,17 @@ Str(string s, int has_b, int lineno, int col_offset, PyArena *arena) "field s is required for Str"); return NULL; } + if (!kind) { + PyErr_SetString(PyExc_ValueError, + "field kind is required for Str"); + return NULL; + } p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = Str_kind; p->v.Str.s = s; - p->v.Str.has_b = has_b; + p->v.Str.kind = kind; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -2889,9 +2894,9 @@ ast2obj_expr(void* _o) if (PyObject_SetAttrString(result, "s", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(o->v.Str.has_b); + value = ast2obj_string(o->v.Str.kind); if (!value) goto failed; - if (PyObject_SetAttrString(result, "has_b", value) == -1) + if (PyObject_SetAttrString(result, "kind", value) == -1) goto failed; Py_DECREF(value); break; @@ -5714,7 +5719,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } if (isinstance) { string s; - int has_b; + string kind; if (PyObject_HasAttrString(obj, "s")) { int res; @@ -5728,18 +5733,19 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str"); return 1; } - if (PyObject_HasAttrString(obj, "has_b")) { + if (PyObject_HasAttrString(obj, "kind")) { int res; - tmp = PyObject_GetAttrString(obj, "has_b"); + tmp = PyObject_GetAttrString(obj, "kind"); if (tmp == NULL) goto failed; - res = obj2ast_int(tmp, &has_b, arena); + res = obj2ast_string(tmp, &kind, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; } else { - has_b = 0; + PyErr_SetString(PyExc_TypeError, "required field \"kind\" missing from Str"); + return 1; } - *out = Str(s, has_b, lineno, col_offset, arena); + *out = Str(s, kind, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } diff --git a/ast27/Python/ast.c b/ast27/Python/ast.c index cdbdfe44..17ee2301 100644 --- a/ast27/Python/ast.c +++ b/ast27/Python/ast.c @@ -1498,10 +1498,20 @@ ast_for_atom(struct compiling *c, const node *n) return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); } case STRING: { - PyObject *str = parsestrplus(c, n); - const char *s = STR(CHILD(n, 0)); + PyObject *kind, *str = parsestrplus(c, n); + const char *raw, *s = STR(CHILD(n, 0)); int quote = Py_CHARMASK(*s); - int has_b = 0; + /* currently Python allows up to 2 string modifiers */ + char *ch, s_kind[3] = {0, 0, 0}; + ch = s_kind; + raw = s; + while (*raw && *raw != '\'' && *raw != '"') { + *ch++ = *raw++; + } + kind = PyUnicode_FromString(s_kind); + if (!kind) { + return NULL; + } if (!str) { #ifdef Py_USING_UNICODE if (PyErr_ExceptionMatches(PyExc_UnicodeError)){ @@ -1526,10 +1536,7 @@ ast_for_atom(struct compiling *c, const node *n) return NULL; } PyArena_AddPyObject(c->c_arena, str); - if (quote == 'b' || quote == 'B') { - has_b = 1; - } - return Str(str, has_b, LINENO(n), n->n_col_offset, c->c_arena); + return Str(str, kind, LINENO(n), n->n_col_offset, c->c_arena); } case NUMBER: { PyObject *pynum = parsenumber(c, STR(ch)); diff --git a/ast3/Include/Python-ast.h b/ast3/Include/Python-ast.h index 38a6e5a6..b56a83bf 100644 --- a/ast3/Include/Python-ast.h +++ b/ast3/Include/Python-ast.h @@ -318,6 +318,7 @@ struct _expr { struct { string s; + string kind; } Str; struct { @@ -601,8 +602,9 @@ expr_ty _Ta3_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int col_offset, PyArena *arena); #define Num(a0, a1, a2, a3) _Ta3_Num(a0, a1, a2, a3) expr_ty _Ta3_Num(object n, int lineno, int col_offset, PyArena *arena); -#define Str(a0, a1, a2, a3) _Ta3_Str(a0, a1, a2, a3) -expr_ty _Ta3_Str(string s, int lineno, int col_offset, PyArena *arena); +#define Str(a0, a1, a2, a3, a4) _Ta3_Str(a0, a1, a2, a3, a4) +expr_ty _Ta3_Str(string s, string kind, int lineno, int col_offset, PyArena + *arena); #define FormattedValue(a0, a1, a2, a3, a4, a5) _Ta3_FormattedValue(a0, a1, a2, a3, a4, a5) expr_ty _Ta3_FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno, int col_offset, PyArena *arena); diff --git a/ast3/Parser/Python.asdl b/ast3/Parser/Python.asdl index 7bde99ca..c4ca9b72 100644 --- a/ast3/Parser/Python.asdl +++ b/ast3/Parser/Python.asdl @@ -77,7 +77,7 @@ module Python | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords) | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? + | Str(string s, string kind) | FormattedValue(expr value, int? conversion, expr? format_spec) | JoinedStr(expr* values) | Bytes(bytes s) diff --git a/ast3/Python/Python-ast.c b/ast3/Python/Python-ast.c index 56313287..b81b5a4d 100644 --- a/ast3/Python/Python-ast.c +++ b/ast3/Python/Python-ast.c @@ -307,8 +307,10 @@ static char *Num_fields[]={ }; static PyTypeObject *Str_type; _Py_IDENTIFIER(s); +_Py_IDENTIFIER(kind); static char *Str_fields[]={ "s", + "kind", }; static PyTypeObject *FormattedValue_type; _Py_IDENTIFIER(conversion); @@ -983,7 +985,7 @@ static int init_types(void) if (!Call_type) return 0; Num_type = make_type("Num", expr_type, Num_fields, 1); if (!Num_type) return 0; - Str_type = make_type("Str", expr_type, Str_fields, 1); + Str_type = make_type("Str", expr_type, Str_fields, 2); if (!Str_type) return 0; FormattedValue_type = make_type("FormattedValue", expr_type, FormattedValue_fields, 3); @@ -2181,7 +2183,7 @@ Num(object n, int lineno, int col_offset, PyArena *arena) } expr_ty -Str(string s, int lineno, int col_offset, PyArena *arena) +Str(string s, string kind, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!s) { @@ -2189,11 +2191,17 @@ Str(string s, int lineno, int col_offset, PyArena *arena) "field s is required for Str"); return NULL; } + if (!kind) { + PyErr_SetString(PyExc_ValueError, + "field kind is required for Str"); + return NULL; + } p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = Str_kind; p->v.Str.s = s; + p->v.Str.kind = kind; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -3448,6 +3456,11 @@ ast2obj_expr(void* _o) if (_PyObject_SetAttrId(result, &PyId_s, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.Str.kind); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1) + goto failed; + Py_DECREF(value); break; case FormattedValue_kind: result = PyType_GenericNew(FormattedValue_type, NULL, NULL); @@ -6766,6 +6779,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } if (isinstance) { string s; + string kind; if (_PyObject_HasAttrId(obj, &PyId_s)) { int res; @@ -6778,7 +6792,18 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str"); return 1; } - *out = Str(s, lineno, col_offset, arena); + if (_PyObject_HasAttrId(obj, &PyId_kind)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_kind); + if (tmp == NULL) goto failed; + res = obj2ast_string(tmp, &kind, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"kind\" missing from Str"); + return 1; + } + *out = Str(s, kind, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } diff --git a/ast3/Python/ast.c b/ast3/Python/ast.c index 7b188477..ec6874aa 100644 --- a/ast3/Python/ast.c +++ b/ast3/Python/ast.c @@ -4971,14 +4971,25 @@ FstringParser_Dealloc(FstringParser *state) static expr_ty make_str_node_and_del(PyObject **str, struct compiling *c, const node* n) { - PyObject *s = *str; + PyObject *kind, *s = *str; + const char *raw = STR(CHILD(n, 0)); + /* currently Python allows up to 2 string modifiers */ + char *ch, s_kind[3] = {0, 0, 0}; + ch = s_kind; + while (*raw && *raw != '\'' && *raw != '"') { + *ch++ = *raw++; + } + kind = PyUnicode_FromString(s_kind); + if (!kind) { + return NULL; + } *str = NULL; assert(PyUnicode_CheckExact(s)); if (PyArena_AddPyObject(c->c_arena, s) < 0) { Py_DECREF(s); return NULL; } - return Str(s, LINENO(n), n->n_col_offset, c->c_arena); + return Str(s, kind, LINENO(n), n->n_col_offset, c->c_arena); } /* Add a non-f-string (that is, a regular literal string). str is