Skip to content
This repository was archived by the owner on Jul 5, 2023. It is now read-only.

Commit f4495f4

Browse files
ilevkivskyiddfisher
authored andcommitted
Distinguish 'x' and b'x' in Python 2 (#17)
Fixes #10 This is a naive fix. It only checks the first string in concatenation at CST level, for ``b`` or ``B`` prefix and sets the ``has_b`` flag in ``Str`` AST node accordingly.
1 parent 8ae1b7f commit f4495f4

File tree

4 files changed

+33
-7
lines changed

4 files changed

+33
-7
lines changed

ast27/Include/Python-ast.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ struct _expr {
286286

287287
struct {
288288
string s;
289+
int has_b;
289290
} Str;
290291

291292
struct {
@@ -503,8 +504,8 @@ expr_ty _Ta27_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty s
503504
expr_ty _Ta27_Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
504505
#define Num(a0, a1, a2, a3) _Ta27_Num(a0, a1, a2, a3)
505506
expr_ty _Ta27_Num(object n, int lineno, int col_offset, PyArena *arena);
506-
#define Str(a0, a1, a2, a3) _Ta27_Str(a0, a1, a2, a3)
507-
expr_ty _Ta27_Str(string s, int lineno, int col_offset, PyArena *arena);
507+
#define Str(a0, a1, a2, a3, a4) _Ta27_Str(a0, a1, a2, a3, a4)
508+
expr_ty _Ta27_Str(string s, int has_b, int lineno, int col_offset, PyArena *arena);
508509
#define Attribute(a0, a1, a2, a3, a4, a5) _Ta27_Attribute(a0, a1, a2, a3, a4, a5)
509510
expr_ty _Ta27_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
510511
col_offset, PyArena *arena);

ast27/Parser/Python.asdl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ module Python version "$Revision$"
7171
expr? starargs, expr? kwargs)
7272
| Repr(expr value)
7373
| Num(object n) -- a number as a PyObject.
74-
| Str(string s) -- need to specify raw, unicode, etc?
74+
| Str(string s, int? has_b) -- need to specify raw, unicode, etc?
7575
-- other literals? bools?
7676

7777
-- the following expression can appear in assignment context

ast27/Python/Python-ast.c

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ static char *Num_fields[]={
252252
static PyTypeObject *Str_type;
253253
static char *Str_fields[]={
254254
"s",
255+
"has_b",
255256
};
256257
static PyTypeObject *Attribute_type;
257258
static char *Attribute_fields[]={
@@ -782,7 +783,7 @@ static int init_types(void)
782783
if (!Repr_type) return 0;
783784
Num_type = make_type("Num", expr_type, Num_fields, 1);
784785
if (!Num_type) return 0;
785-
Str_type = make_type("Str", expr_type, Str_fields, 1);
786+
Str_type = make_type("Str", expr_type, Str_fields, 2);
786787
if (!Str_type) return 0;
787788
Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
788789
if (!Attribute_type) return 0;
@@ -1849,7 +1850,7 @@ Num(object n, int lineno, int col_offset, PyArena *arena)
18491850
}
18501851

18511852
expr_ty
1852-
Str(string s, int lineno, int col_offset, PyArena *arena)
1853+
Str(string s, int has_b, int lineno, int col_offset, PyArena *arena)
18531854
{
18541855
expr_ty p;
18551856
if (!s) {
@@ -1862,6 +1863,7 @@ Str(string s, int lineno, int col_offset, PyArena *arena)
18621863
return NULL;
18631864
p->kind = Str_kind;
18641865
p->v.Str.s = s;
1866+
p->v.Str.has_b = has_b;
18651867
p->lineno = lineno;
18661868
p->col_offset = col_offset;
18671869
return p;
@@ -2887,6 +2889,11 @@ ast2obj_expr(void* _o)
28872889
if (PyObject_SetAttrString(result, "s", value) == -1)
28882890
goto failed;
28892891
Py_DECREF(value);
2892+
value = ast2obj_int(o->v.Str.has_b);
2893+
if (!value) goto failed;
2894+
if (PyObject_SetAttrString(result, "has_b", value) == -1)
2895+
goto failed;
2896+
Py_DECREF(value);
28902897
break;
28912898
case Attribute_kind:
28922899
result = PyType_GenericNew(Attribute_type, NULL, NULL);
@@ -5707,6 +5714,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
57075714
}
57085715
if (isinstance) {
57095716
string s;
5717+
int has_b;
57105718

57115719
if (PyObject_HasAttrString(obj, "s")) {
57125720
int res;
@@ -5720,7 +5728,18 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
57205728
PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str");
57215729
return 1;
57225730
}
5723-
*out = Str(s, lineno, col_offset, arena);
5731+
if (PyObject_HasAttrString(obj, "has_b")) {
5732+
int res;
5733+
tmp = PyObject_GetAttrString(obj, "has_b");
5734+
if (tmp == NULL) goto failed;
5735+
res = obj2ast_int(tmp, &has_b, arena);
5736+
if (res != 0) goto failed;
5737+
Py_XDECREF(tmp);
5738+
tmp = NULL;
5739+
} else {
5740+
has_b = 0;
5741+
}
5742+
*out = Str(s, has_b, lineno, col_offset, arena);
57245743
if (*out == NULL) goto failed;
57255744
return 0;
57265745
}

ast27/Python/ast.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1499,6 +1499,9 @@ ast_for_atom(struct compiling *c, const node *n)
14991499
}
15001500
case STRING: {
15011501
PyObject *str = parsestrplus(c, n);
1502+
const char *s = STR(CHILD(n, 0));
1503+
int quote = Py_CHARMASK(*s);
1504+
int has_b = 0;
15021505
if (!str) {
15031506
#ifdef Py_USING_UNICODE
15041507
if (PyErr_ExceptionMatches(PyExc_UnicodeError)){
@@ -1523,7 +1526,10 @@ ast_for_atom(struct compiling *c, const node *n)
15231526
return NULL;
15241527
}
15251528
PyArena_AddPyObject(c->c_arena, str);
1526-
return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
1529+
if (quote == 'b' || quote == 'B') {
1530+
has_b = 1;
1531+
}
1532+
return Str(str, has_b, LINENO(n), n->n_col_offset, c->c_arena);
15271533
}
15281534
case NUMBER: {
15291535
PyObject *pynum = parsenumber(c, STR(ch));

0 commit comments

Comments
 (0)