#!/usr/bin/env python3 """Check early codeobject.c C89 locals and evaluation order.""" import sys from pathlib import Path if len(sys.argv) != 2: raise SystemExit("usage: vc6-codeobject-c89.test.py codeobject.c") source = Path(sys.argv[1]).read_text() def body_for(signature): start = source.index(signature) brace = source.index("{", start) depth = 0 for pos in range(brace, len(source)): if source[pos] == "{": depth += 1 elif source[pos] == "}": depth -= 1 if depth == 0: return source[brace + 1:pos] raise SystemExit("unterminated function " + signature) body = body_for("intern_string_constants(PyObject *tuple, int *modified)") if body.count("Py_ssize_t i;") != 1 or "for (Py_ssize_t i =" in body: raise SystemExit("intern_string_constants loop index must be declared at function entry") if body.index("Py_ssize_t i;") > body.index("for (i ="): raise SystemExit("intern_string_constants loop index is declared too late") frozen = body[body.index("else if (PyFrozenSet_CheckExact(v))") :] if frozen.index("int tmp_modified;") > frozen.index("if (tmp == NULL)"): raise SystemExit("tmp_modified must be declared before the first statement in the frozen-set block") order = ("PyObject *tmp = PySequence_Tuple(v);", "if (tmp == NULL)", "tmp_modified = 0;", "intern_string_constants(tmp, &tmp_modified)") if [frozen.index(x) for x in order] != sorted(frozen.index(x) for x in order): raise SystemExit("frozen-set tuple construction/error/intern order changed") body = body_for("get_localsplus_counts(PyObject *names, PyObject *kinds,") if body.count("int i;") != 1 or "for (int i =" in body: raise SystemExit("get_localsplus_counts loop index must be declared at function entry") if body.index("int i;") > body.index("for (i ="): raise SystemExit("get_localsplus_counts loop index is declared too late") body = body_for("get_localsplus_names(PyCodeObject *co, _PyLocals_Kind kind, int num)") if "int index;" not in body or "int offset;" not in body: raise SystemExit("get_localsplus_names indices must be declared at function entry") if "for (int offset =" in body or body.index("int index;") > body.index("if (names == NULL)"): raise SystemExit("get_localsplus_names declarations must precede execution") order = ("if (names == NULL)", "index = 0;", "for (offset =", "_PyLocals_Kind k;", "k = _PyLocals_GetKind", "assert(index < num);", "name = PyTuple_GET_ITEM", "Py_INCREF(name);", "PyTuple_SET_ITEM(names, index, name);") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("get_localsplus_names allocation/reference/insertion order changed") body = body_for("_PyCode_Validate(struct _PyCodeConstructor *con)") for name in ("nlocals", "nplainlocals"): if "int " + name + ";" not in body: raise SystemExit("_PyCode_Validate " + name + " must be declared at function entry") if body.index("int nlocals;") > body.index("/* Check argument types */"): raise SystemExit("_PyCode_Validate locals must precede validation") if "int nplainlocals =" in body: raise SystemExit("_PyCode_Validate nplainlocals initializer must stay at its execution point") order = ("/* Check argument types */", "/* Make sure that code is indexable", "get_localsplus_counts(", "nplainlocals = nlocals -") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("_PyCode_Validate validation/count computation order changed") body = body_for("init_code(PyCodeObject *co, struct _PyCodeConstructor *con)") if "int entry_point;" not in body or body.index("int entry_point;") > body.index("get_localsplus_counts("): raise SystemExit("init_code entry_point must be declared before execution") order = ("memcpy(_PyCode_CODE(co)", "entry_point = 0;", "while (entry_point < Py_SIZE(co)", "co->_co_firsttraceable = entry_point;") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("init_code byte copy / entry-point scan order changed") body = body_for("remove_column_info(PyObject *locations)") for name in ("uint8_t *output;", "Py_ssize_t write_offset;"): if name not in body: raise SystemExit("remove_column_info must declare " + name) if body.index("uint8_t *output;") > body.index("if (res == NULL)"): raise SystemExit("remove_column_info output declaration must precede execution") loop = body[body.index("while (offset <") : body.index("\n }\n write_offset =")] if loop.index("int code;") > loop.index("write_offset = output -"): raise SystemExit("remove_column_info loop code local must be declared at block entry") order = ("if (res == NULL)", "output = (uint8_t *)PyBytes_AS_STRING(res);", "while (offset <", "code = (data[offset] >> 3) & 15;", "\n }\n write_offset = output -", "if (_PyBytes_Resize(&res, write_offset))") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("remove_column_info allocation/resizing order changed") body = body_for("_PyCode_New(struct _PyCodeConstructor *con)") for declaration in ("PyObject *replacement_locations;", "Py_ssize_t size;", "PyCodeObject *co;"): if declaration not in body or body.index(declaration) > body.index("/* Ensure that strings are ready Unicode string */"): raise SystemExit("_PyCode_New entry declaration missing or late: " + declaration) if "PyObject *replacement_locations =" in body or "Py_ssize_t size =" in body or "PyCodeObject *co =" in body: raise SystemExit("_PyCode_New initializers must remain at their original execution points") order = ("/* Ensure that strings are ready Unicode string */", "intern_strings(con->names)", "intern_string_constants(con->consts", "replacement_locations = NULL;", "if (!_Py_GetConfig()->code_debug_ranges)", "replacement_locations = remove_column_info", "con->linetable = replacement_locations;", "size = PyBytes_GET_SIZE(con->code)", "co = PyObject_NewVar", "if (co == NULL)", "init_code(co, con);") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("_PyCode_New validation/allocation order changed") if not (body.index("init_code(co, con);") < body.rindex("Py_XDECREF(replacement_locations);") < body.index("return co;")): raise SystemExit("_PyCode_New success cleanup order changed") body = body_for("PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,") for declaration in ("int nvarnames;", "int ncellvars;", "int nfreevars;", "int nlocalsplus;", "int offset;", "int i;", "int j;", "PyObject *local_name;", "int argoffset;", "int cmp;", "_PyLocals_Kind kind;", "struct _PyCodeConstructor con = {0};"): if declaration not in body or body.index(declaration) > body.index("if (varnames == NULL"): raise SystemExit("PyCode_NewWithPosOnlyArgs entry declaration missing or late: " + declaration) if "for (int " in body or "int nvarnames =" in body or "int offset =" in body: raise SystemExit("PyCode_NewWithPosOnlyArgs contains a C99 local declaration") fields = ("filename", "name", "qualname", "flags", "code", "firstlineno", "linetable", "consts", "names", "localsplusnames", "localspluskinds", "argcount", "posonlyargcount", "kwonlyargcount", "stacksize", "exceptiontable") field_positions = [body.index("con." + field + " =") for field in fields] if field_positions != sorted(field_positions): raise SystemExit("code constructor field initialization order changed") order = ("if (varnames == NULL", "nvarnames =", "ncellvars =", "nfreevars =", "nlocalsplus =", "localsplusnames = PyTuple_New", "localspluskinds = PyBytes_FromStringAndSize", "offset = 0;", "for (i = 0;", "for (i = 0;", "for (j = 0;", "for (i = 0;", "_PyTuple_Resize", "con.filename =", "_PyCode_Validate(&con)", "co = _PyCode_New(&con);", "error:", "Py_XDECREF(localsplusnames);", "Py_XDECREF(localspluskinds);") positions = [] for item in order: positions.append(body.index(item, positions[-1] + 1 if positions else 0)) if positions != sorted(positions): raise SystemExit("legacy constructor iteration/validation/cleanup order changed") body = body_for("PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)") if "struct _PyCodeConstructor con = {0};" not in body: raise SystemExit("PyCode_NewEmpty constructor must be zero-initialized at function entry") fields = ("filename", "name", "qualname", "code", "firstlineno", "linetable", "consts", "names", "localsplusnames", "localspluskinds", "exceptiontable", "stacksize") field_positions = [body.index("con." + field + " =") for field in fields] if field_positions != sorted(field_positions): raise SystemExit("PyCode_NewEmpty constructor field order changed") order = ("nulltuple = PyTuple_New", "funcname_ob = PyUnicode_FromString", "filename_ob = PyUnicode_DecodeFSDefault", "code_ob = PyBytes_FromStringAndSize", "linetable_ob = PyBytes_FromStringAndSize", "con.filename =", "con.name =", "con.qualname =", "con.code =", "con.firstlineno =", "con.linetable =", "con.consts =", "con.names =", "con.localsplusnames =", "con.localspluskinds =", "con.exceptiontable =", "con.stacksize =", "result = _PyCode_New(&con);", "failed:", "Py_XDECREF(nulltuple);", "return result;") positions = [body.index(x) for x in order] if positions != sorted(positions): raise SystemExit("PyCode_NewEmpty allocation/cleanup order changed") body = body_for("_PyCode_CreateLineArray(PyCodeObject *co)") for declaration in ("PyCodeAddressRange bounds;", "int size;", "int max_line;"): if declaration not in body or body.index(declaration) > body.index("assert(co->_co_linearray == NULL);"): raise SystemExit("line-array entry declaration missing or late: " + declaration) if body.count("for (int index =") != 0: raise SystemExit("line-array inner loop must not declare its index in the for header") order = ("assert(co->_co_linearray == NULL);", "max_line = 0;", "_PyCode_InitAddressRange(co, &bounds);", "while(_PyLineTable_NextAddressRange(&bounds))", "if (max_line < (1 << 15))", "co->_co_linearray = PyMem_Malloc", "if (co->_co_linearray == NULL)", "co->_co_linearray_entry_size = size;", "_PyCode_InitAddressRange(co, &bounds);", "for (index = start; index < end; index++)") positions = [] for item in order: positions.append(body.index(item, positions[-1] + 1 if positions else 0)) if positions != sorted(positions): raise SystemExit("line-array range/allocation/fill order changed") body = body_for("PyCode_Addr2Line(PyCodeObject *co, int addrq)") if body.count("PyCodeAddressRange bounds;") != 1 or body.index("PyCodeAddressRange bounds;") > body.index("if (addrq < 0)"): raise SystemExit("PyCode_Addr2Line bounds must be declared before execution") order = ("if (addrq < 0)", "return co->co_firstlineno;", "assert(addrq >= 0", "if (co->_co_linearray)", "return _PyCode_LineNumberFromArray", "_PyCode_InitAddressRange(co, &bounds);", "return _PyCode_CheckLineNumber(addrq, &bounds);") positions = [body.index(x) for x in order] if positions != sorted(positions): raise SystemExit("PyCode_Addr2Line early-return/lookup/range initialization order changed") body = body_for("_PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds)") first_statement = body.index("assert(co->co_linetable != NULL);") for declaration in ("const char *linetable;", "Py_ssize_t length;"): if body.count(declaration) != 1 or body.index(declaration) > first_statement: raise SystemExit("_PyCode_InitAddressRange entry declaration missing or late: " + declaration) order = ( "assert(co->co_linetable != NULL);", "\n linetable = PyBytes_AS_STRING(co->co_linetable);", "\n length = PyBytes_GET_SIZE(co->co_linetable);", "_PyLineTable_InitAddressRange(linetable, length, co->co_firstlineno, bounds);", "return bounds->ar_line;", ) positions = [body.index(item) for item in order] if positions != sorted(positions): raise SystemExit("_PyCode_InitAddressRange assertion/byte access/length/range initialization order changed") body = body_for("previous_code_delta(PyCodeAddressRange *bounds)") if body.count("const uint8_t *ptr;") != 1 or body.index("const uint8_t *ptr;") > body.index("if (bounds->ar_start == 0)"): raise SystemExit("previous_code_delta ptr must be declared before execution") if "const uint8_t *ptr =" in body: raise SystemExit("previous_code_delta ptr assignment must remain after the first-entry return") order = ("if (bounds->ar_start == 0)", "return 1;", "ptr = bounds->opaque.lo_next-1;", "while (((*ptr) & 128) == 0)", "ptr--;", "return (((*ptr) & 7) + 1) * sizeof(_Py_CODEUNIT);") positions = [body.index(item) for item in order] if positions != sorted(positions): raise SystemExit("previous_code_delta first-entry return and backward scan order changed") body = body_for("advance_with_locations(PyCodeAddressRange *bounds, int *endline, int *column, int *endcolumn)") first_statement = body.index("ASSERT_VALID_BOUNDS(bounds);") for declaration in ("int first_byte;", "int code;"): if body.count(declaration) != 1 or body.index(declaration) > first_statement: raise SystemExit("advance_with_locations entry declaration missing or late: " + declaration) if "int first_byte =" in body or "int code =" in body: raise SystemExit("advance_with_locations initializers must remain at their execution points") execution_prefix = """ASSERT_VALID_BOUNDS(bounds); first_byte = read_byte(bounds); code = (first_byte >> 3) & 15; bounds->ar_start = bounds->ar_end; bounds->ar_end = bounds->ar_start + ((first_byte & 7) + 1) * sizeof(_Py_CODEUNIT); switch(code) {""" if not body[first_statement:].startswith(execution_prefix): raise SystemExit("advance_with_locations assertion/read/decode/range/dispatch order changed") body = body_for("PyCode_Addr2Location(PyCodeObject *co, int addrq,") if body.count("PyCodeAddressRange bounds;") != 1 or body.index("PyCodeAddressRange bounds;") > body.index("if (addrq < 0)"): raise SystemExit("PyCode_Addr2Location bounds must be declared before execution") order = ( "if (addrq < 0)", "*start_line = *end_line = co->co_firstlineno;", "*start_column = *end_column = 0;", "return 1;", "assert(addrq >= 0 && addrq < _PyCode_NBYTES(co));", "_PyCode_InitAddressRange(co, &bounds);", "_PyCode_CheckLineNumber(addrq, &bounds);", "retreat(&bounds);", "advance_with_locations(&bounds, end_line, start_column, end_column);", "*start_line = bounds.ar_line;", "return 1;", ) positions = [] for item in order: positions.append(body.index(item, positions[-1] + 1 if positions else 0)) if positions != sorted(positions): raise SystemExit("PyCode_Addr2Location early-return/assertion/range initialization/lookup order changed") body = body_for("_PyLineTable_StartsLine(PyCodeAddressRange *range)") for declaration in ("const uint8_t *ptr;", "int code;"): if declaration not in body or body.index(declaration) > body.index("if (range->ar_start <= 0)"): raise SystemExit("StartsLine entry declaration missing or late: " + declaration) order = ("if (range->ar_start <= 0)", "ptr = range->opaque.lo_next;", "do {", "ptr--;") positions = [body.index(x) for x in order] if positions != sorted(positions): raise SystemExit("StartsLine backward-scan order changed") body = body_for("emit_pair(PyObject **bytes, int *offset, int a, int b)") if "unsigned char *lnotab;" not in body or body.index("unsigned char *lnotab;") > body.index("if (*offset + 2 >= len)"): raise SystemExit("emit_pair lnotab pointer must be declared before possible resize") order = ("if (*offset + 2 >= len)", "if (_PyBytes_Resize(bytes, len * 2) < 0)", "lnotab = (unsigned char *) PyBytes_AS_STRING(*bytes);", "lnotab += *offset;", "*lnotab++ = a;", "*lnotab++ = b;", "*offset += 2;") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("emit_pair resize/pointer/write order changed") body = body_for("lineiter_next(lineiterator *li)") for declaration in ("PyObject *start;", "PyObject *end;", "PyObject *line;", "PyObject *result;"): if declaration not in body or body.index(declaration) > body.index("if (!_PyLineTable_NextAddressRange(bounds))"): raise SystemExit("lineiter_next entry declaration missing or late: " + declaration) order = ("if (!_PyLineTable_NextAddressRange(bounds))", "start = NULL;", "end = NULL;", "line = NULL;", "result = PyTuple_New(3);", "start = PyLong_FromLong", "end = PyLong_FromLong", "if (bounds->ar_line < 0)", "if (result == NULL", "PyTuple_SET_ITEM(result, 0, start);", "return result;", "error:", "Py_XDECREF(start);", "Py_XDECREF(result);") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("lineiter_next conversion/insertion/cleanup order changed") # Extra-slot locals must be C89 declarations; preserve the complete execution body. body = body_for('_PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)') entry = 'PyCodeObject *o;\n_PyCodeObjectExtra *co_extra;' if " ".join(body.split()).startswith(" ".join(entry.split())) is False: raise SystemExit("_PyCode_GetExtra locals must be declared at function entry") execution = body for declaration in entry.splitlines(): execution = execution.replace(declaration, "", 1) expected_execution = """if (!PyCode_Check(code)) { PyErr_BadInternalCall(); return -1; } o = (PyCodeObject*) code; co_extra = (_PyCodeObjectExtra*) o->co_extra; if (co_extra == NULL || index < 0 || co_extra->ce_size <= index) { *extra = NULL; return 0; } *extra = co_extra->ce_extras[index]; return 0;""" if " ".join(execution.split()) != " ".join(expected_execution.split()): raise SystemExit("_PyCode_GetExtra branches/allocation/free/assignment order changed") body = body_for('_PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)') entry = 'PyInterpreterState *interp = _PyInterpreterState_GET();\nPyCodeObject *o;\n_PyCodeObjectExtra *co_extra;\nPy_ssize_t i;\nfreefunc free;' if " ".join(body.split()).startswith(" ".join(entry.split())) is False: raise SystemExit("_PyCode_SetExtra locals must be declared at function entry") execution = body for declaration in entry.splitlines(): execution = execution.replace(declaration, "", 1) expected_execution = """if (!PyCode_Check(code) || index < 0 || index >= interp->co_extra_user_count) { PyErr_BadInternalCall(); return -1; } o = (PyCodeObject*) code; co_extra = (_PyCodeObjectExtra *) o->co_extra; if (co_extra == NULL || co_extra->ce_size <= index) { i = (co_extra == NULL ? 0 : co_extra->ce_size); co_extra = PyMem_Realloc( co_extra, sizeof(_PyCodeObjectExtra) + (interp->co_extra_user_count-1) * sizeof(void*)); if (co_extra == NULL) { return -1; } for (; i < interp->co_extra_user_count; i++) { co_extra->ce_extras[i] = NULL; } co_extra->ce_size = interp->co_extra_user_count; o->co_extra = co_extra; } if (co_extra->ce_extras[index] != NULL) { free = interp->co_extra_freefuncs[index]; if (free != NULL) { free(co_extra->ce_extras[index]); } } co_extra->ce_extras[index] = extra; return 0;""" if " ".join(execution.split()) != " ".join(expected_execution.split()): raise SystemExit("_PyCode_SetExtra branches/allocation/free/assignment order changed") body = body_for("deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len)") if "int i;" not in body or "for (int i =" in body: raise SystemExit("deopt_code loop index must be declared before the loop") order = ("for (i = 0; i < len; i++)", "_Py_CODEUNIT instruction = instructions[i];", "int opcode =", "int caches =", "instructions[i] =", "while (caches--)", "instructions[++i] =") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("deopt_code instruction/cache rewrite order changed") body = body_for("_PyCode_GetCode(PyCodeObject *co)") if "PyObject *code;" not in body or body.index("PyObject *code;") > body.index("if (co->_co_code != NULL)"): raise SystemExit("_PyCode_GetCode code object must be declared at entry") if "PyObject *code =" in body: raise SystemExit("_PyCode_GetCode allocation must remain after cache check") order = ("if (co->_co_code != NULL)", "return Py_NewRef(co->_co_code);", "code = PyBytes_FromStringAndSize", "if (code == NULL)", "deopt_code(", "co->_co_code = Py_NewRef(code);", "return code;") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("_PyCode_GetCode cached return/allocation/deopt/cache order changed") body = body_for("code_dealloc(PyCodeObject *co)") if "Py_ssize_t i;" not in body or "for (Py_ssize_t i =" in body: raise SystemExit("code_dealloc loop index must be declared before its loop") order = ("if (co->co_extra != NULL)", "Py_ssize_t i;", "for (i = 0; i < co_extra->ce_size; i++)", "free_extra = interp->co_extra_freefuncs[i];", "free_extra(co_extra->ce_extras[i]);", "PyMem_Free(co_extra);", "Py_XDECREF(co->co_consts);", "PyObject_Free(co);") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("code_dealloc extra cleanup/refcount order changed") body = body_for("code_richcompare(PyObject *self, PyObject *other, int op)") if "int i;" not in body or "for (int i =" in body: raise SystemExit("code_richcompare loop index must be declared before its loop") order = ("if ((op != Py_EQ", "co = (PyCodeObject *)self;", "cp = (PyCodeObject *)other;", "for (i = 0; i < Py_SIZE(co); i++)", "_Py_SET_OPCODE(co_instr", "eq = co_instr == cp_instr;", "i += _PyOpcode_Caches", "consts1 = _PyCode_ConstantKey", "Py_DECREF(consts1);", "Py_DECREF(consts2);", "Py_INCREF(res);", "return res;") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("code_richcompare validation/instruction/constants order changed") body = body_for("code_hash(PyCodeObject *co)") if "Py_hash_t h4, h5;" not in body or body.index("Py_hash_t h4, h5;") > body.index("h0 = PyObject_Hash"): raise SystemExit("code_hash h4/h5 must be declared before hash evaluation") if "Py_hash_t h4 =" in body or "Py_hash_t h5 =" in body: raise SystemExit("code_hash h4/h5 hash calls must remain at their original execution points") order = ("h3 = PyObject_Hash", "h4 = PyObject_Hash", "if (h4 == -1)", "h5 = PyObject_Hash", "if (h5 == -1)", "h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("code_hash error handling/hash combination order changed") body = body_for("code_replace_impl(PyCodeObject *self, int co_argcount,") for declaration in ("PyObject *code;", "PyCodeObject *co;", "PyObject *varnames;", "PyObject *cellvars;", "PyObject *freevars;"): if declaration not in body or body.index(declaration) > body.index("#define CHECK_INT_ARG(ARG)"): raise SystemExit("code_replace_impl entry declaration missing or late: " + declaration) for initializer in ("PyObject *code =", "PyCodeObject *co =", "PyObject *varnames =", "PyObject *cellvars =", "PyObject *freevars ="): if initializer in body: raise SystemExit("code_replace_impl initializer must remain after validation/audit: " + initializer) order = ("CHECK_INT_ARG(co_firstlineno);", "code = NULL;", "if (co_code == NULL)", "if (PySys_Audit", "co = NULL;", "varnames = NULL;", "cellvars = NULL;", "freevars = NULL;", "if (co_varnames == NULL)", "if (co_cellvars == NULL)", "if (co_freevars == NULL)", "co = PyCode_NewWithPosOnlyArgs(", "error:") if [body.index(x) for x in order] != sorted(body.index(x) for x in order): raise SystemExit("code_replace_impl validation/audit/allocation order changed") cleanup = [body.rindex(x) for x in ("Py_XDECREF(code);", "Py_XDECREF(varnames);", "Py_XDECREF(cellvars);", "Py_XDECREF(freevars);")] if cleanup != sorted(cleanup) or cleanup[-1] > body.index("return (PyObject *)co;"): raise SystemExit("code_replace_impl cleanup order changed") print("codeobject C89 declarations and ownership order: PASS")