To: vim_dev@googlegroups.com Subject: Patch 7.3.1056 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 7.3.1056 Problem: Python: possible memory leaks. Solution: Python patch 15. (ZyX) Fix will follow later. Files: src/eval.c, src/if_py_both.h, src/proto/eval.pro *** ../vim-7.3.1055/src/eval.c 2013-05-17 16:03:53.000000000 +0200 --- src/eval.c 2013-05-30 12:11:40.000000000 +0200 *************** *** 412,418 **** static int get_lit_string_tv __ARGS((char_u **arg, typval_T *rettv, int evaluate)); static int get_list_tv __ARGS((char_u **arg, typval_T *rettv, int evaluate)); static int rettv_list_alloc __ARGS((typval_T *rettv)); - static void listitem_free __ARGS((listitem_T *item)); static long list_len __ARGS((list_T *l)); static int list_equal __ARGS((list_T *l1, list_T *l2, int ic, int recursive)); static int dict_equal __ARGS((dict_T *d1, dict_T *d2, int ic, int recursive)); --- 412,417 ---- *************** *** 428,434 **** static int list_join __ARGS((garray_T *gap, list_T *l, char_u *sep, int echo, int copyID)); static int free_unref_items __ARGS((int copyID)); static int rettv_dict_alloc __ARGS((typval_T *rettv)); - static void dict_free __ARGS((dict_T *d, int recurse)); static dictitem_T *dictitem_copy __ARGS((dictitem_T *org)); static void dictitem_remove __ARGS((dict_T *dict, dictitem_T *item)); static dict_T *dict_copy __ARGS((dict_T *orig, int deep, int copyID)); --- 427,432 ---- *************** *** 5955,5961 **** /* * Free a list item. Also clears the value. Does not notify watchers. */ ! static void listitem_free(item) listitem_T *item; { --- 5953,5959 ---- /* * Free a list item. Also clears the value. Does not notify watchers. */ ! void listitem_free(item) listitem_T *item; { *************** *** 7031,7037 **** * Free a Dictionary, including all items it contains. * Ignores the reference count. */ ! static void dict_free(d, recurse) dict_T *d; int recurse; /* Free Lists and Dictionaries recursively. */ --- 7029,7035 ---- * Free a Dictionary, including all items it contains. * Ignores the reference count. */ ! void dict_free(d, recurse) dict_T *d; int recurse; /* Free Lists and Dictionaries recursively. */ *************** *** 8353,8359 **** /* * Call a function with its resolved parameters ! * Return OK when the function can't be called, FAIL otherwise. * Also returns OK when an error was encountered while executing the function. */ static int --- 8351,8357 ---- /* * Call a function with its resolved parameters ! * Return FAIL when the function can't be called, OK otherwise. * Also returns OK when an error was encountered while executing the function. */ static int *** ../vim-7.3.1055/src/if_py_both.h 2013-05-29 22:58:28.000000000 +0200 --- src/if_py_both.h 2013-05-30 12:13:37.000000000 +0200 *************** *** 32,39 **** #define DICTKEY_DECL \ PyObject *dictkey_todecref; #define DICTKEY_GET(err, decref) \ ! if (!(key = StringToChars(keyObject, &dictkey_todecref))) \ { \ if (decref) \ { \ --- 32,46 ---- #define DICTKEY_DECL \ PyObject *dictkey_todecref; + #define DICTKEY_CHECK_EMPTY(err) \ + if (*key == NUL) \ + { \ + PyErr_SetString(PyExc_ValueError, _("empty keys are not allowed")); \ + return err; \ + } + #define DICTKEY_SET_KEY (key = StringToChars(keyObject, &dictkey_todecref)) #define DICTKEY_GET(err, decref) \ ! if (!DICTKEY_SET_KEY) \ { \ if (decref) \ { \ *************** *** 43,53 **** } \ if (decref && !dictkey_todecref) \ dictkey_todecref = keyObject; \ ! if (*key == NUL) \ ! { \ ! PyErr_SetString(PyExc_ValueError, _("empty keys are not allowed")); \ ! return err; \ ! } #define DICTKEY_UNREF \ Py_XDECREF(dictkey_todecref); --- 50,56 ---- } \ if (decref && !dictkey_todecref) \ dictkey_todecref = keyObject; \ ! DICTKEY_CHECK_EMPTY(err) #define DICTKEY_UNREF \ Py_XDECREF(dictkey_todecref); *************** *** 651,659 **** /* Convert the Vim type into a Python type. Create a dictionary that's * used to check for recursive loops. */ ! lookup_dict = PyDict_New(); ! result = VimToPython(our_tv, 1, lookup_dict); ! Py_DECREF(lookup_dict); Py_BEGIN_ALLOW_THREADS --- 654,666 ---- /* Convert the Vim type into a Python type. Create a dictionary that's * used to check for recursive loops. */ ! if (!(lookup_dict = PyDict_New())) ! result = NULL; ! else ! { ! result = VimToPython(our_tv, 1, lookup_dict); ! Py_DECREF(lookup_dict); ! } Py_BEGIN_ALLOW_THREADS *************** *** 1401,1407 **** return NULL; } ! lookup_dict = PyDict_New(); if (list_py_concat(l, obj, lookup_dict) == -1) { Py_DECREF(lookup_dict); --- 1408,1416 ---- return NULL; } ! if (!(lookup_dict = PyDict_New())) ! return NULL; ! if (list_py_concat(l, obj, lookup_dict) == -1) { Py_DECREF(lookup_dict); *************** *** 4023,4034 **** PyObject *valObject; Py_ssize_t iter = 0; ! dict = dict_alloc(); ! if (dict == NULL) ! { ! PyErr_NoMemory(); return -1; - } tv->v_type = VAR_DICT; tv->vval.v_dict = dict; --- 4032,4039 ---- PyObject *valObject; Py_ssize_t iter = 0; ! if (!(dict = dict_alloc())) return -1; tv->v_type = VAR_DICT; tv->vval.v_dict = dict; *************** *** 4038,4046 **** DICTKEY_DECL if (keyObject == NULL || valObject == NULL) return -1; ! DICTKEY_GET(-1, 0) di = dictitem_alloc(key); --- 4043,4059 ---- DICTKEY_DECL if (keyObject == NULL || valObject == NULL) + { + dict_unref(dict); return -1; + } ! if (!DICTKEY_SET_KEY) ! { ! dict_unref(dict); ! return -1; ! } ! DICTKEY_CHECK_EMPTY(-1) di = dictitem_alloc(key); *************** *** 4049,4054 **** --- 4062,4068 ---- if (di == NULL) { PyErr_NoMemory(); + dict_unref(dict); return -1; } di->di_tv.v_lock = 0; *************** *** 4056,4061 **** --- 4070,4076 ---- if (_ConvertFromPyObject(valObject, &di->di_tv, lookup_dict) == -1) { vim_free(di); + dict_unref(dict); return -1; } *************** *** 4063,4072 **** --- 4078,4090 ---- { clear_tv(&di->di_tv); vim_free(di); + dict_unref(dict); PyErr_SetVim(_("failed to add key to dictionary")); return -1; } } + + --dict->dv_refcount; return 0; } *************** *** 4082,4100 **** PyObject *valObject; Py_ssize_t lsize; ! dict = dict_alloc(); ! if (dict == NULL) ! { ! PyErr_NoMemory(); return -1; - } tv->v_type = VAR_DICT; tv->vval.v_dict = dict; list = PyMapping_Items(obj); if (list == NULL) return -1; lsize = PyList_Size(list); while (lsize--) { --- 4100,4117 ---- PyObject *valObject; Py_ssize_t lsize; ! if (!(dict = dict_alloc())) return -1; tv->v_type = VAR_DICT; tv->vval.v_dict = dict; list = PyMapping_Items(obj); if (list == NULL) + { + dict_unref(dict); return -1; + } lsize = PyList_Size(list); while (lsize--) { *************** *** 4104,4109 **** --- 4121,4127 ---- if (litem == NULL) { Py_DECREF(list); + dict_unref(dict); return -1; } *************** *** 4111,4125 **** { Py_DECREF(list); Py_DECREF(litem); return -1; } ! DICTKEY_GET(-1, 1) if (!(valObject = PyTuple_GetItem(litem, 1))) { Py_DECREF(list); Py_DECREF(litem); DICTKEY_UNREF return -1; } --- 4129,4153 ---- { Py_DECREF(list); Py_DECREF(litem); + dict_unref(dict); return -1; } ! if (!DICTKEY_SET_KEY) ! { ! dict_unref(dict); ! Py_DECREF(list); ! Py_DECREF(litem); ! DICTKEY_UNREF ! return -1; ! } ! DICTKEY_CHECK_EMPTY(-1) if (!(valObject = PyTuple_GetItem(litem, 1))) { Py_DECREF(list); Py_DECREF(litem); + dict_unref(dict); DICTKEY_UNREF return -1; } *************** *** 4133,4139 **** if (di == NULL) { Py_DECREF(list); ! Py_DECREF(valObject); PyErr_NoMemory(); return -1; } --- 4161,4167 ---- if (di == NULL) { Py_DECREF(list); ! dict_unref(dict); PyErr_NoMemory(); return -1; } *************** *** 4142,4216 **** if (_ConvertFromPyObject(valObject, &di->di_tv, lookup_dict) == -1) { vim_free(di); Py_DECREF(list); - Py_DECREF(valObject); return -1; } - Py_DECREF(valObject); - if (dict_add(dict, di) == FAIL) { ! clear_tv(&di->di_tv); ! vim_free(di); Py_DECREF(list); PyErr_SetVim(_("failed to add key to dictionary")); return -1; } } Py_DECREF(list); return 0; } static int pyseq_to_tv(PyObject *obj, typval_T *tv, PyObject *lookup_dict) { list_T *l; ! l = list_alloc(); ! if (l == NULL) ! { ! PyErr_NoMemory(); return -1; - } tv->v_type = VAR_LIST; tv->vval.v_list = l; if (list_py_concat(l, obj, lookup_dict) == -1) return -1; return 0; } static int pyiter_to_tv(PyObject *obj, typval_T *tv, PyObject *lookup_dict) { ! PyObject *iterator = PyObject_GetIter(obj); PyObject *item; list_T *l; listitem_T *li; ! l = list_alloc(); ! ! if (l == NULL) ! { ! PyErr_NoMemory(); return -1; - } tv->vval.v_list = l; tv->v_type = VAR_LIST; ! ! if (iterator == NULL) return -1; while ((item = PyIter_Next(iterator))) { li = listitem_alloc(); if (li == NULL) { Py_DECREF(iterator); PyErr_NoMemory(); return -1; --- 4170,4256 ---- if (_ConvertFromPyObject(valObject, &di->di_tv, lookup_dict) == -1) { vim_free(di); + dict_unref(dict); Py_DECREF(list); return -1; } if (dict_add(dict, di) == FAIL) { ! dictitem_free(di); ! dict_unref(dict); Py_DECREF(list); PyErr_SetVim(_("failed to add key to dictionary")); return -1; } } + --dict->dv_refcount; Py_DECREF(list); return 0; } + static list_T * + py_list_alloc() + { + list_T *r; + + if (!(r = list_alloc())) + { + PyErr_NoMemory(); + return NULL; + } + ++r->lv_refcount; + + return r; + } + static int pyseq_to_tv(PyObject *obj, typval_T *tv, PyObject *lookup_dict) { list_T *l; ! if (!(l = py_list_alloc())) return -1; tv->v_type = VAR_LIST; tv->vval.v_list = l; if (list_py_concat(l, obj, lookup_dict) == -1) + { + list_unref(l); return -1; + } + --l->lv_refcount; return 0; } static int pyiter_to_tv(PyObject *obj, typval_T *tv, PyObject *lookup_dict) { ! PyObject *iterator; PyObject *item; list_T *l; listitem_T *li; ! if (!(l = py_list_alloc())) return -1; tv->vval.v_list = l; tv->v_type = VAR_LIST; ! if (!(iterator = PyObject_GetIter(obj))) ! { ! list_unref(l); return -1; + } while ((item = PyIter_Next(iterator))) { li = listitem_alloc(); if (li == NULL) { + list_unref(l); Py_DECREF(iterator); PyErr_NoMemory(); return -1; *************** *** 4219,4224 **** --- 4259,4266 ---- if (_ConvertFromPyObject(item, &li->li_tv, lookup_dict) == -1) { + list_unref(l); + listitem_free(li); Py_DECREF(item); Py_DECREF(iterator); return -1; *************** *** 4230,4235 **** --- 4272,4286 ---- } Py_DECREF(iterator); + + /* Iterator may have finished due to an exception */ + if (PyErr_Occurred()) + { + list_unref(l); + return -1; + } + + --l->lv_refcount; return 0; } *************** *** 4295,4301 **** PyObject *lookup_dict; int r; ! lookup_dict = PyDict_New(); r = _ConvertFromPyObject(obj, tv, lookup_dict); Py_DECREF(lookup_dict); return r; --- 4346,4353 ---- PyObject *lookup_dict; int r; ! if (!(lookup_dict = PyDict_New())) ! return -1; r = _ConvertFromPyObject(obj, tv, lookup_dict); Py_DECREF(lookup_dict); return r; *** ../vim-7.3.1055/src/proto/eval.pro 2013-05-17 16:03:53.000000000 +0200 --- src/proto/eval.pro 2013-05-30 12:11:40.000000000 +0200 *************** *** 49,54 **** --- 49,55 ---- void list_unref __ARGS((list_T *l)); void list_free __ARGS((list_T *l, int recurse)); listitem_T *listitem_alloc __ARGS((void)); + void listitem_free __ARGS((listitem_T *item)); void listitem_remove __ARGS((list_T *l, listitem_T *item)); dictitem_T *dict_lookup __ARGS((hashitem_T *hi)); listitem_T *list_find __ARGS((list_T *l, long n)); *************** *** 65,70 **** --- 66,72 ---- void set_ref_in_item __ARGS((typval_T *tv, int copyID)); dict_T *dict_alloc __ARGS((void)); void dict_unref __ARGS((dict_T *d)); + void dict_free __ARGS((dict_T *d, int recurse)); dictitem_T *dictitem_alloc __ARGS((char_u *key)); void dictitem_free __ARGS((dictitem_T *item)); int dict_add __ARGS((dict_T *d, dictitem_T *item)); *** ../vim-7.3.1055/src/version.c 2013-05-30 11:51:04.000000000 +0200 --- src/version.c 2013-05-30 12:13:57.000000000 +0200 *************** *** 730,731 **** --- 730,733 ---- { /* Add new patch number below this line */ + /**/ + 1056, /**/ -- I have a drinking problem -- I can't afford it. /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///