From 28fe14ffff12745a7340c298b5e6a54ba3b40794 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 1 Apr 2017 00:55:50 -0400 Subject: [PATCH 01/62] Add pop function with an index argument to single-linked list, with tests --- src/sllist.c | 91 ++++++++++++++++++++++++++++++++++++++++++++- tests/llist_test.py | 22 +++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/src/sllist.c b/src/sllist.c index ac28304..4d0388c 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1,4 +1,5 @@ /* Copyright (c) 2011-2013 Adam Jakubek, Rafał Gałczyński + * Copyright (c) 2017 Timothy Savannah * Released under the MIT license (see attached LICENSE file). */ @@ -1161,6 +1162,92 @@ static PyObject* sllist_popright(SLListObject* self) return value; } +static PyObject* sllist_pop(SLListObject* self, PyObject *arg) +{ + SLListNodeObject *del_node; + SLListNodeObject *prev_node; + + PyObject *value; + PyObject *indexObject = NULL; + + Py_ssize_t index; + Py_ssize_t i; + + + if (!PyArg_UnpackTuple(arg, "pop", 0, 1, &indexObject)) + { + return NULL; + } + + /* If we did not get passed the "index" arg, just return popright */ + if ( indexObject == NULL ) + { + return sllist_popright( self ); + } + + if (self->first == Py_None) + { + PyErr_SetString(PyExc_ValueError, "List is empty"); + return NULL; + } + else + { + if (!Py23Int_Check(indexObject)) + { + PyErr_SetString(PyExc_TypeError, "Index must be an integer"); + return NULL; + } + + index = Py23Int_AsSsize_t(indexObject); + } + + + /* Negative index */ + if (index < 0) + index = ((SLListObject*)self)->size + index; + + /* Either a negative greater than index size, or a positive greater than size */ + if ( index < 0 || index >= ((SLListObject*)self)->size ) + { + PyErr_SetString(PyExc_IndexError, "Index out of range"); + return NULL; + } + + /* Start at first node, and walk to the one we will pop */ + prev_node = Py_None; + del_node = (SLListNodeObject*)self->first; + for(i=0; i < index; i++) { + prev_node = del_node; + del_node = del_node->next; + } + + if ( prev_node == Py_None ) + { + /* First node */ + self->first = del_node->next; + } + else + { + /* Any other node */ + prev_node->next = del_node->next; + } + + --self->size; + if ( index == self->size ) + { + /* removeing last node, move last pointer */ + self->last = prev_node; + } + + + Py_INCREF(del_node->value); + value = del_node->value; + + del_node->next = Py_None; + Py_DECREF((PyObject*)del_node); + + return value; +} static PyObject* sllist_iter(PyObject* self) { @@ -1321,8 +1408,8 @@ static PyMethodDef SLListMethods[] = { "nodeat", (PyCFunction)sllist_node_at, METH_O, "Return node at index" }, - { "pop", (PyCFunction)sllist_popright, METH_NOARGS, - "Remove last element from the list and return it" }, + { "pop", (PyCFunction)sllist_pop, METH_VARARGS, + "Remove a given element from the list and return it. Defaults to last" }, { "popleft", (PyCFunction)sllist_popleft, METH_NOARGS, "Remove first element from the list and return it" }, diff --git a/tests/llist_test.py b/tests/llist_test.py index fe9822b..e3e2cd5 100644 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -553,6 +553,28 @@ def test_pop(self): self.assertEqual(list(ll), ref[:-1]) self.assertEqual(del_node.next, None) + ref = py23_range(0, 1024, 4) + ll = sllist(ref) + + result = ll.pop(1) + self.assertEqual(result, ref[1]) + result = ll.pop(1) + self.assertEqual(result, ref[2]) + self.assertEqual(ll.size, len(ref)-2) + + ref = py23_range(0, 1024, 4) + ll = sllist(ref) + result = ll.pop(0) + self.assertEqual(result, ref[0]) + + self.assertEqual(ll.first.value, ref[1]) + for i in range(len(ll)): + ll.pop(0) + self.assertEqual(ll.first, None) + self.assertEqual(ll.last, None) + + + def test_popleft(self): ref = py23_range(0, 1024, 4) ll = sllist(ref) From 4c1b1449bbc1bac5f49ef6afcf9764ccffc3eb1e Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 1 Apr 2017 01:13:29 -0400 Subject: [PATCH 02/62] Add pop with index arg to double-linked list --- src/dllist.c | 95 ++++++++++++++++++++++++++++++++++++++++++++- tests/llist_test.py | 27 +++++++++++++ 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 03eaa03..8ef8180 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -995,6 +995,97 @@ static PyObject* dllist_popright(DLListObject* self) return value; } +static PyObject* dllist_pop(DLListObject* self, PyObject *arg) +{ + DLListNodeObject *del_node; + DLListNodeObject *prev_node; + + PyObject *value; + PyObject *indexObject = NULL; + + Py_ssize_t index; + Py_ssize_t i; + + + if (!PyArg_UnpackTuple(arg, "pop", 0, 1, &indexObject)) + { + return NULL; + } + + /* If we did not get passed the "index" arg, just return popright */ + if ( indexObject == NULL ) + { + return dllist_popright( self ); + } + + if (self->first == Py_None) + { + PyErr_SetString(PyExc_ValueError, "List is empty"); + return NULL; + } + else + { + if (!Py23Int_Check(indexObject)) + { + PyErr_SetString(PyExc_TypeError, "Index must be an integer"); + return NULL; + } + + index = Py23Int_AsSsize_t(indexObject); + } + + + /* Negative index */ + if (index < 0) + index = ((DLListObject*)self)->size + index; + + /* Either a negative greater than index size, or a positive greater than size */ + if ( index < 0 || index >= ((DLListObject*)self)->size ) + { + PyErr_SetString(PyExc_IndexError, "Index out of range"); + return NULL; + } + + /* Start at first node, and walk to the one we will pop */ + prev_node = (DLListObject*)Py_None; + del_node = (DLListNodeObject*)self->first; + for(i=0; i < index; i++) { + prev_node = del_node; + del_node = del_node->next; + } + + if ( prev_node == Py_None ) + { + /* First node */ + self->first = del_node->next; + } + else + { + /* Any other node */ + prev_node->next = del_node->next; + } + if(del_node->next != Py_None) + { + ((DLListNodeObject*)del_node->next)->prev = prev_node; + } + + --self->size; + if ( index == self->size ) + { + /* removeing last node, move last pointer */ + self->last = prev_node; + } + + + Py_INCREF(del_node->value); + value = del_node->value; + + del_node->next = Py_None; + Py_DECREF((PyObject*)del_node); + + return value; +} + static PyObject* dllist_remove(DLListObject* self, PyObject* arg) { DLListNodeObject* del_node; @@ -1280,8 +1371,8 @@ static PyMethodDef DLListMethods[] = "Return node at index" }, { "popleft", (PyCFunction)dllist_popleft, METH_NOARGS, "Remove first element from the list and return it" }, - { "pop", (PyCFunction)dllist_popright, METH_NOARGS, - "Remove last element from the list and return it" }, + { "pop", (PyCFunction)dllist_pop, METH_VARARGS, + "Remove an element by index from the list and return it, or last item if no index provided" }, { "popright", (PyCFunction)dllist_popright, METH_NOARGS, "Remove last element from the list and return it" }, { "remove", (PyCFunction)dllist_remove, METH_O, diff --git a/tests/llist_test.py b/tests/llist_test.py index e3e2cd5..644f112 100644 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -1294,6 +1294,33 @@ def test_pop(self): self.assertEqual(del_node.prev, None) self.assertEqual(del_node.next, None) + ref = py23_range(0, 1024, 4) + ll = dllist(ref) + + result = ll.pop(1) + self.assertEqual(result, ref[1]) + result = ll.pop(1) + self.assertEqual(result, ref[2]) + self.assertEqual(ll.size, len(ref)-2) + + secondNode = ll.nodeat(1) + + self.assertEquals(secondNode.prev, ll.first) + self.assertEquals(ll.first.prev, None) + + ref = py23_range(0, 1024, 4) + ll = dllist(ref) + result = ll.pop(0) + self.assertEqual(result, ref[0]) + + self.assertEqual(ll.first.value, ref[1]) + for i in range(len(ll)): + ll.pop(0) + self.assertEqual(ll.first, None) + self.assertEqual(ll.last, None) + + + def test_popleft(self): ref = py23_range(0, 1024, 4) ll = dllist(ref) From 8181feb76c69cc50783c10123b2cfd189af1ae47 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 1 Apr 2017 01:15:52 -0400 Subject: [PATCH 03/62] Fix warnings on sllist --- src/sllist.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sllist.c b/src/sllist.c index 4d0388c..3db7ba4 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1214,14 +1214,14 @@ static PyObject* sllist_pop(SLListObject* self, PyObject *arg) } /* Start at first node, and walk to the one we will pop */ - prev_node = Py_None; + prev_node = (SLListNodeObject*)Py_None; del_node = (SLListNodeObject*)self->first; for(i=0; i < index; i++) { prev_node = del_node; - del_node = del_node->next; + del_node = (SLListNodeObject*)del_node->next; } - if ( prev_node == Py_None ) + if ( (PyObject*)prev_node == Py_None ) { /* First node */ self->first = del_node->next; @@ -1236,7 +1236,7 @@ static PyObject* sllist_pop(SLListObject* self, PyObject *arg) if ( index == self->size ) { /* removeing last node, move last pointer */ - self->last = prev_node; + self->last = (PyObject*)prev_node; } From 7c5f37538072715fdab41960c05c87b0a3882c89 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 1 Apr 2017 01:18:49 -0400 Subject: [PATCH 04/62] Fix warnings in dllist --- src/dllist.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 8ef8180..b06d948 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1018,7 +1018,7 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) return dllist_popright( self ); } - if (self->first == Py_None) + if ( (PyObject*)self->first == Py_None) { PyErr_SetString(PyExc_ValueError, "List is empty"); return NULL; @@ -1047,33 +1047,33 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) } /* Start at first node, and walk to the one we will pop */ - prev_node = (DLListObject*)Py_None; + prev_node = (DLListNodeObject*)Py_None; del_node = (DLListNodeObject*)self->first; for(i=0; i < index; i++) { prev_node = del_node; - del_node = del_node->next; + del_node = (DLListNodeObject*)del_node->next; } - if ( prev_node == Py_None ) + if ( (PyObject*)prev_node == Py_None ) { /* First node */ - self->first = del_node->next; + self->first = (PyObject*)del_node->next; } else { /* Any other node */ prev_node->next = del_node->next; } - if(del_node->next != Py_None) + if ( (PyObject*)del_node->next != Py_None ) { - ((DLListNodeObject*)del_node->next)->prev = prev_node; + ((DLListNodeObject*)del_node->next)->prev = (PyObject*)prev_node; } --self->size; if ( index == self->size ) { /* removeing last node, move last pointer */ - self->last = prev_node; + self->last = (PyObject*)prev_node; } From 8f7b96554f640a942611c81f2b4155da4d19fbb4 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 1 Apr 2017 01:21:31 -0400 Subject: [PATCH 05/62] Add copyright note --- src/dllist.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dllist.c b/src/dllist.c index b06d948..f55aa06 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1,4 +1,5 @@ /* Copyright (c) 2011-2013 Adam Jakubek, Rafał Gałczyński + * Copyright (c) 2017 Timothy Savannah * Released under the MIT license (see attached LICENSE file). */ From 1acfe1475239dc7bec5ffabc7615aff06865b0b4 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 1 Apr 2017 01:22:52 -0400 Subject: [PATCH 06/62] Make help strings the same for pop function --- src/sllist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sllist.c b/src/sllist.c index 3db7ba4..462bc73 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1409,7 +1409,7 @@ static PyMethodDef SLListMethods[] = "Return node at index" }, { "pop", (PyCFunction)sllist_pop, METH_VARARGS, - "Remove a given element from the list and return it. Defaults to last" }, + "Remove an element by index from the list and return it, or last item if no index provided" }, { "popleft", (PyCFunction)sllist_popleft, METH_NOARGS, "Remove first element from the list and return it" }, From c8e7184a97b18a550bb5d1fbb34474ff608ab2c3 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 1 Apr 2017 01:26:20 -0400 Subject: [PATCH 07/62] Clear last_access_node if we need to in idx-pop, and use dllistnode_delete function for clearing popped node and references --- src/dllist.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index f55aa06..c79db93 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1077,12 +1077,18 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) self->last = (PyObject*)prev_node; } + if (self->last_accessed_node == (PyObject*)del_node) + { + /* invalidate last accessed item */ + self->last_accessed_node = Py_None; + self->last_accessed_idx = -1; + } + Py_INCREF(del_node->value); value = del_node->value; - del_node->next = Py_None; - Py_DECREF((PyObject*)del_node); + dllistnode_delete(del_node); return value; } From 919550215a008dd1d5b2136daf6ea8d465dafc70 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 00:16:24 -0400 Subject: [PATCH 08/62] Add some more tests, test each pop in the full-pop-out, and pop from both directions --- tests/llist_test.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/llist_test.py b/tests/llist_test.py index 644f112..70e373a 100644 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -569,10 +569,25 @@ def test_pop(self): self.assertEqual(ll.first.value, ref[1]) for i in range(len(ll)): - ll.pop(0) + result = ll.pop(0) + self.assertEqual(result, ref[i+1]) + self.assertEqual(ll.first, None) self.assertEqual(ll.last, None) + ref = py23_range(0, 1024, 4) + ll = sllist(ref) + i = len(ll)-1 + while i >= 0: + result = ll.pop(i) + self.assertEqual(result, ref[i]) + i -= 1 + + self.assertEqual(ll.first, None) + self.assertEqual(ll.last, None) + + + def test_popleft(self): @@ -1315,9 +1330,23 @@ def test_pop(self): self.assertEqual(ll.first.value, ref[1]) for i in range(len(ll)): - ll.pop(0) + result = ll.pop(0) + self.assertEqual(result, ref[i+1]) + + self.assertEqual(ll.first, None) + self.assertEqual(ll.last, None) + + ref = py23_range(0, 1024, 4) + ll = dllist(ref) + i = len(ll) - 1 + while i >= 0: + result = ll.pop(i) + self.assertEqual(result, ref[i]) + i -= 1 + self.assertEqual(ll.first, None) self.assertEqual(ll.last, None) + From fc7cf0188e53679ead60f73fffcbc408faada847 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 00:28:12 -0400 Subject: [PATCH 09/62] Greatly simplify indexed pop. Check first if we are popping left or right, and call those functions. This allows us to always skip forward one walk iteration, and remove any logic for first or last. Also, on double-linked list pop, walk from the back if we are popping past middle. These changes bring the linked list random-pop benchmark much closer to the python list performance even further out (like 400 list elements, 200 pops they perform the same. Below that linked list impl's now perform better, above that list performs better (because of walk time) --- src/dllist.c | 77 ++++++++++++++++++++++++++++------------------------ src/sllist.c | 54 ++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index c79db93..38ad3ed 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -999,7 +999,7 @@ static PyObject* dllist_popright(DLListObject* self) static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { DLListNodeObject *del_node; - DLListNodeObject *prev_node; + DLListNodeObject *cur_node; PyObject *value; PyObject *indexObject = NULL; @@ -1019,27 +1019,34 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) return dllist_popright( self ); } - if ( (PyObject*)self->first == Py_None) + if (!Py23Int_Check(indexObject)) { - PyErr_SetString(PyExc_ValueError, "List is empty"); + PyErr_SetString(PyExc_TypeError, "Index must be an integer"); return NULL; } - else - { - if (!Py23Int_Check(indexObject)) - { - PyErr_SetString(PyExc_TypeError, "Index must be an integer"); - return NULL; - } - - index = Py23Int_AsSsize_t(indexObject); - } + index = Py23Int_AsSsize_t(indexObject); /* Negative index */ if (index < 0) index = ((DLListObject*)self)->size + index; + /* If index is 0, popleft */ + if ( index == 0 ) + { + return dllist_popleft( self ); + } + else if ( index + 1 == ((DLListObject*)self)->size ) + { + return dllist_popright(self); + } + + if ( (PyObject*)self->first == Py_None) + { + PyErr_SetString(PyExc_ValueError, "List is empty"); + return NULL; + } + /* Either a negative greater than index size, or a positive greater than size */ if ( index < 0 || index >= ((DLListObject*)self)->size ) { @@ -1047,35 +1054,35 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) return NULL; } - /* Start at first node, and walk to the one we will pop */ - prev_node = (DLListNodeObject*)Py_None; - del_node = (DLListNodeObject*)self->first; - for(i=0; i < index; i++) { - prev_node = del_node; - del_node = (DLListNodeObject*)del_node->next; - } - - if ( (PyObject*)prev_node == Py_None ) + if ( index <= ((DLListObject*)self)->size / 2 ) { - /* First node */ - self->first = (PyObject*)del_node->next; + + /* Start at first node, and walk to the one we will pop */ + cur_node = (DLListNodeObject*)self->first; + del_node = (DLListNodeObject*)cur_node->next; + for(i=1; i < index; i++) { + cur_node = del_node; + del_node = (DLListNodeObject*)del_node->next; + } + + cur_node->next = del_node->next; + + ((DLListNodeObject*)del_node->next)->prev = (PyObject*)cur_node; } else { - /* Any other node */ - prev_node->next = del_node->next; - } - if ( (PyObject*)del_node->next != Py_None ) - { - ((DLListNodeObject*)del_node->next)->prev = (PyObject*)prev_node; + /* Start at last node, and walk back to the one we will pop */ + cur_node = (DLListNodeObject*)self->last; + del_node = (DLListNodeObject*)cur_node->prev; + for(i=((DLListObject*)self)->size - 2; i > index; i--) { + cur_node = del_node; + del_node = (DLListNodeObject*)del_node->prev; + } + ((DLListNodeObject *)cur_node->prev)->next = del_node->next; } + --self->size; - if ( index == self->size ) - { - /* removeing last node, move last pointer */ - self->last = (PyObject*)prev_node; - } if (self->last_accessed_node == (PyObject*)del_node) { diff --git a/src/sllist.c b/src/sllist.c index 462bc73..c98dcb1 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1185,27 +1185,34 @@ static PyObject* sllist_pop(SLListObject* self, PyObject *arg) return sllist_popright( self ); } - if (self->first == Py_None) + if (!Py23Int_Check(indexObject)) { - PyErr_SetString(PyExc_ValueError, "List is empty"); + PyErr_SetString(PyExc_TypeError, "Index must be an integer"); return NULL; } - else - { - if (!Py23Int_Check(indexObject)) - { - PyErr_SetString(PyExc_TypeError, "Index must be an integer"); - return NULL; - } - index = Py23Int_AsSsize_t(indexObject); - } + index = Py23Int_AsSsize_t(indexObject); /* Negative index */ - if (index < 0) + if ( index < 0 ) index = ((SLListObject*)self)->size + index; + if ( index == 0 ) + { + return sllist_popleft(self); + } + else if( index + 1 == ((SLListObject*)self)->size ) + { + return sllist_popright(self); + } + + if ( self->first == Py_None ) + { + PyErr_SetString(PyExc_ValueError, "List is empty"); + return NULL; + } + /* Either a negative greater than index size, or a positive greater than size */ if ( index < 0 || index >= ((SLListObject*)self)->size ) { @@ -1214,30 +1221,17 @@ static PyObject* sllist_pop(SLListObject* self, PyObject *arg) } /* Start at first node, and walk to the one we will pop */ - prev_node = (SLListNodeObject*)Py_None; - del_node = (SLListNodeObject*)self->first; - for(i=0; i < index; i++) { + prev_node = (SLListNodeObject*)self->first; + del_node = (SLListNodeObject*)prev_node->next; + for(i=1; i < index; i++) { prev_node = del_node; del_node = (SLListNodeObject*)del_node->next; } - if ( (PyObject*)prev_node == Py_None ) - { - /* First node */ - self->first = del_node->next; - } - else - { - /* Any other node */ - prev_node->next = del_node->next; - } + /* Unlink this node from the chain */ + prev_node->next = del_node->next; --self->size; - if ( index == self->size ) - { - /* removeing last node, move last pointer */ - self->last = (PyObject*)prev_node; - } Py_INCREF(del_node->value); From fc16922d41672bc5dc4e56aa680c947e177f74f3 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 01:48:28 -0400 Subject: [PATCH 10/62] Add 'middle' to double-linked list --- src/dllist.c | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 3 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 38ad3ed..e8e923c 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -17,6 +17,7 @@ static PyTypeObject DLListType; static PyTypeObject DLListNodeType; static PyTypeObject DLListIteratorType; +static PyObject* dllist_node_at(PyObject* self, PyObject* indexObject); /* DLListNode */ @@ -29,6 +30,7 @@ typedef struct PyObject* list_weakref; } DLListNodeObject; + /* Convenience function for creating list nodes. * Automatically update pointers in neigbours. */ @@ -280,9 +282,11 @@ typedef struct PyObject_HEAD PyObject* first; PyObject* last; + PyObject* middle; PyObject* last_accessed_node; Py_ssize_t last_accessed_idx; Py_ssize_t size; + Py_ssize_t middle_idx; PyObject* weakref_list; } DLListObject; @@ -291,6 +295,58 @@ static Py_ssize_t py_ssize_t_abs(Py_ssize_t x) return (x >= 0) ? x : -x; } +static PyObject *_get_midpoint(DLListObject* self) +{ + long midpoint; + PyObject *midInt; + PyObject *midSsize; + + midpoint = self->size / 2; + + midInt = PyLong_FromLong(midpoint); + + return midInt; +} + +static inline void _set_middle(DLListObject *self, PyObject *middle, Py_ssize_t middle_idx) +{ + self->middle = middle; + self->middle_idx = middle_idx; +} + +static inline void _middle_do_recalc(DLListObject *self) +{ + self->middle_idx = self->size / 2; + self->middle = dllist_node_at(self, (PyObject*)(PyLong_FromLong(self->middle_idx))); +} + +static inline void _middle_moveing_right(DLListObject *self) +{ + + if ( self->middle != Py_None) { + if(self->size % 2 == 0) { + _set_middle(self, ((DLListNodeObject*)self->middle)->next, self->middle_idx + 1); + } + } +} + +static inline void _middle_moveing_left(DLListObject *self) +{ + + if ( self->middle != Py_None) { + if(self->size % 2 == 0) { + _set_middle(self, ((DLListNodeObject*)self->middle)->prev, self->middle_idx - 1); + } + } +} + +static inline void _middle_check_recalc(DLListObject *self) +{ + if( self->size > 10 ) { + _middle_do_recalc(self); + } +} + /* Convenience function for locating list nodes using index. */ static DLListNodeObject* dllist_get_node_internal(DLListObject* self, Py_ssize_t index) @@ -376,6 +432,9 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) self->first = new_node; self->last = new_node; + self->size += 1; + _middle_moveing_right(self); + if (iter_node_obj == last_node_obj) { /* This is needed to terminate loop if self == sequence. */ @@ -384,8 +443,7 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) iter_node_obj = iter_node->next; } - - self->size += ((DLListObject*)sequence)->size; + _middle_check_recalc(self); return 1; } @@ -419,8 +477,11 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) ++self->size; + _middle_moveing_right(self); + Py_DECREF(item); } + _middle_check_recalc(self); return 1; } @@ -518,6 +579,8 @@ static PyObject* dllist_new(PyTypeObject* type, self->first = Py_None; self->last = Py_None; + self->middle = Py_None; + self->middle_idx = -1; self->last_accessed_node = Py_None; self->last_accessed_idx = -1; self->size = 0; @@ -707,14 +770,26 @@ static PyObject* dllist_appendleft(DLListObject* self, PyObject* arg) self->first = (PyObject*)new_node; - if (self->last == Py_None) + if (self->last == Py_None) { self->last = (PyObject*)new_node; + } if (self->last_accessed_idx >= 0) ++self->last_accessed_idx; ++self->size; + if( self->size > 10 ) { + if( self->middle == Py_None ) + { + _middle_do_recalc(self); + } + else + { + _middle_moveing_left(self); + } + } + Py_INCREF((PyObject*)new_node); return (PyObject*)new_node; } @@ -734,6 +809,17 @@ static PyObject* dllist_appendright(DLListObject* self, PyObject* arg) self->first = (PyObject*)new_node; ++self->size; + if( self->size > 10 ) { + if( self->middle == Py_None ) + { + _middle_do_recalc(self); + } + else + { + _middle_moveing_right(self); + } + } + Py_INCREF((PyObject*)new_node); return (PyObject*)new_node; @@ -805,6 +891,11 @@ static PyObject* dllist_insert(DLListObject* self, PyObject* args) } ++self->size; + if( self->size > 10 ) { + /* TODO: Optimize by direction? */ + _middle_do_recalc(self); + } + Py_INCREF((PyObject*)new_node); return (PyObject*)new_node; @@ -881,6 +972,7 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) self->last = new_node; ++self->size; + _middle_moveing_left(self); /* update index of last accessed item */ if (self->last_accessed_idx >= 0) @@ -888,6 +980,10 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) Py_DECREF(item); } + if( self->size > 10 && self->middle == Py_None ) + { + _middle_do_recalc(self); + } Py_RETURN_NONE; } @@ -918,6 +1014,8 @@ static PyObject* dllist_clear(DLListObject* self) self->first = Py_None; self->last = Py_None; + self->middle = Py_None; + self->middle_idx = -1; self->size = 0; Py_RETURN_NONE; @@ -953,6 +1051,15 @@ static PyObject* dllist_popleft(DLListObject* self) } --self->size; + if(self->size <= 10) + { + self->middle = Py_None; + self->middle_idx = -1; + } + else + { + _middle_moveing_right(self); + } Py_INCREF(del_node->value); value = del_node->value; @@ -987,6 +1094,16 @@ static PyObject* dllist_popright(DLListObject* self) } --self->size; + if(self->size <= 10) + { + self->middle = Py_None; + self->middle_idx = -1; + } + else if(self->size % 2 == 0) + { + _middle_moveing_left(self); + } + Py_INCREF(del_node->value); value = del_node->value; @@ -1065,9 +1182,12 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) del_node = (DLListNodeObject*)del_node->next; } + _middle_moveing_right(self); + cur_node->next = del_node->next; ((DLListNodeObject*)del_node->next)->prev = (PyObject*)cur_node; + } else { @@ -1078,11 +1198,20 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) cur_node = del_node; del_node = (DLListNodeObject*)del_node->prev; } + if(self->middle != Py_None) + { + _middle_moveing_left(self); + } ((DLListNodeObject *)cur_node->prev)->next = del_node->next; } --self->size; + if ( self->size <= 10 ) + { + self->middle = Py_None; + self->middle_idx = -1; + } if (self->last_accessed_node == (PyObject*)del_node) { @@ -1147,6 +1276,16 @@ static PyObject* dllist_remove(DLListObject* self, PyObject* arg) self->last_accessed_idx = -1; --self->size; + if( self->size <= 10 ) + { + self->middle = Py_None; + self->middle_idx = -1; + } + else + { + /* TODO: Optimize direction */ + _middle_do_recalc(self); + } Py_INCREF(del_node->value); value = del_node->value; @@ -1207,6 +1346,10 @@ static PyObject* dllist_rotate(DLListObject* self, PyObject* nObject) self->last_accessed_idx = (self->last_accessed_idx + self->size - split_idx) % self->size; } + if( self->size > 10 ) + { + _middle_do_recalc(self); + } Py_RETURN_NONE; } @@ -1402,6 +1545,8 @@ static PyMemberDef DLListMembers[] = "First node" }, { "last", T_OBJECT_EX, offsetof(DLListObject, last), READONLY, "Next node" }, + { "middle", T_OBJECT_EX, offsetof(DLListObject, middle), READONLY, + "Middle node" }, { "size", T_INT, offsetof(DLListObject, size), READONLY, "Number of elements in the list" }, { NULL }, /* sentinel */ From 45a61210fce680a1503156a72ec2c380b786f933 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 02:21:15 -0400 Subject: [PATCH 11/62] We are already relinking at the bottom, so get rid of these. --- src/dllist.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 38ad3ed..ffe807c 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -999,7 +999,6 @@ static PyObject* dllist_popright(DLListObject* self) static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { DLListNodeObject *del_node; - DLListNodeObject *cur_node; PyObject *value; PyObject *indexObject = NULL; @@ -1058,27 +1057,18 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { /* Start at first node, and walk to the one we will pop */ - cur_node = (DLListNodeObject*)self->first; - del_node = (DLListNodeObject*)cur_node->next; + del_node = (DLListNodeObject*) ((DLListNodeObject*) self->first)->next; for(i=1; i < index; i++) { - cur_node = del_node; del_node = (DLListNodeObject*)del_node->next; } - - cur_node->next = del_node->next; - - ((DLListNodeObject*)del_node->next)->prev = (PyObject*)cur_node; } else { /* Start at last node, and walk back to the one we will pop */ - cur_node = (DLListNodeObject*)self->last; - del_node = (DLListNodeObject*)cur_node->prev; + del_node = (DLListNodeObject*) ((DLListNodeObject*) self->last)->prev; for(i=((DLListObject*)self)->size - 2; i > index; i--) { - cur_node = del_node; del_node = (DLListNodeObject*)del_node->prev; } - ((DLListNodeObject *)cur_node->prev)->next = del_node->next; } From 7898e649521a65ce4155f82b28cddc19a5a23f0f Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 13:28:30 -0400 Subject: [PATCH 12/62] Remove the last_accessed node and index from dllist. It is broken somehow. Fixup 'middle', and have it be used by the pop(idx) function for now. This greatly greatly greatly speeds up double-list on large data sets --- src/dllist.c | 253 +++++++++++++++++++++++--------------------- tests/llist_test.py | 32 ++++++ 2 files changed, 166 insertions(+), 119 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index aa93fce..653d418 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -12,6 +12,32 @@ PyObject_HEAD_INIT(type) size, #endif +#include + +#define DO_DEBUG + + +#ifdef DO_DEBUG +static void debugmsg(char *format, ...) +{ + va_list args; + FILE *f, *f2; + + f = fopen("debug.txt", "a"); + + va_start(args, format); + vfprintf(f, format, args); + va_end(args); + + fclose(f); +} +#else +static inline void debugmsg(char *format, ...) +{ + +} +#endif + static PyTypeObject DLListType; static PyTypeObject DLListNodeType; @@ -283,8 +309,6 @@ typedef struct PyObject* first; PyObject* last; PyObject* middle; - PyObject* last_accessed_node; - Py_ssize_t last_accessed_idx; Py_ssize_t size; Py_ssize_t middle_idx; PyObject* weakref_list; @@ -301,7 +325,7 @@ static PyObject *_get_midpoint(DLListObject* self) PyObject *midInt; PyObject *midSsize; - midpoint = self->size / 2; + midpoint = (long)self->size / 2; midInt = PyLong_FromLong(midpoint); @@ -316,30 +340,61 @@ static inline void _set_middle(DLListObject *self, PyObject *middle, Py_ssize_t static inline void _middle_do_recalc(DLListObject *self) { - self->middle_idx = self->size / 2; + self->middle_idx = (long)self->size / 2; self->middle = dllist_node_at(self, (PyObject*)(PyLong_FromLong(self->middle_idx))); } -static inline void _middle_moveing_right(DLListObject *self) +static inline int _middle_should_move(ssize_t size, int afterSizeAdjust) { + if(afterSizeAdjust) { + return !!(size % 2 == 0); + } + else { + return !!(size % 2 != 0); + } +} + +static inline void _middle_moving_right(DLListObject *self, int afterSizeAdjust) +{ if ( self->middle != Py_None) { - if(self->size % 2 == 0) { + if(_middle_should_move(self->size, afterSizeAdjust)) + { _set_middle(self, ((DLListNodeObject*)self->middle)->next, self->middle_idx + 1); } } } -static inline void _middle_moveing_left(DLListObject *self) +static inline void _middle_popped_left_of_middle(DLListObject *self) +{ + if ( self->middle != Py_None ) { + self->middle_idx -= 1; + } +} + + +static inline void _middle_force_moving_right(DLListObject *self) +{ + _set_middle(self, ((DLListNodeObject*)self->middle)->next, self->middle_idx + 1); + +} + +static inline void _middle_moving_left(DLListObject *self, int afterSizeAdjust) { if ( self->middle != Py_None) { - if(self->size % 2 == 0) { + if(_middle_should_move(self->size, afterSizeAdjust)) { _set_middle(self, ((DLListNodeObject*)self->middle)->prev, self->middle_idx - 1); } } } +static inline void _middle_force_moving_left(DLListObject *self) +{ + _set_middle(self, ((DLListNodeObject*)self->middle)->prev, self->middle_idx - 1); +} + + static inline void _middle_check_recalc(DLListObject *self) { if( self->size > 10 ) { @@ -377,16 +432,6 @@ static DLListNodeObject* dllist_get_node_internal(DLListObject* self, reverse_dir = 1; } - /* check if last accessed index is closer */ - if (self->last_accessed_node != Py_None && - self->last_accessed_idx >= 0 && - py_ssize_t_abs(index - self->last_accessed_idx) < middle) - { - node = (DLListNodeObject*)self->last_accessed_node; - start_pos = self->last_accessed_idx; - reverse_dir = (index < self->last_accessed_idx) ? 1 : 0; - } - assert((PyObject*)node != Py_None); if (!reverse_dir) @@ -433,7 +478,7 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) self->last = new_node; self->size += 1; - _middle_moveing_right(self); + _middle_moving_right(self, 1); if (iter_node_obj == last_node_obj) { @@ -477,7 +522,7 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) ++self->size; - _middle_moveing_right(self); + _middle_moving_right(self, 1); Py_DECREF(item); } @@ -581,8 +626,6 @@ static PyObject* dllist_new(PyTypeObject* type, self->last = Py_None; self->middle = Py_None; self->middle_idx = -1; - self->last_accessed_node = Py_None; - self->last_accessed_idx = -1; self->size = 0; self->weakref_list = NULL; @@ -626,14 +669,8 @@ static PyObject* dllist_node_at(PyObject* self, PyObject* indexObject) index = ((DLListObject*)self)->size + index; node = dllist_get_node_internal((DLListObject*)self, index); - if (node != NULL) - { - /* update last accessed node */ - ((DLListObject*)self)->last_accessed_node = (PyObject*)node; - ((DLListObject*)self)->last_accessed_idx = index; - + if ( node != NULL ) Py_INCREF(node); - } return (PyObject*)node; } @@ -774,9 +811,6 @@ static PyObject* dllist_appendleft(DLListObject* self, PyObject* arg) self->last = (PyObject*)new_node; } - if (self->last_accessed_idx >= 0) - ++self->last_accessed_idx; - ++self->size; if( self->size > 10 ) { @@ -786,7 +820,7 @@ static PyObject* dllist_appendleft(DLListObject* self, PyObject* arg) } else { - _middle_moveing_left(self); + _middle_moving_left(self, 1); } } @@ -816,7 +850,7 @@ static PyObject* dllist_appendright(DLListObject* self, PyObject* arg) } else { - _middle_moveing_right(self); + _middle_moving_right(self, 1); } } @@ -885,9 +919,6 @@ static PyObject* dllist_insert(DLListObject* self, PyObject* args) if (self->last == Py_None) self->last = (PyObject*)new_node; - /* invalidate last accessed item */ - self->last_accessed_node = Py_None; - self->last_accessed_idx = -1; } ++self->size; @@ -937,10 +968,6 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) self->size += ((DLListObject*)sequence)->size; - /* update index of last accessed item */ - if (self->last_accessed_idx >= 0) - self->last_accessed_idx += ((DLListObject*)sequence)->size; - Py_RETURN_NONE; } @@ -972,11 +999,7 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) self->last = new_node; ++self->size; - _middle_moveing_left(self); - - /* update index of last accessed item */ - if (self->last_accessed_idx >= 0) - ++self->last_accessed_idx; + _middle_moving_left(self, 1); Py_DECREF(item); } @@ -1008,10 +1031,6 @@ static PyObject* dllist_clear(DLListObject* self) dllistnode_delete(iter_node); } - /* invalidate last accessed item */ - self->last_accessed_node = Py_None; - self->last_accessed_idx = -1; - self->first = Py_None; self->last = Py_None; self->middle = Py_None; @@ -1038,18 +1057,6 @@ static PyObject* dllist_popleft(DLListObject* self) if (self->last == (PyObject*)del_node) self->last = Py_None; - if (self->last_accessed_node != (PyObject*)del_node) - { - if (self->last_accessed_idx >= 0) - --self->last_accessed_idx; - } - else - { - /* invalidate last accessed item */ - self->last_accessed_node = Py_None; - self->last_accessed_idx = -1; - } - --self->size; if(self->size <= 10) { @@ -1058,7 +1065,8 @@ static PyObject* dllist_popleft(DLListObject* self) } else { - _middle_moveing_right(self); + _middle_popped_left_of_middle(self); + _middle_moving_right(self, 1); } Py_INCREF(del_node->value); @@ -1086,22 +1094,15 @@ static PyObject* dllist_popright(DLListObject* self) if (self->first == (PyObject*)del_node) self->first = Py_None; - if (self->last_accessed_node == (PyObject*)del_node) - { - /* invalidate last accessed item */ - self->last_accessed_node = Py_None; - self->last_accessed_idx = -1; - } - --self->size; if(self->size <= 10) { self->middle = Py_None; self->middle_idx = -1; } - else if(self->size % 2 == 0) + else { - _middle_moveing_left(self); + _middle_moving_left(self, 0); } @@ -1113,16 +1114,18 @@ static PyObject* dllist_popright(DLListObject* self) return value; } + static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { DLListNodeObject *del_node; + FILE *f; PyObject *value; PyObject *indexObject = NULL; Py_ssize_t index; Py_ssize_t i; - + Py_ssize_t before_size; if (!PyArg_UnpackTuple(arg, "pop", 0, 1, &indexObject)) { @@ -1143,6 +1146,7 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) index = Py23Int_AsSsize_t(indexObject); + /* Negative index */ if (index < 0) index = ((DLListObject*)self)->size + index; @@ -1170,26 +1174,64 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) return NULL; } - if ( index <= ((DLListObject*)self)->size / 2 ) + if ( index == self->middle_idx ) { - - /* Start at first node, and walk to the one we will pop */ - del_node = (DLListNodeObject*) ((DLListNodeObject*) self->first)->next; - for(i=1; i < index; i++) { - del_node = (DLListNodeObject*)del_node->next; + del_node = (DLListNodeObject*)self->middle; +/* if ( _middle_should_move(self->size, 0) ) + { + _middle_force_moving_right(self); } + else + { + _middle_force_moving_left(self); + self->middle_idx -= 1; + } +*/ + } + else if ( index < ((DLListObject*)self)->size / 2 ) + { + if ( self->middle_idx != -1 && self->middle_idx - index < index ) + { + del_node = (DLListNodeObject*) self->middle; + for(i=0; i < self->middle_idx - index; i++) { + del_node = (DLListNodeObject*)del_node->prev; + } + } + else + { + /* Start at first node, and walk to the one we will pop */ + del_node = (DLListNodeObject*) ((DLListNodeObject*) self->first)->next; + for(i=1; i < index; i++) { + del_node = (DLListNodeObject*)del_node->next; + } + } + before_size = self->middle_idx; +// _middle_moving_left(self, 1); + _middle_popped_left_of_middle(self); + _middle_moving_right(self, 0); - _middle_moveing_right(self); + + /* Todo: conditional these below */ } else { - /* Start at last node, and walk back to the one we will pop */ - del_node = (DLListNodeObject*) ((DLListNodeObject*) self->last)->prev; - for(i=((DLListObject*)self)->size - 2; i > index; i--) { - del_node = (DLListNodeObject*)del_node->prev; + if ( self->middle_idx != -1 && index - self->middle_idx < self->size - index ) + { + del_node = (DLListNodeObject*) self->middle; + for(i=0; i < index - self->middle_idx; i++) { + del_node = (DLListNodeObject*)del_node->next; + } } - _middle_moveing_left(self); + else + { + /* Start at last node, and walk back to the one we will pop */ + del_node = (DLListNodeObject*) ((DLListNodeObject*) self->last)->prev; + for(i=((DLListObject*)self)->size - 2; i > index; i--) { + del_node = (DLListNodeObject*)del_node->prev; + } + } + _middle_moving_left(self, 1); } @@ -1200,19 +1242,17 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) self->middle_idx = -1; } - if (self->last_accessed_node == (PyObject*)del_node) - { - /* invalidate last accessed item */ - self->last_accessed_node = Py_None; - self->last_accessed_idx = -1; - } - Py_INCREF(del_node->value); value = del_node->value; dllistnode_delete(del_node); + if ( index == self->middle_idx ) + { + _middle_check_recalc(self); + } + return value; } @@ -1255,12 +1295,6 @@ static PyObject* dllist_remove(DLListObject* self, PyObject* arg) self->first = del_node->next; if (self->last == arg) self->last = del_node->prev; - if (self->last_accessed_node == arg) - self->last_accessed_node = del_node->prev; - - /* invalidate last accessed item */ - self->last_accessed_node = Py_None; - self->last_accessed_idx = -1; --self->size; if( self->size <= 10 ) @@ -1328,11 +1362,6 @@ static PyObject* dllist_rotate(DLListObject* self, PyObject* nObject) self->first = (PyObject*)new_first; self->last = (PyObject*)new_last; - if (self->last_accessed_idx >= 0) - { - self->last_accessed_idx = - (self->last_accessed_idx + self->size - split_idx) % self->size; - } if( self->size > 10 ) { _middle_do_recalc(self); @@ -1432,10 +1461,6 @@ static PyObject* dllist_get_item(PyObject* self, Py_ssize_t index) Py_XINCREF(value); - /* update last accessed node */ - ((DLListObject*)self)->last_accessed_node = (PyObject*)node; - ((DLListObject*)self)->last_accessed_idx = index; - return value; } @@ -1462,14 +1487,6 @@ static int dllist_set_item(PyObject* self, Py_ssize_t index, PyObject* val) result = dllist_remove(list, (PyObject*)node); - if (prev != Py_None && index > 0) - { - /* Last accessed item was invalidated by dllist_remove. - * We restore it here as the preceding node. */ - list->last_accessed_node = prev; - list->last_accessed_idx = index - 1; - } - Py_XDECREF(result); return (result != NULL) ? 0 : -1; @@ -1486,10 +1503,6 @@ static int dllist_set_item(PyObject* self, Py_ssize_t index, PyObject* val) node->value = val; Py_DECREF(oldval); - /* update last accessed node */ - list->last_accessed_node = (PyObject*)node; - list->last_accessed_idx = index; - return 0; } @@ -1534,6 +1547,8 @@ static PyMemberDef DLListMembers[] = "Next node" }, { "middle", T_OBJECT_EX, offsetof(DLListObject, middle), READONLY, "Middle node" }, + { "middle_idx", T_INT, offsetof(DLListObject, middle_idx), READONLY, + "Middle node index" }, { "size", T_INT, offsetof(DLListObject, size), READONLY, "Number of elements in the list" }, { NULL }, /* sentinel */ diff --git a/tests/llist_test.py b/tests/llist_test.py index 70e373a..ca4a247 100644 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -1297,10 +1297,12 @@ def test_clear(self): self.assertEqual(del_node.next, None) def test_pop(self): + print ( "4") ref = py23_range(0, 1024, 4) ll = dllist(ref) del_node = ll.nodeat(-1) result = ll.pop(); + print ( "5") self.assertEqual(result, ref[-1]) self.assertEqual(len(ll), len(ref) - 1) self.assertEqual(ll.size, len(ref) - 1) @@ -1312,6 +1314,8 @@ def test_pop(self): ref = py23_range(0, 1024, 4) ll = dllist(ref) + print ( "6") + #import pdb; pdb.set_trace() result = ll.pop(1) self.assertEqual(result, ref[1]) result = ll.pop(1) @@ -1323,12 +1327,14 @@ def test_pop(self): self.assertEquals(secondNode.prev, ll.first) self.assertEquals(ll.first.prev, None) + print ( "7") ref = py23_range(0, 1024, 4) ll = dllist(ref) result = ll.pop(0) self.assertEqual(result, ref[0]) self.assertEqual(ll.first.value, ref[1]) + print ( "8") for i in range(len(ll)): result = ll.pop(0) self.assertEqual(result, ref[i+1]) @@ -1339,6 +1345,7 @@ def test_pop(self): ref = py23_range(0, 1024, 4) ll = dllist(ref) i = len(ll) - 1 + print ( "9") while i >= 0: result = ll.pop(i) self.assertEqual(result, ref[i]) @@ -1346,6 +1353,31 @@ def test_pop(self): self.assertEqual(ll.first, None) self.assertEqual(ll.last, None) + + + ref = py23_range(0, 1024, 4) + + import random + + lastIdx = list(ref).index(ref[-1]) + + allIndexes = list(range(lastIdx+1)) + random.shuffle(allIndexes) + + ll = dllist(ref) + + while allIndexes: + print ( "Popping %d out of %d indexes. Value: %s\n\tFirst=%s\n\tMiddle=%s\n\tLast=%s\n\tSize=%d\n" %(allIndexes[0], len(allIndexes), str(ll[allIndexes[0]]), ll.first, ll.middle, ll.last, ll.size)) + nextIndex = allIndexes.pop(0) + + ll.pop(nextIndex) + + for i in range(len(allIndexes)): + if allIndexes[i] > nextIndex: + allIndexes[i] -= 1 + + self.assertEqual(ll.first, None) + self.assertEqual(ll.last, None) From 7cf6f187c99a068272a2c7a4a3c025953df56ac9 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 13:41:27 -0400 Subject: [PATCH 13/62] Move out magic number for when middle index begins into a define --- src/dllist.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 653d418..f1aff58 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -14,6 +14,9 @@ #include + +#define START_MIDDLE_AFTER 10 + #define DO_DEBUG @@ -397,7 +400,7 @@ static inline void _middle_force_moving_left(DLListObject *self) static inline void _middle_check_recalc(DLListObject *self) { - if( self->size > 10 ) { + if( self->size > START_MIDDLE_AFTER ) { _middle_do_recalc(self); } } @@ -813,7 +816,7 @@ static PyObject* dllist_appendleft(DLListObject* self, PyObject* arg) ++self->size; - if( self->size > 10 ) { + if( self->size > START_MIDDLE_AFTER ) { if( self->middle == Py_None ) { _middle_do_recalc(self); @@ -843,7 +846,7 @@ static PyObject* dllist_appendright(DLListObject* self, PyObject* arg) self->first = (PyObject*)new_node; ++self->size; - if( self->size > 10 ) { + if( self->size > START_MIDDLE_AFTER ) { if( self->middle == Py_None ) { _middle_do_recalc(self); @@ -922,7 +925,7 @@ static PyObject* dllist_insert(DLListObject* self, PyObject* args) } ++self->size; - if( self->size > 10 ) { + if( self->size > START_MIDDLE_AFTER ) { /* TODO: Optimize by direction? */ _middle_do_recalc(self); } @@ -1003,7 +1006,7 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) Py_DECREF(item); } - if( self->size > 10 && self->middle == Py_None ) + if( self->size > START_MIDDLE_AFTER && self->middle == Py_None ) { _middle_do_recalc(self); } @@ -1058,7 +1061,7 @@ static PyObject* dllist_popleft(DLListObject* self) self->last = Py_None; --self->size; - if(self->size <= 10) + if(self->size <= START_MIDDLE_AFTER) { self->middle = Py_None; self->middle_idx = -1; @@ -1095,7 +1098,7 @@ static PyObject* dllist_popright(DLListObject* self) self->first = Py_None; --self->size; - if(self->size <= 10) + if(self->size <= START_MIDDLE_AFTER) { self->middle = Py_None; self->middle_idx = -1; @@ -1236,7 +1239,7 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) --self->size; - if ( self->size <= 10 ) + if ( self->size <= START_MIDDLE_AFTER ) { self->middle = Py_None; self->middle_idx = -1; @@ -1297,7 +1300,7 @@ static PyObject* dllist_remove(DLListObject* self, PyObject* arg) self->last = del_node->prev; --self->size; - if( self->size <= 10 ) + if( self->size <= START_MIDDLE_AFTER ) { self->middle = Py_None; self->middle_idx = -1; @@ -1362,7 +1365,7 @@ static PyObject* dllist_rotate(DLListObject* self, PyObject* nObject) self->first = (PyObject*)new_first; self->last = (PyObject*)new_last; - if( self->size > 10 ) + if( self->size > START_MIDDLE_AFTER ) { _middle_do_recalc(self); } @@ -1547,8 +1550,6 @@ static PyMemberDef DLListMembers[] = "Next node" }, { "middle", T_OBJECT_EX, offsetof(DLListObject, middle), READONLY, "Middle node" }, - { "middle_idx", T_INT, offsetof(DLListObject, middle_idx), READONLY, - "Middle node index" }, { "size", T_INT, offsetof(DLListObject, size), READONLY, "Number of elements in the list" }, { NULL }, /* sentinel */ From 4aae917c16c3e9bb9cf4c3d318aaf0786f64d016 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 13:41:36 -0400 Subject: [PATCH 14/62] Add benchmark script --- tests/benchmark.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 tests/benchmark.py diff --git a/tests/benchmark.py b/tests/benchmark.py new file mode 100755 index 0000000..05432ff --- /dev/null +++ b/tests/benchmark.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import os +import time +import random + +from llist import dllist, sllist + +LIST_SIZE = 600 + +NUM_POPS = 300 + +NUM_ITERS = 100 + +def doPops(lst, popIdxs): + + for popIdx in popIdxs: + lst.pop(popIdx) + +def doTime(func): + t1 = time.time() + func() + t2 = time.time() + + return t2 - t1 + +if __name__ == '__main__': + + + if 'LIST_SIZE' in os.environ: + LIST_SIZE = int(os.environ['LIST_SIZE']) + if 'NUM_POPS' in os.environ: + NUM_POPS = int(os.environ['NUM_POPS']) + if 'NUM_ITERS' in os.environ: + NUM_POPS = int(os.environ['NUM_ITERS']) + + primeList = list(range(LIST_SIZE)) + + # Generate a list of random indexes to pop. + # We avoid key errors by making sure we don't have a possible + # maximum larger than the smallest our list will become + + randomPops = [] + for i in range(NUM_POPS): + nextNum = random.randint(0, LIST_SIZE - i - 1) # Cheaply make sure we don't get key errors + + randomPops.append(nextNum) + + headerLine = "Starting: LIST_SIZE=%d NUM_POPS=%d NUM_ITERS=%d" %(LIST_SIZE, NUM_POPS, NUM_ITERS,) + + print ( "%s\n%s\n\n" %(headerLine, '-' * len(headerLine) ) ) + + listTime = 0 + sllistTime = 0 + dllistTime = 0 + + for i in range(NUM_ITERS): + lst1 = primeList[:] + listTime += doTime ( lambda : doPops(lst1, randomPops) ) + + for i in range(NUM_ITERS): + slst = sllist(primeList[:]) + sllistTime += doTime ( lambda : doPops(slst, randomPops) ) + + for i in range(NUM_ITERS): + dlst = dllist(primeList[:]) + dllistTime += doTime ( lambda : doPops(dlst, randomPops) ) + # csllistTime = doTime ( lambda : doPops(cslst, randomPops) ) + csllistTime = 0.0 + + print ( "List time:\t\t%f\t= %f\nSL time:\t\t%f\t= %f\nDL time:\t\t%f\t= %f\n" %(listTime, listTime / NUM_ITERS, sllistTime, sllistTime / NUM_ITERS, dllistTime, dllistTime / NUM_ITERS) ) From 7ab2e91752740eaaf166ecf81f5a78e95323bc11 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 2 Apr 2017 13:41:36 -0400 Subject: [PATCH 15/62] Add benchmark script --- tests/benchmark.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 tests/benchmark.py diff --git a/tests/benchmark.py b/tests/benchmark.py new file mode 100755 index 0000000..05432ff --- /dev/null +++ b/tests/benchmark.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import os +import time +import random + +from llist import dllist, sllist + +LIST_SIZE = 600 + +NUM_POPS = 300 + +NUM_ITERS = 100 + +def doPops(lst, popIdxs): + + for popIdx in popIdxs: + lst.pop(popIdx) + +def doTime(func): + t1 = time.time() + func() + t2 = time.time() + + return t2 - t1 + +if __name__ == '__main__': + + + if 'LIST_SIZE' in os.environ: + LIST_SIZE = int(os.environ['LIST_SIZE']) + if 'NUM_POPS' in os.environ: + NUM_POPS = int(os.environ['NUM_POPS']) + if 'NUM_ITERS' in os.environ: + NUM_POPS = int(os.environ['NUM_ITERS']) + + primeList = list(range(LIST_SIZE)) + + # Generate a list of random indexes to pop. + # We avoid key errors by making sure we don't have a possible + # maximum larger than the smallest our list will become + + randomPops = [] + for i in range(NUM_POPS): + nextNum = random.randint(0, LIST_SIZE - i - 1) # Cheaply make sure we don't get key errors + + randomPops.append(nextNum) + + headerLine = "Starting: LIST_SIZE=%d NUM_POPS=%d NUM_ITERS=%d" %(LIST_SIZE, NUM_POPS, NUM_ITERS,) + + print ( "%s\n%s\n\n" %(headerLine, '-' * len(headerLine) ) ) + + listTime = 0 + sllistTime = 0 + dllistTime = 0 + + for i in range(NUM_ITERS): + lst1 = primeList[:] + listTime += doTime ( lambda : doPops(lst1, randomPops) ) + + for i in range(NUM_ITERS): + slst = sllist(primeList[:]) + sllistTime += doTime ( lambda : doPops(slst, randomPops) ) + + for i in range(NUM_ITERS): + dlst = dllist(primeList[:]) + dllistTime += doTime ( lambda : doPops(dlst, randomPops) ) + # csllistTime = doTime ( lambda : doPops(cslst, randomPops) ) + csllistTime = 0.0 + + print ( "List time:\t\t%f\t= %f\nSL time:\t\t%f\t= %f\nDL time:\t\t%f\t= %f\n" %(listTime, listTime / NUM_ITERS, sllistTime, sllistTime / NUM_ITERS, dllistTime, dllistTime / NUM_ITERS) ) From f3ed019ed26601169a126cf0ea32b8c29bb935a9 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 10:15:05 -0500 Subject: [PATCH 16/62] Remove some noise form llist_test --- tests/llist_test.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/llist_test.py b/tests/llist_test.py index ca4a247..7a66f78 100644 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import gc import sys +import random import unittest from llist import sllist from llist import sllistnode @@ -1297,12 +1298,10 @@ def test_clear(self): self.assertEqual(del_node.next, None) def test_pop(self): - print ( "4") ref = py23_range(0, 1024, 4) ll = dllist(ref) del_node = ll.nodeat(-1) result = ll.pop(); - print ( "5") self.assertEqual(result, ref[-1]) self.assertEqual(len(ll), len(ref) - 1) self.assertEqual(ll.size, len(ref) - 1) @@ -1314,7 +1313,6 @@ def test_pop(self): ref = py23_range(0, 1024, 4) ll = dllist(ref) - print ( "6") #import pdb; pdb.set_trace() result = ll.pop(1) self.assertEqual(result, ref[1]) @@ -1327,14 +1325,12 @@ def test_pop(self): self.assertEquals(secondNode.prev, ll.first) self.assertEquals(ll.first.prev, None) - print ( "7") ref = py23_range(0, 1024, 4) ll = dllist(ref) result = ll.pop(0) self.assertEqual(result, ref[0]) self.assertEqual(ll.first.value, ref[1]) - print ( "8") for i in range(len(ll)): result = ll.pop(0) self.assertEqual(result, ref[i+1]) @@ -1345,7 +1341,6 @@ def test_pop(self): ref = py23_range(0, 1024, 4) ll = dllist(ref) i = len(ll) - 1 - print ( "9") while i >= 0: result = ll.pop(i) self.assertEqual(result, ref[i]) @@ -1357,7 +1352,6 @@ def test_pop(self): ref = py23_range(0, 1024, 4) - import random lastIdx = list(ref).index(ref[-1]) @@ -1367,7 +1361,7 @@ def test_pop(self): ll = dllist(ref) while allIndexes: - print ( "Popping %d out of %d indexes. Value: %s\n\tFirst=%s\n\tMiddle=%s\n\tLast=%s\n\tSize=%d\n" %(allIndexes[0], len(allIndexes), str(ll[allIndexes[0]]), ll.first, ll.middle, ll.last, ll.size)) +# print ( "Popping %d out of %d indexes. Value: %s\n\tFirst=%s\n\tMiddle=%s\n\tLast=%s\n\tSize=%d\n" %(allIndexes[0], len(allIndexes), str(ll[allIndexes[0]]), ll.first, ll.middle, ll.last, ll.size)) nextIndex = allIndexes.pop(0) ll.pop(nextIndex) From e308f6fe5974fd5bfed30aa32da1c1d25d65a101 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 10:23:16 -0500 Subject: [PATCH 17/62] Add more test for node_at --- tests/llist_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/llist_test.py b/tests/llist_test.py index 7a66f78..4efc6b7 100644 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -1364,7 +1364,11 @@ def test_pop(self): # print ( "Popping %d out of %d indexes. Value: %s\n\tFirst=%s\n\tMiddle=%s\n\tLast=%s\n\tSize=%d\n" %(allIndexes[0], len(allIndexes), str(ll[allIndexes[0]]), ll.first, ll.middle, ll.last, ll.size)) nextIndex = allIndexes.pop(0) - ll.pop(nextIndex) + listAccessValue = ll[nextIndex] + + poppedValue = ll.pop(nextIndex) + + self.assertEquals(listAccessValue, poppedValue) for i in range(len(allIndexes)): if allIndexes[i] > nextIndex: From 020e1972977254d515614fdc96996862be756633 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 10:24:35 -0500 Subject: [PATCH 18/62] double-linked list: Use middle on node-access, when available. --- src/dllist.c | 53 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index f1aff58..4fbf968 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -46,7 +46,6 @@ static PyTypeObject DLListType; static PyTypeObject DLListNodeType; static PyTypeObject DLListIteratorType; -static PyObject* dllist_node_at(PyObject* self, PyObject* indexObject); /* DLListNode */ @@ -60,6 +59,7 @@ typedef struct } DLListNodeObject; + /* Convenience function for creating list nodes. * Automatically update pointers in neigbours. */ @@ -317,6 +317,10 @@ typedef struct PyObject* weakref_list; } DLListObject; +static PyObject* dllist_node_at(PyObject* self, PyObject* indexObject); +static DLListNodeObject* dllist_get_node_internal(DLListObject* self, Py_ssize_t index); + + static Py_ssize_t py_ssize_t_abs(Py_ssize_t x) { return (x >= 0) ? x : -x; @@ -343,8 +347,15 @@ static inline void _set_middle(DLListObject *self, PyObject *middle, Py_ssize_t static inline void _middle_do_recalc(DLListObject *self) { - self->middle_idx = (long)self->size / 2; - self->middle = dllist_node_at(self, (PyObject*)(PyLong_FromLong(self->middle_idx))); + Py_ssize_t midIdx; + + midIdx = self->size / 2; + /* Set middle_idx to -1 so we don't use it on the lookup */ + + self->middle_idx = -1; + self->middle = dllist_get_node_internal(self, midIdx); + + self->middle_idx = midIdx; } static inline int _middle_should_move(ssize_t size, int afterSizeAdjust) @@ -410,7 +421,7 @@ static DLListNodeObject* dllist_get_node_internal(DLListObject* self, Py_ssize_t index) { Py_ssize_t i; - Py_ssize_t middle = self->size / 2; + Py_ssize_t midpoint = self->size / 2; DLListNodeObject* node; Py_ssize_t start_pos; int reverse_dir; @@ -420,19 +431,37 @@ static DLListNodeObject* dllist_get_node_internal(DLListObject* self, PyErr_SetString(PyExc_IndexError, "Index out of range"); return NULL; } - /* pick the closest base node */ - if (index <= middle) + if (index <= midpoint) { - node = (DLListNodeObject*)self->first; - start_pos = 0; - reverse_dir = 0; + if ( self->middle_idx != -1 && self->middle_idx - index < index ) + { + node = (DLListNodeObject*)self->middle; + start_pos = self->middle_idx; + reverse_dir = 1; + } + else + { + node = (DLListNodeObject*)self->first; + start_pos = 0; + reverse_dir = 0; + } } else { - node = (DLListNodeObject*)self->last; - start_pos = self->size - 1; - reverse_dir = 1; + if ( self->middle_idx != -1 && index - self->middle_idx < self->size - index ) + { + node = (DLListNodeObject*)self->middle; + start_pos = self->middle_idx; + reverse_dir = 0; + + } + else + { + node = (DLListNodeObject*)self->last; + start_pos = self->size - 1; + reverse_dir = 1; + } } assert((PyObject*)node != Py_None); From 4ef42842896fee9986ebcd25fbca7c92e8c4406a Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 12:06:54 -0500 Subject: [PATCH 19/62] Use consistant generic adjustment method, instead of the maybe-somewhat-optimized-but-hard-to-understand directional mod methods across the board. Cleanup some functions, remove some unused stuff --- src/dllist.c | 284 ++++++++++++++++++++++++--------------------------- 1 file changed, 131 insertions(+), 153 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 4fbf968..f164e86 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -24,7 +24,7 @@ static void debugmsg(char *format, ...) { va_list args; - FILE *f, *f2; + FILE *f; f = fopen("debug.txt", "a"); @@ -326,25 +326,16 @@ static Py_ssize_t py_ssize_t_abs(Py_ssize_t x) return (x >= 0) ? x : -x; } -static PyObject *_get_midpoint(DLListObject* self) -{ - long midpoint; - PyObject *midInt; - PyObject *midSsize; - - midpoint = (long)self->size / 2; - midInt = PyLong_FromLong(midpoint); +/**************** +**** Middle functions +******************** +***/ - return midInt; -} - -static inline void _set_middle(DLListObject *self, PyObject *middle, Py_ssize_t middle_idx) -{ - self->middle = middle; - self->middle_idx = middle_idx; -} +/* _middle_do_recalc - + * Do a recalulation of middle + */ static inline void _middle_do_recalc(DLListObject *self) { Py_ssize_t midIdx; @@ -353,32 +344,44 @@ static inline void _middle_do_recalc(DLListObject *self) /* Set middle_idx to -1 so we don't use it on the lookup */ self->middle_idx = -1; - self->middle = dllist_get_node_internal(self, midIdx); + self->middle = (PyObject *) dllist_get_node_internal(self, midIdx); self->middle_idx = midIdx; } -static inline int _middle_should_move(ssize_t size, int afterSizeAdjust) +/* _middle_postappend_adjust - + * Do an adjustment after a move + * + * sizeAdjust: If called before size is incremented/decremented, pass the difference here + */ +static inline void _middle_postappend_adjust(DLListObject *self, int sizeAdjust) { - if(afterSizeAdjust) { - return !!(size % 2 == 0); - } - else { - return !!(size % 2 != 0); - } -} + Py_ssize_t midpoint; + midpoint = ( (self->size) + sizeAdjust) / 2; -static inline void _middle_moving_right(DLListObject *self, int afterSizeAdjust) -{ - if ( self->middle != Py_None) { - if(_middle_should_move(self->size, afterSizeAdjust)) + if ( self->middle_idx < midpoint ) + { + /* We move at least one, then see if we need to move more */ + for( ; self->middle_idx < midpoint; self->middle_idx++ ) { - _set_middle(self, ((DLListNodeObject*)self->middle)->next, self->middle_idx + 1); + self->middle = ((DLListNodeObject*)self->middle)->next; } } + else if ( self->middle_idx > midpoint ) + { + for( ; self->middle_idx > midpoint ; self->middle_idx-- ) + { + self->middle = ((DLListNodeObject*)self->middle)->prev; + } + } + } - + +/** + * _middle_popped_left_of_middle - When left-of-middle is popped, + * call this to adjust the middle index + */ static inline void _middle_popped_left_of_middle(DLListObject *self) { if ( self->middle != Py_None ) { @@ -386,33 +389,33 @@ static inline void _middle_popped_left_of_middle(DLListObject *self) } } - -static inline void _middle_force_moving_right(DLListObject *self) -{ - _set_middle(self, ((DLListNodeObject*)self->middle)->next, self->middle_idx + 1); - -} - -static inline void _middle_moving_left(DLListObject *self, int afterSizeAdjust) +/** + * _middle_check_recalc - + Do a recalc if we are past START_MIDDLE_AFTER + */ +static inline void _middle_check_recalc(DLListObject *self) { - - if ( self->middle != Py_None) { - if(_middle_should_move(self->size, afterSizeAdjust)) { - _set_middle(self, ((DLListNodeObject*)self->middle)->prev, self->middle_idx - 1); - } + if( self->size > START_MIDDLE_AFTER ) { + _middle_do_recalc(self); } } -static inline void _middle_force_moving_left(DLListObject *self) -{ - _set_middle(self, ((DLListNodeObject*)self->middle)->prev, self->middle_idx - 1); -} - - -static inline void _middle_check_recalc(DLListObject *self) +/** + * _middle_check_adjust_or_recalc - + Check if we are past START_MIDDLE_AFTER, and if so, update middle or recalc + */ +static inline void _middle_check_adjust_or_recalc(DLListObject *self) { - if( self->size > START_MIDDLE_AFTER ) { - _middle_do_recalc(self); + if ( self->size > START_MIDDLE_AFTER ) + { + if ( self->middle != Py_None) + { + _middle_postappend_adjust(self, 0); + } + else + { + _middle_do_recalc(self); + } } } @@ -510,7 +513,6 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) self->last = new_node; self->size += 1; - _middle_moving_right(self, 1); if (iter_node_obj == last_node_obj) { @@ -520,45 +522,47 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) iter_node_obj = iter_node->next; } - _middle_check_recalc(self); return 1; } - - sequence_len = PySequence_Length(sequence); - if (sequence_len == -1) - { - PyErr_SetString(PyExc_ValueError, "Invalid sequence"); - return 0; - } - - for (i = 0; i < sequence_len; ++i) + else { - PyObject* item; - PyObject* new_node; - - item = PySequence_GetItem(sequence, i); - if (item == NULL) + sequence_len = PySequence_Length(sequence); + if (sequence_len == -1) { - PyErr_SetString(PyExc_ValueError, - "Failed to get element from sequence"); + PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return 0; } - new_node = (PyObject*)dllistnode_create( - self->last, NULL, item, (PyObject*)self); + for (i = 0; i < sequence_len; ++i) + { + PyObject* item; + PyObject* new_node; - if (self->first == Py_None) - self->first = new_node; - self->last = new_node; + item = PySequence_GetItem(sequence, i); + if (item == NULL) + { + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return 0; + } - ++self->size; + new_node = (PyObject*)dllistnode_create( + self->last, NULL, item, (PyObject*)self); - _middle_moving_right(self, 1); + if (self->first == Py_None) + self->first = new_node; + self->last = new_node; + + ++self->size; + + + Py_DECREF(item); + } - Py_DECREF(item); } - _middle_check_recalc(self); + + _middle_check_adjust_or_recalc(self); return 1; } @@ -845,16 +849,7 @@ static PyObject* dllist_appendleft(DLListObject* self, PyObject* arg) ++self->size; - if( self->size > START_MIDDLE_AFTER ) { - if( self->middle == Py_None ) - { - _middle_do_recalc(self); - } - else - { - _middle_moving_left(self, 1); - } - } + _middle_check_adjust_or_recalc(self); Py_INCREF((PyObject*)new_node); return (PyObject*)new_node; @@ -875,17 +870,8 @@ static PyObject* dllist_appendright(DLListObject* self, PyObject* arg) self->first = (PyObject*)new_node; ++self->size; - if( self->size > START_MIDDLE_AFTER ) { - if( self->middle == Py_None ) - { - _middle_do_recalc(self); - } - else - { - _middle_moving_right(self, 1); - } - } + _middle_check_adjust_or_recalc(self); Py_INCREF((PyObject*)new_node); return (PyObject*)new_node; @@ -955,7 +941,10 @@ static PyObject* dllist_insert(DLListObject* self, PyObject* args) ++self->size; if( self->size > START_MIDDLE_AFTER ) { - /* TODO: Optimize by direction? */ + /* TODO: Optimize by direction? + We can after the "insert" function is changed to support integer index, + but not with element insert + */ _middle_do_recalc(self); } @@ -1000,46 +989,46 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) self->size += ((DLListObject*)sequence)->size; - Py_RETURN_NONE; - } - - sequence_len = PySequence_Length(sequence); - if (sequence_len == -1) - { - PyErr_SetString(PyExc_ValueError, "Invalid sequence"); - return NULL; } - - for (i = 0; i < sequence_len; ++i) + else { - PyObject* item; - PyObject* new_node; - item = PySequence_GetItem(sequence, i); - if (item == NULL) + sequence_len = PySequence_Length(sequence); + if (sequence_len == -1) { - PyErr_SetString(PyExc_ValueError, - "Failed to get element from sequence"); + PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return NULL; } - new_node = (PyObject*)dllistnode_create( - NULL, self->first, item, (PyObject*)self); + for (i = 0; i < sequence_len; ++i) + { + PyObject* item; + PyObject* new_node; + + item = PySequence_GetItem(sequence, i); + if (item == NULL) + { + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return NULL; + } - self->first = new_node; - if (self->last == Py_None) - self->last = new_node; + new_node = (PyObject*)dllistnode_create( + NULL, self->first, item, (PyObject*)self); + + self->first = new_node; + if (self->last == Py_None) + self->last = new_node; - ++self->size; - _middle_moving_left(self, 1); + ++self->size; - Py_DECREF(item); - } - if( self->size > START_MIDDLE_AFTER && self->middle == Py_None ) - { - _middle_do_recalc(self); + Py_DECREF(item); + } } + + _middle_check_adjust_or_recalc(self); + Py_RETURN_NONE; } @@ -1097,8 +1086,9 @@ static PyObject* dllist_popleft(DLListObject* self) } else { - _middle_popped_left_of_middle(self); - _middle_moving_right(self, 1); + self->middle_idx -= 1; + + _middle_postappend_adjust(self, 0); } Py_INCREF(del_node->value); @@ -1134,7 +1124,7 @@ static PyObject* dllist_popright(DLListObject* self) } else { - _middle_moving_left(self, 0); + _middle_postappend_adjust(self, 0); } @@ -1150,14 +1140,12 @@ static PyObject* dllist_popright(DLListObject* self) static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { DLListNodeObject *del_node; - FILE *f; PyObject *value; PyObject *indexObject = NULL; Py_ssize_t index; Py_ssize_t i; - Py_ssize_t before_size; if (!PyArg_UnpackTuple(arg, "pop", 0, 1, &indexObject)) { @@ -1209,16 +1197,6 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) if ( index == self->middle_idx ) { del_node = (DLListNodeObject*)self->middle; -/* if ( _middle_should_move(self->size, 0) ) - { - _middle_force_moving_right(self); - } - else - { - _middle_force_moving_left(self); - self->middle_idx -= 1; - } -*/ } else if ( index < ((DLListObject*)self)->size / 2 ) { @@ -1237,14 +1215,8 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) del_node = (DLListNodeObject*)del_node->next; } } - before_size = self->middle_idx; -// _middle_moving_left(self, 1); _middle_popped_left_of_middle(self); - _middle_moving_right(self, 0); - - /* Todo: conditional these below */ - } else { @@ -1263,7 +1235,6 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) del_node = (DLListNodeObject*)del_node->prev; } } - _middle_moving_left(self, 1); } @@ -1284,6 +1255,11 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { _middle_check_recalc(self); } + else + { + if ( self->middle != Py_None ) + _middle_postappend_adjust(self, 0); + } return value; } @@ -1514,7 +1490,6 @@ static int dllist_set_item(PyObject* self, Py_ssize_t index, PyObject* val) * del list[index] */ if (val == NULL) { - PyObject* prev = node->prev; PyObject* result; result = dllist_remove(list, (PyObject*)node); @@ -1579,6 +1554,9 @@ static PyMemberDef DLListMembers[] = "Next node" }, { "middle", T_OBJECT_EX, offsetof(DLListObject, middle), READONLY, "Middle node" }, +/* { "middle_idx", T_INT, offsetof(DLListObject, middle_idx), READONLY, + "Middle node index" }, +*/ { "size", T_INT, offsetof(DLListObject, size), READONLY, "Number of elements in the list" }, { NULL }, /* sentinel */ From 3e2a674f566132044b744a40701a337a3bc0c995 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 12:11:19 -0500 Subject: [PATCH 20/62] Add executable bit to files, and change setup.py to using env python --- setup.py | 2 +- tests/llist_test.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 setup.py mode change 100644 => 100755 tests/llist_test.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 6cb024c..ab564e8 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- VERSION='0.4' diff --git a/tests/llist_test.py b/tests/llist_test.py old mode 100644 new mode 100755 From eb71b316e302f60ce81fac3da86eafe5d23f6497 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 13:29:45 -0500 Subject: [PATCH 21/62] Skip a compare and jump straight, and other micro optimizations in pop --- src/dllist.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index f164e86..7815455 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1202,8 +1202,8 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { if ( self->middle_idx != -1 && self->middle_idx - index < index ) { - del_node = (DLListNodeObject*) self->middle; - for(i=0; i < self->middle_idx - index; i++) { + del_node = (DLListNodeObject*) ((DLListNodeObject*) self->middle)->prev; + for(i=1; i < self->middle_idx - index; i++) { del_node = (DLListNodeObject*)del_node->prev; } } @@ -1222,8 +1222,8 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) { if ( self->middle_idx != -1 && index - self->middle_idx < self->size - index ) { - del_node = (DLListNodeObject*) self->middle; - for(i=0; i < index - self->middle_idx; i++) { + del_node = (DLListNodeObject*) ((DLListNodeObject*) self->middle)->next; + for(i=1; i < index - self->middle_idx; i++) { del_node = (DLListNodeObject*)del_node->next; } } @@ -1239,11 +1239,6 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) --self->size; - if ( self->size <= START_MIDDLE_AFTER ) - { - self->middle = Py_None; - self->middle_idx = -1; - } Py_INCREF(del_node->value); @@ -1251,14 +1246,18 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) dllistnode_delete(del_node); - if ( index == self->middle_idx ) + if ( self->size <= START_MIDDLE_AFTER ) { - _middle_check_recalc(self); + self->middle = Py_None; + self->middle_idx = -1; + } + else if ( index != self->middle_idx ) + { + _middle_postappend_adjust(self, 0); } else { - if ( self->middle != Py_None ) - _middle_postappend_adjust(self, 0); + _middle_check_recalc(self); } return value; From 8315705b0cef4e8a02ad9ada545b2ed2aacd5c29 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 13:42:23 -0500 Subject: [PATCH 22/62] Add inline function to check and conditionally clear middle --- src/dllist.c | 61 ++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 7815455..971c2d4 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -419,6 +419,25 @@ static inline void _middle_check_adjust_or_recalc(DLListObject *self) } } + +/** + * _middle_check_on_shrink + After shrinking the list, check if we are small enough that we should clear middle, + and do so if we are. + + return - 1 if middle was cleared, otherwise 0 +*/ +static inline int _middle_check_on_shrink(DLListObject *self) +{ + if ( self->size <= START_MIDDLE_AFTER ) + { + self->middle = Py_None; + self->middle_idx = -1; + return 1; + } + return 0; +} + /* Convenience function for locating list nodes using index. */ static DLListNodeObject* dllist_get_node_internal(DLListObject* self, Py_ssize_t index) @@ -1079,15 +1098,9 @@ static PyObject* dllist_popleft(DLListObject* self) self->last = Py_None; --self->size; - if(self->size <= START_MIDDLE_AFTER) - { - self->middle = Py_None; - self->middle_idx = -1; - } - else + if ( ! _middle_check_on_shrink(self) ) { self->middle_idx -= 1; - _middle_postappend_adjust(self, 0); } @@ -1117,12 +1130,7 @@ static PyObject* dllist_popright(DLListObject* self) self->first = Py_None; --self->size; - if(self->size <= START_MIDDLE_AFTER) - { - self->middle = Py_None; - self->middle_idx = -1; - } - else + if( ! _middle_check_on_shrink(self) ) { _middle_postappend_adjust(self, 0); } @@ -1246,18 +1254,16 @@ static PyObject* dllist_pop(DLListObject* self, PyObject *arg) dllistnode_delete(del_node); - if ( self->size <= START_MIDDLE_AFTER ) - { - self->middle = Py_None; - self->middle_idx = -1; - } - else if ( index != self->middle_idx ) - { - _middle_postappend_adjust(self, 0); - } - else + if ( ! _middle_check_on_shrink(self) ) { - _middle_check_recalc(self); + if ( index != self->middle_idx ) + { + _middle_postappend_adjust(self, 0); + } + else + { + _middle_check_recalc(self); + } } return value; @@ -1304,12 +1310,7 @@ static PyObject* dllist_remove(DLListObject* self, PyObject* arg) self->last = del_node->prev; --self->size; - if( self->size <= START_MIDDLE_AFTER ) - { - self->middle = Py_None; - self->middle_idx = -1; - } - else + if ( ! _middle_check_on_shrink(self) ) { /* TODO: Optimize direction */ _middle_do_recalc(self); From 8970d594c447d4d998ff058604c554bd24a42e51 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 13:56:14 -0500 Subject: [PATCH 23/62] Use check instead of do inline function here --- src/dllist.c | 5 +---- tests/benchmark.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 971c2d4..d63f054 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1370,10 +1370,7 @@ static PyObject* dllist_rotate(DLListObject* self, PyObject* nObject) self->first = (PyObject*)new_first; self->last = (PyObject*)new_last; - if( self->size > START_MIDDLE_AFTER ) - { - _middle_do_recalc(self); - } + _middle_check_recalc(self); Py_RETURN_NONE; } diff --git a/tests/benchmark.py b/tests/benchmark.py index 05432ff..d527df2 100755 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -8,7 +8,7 @@ LIST_SIZE = 600 -NUM_POPS = 300 +NUM_POPS = 400 NUM_ITERS = 100 From 97260aa2d3d419ccebe7cdbdaed795b3598c0f40 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 14:01:51 -0500 Subject: [PATCH 24/62] Remove errant return --- src/dllist.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dllist.c b/src/dllist.c index d63f054..797951c 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -542,7 +542,6 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) iter_node_obj = iter_node->next; } - return 1; } else { From 3cf566087af17038335472e163aeaca4cfe65fc0 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Mon, 3 Apr 2017 14:05:46 -0500 Subject: [PATCH 25/62] Remove dlist hash function which DOES NOT WORK, generates duplicate hashes for different dlists. --- src/dllist.c | 11 ++++++++++- tests/llist_test.py | 22 ++++++++++++---------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 797951c..004757d 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -739,8 +739,17 @@ static PyObject* dllist_str(DLListObject* self) return dllist_to_string(self, PyObject_Str); } +/* NOTE - THIS FUNCTION DOES NOT WORK!! +* +* dllist([1, 5, 9]) has the SAME hash as dllist([5, 1, 9]) +* and thus it is NOT a hash function +*/ static long dllist_hash(DLListObject* self) { + + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + long hash = 0; PyObject* iter_node_obj = self->first; @@ -1587,7 +1596,7 @@ static PyTypeObject DLListType = 0, /* tp_as_number */ DLListSequenceMethods, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)dllist_hash, /* tp_hash */ + 0, /* tp_hash */ 0, /* tp_call */ (reprfunc)dllist_str, /* tp_str */ 0, /* tp_getattro */ diff --git a/tests/llist_test.py b/tests/llist_test.py index 4efc6b7..8739858 100755 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -841,11 +841,12 @@ def test_node_readonly_attributes(self): ll = sllistnode() self.assertRaises(expected_error, setattr, ll, 'next', None) - def test_list_hash(self): - self.assertEqual(hash(sllist()), hash(sllist())) - self.assertEqual(hash(sllist(py23_range(0, 1024, 4))), - hash(sllist(py23_range(0, 1024, 4)))) - self.assertEqual(hash(sllist([0, 2])), hash(sllist([0.0, 2.0]))) +# COMMENTED BECAUSE HASH DOES NOT WORK +# def test_list_hash(self): +# self.assertEqual(hash(sllist()), hash(sllist())) +# self.assertEqual(hash(sllist(py23_range(0, 1024, 4))), +# hash(sllist(py23_range(0, 1024, 4)))) +# self.assertEqual(hash(sllist([0, 2])), hash(sllist([0.0, 2.0]))) class testdllist(unittest.TestCase): @@ -1651,11 +1652,12 @@ def test_node_readonly_attributes(self): self.assertRaises(expected_error, setattr, ll, 'prev', None) self.assertRaises(expected_error, setattr, ll, 'next', None) - def test_list_hash(self): - self.assertEqual(hash(dllist()), hash(dllist())) - self.assertEqual(hash(dllist(py23_range(0, 1024, 4))), - hash(dllist(py23_range(0, 1024, 4)))) - self.assertEqual(hash(dllist([0, 2])), hash(dllist([0.0, 2.0]))) +# COMMENTED BECAUSE HASH DOES NOT WORK +# def test_list_hash(self): +# self.assertEqual(hash(dllist()), hash(dllist())) +# self.assertEqual(hash(dllist(py23_range(0, 1024, 4))), +# hash(dllist(py23_range(0, 1024, 4)))) +# self.assertEqual(hash(dllist([0, 2])), hash(dllist([0.0, 2.0]))) def suite(): From 6018dd8988a0b161eb425afb315b672b53654009 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 15:08:07 -0400 Subject: [PATCH 26/62] Garbage collect before and after every iteration, to take non-work gc time out of equation. Now dl-list consistantly beats list in random-popping --- tests/benchmark.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/benchmark.py b/tests/benchmark.py index d527df2..f95c1f3 100755 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import gc import time import random @@ -10,7 +11,7 @@ NUM_POPS = 400 -NUM_ITERS = 100 +NUM_ITERS = 200 def doPops(lst, popIdxs): @@ -54,17 +55,22 @@ def doTime(func): sllistTime = 0 dllistTime = 0 + gc.collect() for i in range(NUM_ITERS): lst1 = primeList[:] listTime += doTime ( lambda : doPops(lst1, randomPops) ) + gc.collect() for i in range(NUM_ITERS): slst = sllist(primeList[:]) sllistTime += doTime ( lambda : doPops(slst, randomPops) ) + gc.collect() for i in range(NUM_ITERS): dlst = dllist(primeList[:]) dllistTime += doTime ( lambda : doPops(dlst, randomPops) ) + gc.collect() + # csllistTime = doTime ( lambda : doPops(cslst, randomPops) ) csllistTime = 0.0 From fec5facea36a80ca3970f14b5de3817e96282c79 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 15:08:21 -0400 Subject: [PATCH 27/62] Garbage collect before and after every iteration, to take non-work gc time out of equation. Now dl-list consistantly beats list in random-popping --- tests/benchmark.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/benchmark.py b/tests/benchmark.py index 05432ff..f95c1f3 100755 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import gc import time import random @@ -8,9 +9,9 @@ LIST_SIZE = 600 -NUM_POPS = 300 +NUM_POPS = 400 -NUM_ITERS = 100 +NUM_ITERS = 200 def doPops(lst, popIdxs): @@ -54,17 +55,22 @@ def doTime(func): sllistTime = 0 dllistTime = 0 + gc.collect() for i in range(NUM_ITERS): lst1 = primeList[:] listTime += doTime ( lambda : doPops(lst1, randomPops) ) + gc.collect() for i in range(NUM_ITERS): slst = sllist(primeList[:]) sllistTime += doTime ( lambda : doPops(slst, randomPops) ) + gc.collect() for i in range(NUM_ITERS): dlst = dllist(primeList[:]) dllistTime += doTime ( lambda : doPops(dlst, randomPops) ) + gc.collect() + # csllistTime = doTime ( lambda : doPops(cslst, randomPops) ) csllistTime = 0.0 From 36c44a270a59850941161b9b4252baf3071c3c77 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 16:03:47 -0400 Subject: [PATCH 28/62] Mark sllist as unhashable. This method is not safe, and conflicts all over the place. --- src/sllist.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sllist.c b/src/sllist.c index c98dcb1..12f976a 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1350,6 +1350,10 @@ static Py_ssize_t sllist_len(PyObject* self) static long sllist_hash(SLListObject* self) { + /* NOT A VALID HASH METHOD! [1, 5, 8] gives SAME hash as [1, 8, 5], for example, among many others */ + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + long hash = 0; PyObject* iter_node_obj = self->first; @@ -1462,7 +1466,7 @@ static PyTypeObject SLListType = 0, /* tp_as_number */ &SLListSequenceMethods, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)sllist_hash, /* tp_hash */ + 0, /* tp_hash */ 0, /* tp_call */ (reprfunc)sllist_str, /* tp_str */ 0, /* tp_getattro */ From c379a1d5ef0fbf7e2d56d3e439c6fed871b51046 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 16:03:47 -0400 Subject: [PATCH 29/62] Mark sllist as unhashable. This method is not safe, and conflicts all over the place. --- src/sllist.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sllist.c b/src/sllist.c index c98dcb1..12f976a 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1350,6 +1350,10 @@ static Py_ssize_t sllist_len(PyObject* self) static long sllist_hash(SLListObject* self) { + /* NOT A VALID HASH METHOD! [1, 5, 8] gives SAME hash as [1, 8, 5], for example, among many others */ + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + long hash = 0; PyObject* iter_node_obj = self->first; @@ -1462,7 +1466,7 @@ static PyTypeObject SLListType = 0, /* tp_as_number */ &SLListSequenceMethods, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)sllist_hash, /* tp_hash */ + 0, /* tp_hash */ 0, /* tp_call */ (reprfunc)sllist_str, /* tp_str */ 0, /* tp_getattro */ From ca90eaf02977fab1069fef2cf16e0714d807d9f8 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 16:08:23 -0400 Subject: [PATCH 30/62] Just comment out the hash functions --- src/dllist.c | 2 ++ src/sllist.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dllist.c b/src/dllist.c index 004757d..f9a1ac1 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -744,6 +744,7 @@ static PyObject* dllist_str(DLListObject* self) * dllist([1, 5, 9]) has the SAME hash as dllist([5, 1, 9]) * and thus it is NOT a hash function */ +#if 0 static long dllist_hash(DLListObject* self) { @@ -768,6 +769,7 @@ static long dllist_hash(DLListObject* self) return hash; } +#endif static PyObject* dllist_richcompare(DLListObject* self, DLListObject* other, diff --git a/src/sllist.c b/src/sllist.c index 12f976a..b64aeb4 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1347,7 +1347,7 @@ static Py_ssize_t sllist_len(PyObject* self) return ((SLListObject*)self)->size; } - +#if 0 static long sllist_hash(SLListObject* self) { /* NOT A VALID HASH METHOD! [1, 5, 8] gives SAME hash as [1, 8, 5], for example, among many others */ @@ -1372,6 +1372,7 @@ static long sllist_hash(SLListObject* self) return hash; } +#endif static PyMethodDef SLListMethods[] = From d6eead22640cbb1e7f1809ceb7a0845774669c21 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 16:08:23 -0400 Subject: [PATCH 31/62] Just comment out the hash functions --- src/dllist.c | 2 ++ src/sllist.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dllist.c b/src/dllist.c index 004757d..f9a1ac1 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -744,6 +744,7 @@ static PyObject* dllist_str(DLListObject* self) * dllist([1, 5, 9]) has the SAME hash as dllist([5, 1, 9]) * and thus it is NOT a hash function */ +#if 0 static long dllist_hash(DLListObject* self) { @@ -768,6 +769,7 @@ static long dllist_hash(DLListObject* self) return hash; } +#endif static PyObject* dllist_richcompare(DLListObject* self, DLListObject* other, diff --git a/src/sllist.c b/src/sllist.c index 12f976a..b64aeb4 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1347,7 +1347,7 @@ static Py_ssize_t sllist_len(PyObject* self) return ((SLListObject*)self)->size; } - +#if 0 static long sllist_hash(SLListObject* self) { /* NOT A VALID HASH METHOD! [1, 5, 8] gives SAME hash as [1, 8, 5], for example, among many others */ @@ -1372,6 +1372,7 @@ static long sllist_hash(SLListObject* self) return hash; } +#endif static PyMethodDef SLListMethods[] = From d41a5a8a2253f57021350f4b20914ba7a296bb27 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 16:54:37 -0400 Subject: [PATCH 32/62] Add index and rindex methods to both dllist and sllist --- src/dllist.c | 47 ++++++++++++++++++++++++++++++++++++++++ src/sllist.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ tests/llist_test.py | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) diff --git a/src/dllist.c b/src/dllist.c index f9a1ac1..6b165f6 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1520,6 +1520,49 @@ static int dllist_set_item(PyObject* self, Py_ssize_t index, PyObject* val) return 0; } + +static PyObject* dllist_index(DLListObject *self, PyObject *value) +{ + + DLListNodeObject *node; + Py_ssize_t idx; + + node = (DLListNodeObject *) self->first; + idx = 0; + + while ( (PyObject*)node != Py_None ) + { + if( node->value == value ) + return PyLong_FromSsize_t(idx); + + node = (DLListNodeObject *)node->next; + idx += 1; + } + PyErr_Format(PyExc_ValueError, "No such value in list"); + return NULL; +} + +static PyObject* dllist_rindex(DLListObject *self, PyObject *value) +{ + + DLListNodeObject *node; + Py_ssize_t idx; + + node = (DLListNodeObject *) self->last; + idx = self->size - 1; + + while ( (PyObject*)node != Py_None ) + { + if( node->value == value ) + return PyLong_FromSsize_t(idx); + + node = (DLListNodeObject *)node->prev; + idx -= 1; + } + PyErr_Format(PyExc_ValueError, "No such value in list"); + return NULL; +} + static PyMethodDef DLListMethods[] = { { "appendleft", (PyCFunction)dllist_appendleft, METH_O, @@ -1538,6 +1581,10 @@ static PyMethodDef DLListMethods[] = "Append elements from iterable at the right side of the list" }, { "insert", (PyCFunction)dllist_insert, METH_VARARGS, "Inserts element before node" }, + { "index", (PyCFunction)dllist_index, METH_O, + "Returns the first index of a value" }, + { "rindex", (PyCFunction)dllist_rindex, METH_O, + "Returns the last index of a value" }, { "nodeat", (PyCFunction)dllist_node_at, METH_O, "Return node at index" }, { "popleft", (PyCFunction)dllist_popleft, METH_NOARGS, diff --git a/src/sllist.c b/src/sllist.c index b64aeb4..3486915 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1329,6 +1329,53 @@ static PyObject* sllist_to_string(SLListObject* self, return NULL; } +static PyObject* sllist_index(SLListObject *self, PyObject *value) +{ + + SLListNodeObject *node; + Py_ssize_t idx; + + node = (SLListNodeObject *) self->first; + idx = 0; + + while ( (PyObject*)node != Py_None ) + { + if( node->value == value ) + return PyLong_FromSsize_t(idx); + + node = (SLListNodeObject *)node->next; + idx += 1; + } + PyErr_Format(PyExc_ValueError, "No such value in list"); + return NULL; +} + +static PyObject* sllist_rindex(SLListObject *self, PyObject *value) +{ + + SLListNodeObject *node; + Py_ssize_t idx; + Py_ssize_t matchedIdx; + + node = (SLListNodeObject *) self->first; + idx = 0; + matchedIdx = -1; + + while ( (PyObject*)node != Py_None ) + { + if( node->value == value ) + matchedIdx = idx; + + node = (SLListNodeObject *)node->next; + idx += 1; + } + + if ( matchedIdx != -1 ) + return PyLong_FromSsize_t(matchedIdx); + + PyErr_Format(PyExc_ValueError, "No such value in list"); + return NULL; +} static PyObject* sllist_repr(SLListObject* self) { @@ -1407,6 +1454,12 @@ static PyMethodDef SLListMethods[] = { "nodeat", (PyCFunction)sllist_node_at, METH_O, "Return node at index" }, + { "index", (PyCFunction)sllist_index, METH_O, + "Returns the first index of a value" }, + + { "rindex", (PyCFunction)sllist_rindex, METH_O, + "Returns the last index of a value" }, + { "pop", (PyCFunction)sllist_pop, METH_VARARGS, "Remove an element by index from the list and return it, or last item if no index provided" }, diff --git a/tests/llist_test.py b/tests/llist_test.py index 8739858..05c53b4 100755 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -801,6 +801,25 @@ def test_concat_inplace_empty(self): self.assertEqual(filled, sllist(filled_ref + [])) self.assertEqual(len(filled), len(filled_ref)) + def test_index(self): + lst = [1, 5, 10, 5, 9] + + sl = sllist(lst) + + self.assertEqual(sl.index(1), 0) + self.assertEqual(sl.index(5), 1) + self.assertEqual(sl.rindex(5), 3) + self.assertEqual(sl.rindex(9), 4) + + gotException = False + try: + sl.index(2) + except ValueError: + gotException = True + + self.assertEqual(gotException, True) + + def test_repeat(self): ref = py23_range(0, 1024, 4) ll = sllist(ref) @@ -1611,6 +1630,25 @@ def test_concat_inplace_empty(self): self.assertEqual(filled, dllist(filled_ref + [])) self.assertEqual(len(filled), len(filled_ref)) + def test_index(self): + lst = [1, 5, 10, 5, 9] + + dl = dllist(lst) + + self.assertEqual(dl.index(1), 0) + self.assertEqual(dl.index(5), 1) + self.assertEqual(dl.rindex(5), 3) + self.assertEqual(dl.rindex(9), 4) + + gotException = False + try: + dl.index(2) + except ValueError: + gotException = True + + self.assertEqual(gotException, True) + + def test_repeat(self): ref = py23_range(0, 1024, 4) ll = dllist(ref) From bee71e8af6474f9529756143ac1fb5a437327707 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 16:57:38 -0400 Subject: [PATCH 33/62] Fix braces warning --- src/dllist.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 6b165f6..431fd60 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1616,7 +1616,7 @@ static PyMemberDef DLListMembers[] = { NULL }, /* sentinel */ }; -static PySequenceMethods DLListSequenceMethods[] = +static PySequenceMethods DLListSequenceMethods[] = { { dllist_len, /* sq_length */ dllist_concat, /* sq_concat */ @@ -1628,7 +1628,9 @@ static PySequenceMethods DLListSequenceMethods[] = 0, /* sq_contains */ dllist_inplace_concat, /* sq_inplace_concat */ 0, /* sq_inplace_repeat */ -}; +} +} +; static PyTypeObject DLListType = { From 08b6e0390229f0836809f177178541f08d88a633 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 16:58:19 -0400 Subject: [PATCH 34/62] Comment out debug function --- src/dllist.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 431fd60..cdcc16e 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -17,7 +17,7 @@ #define START_MIDDLE_AFTER 10 -#define DO_DEBUG +/* #define DO_DEBUG */ #ifdef DO_DEBUG @@ -35,10 +35,13 @@ static void debugmsg(char *format, ...) fclose(f); } #else -static inline void debugmsg(char *format, ...) + +#define debugmsg(...) +/*static inline void debugmsg(char *format, ...) { } +*/ #endif From 99c9b8639f5353bd8292f9cd1f32075da6419176 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 17:05:14 -0400 Subject: [PATCH 35/62] Add test for 'contains' --- tests/llist_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/llist_test.py b/tests/llist_test.py index 05c53b4..1f0c1fc 100755 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -819,6 +819,17 @@ def test_index(self): self.assertEqual(gotException, True) + def test_contains(self): + + lst = [1, 5, 7] + + sl = sllist(lst) + + self.assertEqual(5 in sl, True) + self.assertEqual(1 in sl, True) + self.assertEqual(7 in sl, True) + self.assertEqual(8 in sl, False) + def test_repeat(self): ref = py23_range(0, 1024, 4) @@ -1648,6 +1659,17 @@ def test_index(self): self.assertEqual(gotException, True) + def test_contains(self): + + lst = [1, 5, 7] + + sl = dllist(lst) + + self.assertEqual(5 in sl, True) + self.assertEqual(1 in sl, True) + self.assertEqual(7 in sl, True) + self.assertEqual(8 in sl, False) + def test_repeat(self): ref = py23_range(0, 1024, 4) From c2a67b64e90d78fe7dc4c321ce7161eb3b972a57 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 5 Apr 2017 17:05:34 -0400 Subject: [PATCH 36/62] Bump version to 0.5.0 and update CHANGES --- CHANGES | 17 +++++++++++++++++ setup.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 82a2ebd..3406254 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,20 @@ +* llist-0.5 + + Work by Tim Savannah: + + - Implement pop(idx) to pop any given index + - Implement "contains" sequence method, so the "in" operator doesn't run the whole list multiple times + - Implement "index" and "rindex" methods to return an index/rindex + - Remove "last_accessed_idx" and "last_accessed" node from dllist, replace with "middle" which is used when + the list size exceeds a certain value (defined as 10). This greatly improves random-access and random-pop performance + on dllist to be comprable or better to that of a base python list + - Remove the "hash" function, which did NOT generate unique hashes (very easy to construct two linked lists with same hash, + such as [1, 5, 7] and [5, 1, 7] or [2, 1] and [3] + - Remove all compiler warnings + - Add some basic benchmarks + - Add some more tests + - Some minor cleanups + * llist-0.4 (2013-01-01) - Python 3.x support diff --git a/setup.py b/setup.py index ab564e8..ff1edc6 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -VERSION='0.4' +VERSION='0.5.0' from distutils.core import setup, Extension From e24d25708334a172d88327027b66b7e88194ecdd Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Fri, 7 Apr 2017 14:51:04 -0400 Subject: [PATCH 37/62] Move types into header files. Converts types to extern in header, and defines within the .c file. Also, make DLL and SLL types have a common header, and introduce llist_types.h to define those common headers (so they can both be transversed as an single-linked list) --- src/dllist.c | 87 +++++++++++++++++----------------------------- src/dllist_types.h | 36 +++++++++++++++++++ src/llist_types.h | 24 +++++++++++++ src/sllist.c | 86 ++++++++++++++++++--------------------------- src/sllist_types.h | 35 +++++++++++++++++++ 5 files changed, 160 insertions(+), 108 deletions(-) create mode 100644 src/dllist_types.h create mode 100644 src/llist_types.h create mode 100644 src/sllist_types.h diff --git a/src/dllist.c b/src/dllist.c index cdcc16e..d9cc572 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -6,6 +6,9 @@ #include #include #include "py23macros.h" +#include "llist_types.h" +#include "dllist_types.h" +#include "sllist_types.h" #ifndef PyVarObject_HEAD_INIT #define PyVarObject_HEAD_INIT(type, size) \ @@ -45,24 +48,6 @@ static void debugmsg(char *format, ...) #endif -static PyTypeObject DLListType; -static PyTypeObject DLListNodeType; -static PyTypeObject DLListIteratorType; - - -/* DLListNode */ - -typedef struct -{ - PyObject_HEAD - PyObject* value; - PyObject* prev; - PyObject* next; - PyObject* list_weakref; -} DLListNodeObject; - - - /* Convenience function for creating list nodes. * Automatically update pointers in neigbours. */ @@ -98,7 +83,7 @@ static DLListNodeObject* dllistnode_create(PyObject* prev, } node = (DLListNodeObject*)PyObject_CallObject( - (PyObject*)&DLListNodeType, args); + (PyObject*)DLListNodeType, args); Py_XDECREF(args); @@ -264,7 +249,7 @@ static PyMemberDef DLListNodeMembers[] = { NULL }, /* sentinel */ }; -static PyTypeObject DLListNodeType = +PyTypeObject DLListNodeType[] = { { PyVarObject_HEAD_INIT(NULL, 0) "llist.dllistnode", /* tp_name */ @@ -304,22 +289,12 @@ static PyTypeObject DLListNodeType = (initproc)dllistnode_init, /* tp_init */ 0, /* tp_alloc */ dllistnode_new, /* tp_new */ +} }; /* DLList */ -typedef struct -{ - PyObject_HEAD - PyObject* first; - PyObject* last; - PyObject* middle; - Py_ssize_t size; - Py_ssize_t middle_idx; - PyObject* weakref_list; -} DLListObject; - static PyObject* dllist_node_at(PyObject* self, PyObject* indexObject); static DLListNodeObject* dllist_get_node_internal(DLListObject* self, Py_ssize_t index); @@ -514,7 +489,7 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, &DLListType)) + if (PyObject_TypeCheck(sequence, DLListType)) { /* Special path for extending with a DLList. * It's not strictly required but it will maintain @@ -782,7 +757,7 @@ static PyObject* dllist_richcompare(DLListObject* self, DLListNodeObject* other_node; int satisfied = 1; - if (!PyObject_TypeCheck(other, &DLListType)) + if (!PyObject_TypeCheck(other, DLListType)) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; @@ -868,7 +843,7 @@ static PyObject* dllist_appendleft(DLListObject* self, PyObject* arg) { DLListNodeObject* new_node; - if (PyObject_TypeCheck(arg, &DLListNodeType)) + if (PyObject_TypeCheck(arg, DLListNodeType)) arg = ((DLListNodeObject*)arg)->value; new_node = dllistnode_create(NULL, self->first, arg, (PyObject*)self); @@ -891,7 +866,7 @@ static PyObject* dllist_appendright(DLListObject* self, PyObject* arg) { DLListNodeObject* new_node; - if (PyObject_TypeCheck(arg, &DLListNodeType)) + if (PyObject_TypeCheck(arg, DLListNodeType)) arg = ((DLListNodeObject*)arg)->value; new_node = dllistnode_create(self->last, NULL, arg, (PyObject*)self); @@ -918,7 +893,7 @@ static PyObject* dllist_insert(DLListObject* self, PyObject* args) if (!PyArg_UnpackTuple(args, "insert", 1, 2, &val, &ref_node)) return NULL; - if (PyObject_TypeCheck(val, &DLListNodeType)) + if (PyObject_TypeCheck(val, DLListNodeType)) val = ((DLListNodeObject*)val)->value; if (ref_node == NULL || ref_node == Py_None) @@ -936,7 +911,7 @@ static PyObject* dllist_insert(DLListObject* self, PyObject* args) PyObject* list_ref; /* insert item before ref_node */ - if (!PyObject_TypeCheck(ref_node, &DLListNodeType)) + if (!PyObject_TypeCheck(ref_node, DLListNodeType)) { PyErr_SetString(PyExc_TypeError, "ref_node argument must be a dllistnode"); @@ -990,7 +965,7 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, &DLListType)) + if (PyObject_TypeCheck(sequence, DLListType)) { /* Special path for extending with a DLList. * It's not strictly required but it will maintain @@ -1288,7 +1263,7 @@ static PyObject* dllist_remove(DLListObject* self, PyObject* arg) PyObject* list_ref; PyObject* value; - if (!PyObject_TypeCheck(arg, &DLListNodeType)) + if (!PyObject_TypeCheck(arg, DLListNodeType)) { PyErr_SetString(PyExc_TypeError, "Argument must be a dllistnode"); return NULL; @@ -1409,7 +1384,7 @@ static PyObject* dllist_iter(PyObject* self) return NULL; } - result = PyObject_CallObject((PyObject*)&DLListIteratorType, args); + result = PyObject_CallObject((PyObject*)DLListIteratorType, args); Py_DECREF(args); @@ -1427,7 +1402,7 @@ static PyObject* dllist_concat(PyObject* self, PyObject* other) DLListObject* new_list; new_list = (DLListObject*)PyObject_CallObject( - (PyObject*)&DLListType, NULL); + (PyObject*)DLListType, NULL); if (!dllist_extend_internal(new_list, self) || !dllist_extend_internal(new_list, other)) @@ -1454,7 +1429,7 @@ static PyObject* dllist_repeat(PyObject* self, Py_ssize_t count) Py_ssize_t i; new_list = (DLListObject*)PyObject_CallObject( - (PyObject*)&DLListType, NULL); + (PyObject*)DLListType, NULL); for (i = 0; i < count; ++i) { @@ -1511,7 +1486,7 @@ static int dllist_set_item(PyObject* self, Py_ssize_t index, PyObject* val) /* The rest of this function handles normal assignment: * list[index] = item */ - if (PyObject_TypeCheck(val, &DLListNodeType)) + if (PyObject_TypeCheck(val, DLListNodeType)) val = ((DLListNodeObject*)val)->value; oldval = node->value; @@ -1635,7 +1610,7 @@ static PySequenceMethods DLListSequenceMethods[] = { } ; -static PyTypeObject DLListType = +PyTypeObject DLListType[] = { { PyVarObject_HEAD_INIT(NULL, 0) "llist.dllist", /* tp_name */ @@ -1677,6 +1652,7 @@ static PyTypeObject DLListType = (initproc)dllist_init, /* tp_init */ 0, /* tp_alloc */ dllist_new, /* tp_new */ +} }; @@ -1707,7 +1683,7 @@ static PyObject* dllistiterator_new(PyTypeObject* type, if (!PyArg_UnpackTuple(args, "__new__", 1, 1, &owner_list)) return NULL; - if (!PyObject_TypeCheck(owner_list, &DLListType)) + if (!PyObject_TypeCheck(owner_list, DLListType)) { PyErr_SetString(PyExc_TypeError, "dllist argument expected"); return NULL; @@ -1751,7 +1727,7 @@ static PyObject* dllistiterator_iternext(PyObject* self) return value; } -static PyTypeObject DLListIteratorType = +PyTypeObject DLListIteratorType[] = { { PyVarObject_HEAD_INIT(NULL, 0) "llist.dllistiterator", /* tp_name */ @@ -1791,25 +1767,26 @@ static PyTypeObject DLListIteratorType = 0, /* tp_init */ 0, /* tp_alloc */ dllistiterator_new, /* tp_new */ +} }; int dllist_init_type(void) { return - ((PyType_Ready(&DLListType) == 0) && - (PyType_Ready(&DLListNodeType) == 0) && - (PyType_Ready(&DLListIteratorType) == 0)) + ((PyType_Ready(DLListType) == 0) && + (PyType_Ready(DLListNodeType) == 0) && + (PyType_Ready(DLListIteratorType) == 0)) ? 1 : 0; } void dllist_register(PyObject* module) { - Py_INCREF(&DLListType); - Py_INCREF(&DLListNodeType); - Py_INCREF(&DLListIteratorType); + Py_INCREF(DLListType); + Py_INCREF(DLListNodeType); + Py_INCREF(DLListIteratorType); - PyModule_AddObject(module, "dllist", (PyObject*)&DLListType); - PyModule_AddObject(module, "dllistnode", (PyObject*)&DLListNodeType); - PyModule_AddObject(module, "dllistiterator", (PyObject*)&DLListIteratorType); + PyModule_AddObject(module, "dllist", (PyObject*)DLListType); + PyModule_AddObject(module, "dllistnode", (PyObject*)DLListNodeType); + PyModule_AddObject(module, "dllistiterator", (PyObject*)DLListIteratorType); } diff --git a/src/dllist_types.h b/src/dllist_types.h new file mode 100644 index 0000000..b6bea5d --- /dev/null +++ b/src/dllist_types.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2017 Timothy Savannah + * Released under the MIT license (see attached LICENSE file). + */ + +#ifndef DLLIST_TYPES_H +#define DLLIST_TYPES_H + +extern PyTypeObject DLListType[]; +extern PyTypeObject DLListNodeType[]; +extern PyTypeObject DLListIteratorType[]; + + +/* DLListNode */ + +typedef struct +{ + PyObject_HEAD + PyObject *value; + PyObject *next; + PyObject *prev; + PyObject *list_weakref; +} DLListNodeObject; + +typedef struct +{ + PyObject_HEAD + PyObject *first; + PyObject *last; + Py_ssize_t size; + PyObject *weakref_list; + PyObject *middle; + Py_ssize_t middle_idx; +} DLListObject; + + +#endif /* DLLIST_TYPES_H */ diff --git a/src/llist_types.h b/src/llist_types.h new file mode 100644 index 0000000..085e2d1 --- /dev/null +++ b/src/llist_types.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2017 Timothy Savannah + * Released under the MIT license (see attached LICENSE file). + */ + +#ifndef LLIST_TYPES_H +#define LLIST_TYPES_H + +/* Common start to both dllist and sllist */ +typedef struct +{ + PyObject_HEAD + PyObject *value; + PyObject *next; +} LListNodeObject; + +typedef struct +{ + PyObject_HEAD + PyObject *first; + PyObject *last; + Py_ssize_t size; +} LListObject; + +#endif /* LLIST_TYPES_H */ diff --git a/src/sllist.c b/src/sllist.c index 3486915..7772fc2 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -6,6 +6,7 @@ #include #include #include "py23macros.h" +#include "sllist_types.h" #ifndef PyVarObject_HEAD_INIT #define PyVarObject_HEAD_INIT(type, size) \ @@ -13,21 +14,6 @@ #endif -static PyTypeObject SLListType; -static PyTypeObject SLListNodeType; -static PyTypeObject SLListIteratorType; - - -/* SLListNode */ - -typedef struct -{ - PyObject_HEAD - PyObject* value; - PyObject* next; - PyObject* list_weakref; -} SLListNodeObject; - static SLListNodeObject* sllistnode_create(PyObject* next, PyObject* value, @@ -60,7 +46,7 @@ static SLListNodeObject* sllistnode_create(PyObject* next, } node = (SLListNodeObject*)PyObject_CallObject( - (PyObject*)&SLListNodeType, args); + (PyObject*)SLListNodeType, args); Py_XDECREF(args); @@ -212,7 +198,7 @@ static PyMemberDef SLListNodeMembers[] = }; -static PyTypeObject SLListNodeType = +PyTypeObject SLListNodeType[] = { { PyVarObject_HEAD_INIT(NULL, 0) "llist.sllistnode", /* tp_name */ @@ -252,6 +238,7 @@ static PyTypeObject SLListNodeType = (initproc)sllistnode_init, /* tp_init */ 0, /* tp_alloc */ sllistnode_new, /* tp_new */ +} }; @@ -260,15 +247,6 @@ static PyTypeObject SLListNodeType = /* SLLIST */ /* ****************************************************************************** */ -typedef struct -{ - PyObject_HEAD - PyObject* first; - PyObject* last; - Py_ssize_t size; - PyObject* weakref_list; -} SLListObject; - static void sllist_dealloc(SLListObject* self) { @@ -318,7 +296,7 @@ static int sllist_extend_internal(SLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, &SLListType)) + if (PyObject_TypeCheck(sequence, SLListType)) { /* Special path for extending with a SLList. * It's not strictly required but it will maintain @@ -426,7 +404,7 @@ static PyObject* sllist_richcompare(SLListObject* self, SLListNodeObject* other_node; int satisfied = 1; - if (!PyObject_TypeCheck(other, &SLListType)) + if (!PyObject_TypeCheck(other, SLListType)) { Py_INCREF(Py_NotImplemented); return Py_NotImplemented; @@ -517,7 +495,7 @@ static SLListNodeObject* sllist_get_prev(SLListObject* self, SLListNodeObject* node = (SLListNodeObject*)self->first; - if (!PyObject_TypeCheck(next, &SLListNodeType)) + if (!PyObject_TypeCheck(next, SLListNodeType)) { PyErr_SetString(PyExc_TypeError, "Argument is not an sllistnode"); return NULL; @@ -554,7 +532,7 @@ static PyObject* sllist_appendleft(SLListObject* self, PyObject* arg) { SLListNodeObject* new_node; - if (PyObject_TypeCheck(arg, &SLListNodeType)) + if (PyObject_TypeCheck(arg, SLListNodeType)) arg = ((SLListNodeObject*)arg)->value; new_node = sllistnode_create(self->first, @@ -577,7 +555,7 @@ static PyObject* sllist_appendright(SLListObject* self, PyObject* arg) { SLListNodeObject* new_node; - if (PyObject_TypeCheck(arg, &SLListNodeType)) + if (PyObject_TypeCheck(arg, SLListNodeType)) arg = ((SLListNodeObject*)arg)->value; new_node = sllistnode_create(Py_None, @@ -610,13 +588,13 @@ static PyObject* sllist_insertafter(SLListObject* self, PyObject* arg) if (!PyArg_UnpackTuple(arg, "insertafter", 2, 2, &value, &before)) return NULL; - if (!PyObject_TypeCheck(before, &SLListNodeType)) + if (!PyObject_TypeCheck(before, SLListNodeType)) { PyErr_SetString(PyExc_TypeError, "Argument is not an sllistnode"); return NULL; } - if (PyObject_TypeCheck(value, &SLListNodeType)) + if (PyObject_TypeCheck(value, SLListNodeType)) value = ((SLListNodeObject*)value)->value; if (((SLListNodeObject*)before)->list_weakref == Py_None) @@ -664,12 +642,12 @@ static PyObject* sllist_insertbefore(SLListObject* self, PyObject* arg) if (!PyArg_UnpackTuple(arg, "insertbefore", 2, 2, &value, &after)) return NULL; - if (!PyObject_TypeCheck(after, &SLListNodeType)) + if (!PyObject_TypeCheck(after, SLListNodeType)) { PyErr_SetString(PyExc_TypeError, "Argument is not an sllistnode"); return NULL; } - if (PyObject_TypeCheck(value, &SLListNodeType)) + if (PyObject_TypeCheck(value, SLListNodeType)) value = ((SLListNodeObject*)value)->value; if (after == Py_None) @@ -718,7 +696,7 @@ static PyObject* sllist_extendleft(SLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, &SLListType)) + if (PyObject_TypeCheck(sequence, SLListType)) { /* Special path for extending with a SLList. * It's not strictly required but it will maintain @@ -850,7 +828,7 @@ static PyObject* sllist_remove(SLListObject* self, PyObject* arg) PyObject* list_ref; PyObject* value; - if (!PyObject_TypeCheck(arg, &SLListNodeType)) + if (!PyObject_TypeCheck(arg, SLListNodeType)) { PyErr_SetString(PyExc_TypeError, "Argument is not an sllistnode"); return NULL; @@ -962,7 +940,7 @@ static PyObject* sllist_concat(PyObject* self, PyObject* other) SLListObject* new_list; new_list = (SLListObject*)PyObject_CallObject( - (PyObject*)&SLListType, NULL); + (PyObject*)SLListType, NULL); if (!sllist_extend_internal(new_list, self) || !sllist_extend_internal(new_list, other)) @@ -991,7 +969,7 @@ static PyObject* sllist_repeat(PyObject* self, Py_ssize_t count) Py_ssize_t i; new_list = (SLListObject*)PyObject_CallObject( - (PyObject*)&SLListType, NULL); + (PyObject*)SLListType, NULL); for (i = 0; i < count; ++i) { @@ -1056,7 +1034,7 @@ static int sllist_set_item(PyObject* self, Py_ssize_t index, PyObject* val) /* The rest of this function handles normal assignment: * list[index] = item */ - if (!PyObject_TypeCheck(val, &SLListNodeType)) { + if (!PyObject_TypeCheck(val, SLListNodeType)) { PyErr_SetString(PyExc_TypeError, "Argument is not an sllistnode"); return -1; } @@ -1264,7 +1242,7 @@ static PyObject* sllist_iter(PyObject* self) return NULL; } - result = PyObject_CallObject((PyObject*)&SLListIteratorType, args); + result = PyObject_CallObject((PyObject*)SLListIteratorType, args); Py_DECREF(args); @@ -1505,7 +1483,7 @@ static PySequenceMethods SLListSequenceMethods = 0, /* sq_inplace_repeat */ }; -static PyTypeObject SLListType = +PyTypeObject SLListType[] = { { PyVarObject_HEAD_INIT(NULL, 0) "llist.sllist", /* tp_name */ @@ -1547,6 +1525,7 @@ static PyTypeObject SLListType = (initproc)sllist_init, /* tp_init */ 0, /* tp_alloc */ sllist_new, /* tp_new */ +} }; @@ -1577,7 +1556,7 @@ static PyObject* sllistiterator_new(PyTypeObject* type, if (!PyArg_UnpackTuple(args, "__new__", 1, 1, &owner_list)) return NULL; - if (!PyObject_TypeCheck(owner_list, &SLListType)) + if (!PyObject_TypeCheck(owner_list, SLListType)) { PyErr_SetString(PyExc_TypeError, "sllist argument expected"); return NULL; @@ -1624,7 +1603,7 @@ static PyObject* sllistiterator_iternext(PyObject* self) -static PyTypeObject SLListIteratorType = +PyTypeObject SLListIteratorType[] = { { PyVarObject_HEAD_INIT(NULL, 0) "llist.sllistiterator", /* tp_name */ @@ -1664,6 +1643,7 @@ static PyTypeObject SLListIteratorType = 0, /* tp_init */ 0, /* tp_alloc */ sllistiterator_new, /* tp_new */ +} }; @@ -1671,19 +1651,19 @@ static PyTypeObject SLListIteratorType = int sllist_init_type(void) { return - ((PyType_Ready(&SLListType) == 0) && - (PyType_Ready(&SLListNodeType) == 0) && - (PyType_Ready(&SLListIteratorType) == 0)) + ((PyType_Ready(SLListType) == 0) && + (PyType_Ready(SLListNodeType) == 0) && + (PyType_Ready(SLListIteratorType) == 0)) ? 1 : 0; } void sllist_register(PyObject* module) { - Py_INCREF(&SLListType); - Py_INCREF(&SLListNodeType); - Py_INCREF(&SLListIteratorType); + Py_INCREF(SLListType); + Py_INCREF(SLListNodeType); + Py_INCREF(SLListIteratorType); - PyModule_AddObject(module, "sllist", (PyObject*)&SLListType); - PyModule_AddObject(module, "sllistnode", (PyObject*)&SLListNodeType); - PyModule_AddObject(module, "sllistiterator", (PyObject*)&SLListIteratorType); + PyModule_AddObject(module, "sllist", (PyObject*)SLListType); + PyModule_AddObject(module, "sllistnode", (PyObject*)SLListNodeType); + PyModule_AddObject(module, "sllistiterator", (PyObject*)SLListIteratorType); } diff --git a/src/sllist_types.h b/src/sllist_types.h new file mode 100644 index 0000000..adeb3cc --- /dev/null +++ b/src/sllist_types.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2017 Timothy Savannah + * Released under the MIT license (see attached LICENSE file). + */ + +#ifndef SLLIST_TYPES_H +#define SLLIST_TYPES_H + +int sllist_init_type(void); +void sllist_register(PyObject* module); + +extern PyTypeObject SLListType[]; +extern PyTypeObject SLListNodeType[]; +extern PyTypeObject SLListIteratorType[]; + + +/* SLListNode */ + +typedef struct +{ + PyObject_HEAD + PyObject *value; + PyObject *next; + PyObject *list_weakref; +} SLListNodeObject; + +typedef struct +{ + PyObject_HEAD + PyObject *first; + PyObject *last; + Py_ssize_t size; + PyObject *weakref_list; +} SLListObject; + +#endif /* SLLIST_TYPES_H */ From d82f3623d072173333db090ded4477cc2c8e5e26 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Fri, 7 Apr 2017 15:34:27 -0400 Subject: [PATCH 38/62] Improve extend and extendleft, higher performance all-around, and also now DLList can extend directly with SLList and SLList can extend directly with a DLList, at a much much higher performance rate than before --- src/dllist.c | 51 +++++++++++++++++++------------------------------ src/sllist.c | 54 +++++++++++++++++++++------------------------------- 2 files changed, 42 insertions(+), 63 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index d9cc572..6b12e00 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -489,17 +489,20 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, DLListType)) + if (PyObject_TypeCheck(sequence, DLListType) || PyObject_TypeCheck(sequence, SLListType)) { - /* Special path for extending with a DLList. - * It's not strictly required but it will maintain - * the last accessed item. */ - PyObject* iter_node_obj = ((DLListObject*)sequence)->first; - PyObject* last_node_obj = self->last; + /* Special path for extending with a LList (double or single) + * It's not strictly required but greatly improves performance + */ + PyObject* iter_node_obj = ((LListObject*)sequence)->first; - while (iter_node_obj != Py_None) + sequence_len = ((LListObject*)sequence)->size; + + self->size += sequence_len; + + while (sequence_len--) { - DLListNodeObject* iter_node = (DLListNodeObject*)iter_node_obj; + LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; new_node = (PyObject*)dllistnode_create( @@ -509,14 +512,6 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) self->first = new_node; self->last = new_node; - self->size += 1; - - if (iter_node_obj == last_node_obj) - { - /* This is needed to terminate loop if self == sequence. */ - break; - } - iter_node_obj = iter_node->next; } @@ -965,17 +960,18 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, DLListType)) + if (PyObject_TypeCheck(sequence, DLListType) || PyObject_TypeCheck(sequence, SLListType)) { - /* Special path for extending with a DLList. - * It's not strictly required but it will maintain - * the last accessed item. */ - PyObject* iter_node_obj = ((DLListObject*)sequence)->first; - PyObject* last_node_obj = ((DLListObject*)sequence)->last; + /* Special path for extending with a LList. Better performance. */ + PyObject* iter_node_obj = ((LListObject*)sequence)->first; - while (iter_node_obj != Py_None) + sequence_len = ((LListObject*)sequence)->size; + + self->size += sequence_len; + + while (sequence_len--) { - DLListNodeObject* iter_node = (DLListNodeObject*)iter_node_obj; + LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; new_node = (PyObject*)dllistnode_create( @@ -985,16 +981,9 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) if (self->last == Py_None) self->last = new_node; - if (iter_node_obj == last_node_obj) - { - /* This is needed to terminate loop if self == sequence. */ - break; - } - iter_node_obj = iter_node->next; } - self->size += ((DLListObject*)sequence)->size; } else diff --git a/src/sllist.c b/src/sllist.c index 7772fc2..af4d4b0 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -6,7 +6,9 @@ #include #include #include "py23macros.h" +#include "llist_types.h" #include "sllist_types.h" +#include "dllist_types.h" #ifndef PyVarObject_HEAD_INIT #define PyVarObject_HEAD_INIT(type, size) \ @@ -296,17 +298,20 @@ static int sllist_extend_internal(SLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, SLListType)) + if (PyObject_TypeCheck(sequence, SLListType) || PyObject_TypeCheck(sequence, DLListType)) { - /* Special path for extending with a SLList. - * It's not strictly required but it will maintain - * the last accessed item. */ - PyObject* iter_node_obj = ((SLListObject *)sequence)->first; - PyObject* last_node_obj = self->last; + /* Special path for extending with a LList (double or single) + * It's not strictly required but greatly improves performance + */ + PyObject* iter_node_obj = ((LListObject *)sequence)->first; - while (iter_node_obj != Py_None) + sequence_len = ((LListObject *)sequence)->size; + + self->size += sequence_len; + + while (sequence_len--) { - SLListNodeObject* iter_node = (SLListNodeObject*)iter_node_obj; + LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; new_node = (PyObject*)sllistnode_create( @@ -319,17 +324,9 @@ static int sllist_extend_internal(SLListObject* self, PyObject* sequence) self->first = new_node; self->last = new_node; - if (iter_node_obj == last_node_obj) - { - /* This is needed to terminate loop if self == sequence. */ - break; - } - iter_node_obj = iter_node->next; } - self->size += ((SLListObject*)sequence)->size; - return 1; } @@ -696,17 +693,18 @@ static PyObject* sllist_extendleft(SLListObject* self, PyObject* sequence) Py_ssize_t i; Py_ssize_t sequence_len; - if (PyObject_TypeCheck(sequence, SLListType)) + if (PyObject_TypeCheck(sequence, SLListType) || PyObject_TypeCheck(sequence, DLListType)) { - /* Special path for extending with a SLList. - * It's not strictly required but it will maintain - * the last accessed item. */ - PyObject* iter_node_obj = ((SLListObject*)sequence)->first; - PyObject* last_node_obj = ((SLListObject*)sequence)->last; + /* Special path for extending with a LList. Better performance. */ + PyObject* iter_node_obj = ((LListObject*)sequence)->first; - while (iter_node_obj != Py_None) + sequence_len = ((LListObject*)sequence)->size; + + self->size += sequence_len; + + while (sequence_len--) { - SLListNodeObject* iter_node = (SLListNodeObject*)iter_node_obj; + LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; new_node = (PyObject*)sllistnode_create( @@ -716,17 +714,9 @@ static PyObject* sllist_extendleft(SLListObject* self, PyObject* sequence) if (self->last == Py_None) self->last = new_node; - if (iter_node_obj == last_node_obj) - { - /* This is needed to terminate loop if self == sequence. */ - break; - } - iter_node_obj = iter_node->next; } - self->size += ((SLListObject*)sequence)->size; - Py_RETURN_NONE; } From f9f431062c9af87e81322d9478d93f32398f50ea Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Fri, 7 Apr 2017 15:34:37 -0400 Subject: [PATCH 39/62] Add benchmark_extend --- tests/benchmark_extend.py | 162 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100755 tests/benchmark_extend.py diff --git a/tests/benchmark_extend.py b/tests/benchmark_extend.py new file mode 100755 index 0000000..3579e6f --- /dev/null +++ b/tests/benchmark_extend.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +import os +import gc +import time +import random + +from llist import dllist, sllist + +LIST1_SIZE = 600 + +LIST2_SIZE = 400 + +NUM_ITERS = 200 + +def doExtend(lst, otherList): + + return lst + otherList + +def doIExtend(lst, otherList): + + lst += otherList + return lst + +def doExtendLeft(lst, otherList): + + lst.extendleft(otherList) + return lst + +def doTime(func): + t1 = time.time() + func() + t2 = time.time() + + return t2 - t1 + +if __name__ == '__main__': + + + if 'LIST1_SIZE' in os.environ: + LIST1_SIZE = int(os.environ['LIST1_SIZE']) + if 'LIST2_SIZE' in os.environ: + LIST2_SIZE = int(os.environ['LIST2_SIZE']) + if 'NUM_ITERS' in os.environ: + LIST2_SIZE = int(os.environ['NUM_ITERS']) + + primeList = list(range(LIST1_SIZE)) + + # Generate a list of random indexes to pop. + # We avoid key errors by making sure we don't have a possible + # maximum larger than the smallest our list will become + + primeList2 = list(range(LIST2_SIZE)) + + headerLine = "Starting: LIST1_SIZE=%d LIST2_SIZE=%d NUM_ITERS=%d" %(LIST1_SIZE, LIST2_SIZE, NUM_ITERS,) + + print ( "%s\n%s\n\n" %(headerLine, '-' * len(headerLine) ) ) + + listTime = 0 + sllistTime = 0 + dllistTime = 0 + bllistTime = 0 + b2llistTime = 0 + + listITime = 0 + sllistITime = 0 + dllistITime = 0 + bllistITime = 0 + b2llistITime = 0 + + leftbllistTime = 0 + leftb2llistTime = 0 + + lst1 = primeList[:] + lst2 = primeList2[:] + gc.collect() + for i in range(NUM_ITERS): + listTime += doTime ( lambda : doExtend(lst1, lst2) ) + gc.collect() + + for i in range(NUM_ITERS): + lst1 = primeList[:] + lst2 = primeList2[:] + gc.collect() + listITime += doTime ( lambda : doIExtend(lst1, lst2) ) + gc.collect() + + slst1 = sllist(primeList[:]) + slst2 = sllist(primeList2[:]) + gc.collect() + for i in range(NUM_ITERS): + sllistTime += doTime ( lambda : doExtend(slst1, slst2) ) + gc.collect() + + for i in range(NUM_ITERS): + slst1 = sllist(primeList[:]) + slst2 = sllist(primeList2[:]) + gc.collect() + sllistITime += doTime ( lambda : doIExtend(slst1, slst2) ) + gc.collect() + + + dlst1 = dllist(primeList[:]) + dlst2 = dllist(primeList2[:]) + gc.collect() + for i in range(NUM_ITERS): + dllistTime += doTime ( lambda : doExtend(dlst1, dlst2) ) + gc.collect() + + for i in range(NUM_ITERS): + dlst1 = dllist(primeList[:]) + dlst2 = dllist(primeList2[:]) + gc.collect() + dllistITime += doTime ( lambda : doIExtend(dlst1, dlst2) ) + gc.collect() + + dlst1 = dllist(primeList[:]) + slst2 = sllist(primeList2[:]) + gc.collect() + for i in range(NUM_ITERS): + bllistTime += doTime ( lambda : doExtend(dlst1, slst2) ) + gc.collect() + + for i in range(NUM_ITERS): + dlst1 = dllist(primeList[:]) + slst2 = sllist(primeList2[:]) + gc.collect() + bllistITime += doTime ( lambda : doIExtend(dlst1, slst2) ) + gc.collect() + + slst1 = sllist(primeList[:]) + dlst2 = dllist(primeList2[:]) + gc.collect() + for i in range(NUM_ITERS): + b2llistTime += doTime ( lambda : doExtend(slst1, dlst2) ) + gc.collect() + + for i in range(NUM_ITERS): + slst1 = sllist(primeList[:]) + dlst2 = dllist(primeList2[:]) + gc.collect() + b2llistITime += doTime ( lambda : doIExtend(slst1, dlst2) ) + gc.collect() + + for i in range(NUM_ITERS): + dlst1 = dllist(primeList[:]) + slst2 = sllist(primeList2[:]) + gc.collect() + leftbllistTime += doTime ( lambda : doExtendLeft(dlst1, slst2) ) + gc.collect() + + for i in range(NUM_ITERS): + slst1 = sllist(primeList[:]) + dlst2 = dllist(primeList2[:]) + gc.collect() + leftb2llistTime += doTime ( lambda : doExtendLeft(slst1, dlst2) ) + gc.collect() + + print ( "List time:\t\t%f\t= %f\nSL time:\t\t%f\t= %f\nDL time:\t\t%f\t= %f\nBL time:\t\t%f\t= %f\nBL2 time:\t\t%f\t= %f\n" %(listTime, listTime / NUM_ITERS, sllistTime, sllistTime / NUM_ITERS, dllistTime, dllistTime / NUM_ITERS, bllistTime, bllistTime / NUM_ITERS, b2llistTime, b2llistTime / NUM_ITERS) ) + print ( "List itime:\t\t%f\t= %f\nSL itime:\t\t%f\t= %f\nDL itime:\t\t%f\t= %f\nBL time:\t\t%f\t= %f\nBL2 time:\t\t%f\t= %f\n" %(listITime, listITime / NUM_ITERS, sllistITime, sllistITime / NUM_ITERS, dllistITime, dllistITime / NUM_ITERS, bllistITime, bllistITime / NUM_ITERS, b2llistITime, b2llistITime / NUM_ITERS) ) + + print ( "leftB time:\t\t%f\t= %f\nleftB2 time:\t\t%f\t= %f\n" %(leftbllistTime, leftbllistTime / NUM_ITERS, leftb2llistTime, leftb2llistTime / NUM_ITERS) ) From 05aa302ffc1454c152a168c614661bb51bbcb4de Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Fri, 7 Apr 2017 15:37:03 -0400 Subject: [PATCH 40/62] Update ChangeLog --- CHANGES | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES b/CHANGES index 3406254..4c58fa5 100644 --- a/CHANGES +++ b/CHANGES @@ -15,6 +15,12 @@ - Add some more tests - Some minor cleanups + - Move types into headers, make generic LList node and list structures, which are common to both double and single linked lists. + - Allow a double-linked list to extend with a single-linked list, and a single-linked list to extend with a double (for much higher performance) + + - Some general optimizations + + * llist-0.4 (2013-01-01) - Python 3.x support From 801120429a2b4eedae00e8acd1e18d61c7d058b9 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sat, 8 Apr 2017 20:17:14 -0400 Subject: [PATCH 41/62] Dlist implementation of slicing / subscript --- src/dllist.c | 358 +++++++++++++++++++++++++++++++++++++++++++- tests/llist_test.py | 47 ++++++ 2 files changed, 402 insertions(+), 3 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 6b12e00..6bb1a9a 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -15,8 +15,21 @@ PyObject_HEAD_INIT(type) size, #endif +#include + #include +#ifdef __GNUC__ + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + +#else + +#define likely(x) (x) +#define unlikely(x) (x) + +#endif #define START_MIDDLE_AFTER 10 @@ -46,7 +59,8 @@ static void debugmsg(char *format, ...) } */ #endif - + + /* Convenience function for creating list nodes. * Automatically update pointers in neigbours. @@ -220,6 +234,32 @@ static int dllistnode_init(DLListNodeObject* self, return 0; } +/* + * dllistnode_make - A slightly cheaper version that doesn't set next or prev + */ +static inline DLListNodeObject *dllistnode_make(DLListObject *dllist, PyObject *value) +{ + DLListNodeObject *ret; + + ret = (DLListNodeObject*)DLListNodeType->tp_alloc(DLListNodeType, 0); + if (ret == NULL) + return NULL; + + /* A single reference to Py_None is held for the whole + * lifetime of a node. */ + Py_INCREF(Py_None); + + ret->value = value; + + Py_INCREF(ret->value); + + ret->list_weakref = PyWeakref_NewRef((PyObject *)dllist, NULL); + Py_INCREF(ret->list_weakref); + + return ret; +} + + static PyObject* dllistnode_call(DLListNodeObject* self, PyObject* args, PyObject* kw) @@ -1530,6 +1570,310 @@ static PyObject* dllist_rindex(DLListObject *self, PyObject *value) return NULL; } +static inline int _normalize_indexes(Py_ssize_t size, Py_ssize_t *idx_start, Py_ssize_t *idx_end) +{ + + if ( unlikely(*idx_start < 0 )) + { + *idx_start = size - *idx_start; + if ( unlikely(*idx_start < 0 )) + return 0; + } + if ( unlikely(*idx_end < 0 )) + { + *idx_end = size - *idx_end; + if ( unlikely(*idx_end < 0 )) + return 0; + } + + if( unlikely(*idx_end >= size )) + *idx_end = size - 1; + + if ( unlikely(*idx_start >= size || *idx_start >= *idx_end)) + return 0; + + if( unlikely(*idx_start >= *idx_end )) + return 0; + + return 1; +} + + +static inline Py_ssize_t py_ssize_min(Py_ssize_t a, Py_ssize_t b) +{ + if ( a < b ) + return a; + return b; +} + + +#define calc_end_difference_step(_start, _end, _step) (((_end - _start - 1) % _step) + 1) + +/* + * dllist_slice - Slice function assuming normalized indexes. + * + * For potentially non-normalized, use dllist_simpleslice + * or call _normalize_indexes + * + * self - DLList to slice from + * start_idx - Start of slicing (normalized) + * end_idx - End of slicing (normalized) + * step - Slice step + * sliceLength - Length of resulting slice. Pass -1 to calculate. + */ +static PyObject *dllist_slice(DLListObject *self, Py_ssize_t start_idx, Py_ssize_t end_idx, Py_ssize_t step, Py_ssize_t sliceLength) +{ + DLListObject *ret; + DLListNodeObject *cur; + DLListNodeObject *new_node; + Py_ssize_t i; + Py_ssize_t remaining; + int direction = 0; + int pre_walk_direction; + + Py_ssize_t stepI; + + Py_ssize_t diff_start_left; + Py_ssize_t diff_end_right; + + ret = (DLListObject *)dllist_new(DLListType, Py_None, Py_None); + + if ( sliceLength == -1 ) + { + sliceLength = end_idx - start_idx; /* Reusing end_idx as max */ + if(step > 1 ) + { + sliceLength = sliceLength / step + ( end_idx % step ? 1 : 0 ); + } + } + + + if ( step > 1 ) + { + Py_ssize_t tmp_slice_resize; + tmp_slice_resize = start_idx + (step * sliceLength); + if ( tmp_slice_resize < end_idx ) + end_idx = tmp_slice_resize; + } + + + + if ( sliceLength == 0 ) + return (PyObject *)ret; + + + diff_start_left = start_idx; + diff_end_right = self->size - 1 - end_idx; + + remaining = py_ssize_min(diff_start_left, diff_end_right); + if (self->middle_idx != -1 ) + { + Py_ssize_t diff_mid_left; + Py_ssize_t diff_mid_right; + + diff_mid_left = py_ssize_t_abs( self->middle_idx - start_idx ); + diff_mid_right = py_ssize_t_abs ( end_idx - self->middle_idx ); + + remaining = py_ssize_min(remaining, diff_mid_left); + remaining = py_ssize_min(remaining, diff_mid_right); + + if ( remaining == diff_mid_left ) + { + direction = 1; + if ( start_idx < self->middle_idx ) + { + pre_walk_direction = -1; + } + else + { + pre_walk_direction = 1; + } + remaining = diff_mid_left; + cur = (DLListNodeObject *)self->middle; + } + else if( remaining == diff_mid_right ) + { + direction = -1; + if ( end_idx < self->middle_idx ) + { + pre_walk_direction = -1; + remaining += calc_end_difference_step(start_idx, end_idx, step); + } + else if ( end_idx > self->middle_idx ) + { + pre_walk_direction = 1; + remaining -= calc_end_difference_step(start_idx, end_idx, step); + if ( remaining < 0 ) + { + /* If here, we had to change direction because of step-end */ + pre_walk_direction = -1; + remaining *= -1; + } + } + else + { + pre_walk_direction = -1; + remaining += calc_end_difference_step(start_idx, end_idx, step); + //remaining += 1; + } + + cur = (DLListNodeObject *)self->middle; + } + } + + if ( direction == 0 ) + { + if ( remaining == diff_start_left ) + { + direction = 1; + pre_walk_direction = 1; + cur = (DLListNodeObject *)self->first; + } + else /* diff_end_right */ + { + direction = -1; + pre_walk_direction = -1; + remaining += calc_end_difference_step(start_idx, end_idx, step); + cur = (DLListNodeObject *)self->last; + } + } + + + if ( pre_walk_direction == 1 ) + { + for(i=0; i < remaining; i++) + cur = (DLListNodeObject*)cur->next; + } + else + { + for(i=remaining; i > 0; i--) + cur = (DLListNodeObject*)cur->prev; + } + + new_node = dllistnode_make(self, cur->value); + new_node->prev = new_node->next = Py_None; + + ret->first = ret->last = (PyObject *)new_node; + ret->size = 1; + + + if ( direction == 1 ) + { + DLListNodeObject *prev; + prev = new_node; + + + while ( ret->size < sliceLength ) + { + stepI = step; + while(stepI--) + cur = (DLListNodeObject*)cur->next; + + new_node = dllistnode_make(self, cur->value); + + new_node->prev = (PyObject *)prev; + prev->next = (PyObject *)new_node; + prev = new_node; + + ret->last = (PyObject *)new_node; + + ret->size += 1; + + + } + ((DLListNodeObject*)ret->last)->next = Py_None; + + } + else + { + DLListNodeObject *next; + + next = new_node; + + + while ( ret->size < sliceLength ) + { + stepI = step; + while(stepI--) + cur = (DLListNodeObject*)cur->prev; + + new_node = dllistnode_make(self, cur->value); + + new_node->next = (PyObject *)next; + next->prev = (PyObject *)new_node; + next = new_node; + + ret->first = (PyObject *)new_node; + + ret->size += 1; + + } + ((DLListNodeObject*)ret->last)->next = Py_None; + ((DLListNodeObject*)ret->first)->prev = Py_None; + + } + + _middle_check_recalc(ret); + + return (PyObject*) ret; + +} + +/* + * dllist_simpleslice - Simple (PySequence) slices. Can take non-normalized indexes (like negatives) + */ +static PyObject *dllist_simpleslice(DLListObject *self, Py_ssize_t idx_start, Py_ssize_t idx_end) +{ + if( !_normalize_indexes(self->size, &idx_start, &idx_end) ) + { + DLListObject *ret = (DLListObject *)dllist_new(DLListType, NULL, NULL); + return (PyObject *)ret; + } + + return dllist_slice(self, idx_start, idx_end, 1, -1); +} + +static PyObject *dllist_subscript(DLListObject *self, PyObject *item) +{ + Py_ssize_t i; + + if ( PyIndex_Check(item) ) + { + i = PyNumber_AsSsize_t(item, PyExc_IndexError); + + if ( i == -1 && PyErr_Occurred() ) + return NULL; + + if ( i < 0 ) + i += self->size; + + return dllist_get_item((PyObject *)self, i); + } + + if ( PySlice_Check(item) ) + { + + + Py_ssize_t start, stop, step, sliceLength; + + if ( PySlice_GetIndicesEx( (PyObject *)item, self->size, &start, &stop, &step, &sliceLength ) ) + return NULL; /* Error */ + + return dllist_slice(self, start, stop, step, sliceLength); + } + else + { + + PyErr_Format(PyExc_TypeError, "Indicies must be integers, not %s", item->ob_type->tp_name); + return NULL; + } + +} + + + + + static PyMethodDef DLListMethods[] = { { "appendleft", (PyCFunction)dllist_appendleft, METH_O, @@ -1589,7 +1933,7 @@ static PySequenceMethods DLListSequenceMethods[] = { dllist_concat, /* sq_concat */ dllist_repeat, /* sq_repeat */ dllist_get_item, /* sq_item */ - 0, /* sq_slice */ + dllist_simpleslice, /* sq_slice */ dllist_set_item, /* sq_ass_item */ 0, /* sq_ass_slice */ 0, /* sq_contains */ @@ -1599,6 +1943,14 @@ static PySequenceMethods DLListSequenceMethods[] = { } ; +static PyMappingMethods DLListMappingMethods[] = { +{ + dllist_len, /* mp_length */ + (binaryfunc)dllist_subscript, /* mp_subscript */ + 0, /* mp_ass_subscript */ +} +}; + PyTypeObject DLListType[] = { { PyVarObject_HEAD_INIT(NULL, 0) @@ -1613,7 +1965,7 @@ PyTypeObject DLListType[] = { (reprfunc)dllist_repr, /* tp_repr */ 0, /* tp_as_number */ DLListSequenceMethods, /* tp_as_sequence */ - 0, /* tp_as_mapping */ + DLListMappingMethods, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ (reprfunc)dllist_str, /* tp_str */ diff --git a/tests/llist_test.py b/tests/llist_test.py index 1f0c1fc..2bb8fa9 100755 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -1670,6 +1670,53 @@ def test_contains(self): self.assertEqual(7 in sl, True) self.assertEqual(8 in sl, False) + def test_slice(self): + + lst = list(range(100)) + dlst = dllist(lst) + + self.assertEqual(lst[0:20], list(dlst[0:20])) + self.assertEqual(lst[40:60], list(dlst[40:60])) + self.assertEqual(lst[60:40], list(dlst[60:40])) + self.assertEqual(lst[:-1], list(dlst[:-1])) + self.assertEqual(lst[-20:], list(dlst[-20:])) + self.assertEqual(lst[-20:-5], list(dlst[-20:-5])) + self.assertEqual(lst[-5:-20], list(dlst[-5:-20])) + self.assertEqual(lst[-70:50], list(dlst[-70:50])) + self.assertEqual(lst[5:500], list(dlst[5:500])) + self.assertEqual(lst[:], list(dlst[:])) + + slst = list(range(8)) + sdlst = dllist(slst) + + self.assertEqual(slst[2:5], list(sdlst[2:5])) + self.assertEqual(slst[-3:-1], list(sdlst[-3:-1])) + + for i in range(100): + for j in range(100): + self.assertEqual(lst[i:j], list(dlst[i:j])) + + # Test if version of python (2.7+ , 3.? + ) supports step in slices + try: + lst[0:10:2] + except: + # If not supported, test is over + return + + self.assertEqual(lst[0:20:2], list(dlst[0:20:2])) + self.assertEqual(lst[0:21:2], list(dlst[0:21:2])) + self.assertEqual(lst[50:80:6], list(dlst[50:80:6])) + + for i in range(100): + for j in range(100): + for s in range(1, 100, 1): + try: + self.assertEqual(lst[i:j:s], list(dlst[i:j:s])) + except AssertionError as ae: + sys.stderr.write("Failed on [ %d : %d : %d ]\n" %(i, j, s)) + raise ae + + def test_repeat(self): ref = py23_range(0, 1024, 4) From 058d829966b1f2e525dba2d3378cf79aa946407c Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 21:38:23 -0400 Subject: [PATCH 42/62] Slist implementation of slicing / subscript --- src/sllist.c | 216 +++++++++++++++++++++++++++++++++++++++++++- tests/llist_test.py | 59 +++++++++++- 2 files changed, 269 insertions(+), 6 deletions(-) diff --git a/src/sllist.c b/src/sllist.c index af4d4b0..d3ce751 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -15,6 +15,17 @@ PyObject_HEAD_INIT(type) size, #endif +#ifdef __GNUC__ + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + +#else + +#define likely(x) (x) +#define unlikely(x) (x) + +#endif static SLListNodeObject* sllistnode_create(PyObject* next, @@ -1389,6 +1400,198 @@ static long sllist_hash(SLListObject* self) } #endif +static inline int _normalize_indexes(Py_ssize_t size, Py_ssize_t *idx_start, Py_ssize_t *idx_end) +{ + + if ( unlikely(*idx_start < 0 )) + { + *idx_start = size - *idx_start; + if ( unlikely(*idx_start < 0 )) + return 0; + } + if ( unlikely(*idx_end < 0 )) + { + *idx_end = size - *idx_end; + if ( unlikely(*idx_end < 0 )) + return 0; + } + + if( unlikely(*idx_end >= size )) + *idx_end = size - 1; + + if ( unlikely(*idx_start >= size || *idx_start >= *idx_end)) + return 0; + + if( unlikely(*idx_start >= *idx_end )) + return 0; + + return 1; +} +/* + * sllistnode_make - A slightly cheaper version that doesn't set next or prev + */ +static inline SLListNodeObject *sllistnode_make(SLListObject *sllist, PyObject *value) +{ + SLListNodeObject *ret; + + ret = (SLListNodeObject*)SLListNodeType->tp_alloc(SLListNodeType, 0); + if (ret == NULL) + return NULL; + + /* A single reference to Py_None is held for the whole + * lifetime of a node. */ + Py_INCREF(Py_None); + + ret->value = value; + + Py_INCREF(ret->value); + + ret->list_weakref = PyWeakref_NewRef((PyObject *)sllist, NULL); + Py_INCREF(ret->list_weakref); + + return ret; +} + + + +#define calc_end_difference_step(_start, _end, _step) (((_end - _start - 1) % _step) + 1) + +/* + * sllist_slice - Slice function assuming normalized indexes. + * + * For potentially non-normalized, use sllist_simpleslice + * or call _normalize_indexes + * + * self - SLList to slice from + * start_idx - Start of slicing (normalized) + * end_idx - End of slicing (normalized) + * step - Slice step + * sliceLength - Length of resulting slice. Pass -1 to calculate. + */ +static PyObject *sllist_slice(SLListObject *self, Py_ssize_t start_idx, Py_ssize_t end_idx, Py_ssize_t step, Py_ssize_t sliceLength) +{ + SLListObject *ret; + SLListNodeObject *cur; + SLListNodeObject *new_node; + Py_ssize_t i; + + Py_ssize_t stepI; + + ret = (SLListObject *)sllist_new(SLListType, Py_None, Py_None); + + if ( sliceLength == -1 ) + { + sliceLength = end_idx - start_idx; /* Reusing end_idx as max */ + if(step > 1 ) + { + sliceLength = sliceLength / step + ( end_idx % step ? 1 : 0 ); + } + } + + + if ( step > 1 ) + { + Py_ssize_t tmp_slice_resize; + tmp_slice_resize = start_idx + (step * sliceLength); + if ( tmp_slice_resize < end_idx ) + end_idx = tmp_slice_resize; + } + + + + if ( sliceLength == 0 ) + return (PyObject *)ret; + + + cur = (SLListNodeObject *)self->first; + for(i=0; i < start_idx; i++) + cur = (SLListNodeObject*)cur->next; + + new_node = sllistnode_make(self, cur->value); + new_node->next = Py_None; + + ret->first = ret->last = (PyObject *)new_node; + ret->size = 1; + + + SLListNodeObject *prev; + prev = new_node; + + + while ( ret->size < sliceLength ) + { + stepI = step; + while(stepI--) + cur = (SLListNodeObject*)cur->next; + + new_node = sllistnode_make(self, cur->value); + + prev->next = (PyObject *)new_node; + prev = new_node; + + ret->last = (PyObject *)new_node; + + ret->size += 1; + + + } + ((SLListNodeObject*)ret->last)->next = Py_None; + + return (PyObject*) ret; + +} + +/* + * sllist_simpleslice - Simple (PySequence) slices. Can take non-normalized indexes (like negatives) + */ +static PyObject *sllist_simpleslice(SLListObject *self, Py_ssize_t idx_start, Py_ssize_t idx_end) +{ + if( !_normalize_indexes(self->size, &idx_start, &idx_end) ) + { + SLListObject *ret = (SLListObject *)sllist_new(SLListType, NULL, NULL); + return (PyObject *)ret; + } + + return sllist_slice(self, idx_start, idx_end, 1, -1); +} + +static PyObject *sllist_subscript(SLListObject *self, PyObject *item) +{ + Py_ssize_t i; + + if ( PyIndex_Check(item) ) + { + i = PyNumber_AsSsize_t(item, PyExc_IndexError); + + if ( i == -1 && PyErr_Occurred() ) + return NULL; + + if ( i < 0 ) + i += self->size; + + return sllist_get_item((PyObject *)self, i); + } + + if ( PySlice_Check(item) ) + { + + + Py_ssize_t start, stop, step, sliceLength; + + if ( PySlice_GetIndicesEx( (PyObject *)item, self->size, &start, &stop, &step, &sliceLength ) ) + return NULL; /* Error */ + + return sllist_slice(self, start, stop, step, sliceLength); + } + else + { + + PyErr_Format(PyExc_TypeError, "Indicies must be integers, not %s", item->ob_type->tp_name); + return NULL; + } + +} + static PyMethodDef SLListMethods[] = { @@ -1465,7 +1668,7 @@ static PySequenceMethods SLListSequenceMethods = sllist_concat, /* sq_concat */ sllist_repeat, /* sq_repeat */ sllist_get_item, /* sq_item */ - 0, /* sq_slice; */ + sllist_simpleslice, /* sq_slice; */ sllist_set_item, /* sq_ass_item */ 0, /* sq_ass_slice */ 0, /* sq_contains */ @@ -1473,6 +1676,15 @@ static PySequenceMethods SLListSequenceMethods = 0, /* sq_inplace_repeat */ }; +static PyMappingMethods SLListMappingMethods[] = { +{ + sllist_len, /* mp_length */ + (binaryfunc)sllist_subscript, /* mp_subscript */ + 0, /* mp_ass_subscript */ +} +}; + + PyTypeObject SLListType[] = { { PyVarObject_HEAD_INIT(NULL, 0) @@ -1487,7 +1699,7 @@ PyTypeObject SLListType[] = { (reprfunc)sllist_repr, /* tp_repr */ 0, /* tp_as_number */ &SLListSequenceMethods, /* tp_as_sequence */ - 0, /* tp_as_mapping */ + SLListMappingMethods, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ (reprfunc)sllist_str, /* tp_str */ diff --git a/tests/llist_test.py b/tests/llist_test.py index 2bb8fa9..ce4e3b3 100755 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -588,6 +588,57 @@ def test_pop(self): self.assertEqual(ll.last, None) + def test_slice(self): + + lst = list(range(100)) + slst = sllist(lst) + + self.assertEqual(lst[0:20], list(slst[0:20])) + self.assertEqual(lst[40:60], list(slst[40:60])) + self.assertEqual(lst[60:40], list(slst[60:40])) + self.assertEqual(lst[:-1], list(slst[:-1])) + self.assertEqual(lst[-20:], list(slst[-20:])) + self.assertEqual(lst[-20:-5], list(slst[-20:-5])) + self.assertEqual(lst[-5:-20], list(slst[-5:-20])) + self.assertEqual(lst[-70:50], list(slst[-70:50])) + self.assertEqual(lst[5:500], list(slst[5:500])) + self.assertEqual(lst[:], list(slst[:])) + + smlst = list(range(8)) + smslst = sllist(smlst) + + self.assertEqual(smlst[2:5], list(smslst[2:5])) + self.assertEqual(smlst[-3:-1], list(smslst[-3:-1])) + + for i in range(100): + for j in range(100): + try: + self.assertEqual(lst[i:j], list(slst[i:j])) + except AssertionError as ae: + import pdb; pdb.set_trace() + sys.stderr.write("Failed on [ %d : %d ]\n" %(i, j)) + raise ae + + # Test if version of python (2.7+ , 3.? + ) supports step in slices + try: + lst[0:10:2] + except: + # If not supported, test is over + return + + self.assertEqual(lst[0:20:2], list(slst[0:20:2])) + self.assertEqual(lst[0:21:2], list(slst[0:21:2])) + self.assertEqual(lst[50:80:6], list(slst[50:80:6])) + + for i in range(30): + for j in range(30): + for s in range(1, 30, 1): + try: + self.assertEqual(lst[i:j:s], list(slst[i:j:s])) + except AssertionError as ae: + sys.stderr.write("Failed on [ %d : %d : %d ]\n" %(i, j, s)) + raise ae + @@ -1686,11 +1737,11 @@ def test_slice(self): self.assertEqual(lst[5:500], list(dlst[5:500])) self.assertEqual(lst[:], list(dlst[:])) - slst = list(range(8)) - sdlst = dllist(slst) + smlst = list(range(8)) + smdlst = dllist(smlst) - self.assertEqual(slst[2:5], list(sdlst[2:5])) - self.assertEqual(slst[-3:-1], list(sdlst[-3:-1])) + self.assertEqual(smlst[2:5], list(smdlst[2:5])) + self.assertEqual(smlst[-3:-1], list(smdlst[-3:-1])) for i in range(100): for j in range(100): From ab85a8121e4f3ce5eb610639bebcb35e0dea4e08 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 21:54:35 -0400 Subject: [PATCH 43/62] Move some common stuff to llist.h and llist_types.h --- src/dllist.c | 79 ++-------------------------------------------- src/dllist_types.h | 2 +- src/llist.h | 69 ++++++++++++++++++++++++++++++++++++++++ src/llist_types.h | 24 ++++++++++++++ src/sllist.c | 62 ++---------------------------------- src/sllist_types.h | 2 +- 6 files changed, 99 insertions(+), 139 deletions(-) create mode 100644 src/llist.h diff --git a/src/dllist.c b/src/dllist.c index 6bb1a9a..9e9fa1f 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -6,6 +6,7 @@ #include #include #include "py23macros.h" +#include "llist.h" #include "llist_types.h" #include "dllist_types.h" #include "sllist_types.h" @@ -19,18 +20,6 @@ #include -#ifdef __GNUC__ - -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) - -#else - -#define likely(x) (x) -#define unlikely(x) (x) - -#endif - #define START_MIDDLE_AFTER 10 /* #define DO_DEBUG */ @@ -239,24 +228,7 @@ static int dllistnode_init(DLListNodeObject* self, */ static inline DLListNodeObject *dllistnode_make(DLListObject *dllist, PyObject *value) { - DLListNodeObject *ret; - - ret = (DLListNodeObject*)DLListNodeType->tp_alloc(DLListNodeType, 0); - if (ret == NULL) - return NULL; - - /* A single reference to Py_None is held for the whole - * lifetime of a node. */ - Py_INCREF(Py_None); - - ret->value = value; - - Py_INCREF(ret->value); - - ret->list_weakref = PyWeakref_NewRef((PyObject *)dllist, NULL); - Py_INCREF(ret->list_weakref); - - return ret; + return (DLListNodeObject *)llistnode_make(DLListNodeType, (PyObject *)dllist, value); } @@ -339,12 +311,6 @@ static PyObject* dllist_node_at(PyObject* self, PyObject* indexObject); static DLListNodeObject* dllist_get_node_internal(DLListObject* self, Py_ssize_t index); -static Py_ssize_t py_ssize_t_abs(Py_ssize_t x) -{ - return (x >= 0) ? x : -x; -} - - /**************** **** Middle functions ******************** @@ -1570,45 +1536,6 @@ static PyObject* dllist_rindex(DLListObject *self, PyObject *value) return NULL; } -static inline int _normalize_indexes(Py_ssize_t size, Py_ssize_t *idx_start, Py_ssize_t *idx_end) -{ - - if ( unlikely(*idx_start < 0 )) - { - *idx_start = size - *idx_start; - if ( unlikely(*idx_start < 0 )) - return 0; - } - if ( unlikely(*idx_end < 0 )) - { - *idx_end = size - *idx_end; - if ( unlikely(*idx_end < 0 )) - return 0; - } - - if( unlikely(*idx_end >= size )) - *idx_end = size - 1; - - if ( unlikely(*idx_start >= size || *idx_start >= *idx_end)) - return 0; - - if( unlikely(*idx_start >= *idx_end )) - return 0; - - return 1; -} - - -static inline Py_ssize_t py_ssize_min(Py_ssize_t a, Py_ssize_t b) -{ - if ( a < b ) - return a; - return b; -} - - -#define calc_end_difference_step(_start, _end, _step) (((_end - _start - 1) % _step) + 1) - /* * dllist_slice - Slice function assuming normalized indexes. * @@ -1872,8 +1799,6 @@ static PyObject *dllist_subscript(DLListObject *self, PyObject *item) - - static PyMethodDef DLListMethods[] = { { "appendleft", (PyCFunction)dllist_appendleft, METH_O, diff --git a/src/dllist_types.h b/src/dllist_types.h index b6bea5d..14dd36d 100644 --- a/src/dllist_types.h +++ b/src/dllist_types.h @@ -15,10 +15,10 @@ extern PyTypeObject DLListIteratorType[]; typedef struct { PyObject_HEAD + PyObject *list_weakref; PyObject *value; PyObject *next; PyObject *prev; - PyObject *list_weakref; } DLListNodeObject; typedef struct diff --git a/src/llist.h b/src/llist.h new file mode 100644 index 0000000..bf09b14 --- /dev/null +++ b/src/llist.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 Timothy Savannah, + * all rights reserved under terms of MIT license (see LICENSE) + */ + +#ifndef LLIST_H +#define LLIST_H + + +#ifdef __GNUC__ + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + +#else + +#define likely(x) (x) +#define unlikely(x) (x) + +#endif + + +static inline Py_ssize_t py_ssize_t_abs(Py_ssize_t x) +{ + return (x >= 0) ? x : -x; +} + +static inline Py_ssize_t py_ssize_min(Py_ssize_t a, Py_ssize_t b) +{ + if ( a < b ) + return a; + return b; +} + + +/* Calc difference between given end and actual end with step */ +#define calc_end_difference_step(_start, _end, _step) (((_end - _start - 1) % _step) + 1) + + +static inline int _normalize_indexes(Py_ssize_t size, Py_ssize_t *idx_start, Py_ssize_t *idx_end) +{ + + if ( unlikely(*idx_start < 0 )) + { + *idx_start = size - *idx_start; + if ( unlikely(*idx_start < 0 )) + return 0; + } + if ( unlikely(*idx_end < 0 )) + { + *idx_end = size - *idx_end; + if ( unlikely(*idx_end < 0 )) + return 0; + } + + if( unlikely(*idx_end >= size )) + *idx_end = size - 1; + + if ( unlikely(*idx_start >= size || *idx_start >= *idx_end)) + return 0; + + if( unlikely(*idx_start >= *idx_end )) + return 0; + + return 1; +} + + +#endif diff --git a/src/llist_types.h b/src/llist_types.h index 085e2d1..05a5315 100644 --- a/src/llist_types.h +++ b/src/llist_types.h @@ -9,6 +9,7 @@ typedef struct { PyObject_HEAD + PyObject *list_weakref; PyObject *value; PyObject *next; } LListNodeObject; @@ -21,4 +22,27 @@ typedef struct Py_ssize_t size; } LListObject; + +static inline PyObject *llistnode_make(PyTypeObject *llist_type, PyObject *llist, PyObject *value) +{ + LListNodeObject *ret; + + ret = (LListNodeObject *)llist_type->tp_alloc(llist_type, 0); + if ( ret == NULL ) + return NULL; + + /* A single reference to Py_None is held for the whole + * lifetime of a node. */ + Py_INCREF(Py_None); + + ret->value = value; + + Py_INCREF(ret->value); + + ret->list_weakref = PyWeakref_NewRef((PyObject *)llist, NULL); + Py_INCREF(ret->list_weakref); + + return (PyObject *)ret; +} + #endif /* LLIST_TYPES_H */ diff --git a/src/sllist.c b/src/sllist.c index d3ce751..8ccca3d 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -6,6 +6,7 @@ #include #include #include "py23macros.h" +#include "llist.h" #include "llist_types.h" #include "sllist_types.h" #include "dllist_types.h" @@ -15,18 +16,6 @@ PyObject_HEAD_INIT(type) size, #endif -#ifdef __GNUC__ - -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) - -#else - -#define likely(x) (x) -#define unlikely(x) (x) - -#endif - static SLListNodeObject* sllistnode_create(PyObject* next, PyObject* value, @@ -1400,62 +1389,15 @@ static long sllist_hash(SLListObject* self) } #endif -static inline int _normalize_indexes(Py_ssize_t size, Py_ssize_t *idx_start, Py_ssize_t *idx_end) -{ - - if ( unlikely(*idx_start < 0 )) - { - *idx_start = size - *idx_start; - if ( unlikely(*idx_start < 0 )) - return 0; - } - if ( unlikely(*idx_end < 0 )) - { - *idx_end = size - *idx_end; - if ( unlikely(*idx_end < 0 )) - return 0; - } - - if( unlikely(*idx_end >= size )) - *idx_end = size - 1; - - if ( unlikely(*idx_start >= size || *idx_start >= *idx_end)) - return 0; - - if( unlikely(*idx_start >= *idx_end )) - return 0; - - return 1; -} /* * sllistnode_make - A slightly cheaper version that doesn't set next or prev */ static inline SLListNodeObject *sllistnode_make(SLListObject *sllist, PyObject *value) { - SLListNodeObject *ret; - - ret = (SLListNodeObject*)SLListNodeType->tp_alloc(SLListNodeType, 0); - if (ret == NULL) - return NULL; - - /* A single reference to Py_None is held for the whole - * lifetime of a node. */ - Py_INCREF(Py_None); - - ret->value = value; - - Py_INCREF(ret->value); - - ret->list_weakref = PyWeakref_NewRef((PyObject *)sllist, NULL); - Py_INCREF(ret->list_weakref); - - return ret; + return (SLListNodeObject *)llistnode_make(SLListNodeType, (PyObject *)sllist, value); } - -#define calc_end_difference_step(_start, _end, _step) (((_end - _start - 1) % _step) + 1) - /* * sllist_slice - Slice function assuming normalized indexes. * diff --git a/src/sllist_types.h b/src/sllist_types.h index adeb3cc..05f89c2 100644 --- a/src/sllist_types.h +++ b/src/sllist_types.h @@ -18,9 +18,9 @@ extern PyTypeObject SLListIteratorType[]; typedef struct { PyObject_HEAD + PyObject *list_weakref; PyObject *value; PyObject *next; - PyObject *list_weakref; } SLListNodeObject; typedef struct From c99fad11443063eecd35fd9f52303f96188671e2 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 21:54:47 -0400 Subject: [PATCH 44/62] Add tilda files to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e43c43a..7baad79 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist *.pyc *.so *.swp +*~ From 324bb6014809d7447c8e7ed69f478dfa41a603fa Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 21:55:48 -0400 Subject: [PATCH 45/62] Update Changes --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index 4c58fa5..dab04d6 100644 --- a/CHANGES +++ b/CHANGES @@ -18,6 +18,9 @@ - Move types into headers, make generic LList node and list structures, which are common to both double and single linked lists. - Allow a double-linked list to extend with a single-linked list, and a single-linked list to extend with a double (for much higher performance) + - Implement mappings on sllist and dllist + - Implement slicing (including with step) on both sllist and dllist + - Some general optimizations From febdbf764eead75ea59cb453f1cc149214d723f8 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 22:00:26 -0400 Subject: [PATCH 46/62] Some minor cleanups --- src/dllist.c | 9 ++++----- src/sllist.c | 7 +++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 9e9fa1f..c93e476 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1615,7 +1615,7 @@ static PyObject *dllist_slice(DLListObject *self, Py_ssize_t start_idx, Py_ssize { pre_walk_direction = 1; } - remaining = diff_mid_left; + cur = (DLListNodeObject *)self->middle; } else if( remaining == diff_mid_right ) @@ -1641,7 +1641,7 @@ static PyObject *dllist_slice(DLListObject *self, Py_ssize_t start_idx, Py_ssize { pre_walk_direction = -1; remaining += calc_end_difference_step(start_idx, end_idx, step); - //remaining += 1; + } cur = (DLListNodeObject *)self->middle; @@ -1762,10 +1762,11 @@ static PyObject *dllist_simpleslice(DLListObject *self, Py_ssize_t idx_start, Py static PyObject *dllist_subscript(DLListObject *self, PyObject *item) { - Py_ssize_t i; if ( PyIndex_Check(item) ) { + Py_ssize_t i; + i = PyNumber_AsSsize_t(item, PyExc_IndexError); if ( i == -1 && PyErr_Occurred() ) @@ -1779,8 +1780,6 @@ static PyObject *dllist_subscript(DLListObject *self, PyObject *item) if ( PySlice_Check(item) ) { - - Py_ssize_t start, stop, step, sliceLength; if ( PySlice_GetIndicesEx( (PyObject *)item, self->size, &start, &stop, &step, &sliceLength ) ) diff --git a/src/sllist.c b/src/sllist.c index 8ccca3d..76a4924 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1390,7 +1390,7 @@ static long sllist_hash(SLListObject* self) #endif /* - * sllistnode_make - A slightly cheaper version that doesn't set next or prev + * sllistnode_make - A slightly cheaper version that doesn't set next */ static inline SLListNodeObject *sllistnode_make(SLListObject *sllist, PyObject *value) { @@ -1499,10 +1499,11 @@ static PyObject *sllist_simpleslice(SLListObject *self, Py_ssize_t idx_start, Py static PyObject *sllist_subscript(SLListObject *self, PyObject *item) { - Py_ssize_t i; if ( PyIndex_Check(item) ) { + Py_ssize_t i; + i = PyNumber_AsSsize_t(item, PyExc_IndexError); if ( i == -1 && PyErr_Occurred() ) @@ -1516,8 +1517,6 @@ static PyObject *sllist_subscript(SLListObject *self, PyObject *item) if ( PySlice_Check(item) ) { - - Py_ssize_t start, stop, step, sliceLength; if ( PySlice_GetIndicesEx( (PyObject *)item, self->size, &start, &stop, &step, &sliceLength ) ) From 4acb06a761f09cc08a97e6de9b420d38896e6097 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 22:10:46 -0400 Subject: [PATCH 47/62] Update speed_test - Fix for python3, and fix popleft... was actually doing a popright on sllist and dllist --- tests/speed_test.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/speed_test.py b/tests/speed_test.py index 521e48a..c226552 100644 --- a/tests/speed_test.py +++ b/tests/speed_test.py @@ -25,12 +25,8 @@ def pop(c): def popleft(c): - if isinstance(c, deque): - for i in range(num): - c.popleft() - else: - for i in range(num): - c.pop() + for i in range(num): + c.popleft() def remove(c): @@ -47,8 +43,8 @@ def remove(c): start = time.time() operation(c) elapsed = time.time() - start - print "Completed %s/%s in \t\t%.8f seconds:\t %.1f ops/sec" % ( + print ( "Completed %s/%s in \t\t%.8f seconds:\t %.1f ops/sec" % ( container.__name__, operation.__name__, elapsed, - num / elapsed) + num / elapsed) ) From d95db50d75ddb9e23d36c1d585f04c9420bffa15 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 22:11:32 -0400 Subject: [PATCH 48/62] speed_text: fix columns to always line up and make speed_test executable --- tests/speed_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) mode change 100644 => 100755 tests/speed_test.py diff --git a/tests/speed_test.py b/tests/speed_test.py old mode 100644 new mode 100755 index c226552..9f2c399 --- a/tests/speed_test.py +++ b/tests/speed_test.py @@ -43,8 +43,7 @@ def remove(c): start = time.time() operation(c) elapsed = time.time() - start - print ( "Completed %s/%s in \t\t%.8f seconds:\t %.1f ops/sec" % ( - container.__name__, - operation.__name__, - elapsed, - num / elapsed) ) + + col1 = "Completed %s/%s in" %(container.__name__, operation.__name__) + col2 = "%.8f seconds: %.1f" %(elapsed, num / elapsed ) + print ( "%s%s %s%s ops/s" %(col1, (35 - len(col1)) * ' ', col2, (36 - len(col2)) * ' ') ) From 2702fa90f46e5b46fd1c613312cb1c605e937f11 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 22:26:17 -0400 Subject: [PATCH 49/62] Implement contains method --- src/dllist.c | 21 ++++++++++++++++++++- src/sllist.c | 20 +++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index c93e476..b5c928b 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1494,6 +1494,25 @@ static int dllist_set_item(PyObject* self, Py_ssize_t index, PyObject* val) } +static int dllist_contains(PyObject *self, PyObject *value) +{ + DLListNodeObject *node; + + node = (DLListNodeObject *) ((DLListObject*)self)->first; + + while ( node != Py_None ) + { + if( node->value == value ) + return 1; + + node = (DLListNodeObject *)node->next; + } + + return 0; +} + + + static PyObject* dllist_index(DLListObject *self, PyObject *value) { @@ -1860,7 +1879,7 @@ static PySequenceMethods DLListSequenceMethods[] = { dllist_simpleslice, /* sq_slice */ dllist_set_item, /* sq_ass_item */ 0, /* sq_ass_slice */ - 0, /* sq_contains */ + dllist_contains, /* sq_contains */ dllist_inplace_concat, /* sq_inplace_concat */ 0, /* sq_inplace_repeat */ } diff --git a/src/sllist.c b/src/sllist.c index 76a4924..9a2301e 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1297,6 +1297,24 @@ static PyObject* sllist_to_string(SLListObject* self, return NULL; } +static int sllist_contains(PyObject *self, PyObject *value) +{ + SLListNodeObject *node; + + node = (SLListNodeObject *) ((SLListObject*)self)->first; + + while ( node != Py_None ) + { + if( node->value == value ) + return 1; + + node = (SLListNodeObject *)node->next; + } + + return 0; +} + + static PyObject* sllist_index(SLListObject *self, PyObject *value) { @@ -1612,7 +1630,7 @@ static PySequenceMethods SLListSequenceMethods = sllist_simpleslice, /* sq_slice; */ sllist_set_item, /* sq_ass_item */ 0, /* sq_ass_slice */ - 0, /* sq_contains */ + sllist_contains, /* sq_contains */ sllist_inplace_concat, /* sq_inplace_concat */ 0, /* sq_inplace_repeat */ }; From 6f00302f3ba77895555ce05417279dac1af66016 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 22:26:42 -0400 Subject: [PATCH 50/62] Add two more benchmarks --- tests/benchmark_contains.py | 81 +++++++++++++++++++++++++++++++++ tests/benchmark_slice.py | 90 +++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100755 tests/benchmark_contains.py create mode 100755 tests/benchmark_slice.py diff --git a/tests/benchmark_contains.py b/tests/benchmark_contains.py new file mode 100755 index 0000000..f2b782c --- /dev/null +++ b/tests/benchmark_contains.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +import os +import gc +import time +import random + +from llist import dllist, sllist + +LIST_SIZE = 700 + +NUM_CONTAINS = 1500 + +NUM_ITERS = 200 + +def doContains(lst, containsItems): + + for containsItem in containsItems: + return containsItem in lst + +def doTime(func): + t1 = time.time() + func() + t2 = time.time() + + return t2 - t1 + +if __name__ == '__main__': + + + if 'LIST_SIZE' in os.environ: + LIST_SIZE = int(os.environ['LIST_SIZE']) + if 'NUM_CONTAINS' in os.environ: + NUM_CONTAINS = int(os.environ['NUM_CONTAINS']) + if 'NUM_ITERS' in os.environ: + NUM_CONTAINS = int(os.environ['NUM_ITERS']) + + primeList = list(range(LIST_SIZE)) + + # Generate a list of random indexes to pop. + # We avoid key errors by making sure we don't have a possible + # maximum larger than the smallest our list will become + + randomItems = [] + for j in range(NUM_ITERS): + theseItems = [] + for i in range(NUM_CONTAINS): + nextNum = random.randint(0, LIST_SIZE - 1) # Cheaply make sure we don't get key errors + + theseItems.append( primeList[nextNum] ) + randomItems.append(theseItems) + + headerLine = "Starting: LIST_SIZE=%d NUM_CONTAINS=%d NUM_ITERS=%d" %(LIST_SIZE, NUM_CONTAINS, NUM_ITERS,) + + print ( "%s\n%s\n\n" %(headerLine, '-' * len(headerLine) ) ) + + listTime = 0 + sllistTime = 0 + dllistTime = 0 + + gc.collect() + for i in range(NUM_ITERS): + dlst = dllist(primeList[:]) + dllistTime += doTime ( lambda : doContains(dlst, randomItems[i]) ) + gc.collect() + + for i in range(NUM_ITERS): + lst1 = primeList[:] + listTime += doTime ( lambda : doContains(lst1, randomItems[i]) ) + gc.collect() + + for i in range(NUM_ITERS): + slst = sllist(primeList[:]) + sllistTime += doTime ( lambda : doContains(slst, randomItems[i]) ) + gc.collect() + + + # csllistTime = doTime ( lambda : doPops(cslst, randomPops) ) + csllistTime = 0.0 + + print ( "List time:\t\t%f\t= %f\nSL time:\t\t%f\t= %f\nDL time:\t\t%f\t= %f\n" %(listTime, listTime / NUM_ITERS, sllistTime, sllistTime / NUM_ITERS, dllistTime, dllistTime / NUM_ITERS) ) diff --git a/tests/benchmark_slice.py b/tests/benchmark_slice.py new file mode 100755 index 0000000..cc87f94 --- /dev/null +++ b/tests/benchmark_slice.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +import os +import gc +import time +import random + +from llist import dllist, sllist + +LIST_SIZE = 700 + +NUM_SLICES = 500 + +NUM_ITERS = 200 + +def doSlice(lst, slices): + for thisSlice in slices: + try: + x = lst[thisSlice[0] : thisSlice[1]] + except TypeError as e: + import pdb; pdb.set_trace() + +def doTime(func): + t1 = time.time() + func() + t2 = time.time() + + return t2 - t1 + +if __name__ == '__main__': + + + if 'LIST_SIZE' in os.environ: + LIST_SIZE = int(os.environ['LIST_SIZE']) + if 'NUM_SLICES' in os.environ: + NUM_SLICES = int(os.environ['NUM_SLICES']) + if 'NUM_ITERS' in os.environ: + NUM_SLICES = int(os.environ['NUM_ITERS']) + + primeList = list(range(LIST_SIZE)) + + # Generate a list of random indexes to pop. + # We avoid key errors by making sure we don't have a possible + # maximum larger than the smallest our list will become + + randomSlices = [] + for j in range(NUM_ITERS): + theseSlices = [] + for i in range(NUM_SLICES): + num1 = random.randint(0, LIST_SIZE - 1) + num2 = random.randint(0, LIST_SIZE - 1) + if num1 > num2: + thisSlice = (num2, num1) + else: + thisSlice = (num1, num2) + + theseSlices.append( thisSlice ) + randomSlices.append(theseSlices) + + headerLine = "Starting: LIST_SIZE=%d NUM_SLICES=%d NUM_ITERS=%d" %(LIST_SIZE, NUM_SLICES, NUM_ITERS,) + + print ( "%s\n%s\n\n" %(headerLine, '-' * len(headerLine) ) ) + + listTime = 0 + sllistTime = 0 + dllistTime = 0 + + gc.collect() + for i in range(NUM_ITERS): + lst1 = primeList[:] + listTime += doTime ( lambda : doSlice(lst1, randomSlices[i]) ) + gc.collect() + + gc.collect() + for i in range(NUM_ITERS): + dlst = dllist(primeList[:]) + dllistTime += doTime ( lambda : doSlice(dlst, randomSlices[i]) ) + gc.collect() + + + for i in range(NUM_ITERS): + slst = sllist(primeList[:]) + sllistTime += doTime ( lambda : doSlice(slst, randomSlices[i]) ) + gc.collect() + + + # csllistTime = doTime ( lambda : doPops(cslst, randomPops) ) + csllistTime = 0.0 + + print ( "List time:\t\t%f\t= %f\nSL time:\t\t%f\t= %f\nDL time:\t\t%f\t= %f\n" %(listTime, listTime / NUM_ITERS, sllistTime, sllistTime / NUM_ITERS, dllistTime, dllistTime / NUM_ITERS) ) From 3d3d416e3eac0702f072fbbde727ca3104e9eb32 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 22:27:56 -0400 Subject: [PATCH 51/62] Fix warning --- src/dllist.c | 2 +- src/sllist.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index b5c928b..eb84887 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -1500,7 +1500,7 @@ static int dllist_contains(PyObject *self, PyObject *value) node = (DLListNodeObject *) ((DLListObject*)self)->first; - while ( node != Py_None ) + while ( (PyObject *)node != Py_None ) { if( node->value == value ) return 1; diff --git a/src/sllist.c b/src/sllist.c index 9a2301e..407cf5f 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1303,7 +1303,7 @@ static int sllist_contains(PyObject *self, PyObject *value) node = (SLListNodeObject *) ((SLListObject*)self)->first; - while ( node != Py_None ) + while ( (PyObject *)node != Py_None ) { if( node->value == value ) return 1; From bba344a8ca085f492a0a3735656b31e2053f5861 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 22:42:12 -0400 Subject: [PATCH 52/62] Greatly improve performance with extending on dllist --- src/dllist.c | 89 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index eb84887..e30d6ff 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -511,15 +511,24 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; - new_node = (PyObject*)dllistnode_create( - self->last, NULL, iter_node->value, (PyObject*)self); - - if (self->first == Py_None) + new_node = (PyObject *)dllistnode_make(self, iter_node->value); + if ( unlikely( self->first == Py_None) ) + { self->first = new_node; - self->last = new_node; + self->last = new_node; + ((DLListNodeObject *)new_node)->prev = Py_None; + } + else + { + ((DLListNodeObject *)self->last)->next = new_node; + ((DLListNodeObject *)new_node)->prev = self->last; + self->last = new_node; + } iter_node_obj = iter_node->next; } + if( likely( self->last != Py_None) ) + ((DLListNodeObject *)self->last)->next = Py_None; } else @@ -531,31 +540,39 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) return 0; } - for (i = 0; i < sequence_len; ++i) + self->size += sequence_len; + for(i=0; i < sequence_len; i++) { PyObject* item; PyObject* new_node; item = PySequence_GetItem(sequence, i); - if (item == NULL) + if ( unlikely(item == NULL) ) { PyErr_SetString(PyExc_ValueError, "Failed to get element from sequence"); return 0; } - new_node = (PyObject*)dllistnode_create( - self->last, NULL, item, (PyObject*)self); - - if (self->first == Py_None) + new_node = (PyObject *)dllistnode_make(self, item); + if ( unlikely(self->first == Py_None) ) + { self->first = new_node; - self->last = new_node; - - ++self->size; + self->last = new_node; + ((DLListNodeObject *)new_node)->prev = Py_None; + } + else + { + ((DLListNodeObject *)self->last)->next = new_node; + ((DLListNodeObject *)new_node)->prev = self->last; + self->last = new_node; + } Py_DECREF(item); } + if(self->last != Py_None) + ((DLListNodeObject *)self->last)->next = Py_None; } @@ -980,15 +997,24 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; - new_node = (PyObject*)dllistnode_create( - NULL, self->first, iter_node->value, (PyObject*)self); - - self->first = new_node; - if (self->last == Py_None) + new_node = (PyObject *)dllistnode_make(self, iter_node->value); + if ( unlikely(self->first == Py_None) ) + { + self->first = new_node; self->last = new_node; + ((DLListNodeObject *)new_node)->next = Py_None; + } + else + { + ((DLListNodeObject *)self->first)->prev = new_node; + ((DLListNodeObject *)new_node)->next = self->first; + self->first = new_node; + } iter_node_obj = iter_node->next; } + if ( likely(self->first != Py_None) ) + ((DLListNodeObject*)self->first)->prev = Py_None; } @@ -1002,30 +1028,39 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) return NULL; } + self->size += sequence_len; + for (i = 0; i < sequence_len; ++i) { PyObject* item; PyObject* new_node; item = PySequence_GetItem(sequence, i); - if (item == NULL) + if ( unlikely(item == NULL) ) { PyErr_SetString(PyExc_ValueError, "Failed to get element from sequence"); return NULL; } - new_node = (PyObject*)dllistnode_create( - NULL, self->first, item, (PyObject*)self); - - self->first = new_node; - if (self->last == Py_None) + new_node = (PyObject *)dllistnode_make(self, item); + if ( unlikely(self->first == Py_None) ) + { + self->first = new_node; self->last = new_node; - - ++self->size; + ((DLListNodeObject *)new_node)->next = Py_None; + } + else + { + ((DLListNodeObject *)self->first)->prev = new_node; + ((DLListNodeObject *)new_node)->next = self->first; + self->first = new_node; + } Py_DECREF(item); } + if ( likely(self->first != Py_None) ) + ((DLListNodeObject*)self->first)->prev = Py_None; } From 9ea5e9edd6fa731342e808bf33b3caabdb8dc743 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 23:08:18 -0400 Subject: [PATCH 53/62] Greatly improve performance of extending on sllist --- src/sllist.c | 172 ++++++++++++++++++++++++++++----------------------- 1 file changed, 96 insertions(+), 76 deletions(-) diff --git a/src/sllist.c b/src/sllist.c index 407cf5f..0d5540f 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -120,6 +120,15 @@ static PyObject* sllistnode_new(PyTypeObject* type, } +/* + * sllistnode_make - A slightly cheaper version that doesn't set next + */ +static inline SLListNodeObject *sllistnode_make(SLListObject *sllist, PyObject *value) +{ + return (SLListNodeObject *)llistnode_make(SLListNodeType, (PyObject *)sllist, value); +} + + static PyObject* sllistnode_repr(SLListNodeObject* self) { @@ -314,59 +323,67 @@ static int sllist_extend_internal(SLListObject* self, PyObject* sequence) LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; - new_node = (PyObject*)sllistnode_create( - Py_None, iter_node->value, (PyObject*)self); - - if (self->last != Py_None) - ((SLListNodeObject*)self->last)->next = new_node; - - if (self->first == Py_None) + new_node = (PyObject *)sllistnode_make(self, iter_node->value); + if ( unlikely( self->first == Py_None) ) + { self->first = new_node; - self->last = new_node; + self->last = new_node; + } + else + { + ((SLListNodeObject *)self->last)->next = new_node; + self->last = new_node; + } iter_node_obj = iter_node->next; } + if( likely(self->last != Py_None) ) + ((SLListNodeObject *)self->last)->next = Py_None; - return 1; - } - sequence_len = PySequence_Length(sequence); - if (sequence_len == -1) - { - PyErr_SetString(PyExc_ValueError, "Invalid sequence"); - return 0; } - - for (i = 0; i < sequence_len; ++i) + else { - PyObject* item; - PyObject* new_node; - - item = PySequence_GetItem(sequence, i); - if (item == NULL) + sequence_len = PySequence_Length(sequence); + if ( unlikely(sequence_len == -1) ) { - PyErr_SetString(PyExc_ValueError, - "Failed to get element from sequence"); + PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return 0; } - new_node = (PyObject*)sllistnode_create(Py_None, - item, - (PyObject*)self); - + self->size += sequence_len; - if(self->first == Py_None) - self->first = new_node; - else - ((SLListNodeObject*)self->last)->next = new_node; + for (i = 0; i < sequence_len; ++i) + { + PyObject* item; + PyObject* new_node; - self->last = new_node; + item = PySequence_GetItem(sequence, i); + if ( unlikely(item == NULL) ) + { + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return 0; + } + + new_node = (PyObject *)sllistnode_make(self, item); + if ( unlikely( self->first == Py_None) ) + { + self->first = new_node; + self->last = new_node; + } + else + { + ((SLListNodeObject *)self->last)->next = new_node; + self->last = new_node; + } - ++self->size; + Py_DECREF(item); + } + if( likely(self->last != Py_None) ) + ((SLListNodeObject *)self->last)->next = Py_None; - Py_DECREF(item); } - return 1; } @@ -707,52 +724,64 @@ static PyObject* sllist_extendleft(SLListObject* self, PyObject* sequence) LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; PyObject* new_node; - new_node = (PyObject*)sllistnode_create( - self->first, iter_node->value, (PyObject*)self); - - self->first = new_node; - if (self->last == Py_None) + new_node = (PyObject *)sllistnode_make(self, iter_node->value); + if ( unlikely(self->first == Py_None) ) + { + self->first = new_node; self->last = new_node; + ((SLListNodeObject *)new_node)->next = Py_None; + } + else + { + ((SLListNodeObject *)new_node)->next = self->first; + self->first = new_node; + } iter_node_obj = iter_node->next; } - Py_RETURN_NONE; - } - - sequence_len = PySequence_Length(sequence); - if (sequence_len == -1) - { - PyErr_SetString(PyExc_ValueError, "Invalid sequence"); - return NULL; } - - for (i = 0; i < sequence_len; ++i) + else { - PyObject* item; - PyObject* new_node; - - item = PySequence_GetItem(sequence, i); - if (item == NULL) + sequence_len = PySequence_Length(sequence); + if ( unlikely( sequence_len == -1) ) { - PyErr_SetString(PyExc_ValueError, - "Failed to get element from sequence"); + PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return NULL; } - new_node = (PyObject*)sllistnode_create(self->first, - item, - (PyObject*)self); + self->size += sequence_len; + + for (i = 0; i < sequence_len; ++i) + { + PyObject* item; + PyObject* new_node; - self->first = new_node; - if (self->last == Py_None) - self->last = new_node; + item = PySequence_GetItem(sequence, i); + if ( unlikely(item == NULL) ) + { + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return NULL; + } + + new_node = (PyObject *)sllistnode_make(self, item); + if ( unlikely(self->first == Py_None) ) + { + self->first = new_node; + self->last = new_node; + ((SLListNodeObject *)new_node)->next = Py_None; + } + else + { + ((SLListNodeObject *)new_node)->next = self->first; + self->first = new_node; + } - ++self->size; + Py_DECREF(item); + } - Py_DECREF(item); } - Py_RETURN_NONE; } @@ -1407,15 +1436,6 @@ static long sllist_hash(SLListObject* self) } #endif -/* - * sllistnode_make - A slightly cheaper version that doesn't set next - */ -static inline SLListNodeObject *sllistnode_make(SLListObject *sllist, PyObject *value) -{ - return (SLListNodeObject *)llistnode_make(SLListNodeType, (PyObject *)sllist, value); -} - - /* * sllist_slice - Slice function assuming normalized indexes. * From 4d452b8d3d983950478ef2ff33290c31bbcde164 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Sun, 9 Apr 2017 23:46:34 -0400 Subject: [PATCH 54/62] Simplify extending by pulling out the first set (removes the conditional from loop) --- src/dllist.c | 189 +++++++++++++++++++++++++++++++++------------------ src/sllist.c | 159 ++++++++++++++++++++++++++++--------------- 2 files changed, 229 insertions(+), 119 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index e30d6ff..41d88f4 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -492,7 +492,7 @@ static DLListNodeObject* dllist_get_node_internal(DLListObject* self, * the list with elements from a sequence. */ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) { - Py_ssize_t i; + Py_ssize_t i = 0; Py_ssize_t sequence_len; if (PyObject_TypeCheck(sequence, DLListType) || PyObject_TypeCheck(sequence, SLListType)) @@ -500,30 +500,40 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) /* Special path for extending with a LList (double or single) * It's not strictly required but greatly improves performance */ - PyObject* iter_node_obj = ((LListObject*)sequence)->first; + PyObject *iter_node_obj = ((LListObject*)sequence)->first; + LListNodeObject *iter_node; + PyObject *new_node; sequence_len = ((LListObject*)sequence)->size; + + if ( unlikely(sequence_len == 0) ) + return 1; self->size += sequence_len; - while (sequence_len--) + if ( self->size == sequence_len ) { - LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; - PyObject* new_node; + iter_node = (LListNodeObject*)iter_node_obj; new_node = (PyObject *)dllistnode_make(self, iter_node->value); - if ( unlikely( self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - ((DLListNodeObject *)new_node)->prev = Py_None; - } - else - { - ((DLListNodeObject *)self->last)->next = new_node; - ((DLListNodeObject *)new_node)->prev = self->last; - self->last = new_node; - } + self->first = new_node; + self->last = new_node; + ((DLListNodeObject *)new_node)->prev = Py_None; + + iter_node_obj = iter_node->next; + sequence_len -= 1; + } + + while (sequence_len-- > 0) + { + iter_node = (LListNodeObject*)iter_node_obj; + + new_node = (PyObject *)dllistnode_make(self, iter_node->value); + + ((DLListNodeObject *)self->last)->next = new_node; + ((DLListNodeObject *)new_node)->prev = self->last; + self->last = new_node; + iter_node_obj = iter_node->next; } @@ -534,19 +544,23 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) else { sequence_len = PySequence_Length(sequence); - if (sequence_len == -1) + if ( unlikely(sequence_len == -1) ) { PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return 0; } + if ( unlikely(sequence_len == 0) ) + return 1; + + PyObject *item; + PyObject *new_node; self->size += sequence_len; - for(i=0; i < sequence_len; i++) + + if ( self->size == sequence_len ) { - PyObject* item; - PyObject* new_node; - item = PySequence_GetItem(sequence, i); + item = PySequence_GetItem(sequence, 0); if ( unlikely(item == NULL) ) { PyErr_SetString(PyExc_ValueError, @@ -555,22 +569,36 @@ static int dllist_extend_internal(DLListObject* self, PyObject* sequence) } new_node = (PyObject *)dllistnode_make(self, item); - if ( unlikely(self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - ((DLListNodeObject *)new_node)->prev = Py_None; - } - else + self->first = new_node; + self->last = new_node; + ((DLListNodeObject *)new_node)->prev = Py_None; + + Py_DECREF(item); + + i = 1; + } + + for(; i < sequence_len; i++) + { + + item = PySequence_GetItem(sequence, i); + if ( unlikely(item == NULL) ) { - ((DLListNodeObject *)self->last)->next = new_node; - ((DLListNodeObject *)new_node)->prev = self->last; - self->last = new_node; + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return 0; } + new_node = (PyObject *)dllistnode_make(self, item); + + ((DLListNodeObject *)self->last)->next = new_node; + ((DLListNodeObject *)new_node)->prev = self->last; + self->last = new_node; + Py_DECREF(item); } + if(self->last != Py_None) ((DLListNodeObject *)self->last)->next = Py_None; @@ -980,36 +1008,46 @@ static PyObject* dllist_insert(DLListObject* self, PyObject* args) static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) { - Py_ssize_t i; + Py_ssize_t i = 0; Py_ssize_t sequence_len; if (PyObject_TypeCheck(sequence, DLListType) || PyObject_TypeCheck(sequence, SLListType)) { /* Special path for extending with a LList. Better performance. */ PyObject* iter_node_obj = ((LListObject*)sequence)->first; + LListNodeObject *iter_node; + PyObject *new_node; sequence_len = ((LListObject*)sequence)->size; + if ( unlikely(sequence_len == 0) ) + Py_RETURN_NONE; self->size += sequence_len; - while (sequence_len--) + if ( self->size == sequence_len ) { - LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; - PyObject* new_node; + iter_node = (LListNodeObject*)iter_node_obj; new_node = (PyObject *)dllistnode_make(self, iter_node->value); - if ( unlikely(self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - ((DLListNodeObject *)new_node)->next = Py_None; - } - else - { - ((DLListNodeObject *)self->first)->prev = new_node; - ((DLListNodeObject *)new_node)->next = self->first; - self->first = new_node; - } + self->first = new_node; + self->last = new_node; + ((DLListNodeObject *)new_node)->next = Py_None; + + iter_node_obj = iter_node->next; + + sequence_len -= 1; + } + + while (sequence_len-- > 0) + { + iter_node = (LListNodeObject*)iter_node_obj; + + new_node = (PyObject *)dllistnode_make(self, iter_node->value); + + ((DLListNodeObject *)self->first)->prev = new_node; + ((DLListNodeObject *)new_node)->next = self->first; + self->first = new_node; + iter_node_obj = iter_node->next; } @@ -1022,18 +1060,43 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) { sequence_len = PySequence_Length(sequence); - if (sequence_len == -1) + if ( unlikely(sequence_len == -1) ) { PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return NULL; } + if ( unlikely(sequence_len == 0) ) + Py_RETURN_NONE; + + PyObject *item; + PyObject *new_node; + DLListNodeObject *first; + + if ( self->size == 0 ) + { + item = PySequence_GetItem(sequence, 0); + if ( unlikely(item == NULL) ) + { + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return NULL; + } + + new_node = (PyObject *)dllistnode_make(self, item); + self->first = new_node; + self->last = new_node; + ((DLListNodeObject *)new_node)->next = Py_None; + + Py_DECREF(item); + i = 1; + } + self->size += sequence_len; - for (i = 0; i < sequence_len; ++i) + first = (DLListNodeObject *)self->first; + for (; i < sequence_len; ++i) { - PyObject* item; - PyObject* new_node; item = PySequence_GetItem(sequence, i); if ( unlikely(item == NULL) ) @@ -1044,23 +1107,17 @@ static PyObject* dllist_extendleft(DLListObject* self, PyObject* sequence) } new_node = (PyObject *)dllistnode_make(self, item); - if ( unlikely(self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - ((DLListNodeObject *)new_node)->next = Py_None; - } - else - { - ((DLListNodeObject *)self->first)->prev = new_node; - ((DLListNodeObject *)new_node)->next = self->first; - self->first = new_node; - } + + first->prev = new_node; + ((DLListNodeObject *)new_node)->next = (PyObject *)first; + self->first = new_node; + first = (DLListNodeObject *)new_node; + Py_DECREF(item); } - if ( likely(self->first != Py_None) ) - ((DLListNodeObject*)self->first)->prev = Py_None; + if ( likely( (PyObject *)first != Py_None) ) + first->prev = Py_None; } diff --git a/src/sllist.c b/src/sllist.c index 0d5540f..8dda374 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -315,25 +315,33 @@ static int sllist_extend_internal(SLListObject* self, PyObject* sequence) PyObject* iter_node_obj = ((LListObject *)sequence)->first; sequence_len = ((LListObject *)sequence)->size; + if ( unlikely(sequence_len == 0) ) + return 1; + + LListNodeObject *iter_node; + PyObject *new_node; self->size += sequence_len; + if(self->size == sequence_len) + { + iter_node = (LListNodeObject*)iter_node_obj; + + new_node = (PyObject *)sllistnode_make(self, iter_node->value); + self->first = new_node; + self->last = new_node; - while (sequence_len--) + iter_node_obj = iter_node->next; + sequence_len -= 1; + } + while (sequence_len-- > 0) { - LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; - PyObject* new_node; + iter_node = (LListNodeObject*)iter_node_obj; new_node = (PyObject *)sllistnode_make(self, iter_node->value); - if ( unlikely( self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - } - else - { - ((SLListNodeObject *)self->last)->next = new_node; - self->last = new_node; - } + + ((SLListNodeObject *)self->last)->next = new_node; + self->last = new_node; + iter_node_obj = iter_node->next; } @@ -350,10 +358,36 @@ static int sllist_extend_internal(SLListObject* self, PyObject* sequence) PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return 0; } + if ( unlikely(sequence_len == 0) ) + return 1; + + PyObject *item; + PyObject *new_node; self->size += sequence_len; + i = 0; + + if ( self->size == sequence_len ) + { + + item = PySequence_GetItem(sequence, 0); + if ( unlikely(item == NULL) ) + { + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return 0; + } + + new_node = (PyObject *)sllistnode_make(self, item); - for (i = 0; i < sequence_len; ++i) + self->first = new_node; + self->last = new_node; + + Py_DECREF(item); + i += 1; + } + + for (; i < sequence_len; ++i) { PyObject* item; PyObject* new_node; @@ -367,16 +401,10 @@ static int sllist_extend_internal(SLListObject* self, PyObject* sequence) } new_node = (PyObject *)sllistnode_make(self, item); - if ( unlikely( self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - } - else - { - ((SLListNodeObject *)self->last)->next = new_node; - self->last = new_node; - } + + ((SLListNodeObject *)self->last)->next = new_node; + self->last = new_node; + Py_DECREF(item); } @@ -717,25 +745,35 @@ static PyObject* sllist_extendleft(SLListObject* self, PyObject* sequence) sequence_len = ((LListObject*)sequence)->size; + if ( unlikely( sequence_len == 0 ) ) + Py_RETURN_NONE; + + LListNodeObject *iter_node; + PyObject *new_node; + self->size += sequence_len; + if ( self->size == sequence_len ) + { + iter_node = (LListNodeObject*)iter_node_obj; + + new_node = (PyObject *)sllistnode_make(self, iter_node->value); + + self->first = new_node; + self->last = new_node; + ((SLListNodeObject *)new_node)->next = Py_None; - while (sequence_len--) + iter_node_obj = iter_node->next; + sequence_len -= 1; + } + + while (sequence_len-- > 0) { - LListNodeObject* iter_node = (LListNodeObject*)iter_node_obj; - PyObject* new_node; + iter_node = (LListNodeObject*)iter_node_obj; new_node = (PyObject *)sllistnode_make(self, iter_node->value); - if ( unlikely(self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - ((SLListNodeObject *)new_node)->next = Py_None; - } - else - { - ((SLListNodeObject *)new_node)->next = self->first; - self->first = new_node; - } + + ((SLListNodeObject *)new_node)->next = self->first; + self->first = new_node; iter_node_obj = iter_node->next; } @@ -749,15 +787,17 @@ static PyObject* sllist_extendleft(SLListObject* self, PyObject* sequence) PyErr_SetString(PyExc_ValueError, "Invalid sequence"); return NULL; } + if ( unlikely( sequence_len == 0 ) ) + Py_RETURN_NONE; - self->size += sequence_len; + PyObject *item; + PyObject *new_node; + i = 0; - for (i = 0; i < sequence_len; ++i) + self->size += sequence_len; + if ( self->size == sequence_len ) { - PyObject* item; - PyObject* new_node; - - item = PySequence_GetItem(sequence, i); + item = PySequence_GetItem(sequence, 0); if ( unlikely(item == NULL) ) { PyErr_SetString(PyExc_ValueError, @@ -766,18 +806,31 @@ static PyObject* sllist_extendleft(SLListObject* self, PyObject* sequence) } new_node = (PyObject *)sllistnode_make(self, item); - if ( unlikely(self->first == Py_None) ) - { - self->first = new_node; - self->last = new_node; - ((SLListNodeObject *)new_node)->next = Py_None; - } - else + + self->first = new_node; + self->last = new_node; + ((SLListNodeObject *)new_node)->next = Py_None; + + Py_DECREF(item); + i += 1; + } + + for (; i < sequence_len; ++i) + { + item = PySequence_GetItem(sequence, i); + if ( unlikely(item == NULL) ) { - ((SLListNodeObject *)new_node)->next = self->first; - self->first = new_node; + PyErr_SetString(PyExc_ValueError, + "Failed to get element from sequence"); + return NULL; } + new_node = (PyObject *)sllistnode_make(self, item); + + ((SLListNodeObject *)new_node)->next = self->first; + self->first = new_node; + + Py_DECREF(item); } From 593f2cd2eaa698456f622122d8c61a2016bf970f Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 09:32:29 -0400 Subject: [PATCH 55/62] Rename this fork to cllist, bump to version 1.0.0 to prepare for release --- CHANGES | 4 +++- setup.py | 24 +++++++++++++++--------- src/{llist.c => cllist.c} | 23 +++++++++++++---------- src/dllist.h | 1 + tests/benchmark.py | 2 +- tests/benchmark_contains.py | 2 +- tests/benchmark_extend.py | 2 +- tests/benchmark_slice.py | 2 +- tests/llist_test.py | 8 ++++---- tests/speed_test.py | 2 +- 10 files changed, 41 insertions(+), 29 deletions(-) rename src/{llist.c => cllist.c} (67%) diff --git a/CHANGES b/CHANGES index dab04d6..973d47d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,6 @@ -* llist-0.5 +* cllist-1.0 + + Forked from python-llist to python-cllist Work by Tim Savannah: diff --git a/setup.py b/setup.py index ff1edc6..00b3e63 100755 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -VERSION='0.5.0' +VERSION='1.0.0' from distutils.core import setup, Extension -sources = ['src/llist.c', +sources = ['src/cllist.c', 'src/dllist.c', 'src/sllist.c', ] @@ -13,16 +13,17 @@ setup(name='llist', description='Linked list data structures for Python', long_description=open('README').read(), - author='Adam Jakubek, Rafał Gałczyński', - author_email='ajakubek@gmail.com, rafal.galczynski@gmail.com', + author='Timothy Savannah, Adam Jakubek, Rafał Gałczyński', + author_email='kata198@gmail.com, ajakubek@gmail.com, rafal.galczynski@gmail.com', + maintainer_email='kata198@gmail.com', version=VERSION, - url='https://github.com/ajakubek/python-llist', - download_url='http://pypi.python.org/pypi/llist/%s' % VERSION, + url='https://github.com/kata198/python-cllist', + download_url='http://pypi.python.org/pypi/cllist/%s' % VERSION, license='MIT', - keywords='linked list, list', - ext_modules=[Extension('llist', sources)], + keywords=['llist', 'linked', 'double', 'single', 'linked list'], + ext_modules=[Extension('cllist', sources)], classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: MacOS :: MacOS X', @@ -30,7 +31,12 @@ 'Operating System :: POSIX', 'Programming Language :: C', 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Software Development :: Libraries :: Python Modules', ], diff --git a/src/llist.c b/src/cllist.c similarity index 67% rename from src/llist.c rename to src/cllist.c index 0b06f9c..074ee88 100644 --- a/src/llist.c +++ b/src/cllist.c @@ -1,4 +1,5 @@ /* Copyright (c) 2011-2013 Adam Jakubek, Rafał Gałczyński + * Copyright (c) 2017 Timothy Savannah * Released under the MIT license (see attached LICENSE file). */ @@ -7,7 +8,7 @@ #include "sllist.h" #include "dllist.h" -static PyMethodDef llist_methods[] = +static PyMethodDef cllist_methods[] = { { NULL } /* sentinel */ }; @@ -16,14 +17,16 @@ static PyMethodDef llist_methods[] = #define PyMODINIT_FUNC void #endif +#define docstr ("C-extension providing single and double linked lists.") + #if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef llist_moduledef = { +static struct PyModuleDef cllist_moduledef = { PyModuleDef_HEAD_INIT, - "llist", /* m_name */ - "Singly and doubly linked lists.", /* m_doc */ + "cllist", /* m_name */ + docstr, /* m_doc */ -1, /* m_size */ - llist_methods, /* m_methods */ + cllist_methods, /* m_methods */ NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ @@ -31,7 +34,7 @@ static struct PyModuleDef llist_moduledef = { }; PyMODINIT_FUNC -PyInit_llist(void) +PyInit_cllist(void) { PyObject* m; @@ -40,7 +43,7 @@ PyInit_llist(void) if (!dllist_init_type()) return NULL; - m = PyModule_Create(&llist_moduledef); + m = PyModule_Create(&cllist_moduledef); sllist_register(m); dllist_register(m); @@ -51,7 +54,7 @@ PyInit_llist(void) #else PyMODINIT_FUNC -initllist(void) +initcllist(void) { PyObject* m; @@ -60,8 +63,8 @@ initllist(void) if (!dllist_init_type()) return; - m = Py_InitModule3("llist", llist_methods, - "Singly and doubly linked lists."); + m = Py_InitModule3("cllist", cllist_methods, + docstr); sllist_register(m); dllist_register(m); diff --git a/src/dllist.h b/src/dllist.h index d67bca2..5cdd256 100644 --- a/src/dllist.h +++ b/src/dllist.h @@ -1,4 +1,5 @@ /* Copyright (c) 2011-2013 Adam Jakubek, Rafał Gałczyński + * Copyright (c) 2017 Timothy Savannah * Released under the MIT license (see attached LICENSE file). */ diff --git a/tests/benchmark.py b/tests/benchmark.py index f95c1f3..78acc36 100755 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -5,7 +5,7 @@ import time import random -from llist import dllist, sllist +from cllist import dllist, sllist LIST_SIZE = 600 diff --git a/tests/benchmark_contains.py b/tests/benchmark_contains.py index f2b782c..00b037f 100755 --- a/tests/benchmark_contains.py +++ b/tests/benchmark_contains.py @@ -5,7 +5,7 @@ import time import random -from llist import dllist, sllist +from cllist import dllist, sllist LIST_SIZE = 700 diff --git a/tests/benchmark_extend.py b/tests/benchmark_extend.py index 3579e6f..9a3a26b 100755 --- a/tests/benchmark_extend.py +++ b/tests/benchmark_extend.py @@ -5,7 +5,7 @@ import time import random -from llist import dllist, sllist +from cllist import dllist, sllist LIST1_SIZE = 600 diff --git a/tests/benchmark_slice.py b/tests/benchmark_slice.py index cc87f94..dccde5e 100755 --- a/tests/benchmark_slice.py +++ b/tests/benchmark_slice.py @@ -5,7 +5,7 @@ import time import random -from llist import dllist, sllist +from cllist import dllist, sllist LIST_SIZE = 700 diff --git a/tests/llist_test.py b/tests/llist_test.py index ce4e3b3..522c1f3 100755 --- a/tests/llist_test.py +++ b/tests/llist_test.py @@ -4,10 +4,10 @@ import sys import random import unittest -from llist import sllist -from llist import sllistnode -from llist import dllist -from llist import dllistnode +from cllist import sllist +from cllist import sllistnode +from cllist import dllist +from cllist import dllistnode gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_STATS) diff --git a/tests/speed_test.py b/tests/speed_test.py index 9f2c399..0e80193 100755 --- a/tests/speed_test.py +++ b/tests/speed_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from collections import deque -from llist import sllist, dllist +from cllist import sllist, dllist import time # import gc # gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_STATS) From f4faf4e3984c5d24bef0b85f39423f74e6a668ee Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 10:04:39 -0400 Subject: [PATCH 56/62] Add __version__ and __version_tuple__ --- CHANGES | 2 ++ src/cllist.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/llist.h | 12 +++++++ 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 973d47d..b388c0a 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,8 @@ - Implement mappings on sllist and dllist - Implement slicing (including with step) on both sllist and dllist + - Add __version__ and __version_tuple__ + - Some general optimizations diff --git a/src/cllist.c b/src/cllist.c index 074ee88..79ded54 100644 --- a/src/cllist.c +++ b/src/cllist.c @@ -5,6 +5,7 @@ #include +#include "llist.h" #include "sllist.h" #include "dllist.h" @@ -19,6 +20,91 @@ static PyMethodDef cllist_methods[] = #define docstr ("C-extension providing single and double linked lists.") +#if PY_MAJOR_VERSION >= 3 +#define PyString_FromString PyUnicode_FromString +#endif + +static PyObject *get_version_tuple_object(long major, long minor, long patch) +{ + + PyObject *majorO, *minorO, *patchO; + PyObject *versionTupleObj; + + majorO = PyLong_FromLong(major); + minorO = PyLong_FromLong(minor); + patchO = PyLong_FromLong(patch); + + Py_INCREF(majorO); + Py_INCREF(minorO); + Py_INCREF(patchO); + + versionTupleObj = PyTuple_New(3); + + PyTuple_SetItem(versionTupleObj, 0, majorO); + PyTuple_SetItem(versionTupleObj, 1, minorO); + PyTuple_SetItem(versionTupleObj, 2, patchO); + + Py_INCREF(versionTupleObj); + + return versionTupleObj; +} + +static PyObject *get_version_object(const char *versionStr) +{ + PyObject *versionObj; + + versionObj = PyString_FromString(versionStr); + + Py_INCREF(versionObj); + + return versionObj; +} + + +static void free_version_tuple_object(PyObject *versionTupleObj) +{ + PyObject *majorO, *minorO, *patchO; + + majorO = PyTuple_GetItem(versionTupleObj, 0); + minorO = PyTuple_GetItem(versionTupleObj, 1); + patchO = PyTuple_GetItem(versionTupleObj, 2); + + Py_DECREF(majorO); + Py_DECREF(minorO); + Py_DECREF(patchO); + + Py_DECREF(versionTupleObj); + +} + +static void free_version_object(PyObject *versionObj) +{ + + Py_DECREF(versionObj); +} + +PyObject *mod_version_tuple; +PyObject *mod_version; + + + +static void cllist_setup_module(PyObject *mod) +{ + mod_version_tuple = get_version_tuple_object(LLIST_VERSION_MAJOR, LLIST_VERSION_MINOR, LLIST_VERSION_PATCH); + mod_version = get_version_object(LLIST_VERSION_STR); + + PyModule_AddObject(mod, "__version_tuple__", mod_version_tuple); + PyModule_AddObject(mod, "__version__", mod_version); + +} + +static void cllist_free(void *modPtr) +{ + free_version_object(mod_version); + free_version_tuple_object(mod_version_tuple); +} + + #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef cllist_moduledef = { @@ -30,7 +116,7 @@ static struct PyModuleDef cllist_moduledef = { NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ - NULL, /* m_free */ + cllist_free, /* m_free */ }; PyMODINIT_FUNC @@ -45,6 +131,8 @@ PyInit_cllist(void) m = PyModule_Create(&cllist_moduledef); + cllist_setup_module(m); + sllist_register(m); dllist_register(m); @@ -66,6 +154,7 @@ initcllist(void) m = Py_InitModule3("cllist", cllist_methods, docstr); + cllist_setup_module(m); sllist_register(m); dllist_register(m); } diff --git a/src/llist.h b/src/llist.h index bf09b14..653dc5d 100644 --- a/src/llist.h +++ b/src/llist.h @@ -6,6 +6,17 @@ #ifndef LLIST_H #define LLIST_H +#include + + +#define LLIST_VERSION_MAJOR 1 +#define LLIST_VERSION_MINOR 0 +#define LLIST_VERSION_PATCH 0 +#define LLIST_VERSION_STR ("1.0.0") + +extern PyObject *mod_version; +extern PyObject *mod_version_tuple; + #ifdef __GNUC__ @@ -66,4 +77,5 @@ static inline int _normalize_indexes(Py_ssize_t size, Py_ssize_t *idx_start, Py_ } + #endif From 6ff909e210d4efd32494565dfcd273f4b3e135f1 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 10:13:29 -0400 Subject: [PATCH 57/62] Fix warnings and fix normalize_indexes for python2 Also, move debug function to llist.h --- src/dllist.c | 33 ++++----------------------------- src/llist.h | 43 ++++++++++++++++++++++++++++++++++++++++--- src/sllist.c | 6 +++--- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/dllist.c b/src/dllist.c index 41d88f4..80ffb81 100644 --- a/src/dllist.c +++ b/src/dllist.c @@ -22,32 +22,6 @@ #define START_MIDDLE_AFTER 10 -/* #define DO_DEBUG */ - - -#ifdef DO_DEBUG -static void debugmsg(char *format, ...) -{ - va_list args; - FILE *f; - - f = fopen("debug.txt", "a"); - - va_start(args, format); - vfprintf(f, format, args); - va_end(args); - - fclose(f); -} -#else - -#define debugmsg(...) -/*static inline void debugmsg(char *format, ...) -{ - -} -*/ -#endif @@ -1862,6 +1836,7 @@ static PyObject *dllist_slice(DLListObject *self, Py_ssize_t start_idx, Py_ssize */ static PyObject *dllist_simpleslice(DLListObject *self, Py_ssize_t idx_start, Py_ssize_t idx_end) { + debugmsg("Calling simpleslice: %p %ld %ld\n", self, idx_start, idx_end); if( !_normalize_indexes(self->size, &idx_start, &idx_end) ) { DLListObject *ret = (DLListObject *)dllist_new(DLListType, NULL, NULL); @@ -1893,7 +1868,7 @@ static PyObject *dllist_subscript(DLListObject *self, PyObject *item) { Py_ssize_t start, stop, step, sliceLength; - if ( PySlice_GetIndicesEx( (PyObject *)item, self->size, &start, &stop, &step, &sliceLength ) ) + if ( PySlice_GetIndicesEx( (GET_INDICES_TYPE *)item, self->size, &start, &stop, &step, &sliceLength ) ) return NULL; /* Error */ return dllist_slice(self, start, stop, step, sliceLength); @@ -1901,7 +1876,7 @@ static PyObject *dllist_subscript(DLListObject *self, PyObject *item) else { - PyErr_Format(PyExc_TypeError, "Indicies must be integers, not %s", item->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Indices must be integers, not %s", item->ob_type->tp_name); return NULL; } @@ -1968,7 +1943,7 @@ static PySequenceMethods DLListSequenceMethods[] = { dllist_concat, /* sq_concat */ dllist_repeat, /* sq_repeat */ dllist_get_item, /* sq_item */ - dllist_simpleslice, /* sq_slice */ + (ssizessizeargfunc)dllist_simpleslice, /* sq_slice */ dllist_set_item, /* sq_ass_item */ 0, /* sq_ass_slice */ dllist_contains, /* sq_contains */ diff --git a/src/llist.h b/src/llist.h index 653dc5d..96d8cd9 100644 --- a/src/llist.h +++ b/src/llist.h @@ -30,6 +30,42 @@ extern PyObject *mod_version_tuple; #endif +/* #define DO_DEBUG */ + + +#ifdef DO_DEBUG +static void debugmsg(char *format, ...) +{ + va_list args; + FILE *f; + + f = fopen("debug.txt", "a"); + + va_start(args, format); + vfprintf(f, format, args); + va_end(args); + + fclose(f); +} +#else + +#define debugmsg(...) +/*static inline void debugmsg(char *format, ...) +{ + +} +*/ +#endif + +#if PY_MAJOR_VERSION >= 3 + +#define GET_INDICES_TYPE PyObject + +#else + +#define GET_INDICES_TYPE PySliceObject + +#endif static inline Py_ssize_t py_ssize_t_abs(Py_ssize_t x) { @@ -64,15 +100,16 @@ static inline int _normalize_indexes(Py_ssize_t size, Py_ssize_t *idx_start, Py_ return 0; } - if( unlikely(*idx_end >= size )) - *idx_end = size - 1; + if ( unlikely(*idx_end >= size )) + *idx_end = size; if ( unlikely(*idx_start >= size || *idx_start >= *idx_end)) return 0; - if( unlikely(*idx_start >= *idx_end )) + if ( unlikely(*idx_start >= *idx_end )) return 0; + return 1; } diff --git a/src/sllist.c b/src/sllist.c index 8dda374..90b52fe 100644 --- a/src/sllist.c +++ b/src/sllist.c @@ -1610,7 +1610,7 @@ static PyObject *sllist_subscript(SLListObject *self, PyObject *item) { Py_ssize_t start, stop, step, sliceLength; - if ( PySlice_GetIndicesEx( (PyObject *)item, self->size, &start, &stop, &step, &sliceLength ) ) + if ( PySlice_GetIndicesEx( (GET_INDICES_TYPE *)item, self->size, &start, &stop, &step, &sliceLength ) ) return NULL; /* Error */ return sllist_slice(self, start, stop, step, sliceLength); @@ -1618,7 +1618,7 @@ static PyObject *sllist_subscript(SLListObject *self, PyObject *item) else { - PyErr_Format(PyExc_TypeError, "Indicies must be integers, not %s", item->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Indices must be integers, not %s", item->ob_type->tp_name); return NULL; } @@ -1700,7 +1700,7 @@ static PySequenceMethods SLListSequenceMethods = sllist_concat, /* sq_concat */ sllist_repeat, /* sq_repeat */ sllist_get_item, /* sq_item */ - sllist_simpleslice, /* sq_slice; */ + (ssizessizeargfunc) sllist_simpleslice, /* sq_slice; */ sllist_set_item, /* sq_ass_item */ 0, /* sq_ass_slice */ sllist_contains, /* sq_contains */ From 3f45f02a8ea4997b6af5087ef304d93f64b341c1 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 10:19:30 -0400 Subject: [PATCH 58/62] Rename READMEs to match markdown (for github) and rst (for pypi). Also fixup manifest --- MANIFEST.in | 7 ++++++- README => README.md | 0 README.rst | 36 ++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) rename README => README.md (100%) create mode 100644 README.rst diff --git a/MANIFEST.in b/MANIFEST.in index 812b7fa..dff971f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,9 @@ -include Makefile CHANGES LICENSE MANIFEST MANIFEST.in +include Makefile +include CHANGES +include LICENSE +include MANIFEST.in +include README.rst +include README.md recursive-include src *.c *.h recursive-include docs *.py *.rst include docs/Makefile docs/make.bat diff --git a/README b/README.md similarity index 100% rename from README rename to README.md diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..36da05b --- /dev/null +++ b/README.rst @@ -0,0 +1,36 @@ +llist - linked lists for CPython +================================ + +llist is an extension module for CPython providing basic linked list +data structures. +Collections implemented in the llist module perform well in problems +which rely on fast insertions and/or deletions of elements in +the middle of a sequence. +For this kind of workload, they can be significantly faster than +collections.deque or standard Python lists. + +This extension requires CPython 2.5 or newer (3.x is supported). +If you are looking for an implementation of linked lists in pure Python, +visit http://github.com/rgsoda/pypy-llist/ +The pypy-llist module has the same API as this extension, but is +significantly slower in CPython. + +Currently llist provides the following types of linked lists: + - dllist - a doubly linked list + - sllist - a singly linked list + +Full documentation of these classes is available at: +http://packages.python.org/llist/ + +To install this package, either run "pip install llist", +or download it manually from http://pypi.python.org/pypi +then unpack the sources and compile them with "python setup.py install". + +The most current development version is available at: +https://github.com/ajakubek/python-llist/ + +Bugs can be reported at: +https://github.com/ajakubek/python-llist/issues + +This software is distributed under the MIT license. +Please see the LICENSE file included in the package for details. diff --git a/setup.py b/setup.py index 00b3e63..02998be 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup(name='llist', description='Linked list data structures for Python', - long_description=open('README').read(), + long_description=open('README.rst').read(), author='Timothy Savannah, Adam Jakubek, Rafał Gałczyński', author_email='kata198@gmail.com, ajakubek@gmail.com, rafal.galczynski@gmail.com', maintainer_email='kata198@gmail.com', From 22d953c748867e159f6c420aef124cabb13cf15a Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 10:22:16 -0400 Subject: [PATCH 59/62] Add pydocs --- MANIFEST.in | 1 + pydocs/cllist.dllist.html | 109 ++++++++++++ pydocs/cllist.html | 341 ++++++++++++++++++++++++++++++++++++++ pydocs/cllist.sllist.html | 108 ++++++++++++ pydocs/index.html | 1 + 5 files changed, 560 insertions(+) create mode 100644 pydocs/cllist.dllist.html create mode 100644 pydocs/cllist.html create mode 100644 pydocs/cllist.sllist.html create mode 120000 pydocs/index.html diff --git a/MANIFEST.in b/MANIFEST.in index dff971f..216cad4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,6 +6,7 @@ include README.rst include README.md recursive-include src *.c *.h recursive-include docs *.py *.rst +recursive-include pydocs *.html include docs/Makefile docs/make.bat graft docs/_static graft docs/_templates diff --git a/pydocs/cllist.dllist.html b/pydocs/cllist.dllist.html new file mode 100644 index 0000000..e0e49ca --- /dev/null +++ b/pydocs/cllist.dllist.html @@ -0,0 +1,109 @@ + + +Python: class dllist + + +

+ + + + + + + +
 
+cllist.dllist = class dllist(__builtin__.object)
   Doubly linked list
 
 Methods defined here:
+
__add__(...)
x.__add__(y) <==> x+y
+ +
__contains__(...)
x.__contains__(y) <==> y in x
+ +
__delitem__(...)
x.__delitem__(y) <==> del x[y]
+ +
__eq__(...)
x.__eq__(y) <==> x==y
+ +
__ge__(...)
x.__ge__(y) <==> x>=y
+ +
__getitem__(...)
x.__getitem__(y) <==> x[y]
+ +
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
+ 
+Use of negative indices is not supported.
+ +
__gt__(...)
x.__gt__(y) <==> x>y
+ +
__iadd__(...)
x.__iadd__(y) <==> x+=y
+ +
__init__(...)
x.__init__(...) initializes x; see help(type(x)) for signature
+ +
__iter__(...)
x.__iter__() <==> iter(x)
+ +
__le__(...)
x.__le__(y) <==> x<=y
+ +
__len__(...)
x.__len__() <==> len(x)
+ +
__lt__(...)
x.__lt__(y) <==> x<y
+ +
__mul__(...)
x.__mul__(n) <==> x*n
+ +
__ne__(...)
x.__ne__(y) <==> x!=y
+ +
__repr__(...)
x.__repr__() <==> repr(x)
+ +
__rmul__(...)
x.__rmul__(n) <==> n*x
+ +
__setitem__(...)
x.__setitem__(i, y) <==> x[i]=y
+ +
__str__(...)
x.__str__() <==> str(x)
+ +
append(...)
Append element at the end of the list
+ +
appendleft(...)
Append element at the beginning of the list
+ +
appendright(...)
Append element at the end of the list
+ +
clear(...)
Remove all elements from the list
+ +
extend(...)
Append elements from iterable at the right side of the list
+ +
extendleft(...)
Append elements from iterable at the left side of the list
+ +
extendright(...)
Append elements from iterable at the right side of the list
+ +
index(...)
Returns the first index of a value
+ +
insert(...)
Inserts element before node
+ +
nodeat(...)
Return node at index
+ +
pop(...)
Remove an element by index from the list and return it, or last item if no index provided
+ +
popleft(...)
Remove first element from the list and return it
+ +
popright(...)
Remove last element from the list and return it
+ +
remove(...)
Remove element from the list
+ +
rindex(...)
Returns the last index of a value
+ +
rotate(...)
Rotate the list n steps to the right
+ +
+Data descriptors defined here:
+
first
+
First node
+
+
last
+
Next node
+
+
middle
+
Middle node
+
+
size
+
Number of elements in the list
+
+
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +
+ \ No newline at end of file diff --git a/pydocs/cllist.html b/pydocs/cllist.html new file mode 100644 index 0000000..e70c377 --- /dev/null +++ b/pydocs/cllist.html @@ -0,0 +1,341 @@ + + +Python: module cllist + + + + + +
 
+ 
cllist (version 1.0.0)		index
/home/media/projects/python-llist/myenv2/lib/python2.7/site-packages/cllist.so
+

C-extension providing single and double linked lists.

+

+ + + + + +
 
+Classes
       
+
__builtin__.object +
+
+
llist.dllist +
llist.dllistiterator +
llist.dllistnode +
llist.sllist +
llist.sllistiterator +
llist.sllistnode +
+
+
+

+ + + + + + + +
 
+class dllist(__builtin__.object)
   Doubly linked list
 
 Methods defined here:
+
__add__(...)
x.__add__(y) <==> x+y
+ +
__contains__(...)
x.__contains__(y) <==> y in x
+ +
__delitem__(...)
x.__delitem__(y) <==> del x[y]
+ +
__eq__(...)
x.__eq__(y) <==> x==y
+ +
__ge__(...)
x.__ge__(y) <==> x>=y
+ +
__getitem__(...)
x.__getitem__(y) <==> x[y]
+ +
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
+ 
+Use of negative indices is not supported.
+ +
__gt__(...)
x.__gt__(y) <==> x>y
+ +
__iadd__(...)
x.__iadd__(y) <==> x+=y
+ +
__init__(...)
x.__init__(...) initializes x; see help(type(x)) for signature
+ +
__iter__(...)
x.__iter__() <==> iter(x)
+ +
__le__(...)
x.__le__(y) <==> x<=y
+ +
__len__(...)
x.__len__() <==> len(x)
+ +
__lt__(...)
x.__lt__(y) <==> x<y
+ +
__mul__(...)
x.__mul__(n) <==> x*n
+ +
__ne__(...)
x.__ne__(y) <==> x!=y
+ +
__repr__(...)
x.__repr__() <==> repr(x)
+ +
__rmul__(...)
x.__rmul__(n) <==> n*x
+ +
__setitem__(...)
x.__setitem__(i, y) <==> x[i]=y
+ +
__str__(...)
x.__str__() <==> str(x)
+ +
append(...)
Append element at the end of the list
+ +
appendleft(...)
Append element at the beginning of the list
+ +
appendright(...)
Append element at the end of the list
+ +
clear(...)
Remove all elements from the list
+ +
extend(...)
Append elements from iterable at the right side of the list
+ +
extendleft(...)
Append elements from iterable at the left side of the list
+ +
extendright(...)
Append elements from iterable at the right side of the list
+ +
index(...)
Returns the first index of a value
+ +
insert(...)
Inserts element before node
+ +
nodeat(...)
Return node at index
+ +
pop(...)
Remove an element by index from the list and return it, or last item if no index provided
+ +
popleft(...)
Remove first element from the list and return it
+ +
popright(...)
Remove last element from the list and return it
+ +
remove(...)
Remove element from the list
+ +
rindex(...)
Returns the last index of a value
+ +
rotate(...)
Rotate the list n steps to the right
+ +
+Data descriptors defined here:
+
first
+
First node
+
+
last
+
Next node
+
+
middle
+
Middle node
+
+
size
+
Number of elements in the list
+
+
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +

+ + + + + + + +
 
+class dllistiterator(__builtin__.object)
   Doubly linked list iterator
 
 Methods defined here:
+
next(...)
x.next() -> the next value, or raise StopIteration
+ +
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +

+ + + + + + + +
 
+class dllistnode(__builtin__.object)
   Doubly linked list node
 
 Methods defined here:
+
__call__(...)
x.__call__(...) <==> x(...)
+ +
__init__(...)
x.__init__(...) initializes x; see help(type(x)) for signature
+ +
__repr__(...)
x.__repr__() <==> repr(x)
+ +
__str__(...)
x.__str__() <==> str(x)
+ +
+Data descriptors defined here:
+
next
+
Next node
+
+
prev
+
Previous node
+
+
value
+
Value stored in node
+
+
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +

+ + + + + + + +
 
+class sllist(__builtin__.object)
   Singly linked list
 
 Methods defined here:
+
__add__(...)
x.__add__(y) <==> x+y
+ +
__contains__(...)
x.__contains__(y) <==> y in x
+ +
__delitem__(...)
x.__delitem__(y) <==> del x[y]
+ +
__eq__(...)
x.__eq__(y) <==> x==y
+ +
__ge__(...)
x.__ge__(y) <==> x>=y
+ +
__getitem__(...)
x.__getitem__(y) <==> x[y]
+ +
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
+ 
+Use of negative indices is not supported.
+ +
__gt__(...)
x.__gt__(y) <==> x>y
+ +
__iadd__(...)
x.__iadd__(y) <==> x+=y
+ +
__init__(...)
x.__init__(...) initializes x; see help(type(x)) for signature
+ +
__iter__(...)
x.__iter__() <==> iter(x)
+ +
__le__(...)
x.__le__(y) <==> x<=y
+ +
__len__(...)
x.__len__() <==> len(x)
+ +
__lt__(...)
x.__lt__(y) <==> x<y
+ +
__mul__(...)
x.__mul__(n) <==> x*n
+ +
__ne__(...)
x.__ne__(y) <==> x!=y
+ +
__repr__(...)
x.__repr__() <==> repr(x)
+ +
__rmul__(...)
x.__rmul__(n) <==> n*x
+ +
__setitem__(...)
x.__setitem__(i, y) <==> x[i]=y
+ +
__str__(...)
x.__str__() <==> str(x)
+ +
append(...)
Append element at the end of the list
+ +
appendleft(...)
Append element at the beginning of the list
+ +
appendright(...)
Append element at the end of the list
+ +
clear(...)
Remove all elements from the list
+ +
extend(...)
Append elements from iterable at the right side of the list
+ +
extendleft(...)
Append elements from iterable at the left side of the list
+ +
extendright(...)
Append elements from iterable at the right side of the list
+ +
index(...)
Returns the first index of a value
+ +
insertafter(...)
Inserts element after node
+ +
insertbefore(...)
Inserts element before node
+ +
nodeat(...)
Return node at index
+ +
pop(...)
Remove an element by index from the list and return it, or last item if no index provided
+ +
popleft(...)
Remove first element from the list and return it
+ +
popright(...)
Remove last element from the list and return it
+ +
remove(...)
Remove element from the list
+ +
rindex(...)
Returns the last index of a value
+ +
rotate(...)
Rotate the list n steps to the right
+ +
+Data descriptors defined here:
+
first
+
First node
+
+
last
+
Next node
+
+
size
+
size
+
+
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +

+ + + + + + + +
 
+class sllistiterator(__builtin__.object)
   Singly linked list iterator
 
 Methods defined here:
+
next(...)
x.next() -> the next value, or raise StopIteration
+ +
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +

+ + + + + + + +
 
+class sllistnode(__builtin__.object)
   Singly linked list node
 
 Methods defined here:
+
__call__(...)
x.__call__(...) <==> x(...)
+ +
__init__(...)
x.__init__(...) initializes x; see help(type(x)) for signature
+ +
__repr__(...)
x.__repr__() <==> repr(x)
+ +
__str__(...)
x.__str__() <==> str(x)
+ +
+Data descriptors defined here:
+
next
+
next node
+
+
value
+
value
+
+
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +

+ + + + + +
 
+Data
       __version__ = '1.0.0'
+__version_tuple__ = (1L, 0L, 0L)
+ \ No newline at end of file diff --git a/pydocs/cllist.sllist.html b/pydocs/cllist.sllist.html new file mode 100644 index 0000000..b55a401 --- /dev/null +++ b/pydocs/cllist.sllist.html @@ -0,0 +1,108 @@ + + +Python: class sllist + + +

+ + + + + + + +
 
+cllist.sllist = class sllist(__builtin__.object)
   Singly linked list
 
 Methods defined here:
+
__add__(...)
x.__add__(y) <==> x+y
+ +
__contains__(...)
x.__contains__(y) <==> y in x
+ +
__delitem__(...)
x.__delitem__(y) <==> del x[y]
+ +
__eq__(...)
x.__eq__(y) <==> x==y
+ +
__ge__(...)
x.__ge__(y) <==> x>=y
+ +
__getitem__(...)
x.__getitem__(y) <==> x[y]
+ +
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
+ 
+Use of negative indices is not supported.
+ +
__gt__(...)
x.__gt__(y) <==> x>y
+ +
__iadd__(...)
x.__iadd__(y) <==> x+=y
+ +
__init__(...)
x.__init__(...) initializes x; see help(type(x)) for signature
+ +
__iter__(...)
x.__iter__() <==> iter(x)
+ +
__le__(...)
x.__le__(y) <==> x<=y
+ +
__len__(...)
x.__len__() <==> len(x)
+ +
__lt__(...)
x.__lt__(y) <==> x<y
+ +
__mul__(...)
x.__mul__(n) <==> x*n
+ +
__ne__(...)
x.__ne__(y) <==> x!=y
+ +
__repr__(...)
x.__repr__() <==> repr(x)
+ +
__rmul__(...)
x.__rmul__(n) <==> n*x
+ +
__setitem__(...)
x.__setitem__(i, y) <==> x[i]=y
+ +
__str__(...)
x.__str__() <==> str(x)
+ +
append(...)
Append element at the end of the list
+ +
appendleft(...)
Append element at the beginning of the list
+ +
appendright(...)
Append element at the end of the list
+ +
clear(...)
Remove all elements from the list
+ +
extend(...)
Append elements from iterable at the right side of the list
+ +
extendleft(...)
Append elements from iterable at the left side of the list
+ +
extendright(...)
Append elements from iterable at the right side of the list
+ +
index(...)
Returns the first index of a value
+ +
insertafter(...)
Inserts element after node
+ +
insertbefore(...)
Inserts element before node
+ +
nodeat(...)
Return node at index
+ +
pop(...)
Remove an element by index from the list and return it, or last item if no index provided
+ +
popleft(...)
Remove first element from the list and return it
+ +
popright(...)
Remove last element from the list and return it
+ +
remove(...)
Remove element from the list
+ +
rindex(...)
Returns the last index of a value
+ +
rotate(...)
Rotate the list n steps to the right
+ +
+Data descriptors defined here:
+
first
+
First node
+
+
last
+
Next node
+
+
size
+
size
+
+
+Data and other attributes defined here:
+
__new__ = <built-in method __new__ of type object>
T.__new__(S, ...) -> a new object with type S, a subtype of T
+ +
+ \ No newline at end of file diff --git a/pydocs/index.html b/pydocs/index.html new file mode 120000 index 0000000..f7bbc6f --- /dev/null +++ b/pydocs/index.html @@ -0,0 +1 @@ +cllist.html \ No newline at end of file From 0b6b64965b761c70ba45e39d1b8e1a5271e88236 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 10:25:08 -0400 Subject: [PATCH 60/62] Fixup setup.py --- setup.py | 81 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/setup.py b/setup.py index 02998be..f7c3d48 100755 --- a/setup.py +++ b/setup.py @@ -3,41 +3,58 @@ VERSION='1.0.0' -from distutils.core import setup, Extension +import os + +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension sources = ['src/cllist.c', 'src/dllist.c', 'src/sllist.c', ] -setup(name='llist', - description='Linked list data structures for Python', - long_description=open('README.rst').read(), - author='Timothy Savannah, Adam Jakubek, Rafał Gałczyński', - author_email='kata198@gmail.com, ajakubek@gmail.com, rafal.galczynski@gmail.com', - maintainer_email='kata198@gmail.com', - version=VERSION, - url='https://github.com/kata198/python-cllist', - download_url='http://pypi.python.org/pypi/cllist/%s' % VERSION, - license='MIT', - keywords=['llist', 'linked', 'double', 'single', 'linked list'], - ext_modules=[Extension('cllist', sources)], - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Programming Language :: C', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - ) +if __name__ == '__main__': + + dirName = os.path.dirname(__file__) + if dirName and os.getcwd() != dirName: + os.chdir(dirName) + + try: + with open('README.rst', 'rt') as f: + long_description = f.read() + except Exception as e: + long_description = 'Error reading description: ' + str(e) + + setup(name='llist', + description='C-implemented linked-list module for Python', + long_description=long_description, + author='Timothy Savannah, Adam Jakubek, Rafał Gałczyński', + author_email='kata198@gmail.com, ajakubek@gmail.com, rafal.galczynski@gmail.com', + maintainer_email='kata198@gmail.com', + version=VERSION, + url='https://github.com/kata198/python-cllist', + download_url='http://pypi.python.org/pypi/cllist/%s' % VERSION, + license='MIT', + keywords=['llist', 'linked', 'double', 'single', 'linked list'], + ext_modules=[Extension('cllist', sources)], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Programming Language :: C', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + ) From 11562921d6bd5ca02519be4e7210bf2535c43819 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 10:35:15 -0400 Subject: [PATCH 61/62] Add my name as an author --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index b09b4b4..5d988a3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,7 @@ .. module:: llist :synopsis: Linked list datatypes for Python +.. moduleauthor:: Timothy Savannah .. moduleauthor:: Adam Jakubek .. moduleauthor:: Rafał Gałczyński From 1fe681080330b9120521b7dde5f5112c89b4ffa8 Mon Sep 17 00:00:00 2001 From: Tim Savannah Date: Wed, 12 Apr 2017 10:36:35 -0400 Subject: [PATCH 62/62] Update READMEs --- README.md | 65 +++++++++++++++++++++++++++++++++++------------------- README.rst | 65 +++++++++++++++++++++++++++++++++++------------------- 2 files changed, 84 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 36da05b..91bd042 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,55 @@ -llist - linked lists for CPython -================================ +cllist - C-implemented Linked Lists for Python +============================================== -llist is an extension module for CPython providing basic linked list +cllist is an extension module for CPython providing basic linked list data structures. + Collections implemented in the llist module perform well in problems -which rely on fast insertions and/or deletions of elements in +which rely on fast insertions, pops, and removal of elements in the middle of a sequence. + For this kind of workload, they can be significantly faster than collections.deque or standard Python lists. -This extension requires CPython 2.5 or newer (3.x is supported). -If you are looking for an implementation of linked lists in pure Python, -visit http://github.com/rgsoda/pypy-llist/ -The pypy-llist module has the same API as this extension, but is -significantly slower in CPython. -Currently llist provides the following types of linked lists: - - dllist - a doubly linked list - - sllist - a singly linked list +This project was forked from https://github.com/ajakubek/python-llist + +and adds many features and enhancements to the original, under a new name "cllist". + +The cllist version now lives at https://github.com/kata198/python-cllist + +If you were using the previous module, you can change to this new module, it is completely backwards (but not forwards) compatible. + + + +Single Linked List +------------------ + +Singly linked lists are provided by the "sllist" module. This is your basic single-linked list, and might be useful for some scenarios. + +A single linked list is far less efficient at everything than the double-linked list implementation. + + +Double Linked List +------------------ + +A double-linked list is provided by the "dllist" module. + +This provides great performance when doing pops and insertions at random (in the middle), or at either end. + +This implementation has been enhanced by implementing a "middle" marker. + +This "middle" marker is used when the list size exceeds 10 elements, and is used in all operations which involve walking the list, + +which ensures that AT MOST N/4 elements will need to be walked (so the shortest distance from either start, middle, or end is calculated, and walked from there). + +This additional feature makes this linked list much more efficient on larger data sets than a standard double-linked list implementation. -Full documentation of these classes is available at: -http://packages.python.org/llist/ -To install this package, either run "pip install llist", -or download it manually from http://pypi.python.org/pypi -then unpack the sources and compile them with "python setup.py install". +Why use a linked list? +---------------------- -The most current development version is available at: -https://github.com/ajakubek/python-llist/ +A linked list should be used when you are going to be adding or removing elements in the middle of the dataset. A standard python list forces the entire list +to be reallocated and copied when such happens, whereas this can do so without reallocating. -Bugs can be reported at: -https://github.com/ajakubek/python-llist/issues -This software is distributed under the MIT license. -Please see the LICENSE file included in the package for details. diff --git a/README.rst b/README.rst index 36da05b..91bd042 100644 --- a/README.rst +++ b/README.rst @@ -1,36 +1,55 @@ -llist - linked lists for CPython -================================ +cllist - C-implemented Linked Lists for Python +============================================== -llist is an extension module for CPython providing basic linked list +cllist is an extension module for CPython providing basic linked list data structures. + Collections implemented in the llist module perform well in problems -which rely on fast insertions and/or deletions of elements in +which rely on fast insertions, pops, and removal of elements in the middle of a sequence. + For this kind of workload, they can be significantly faster than collections.deque or standard Python lists. -This extension requires CPython 2.5 or newer (3.x is supported). -If you are looking for an implementation of linked lists in pure Python, -visit http://github.com/rgsoda/pypy-llist/ -The pypy-llist module has the same API as this extension, but is -significantly slower in CPython. -Currently llist provides the following types of linked lists: - - dllist - a doubly linked list - - sllist - a singly linked list +This project was forked from https://github.com/ajakubek/python-llist + +and adds many features and enhancements to the original, under a new name "cllist". + +The cllist version now lives at https://github.com/kata198/python-cllist + +If you were using the previous module, you can change to this new module, it is completely backwards (but not forwards) compatible. + + + +Single Linked List +------------------ + +Singly linked lists are provided by the "sllist" module. This is your basic single-linked list, and might be useful for some scenarios. + +A single linked list is far less efficient at everything than the double-linked list implementation. + + +Double Linked List +------------------ + +A double-linked list is provided by the "dllist" module. + +This provides great performance when doing pops and insertions at random (in the middle), or at either end. + +This implementation has been enhanced by implementing a "middle" marker. + +This "middle" marker is used when the list size exceeds 10 elements, and is used in all operations which involve walking the list, + +which ensures that AT MOST N/4 elements will need to be walked (so the shortest distance from either start, middle, or end is calculated, and walked from there). + +This additional feature makes this linked list much more efficient on larger data sets than a standard double-linked list implementation. -Full documentation of these classes is available at: -http://packages.python.org/llist/ -To install this package, either run "pip install llist", -or download it manually from http://pypi.python.org/pypi -then unpack the sources and compile them with "python setup.py install". +Why use a linked list? +---------------------- -The most current development version is available at: -https://github.com/ajakubek/python-llist/ +A linked list should be used when you are going to be adding or removing elements in the middle of the dataset. A standard python list forces the entire list +to be reallocated and copied when such happens, whereas this can do so without reallocating. -Bugs can be reported at: -https://github.com/ajakubek/python-llist/issues -This software is distributed under the MIT license. -Please see the LICENSE file included in the package for details.