Changeset 234

Timestamp:

01/09/11 19:37:51 (2 years ago)

Author:

mbutscher

Message:

branches/stable-2.0:
* Bug fixed: Windows: "Invalid handle" error on

command line bridge insertion plugins

branches/mbutscher/work:
* Write last writing program version into wiki db
* Several bug fixes with whoosh (index search)
* Index search: Highlight found terms and jump to

one found term on double-click

* Bug fixed: Windows: "Invalid handle" error on

command line bridge insertion plugins

Location:

branches

Files:

• mbutscher/work/extensions/GnuplotClBridge.py (modified) (1 diff)
• mbutscher/work/extensions/GraphvizClBridge.py (modified) (1 diff)
• mbutscher/work/extensions/GraphvizStructureView.py (modified) (1 diff)
• mbutscher/work/extensions/MimeTexCGIBridge.py (modified) (1 diff)
• mbutscher/work/extensions/PloticusClBridge.py (modified) (1 diff)
• mbutscher/work/lib/pwiki/DocPages.py (modified) (3 diffs)
• mbutscher/work/lib/pwiki/PersonalWikiFrame.py (modified) (1 diff)
• mbutscher/work/lib/pwiki/SearchAndReplace.py (modified) (3 diffs)
• mbutscher/work/lib/pwiki/SearchAndReplaceDialogs.py (modified) (6 diffs)
• mbutscher/work/lib/pwiki/wikidata/WikiDataManager.py (modified) (6 diffs)
• mbutscher/work/lib/pwiki/wikidata/compact_sqlite/DbStructure.py (modified) (2 diffs)
• mbutscher/work/lib/pwiki/wikidata/original_gadfly/DbStructure.py (modified) (2 diffs)
• mbutscher/work/lib/pwiki/wikidata/original_sqlite/DbStructure.py (modified) (2 diffs)
• mbutscher/work/lib/whoosh/analysis.py (modified) (22 diffs)
• mbutscher/work/lib/whoosh/fields.py (modified) (2 diffs)
• mbutscher/work/lib/whoosh/filedb/fileindex.py (modified) (2 diffs)
• mbutscher/work/lib/whoosh/filedb/filepostings.py (modified) (1 diff)
• mbutscher/work/lib/whoosh/formats.py (modified) (1 diff)
• mbutscher/work/lib/whoosh/highlight.py (modified) (7 diffs)
• mbutscher/work/lib/whoosh/lang/phonetic.py (added)
• mbutscher/work/lib/whoosh/lang/porter2.py (modified) (4 diffs)
• mbutscher/work/lib/whoosh/matching.py (modified) (4 diffs)
• mbutscher/work/lib/whoosh/qparser/common.py (added)
• mbutscher/work/lib/whoosh/qparser/default.py (modified) (4 diffs)
• mbutscher/work/lib/whoosh/qparser/plugins.py (added)
• mbutscher/work/lib/whoosh/qparser/syntax.py (added)
• mbutscher/work/lib/whoosh/query.py (modified) (13 diffs)
• mbutscher/work/lib/whoosh/reading.py (modified) (1 diff)
• mbutscher/work/lib/whoosh/scoring.py (modified) (1 diff)
• mbutscher/work/lib/whoosh/searching.py (modified) (3 diffs)
• mbutscher/work/lib/whoosh/support/bench.py (modified) (7 diffs)
• mbutscher/work/lib/whoosh/support/unicode.py (modified) (2 diffs)
• stable-2.0/extensions/GnuplotClBridge.py (modified) (1 diff)
• stable-2.0/extensions/GraphvizClBridge.py (modified) (1 diff)
• stable-2.0/extensions/GraphvizStructureView.py (modified) (1 diff)
• stable-2.0/extensions/MimeTexCGIBridge.py (modified) (1 diff)
• stable-2.0/extensions/PloticusClBridge.py (modified) (1 diff)

branches/mbutscher/work/extensions/GnuplotClBridge.py

@@ -117 +117 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
             
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
+
             if u"noerror" in [a.strip() for a in insToken.appendices]:
                 childErr.read()

branches/mbutscher/work/extensions/GraphvizClBridge.py

@@ -124 +124 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
+
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
 
             if u"noerror" in [a.strip() for a in insToken.appendices]:

branches/mbutscher/work/extensions/GraphvizStructureView.py

@@ -342 +342 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
+
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
 
             if u"noerror" in [a.strip() for a in insParams]:

branches/mbutscher/work/extensions/MimeTexCGIBridge.py

@@ -103 +103 @@
         # Run MimeTeX process
         popenObject = subprocess.Popen(cmdline, shell=True,
-                stdout=subprocess.PIPE)
+                 stdout=subprocess.PIPE, stdin=subprocess.PIPE,
+                 stderr=subprocess.PIPE)
+
         childOut = popenObject.stdout
+        
+        # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+        # why this is necessary
+        popenObject.stdin.close()
+        popenObject.stderr.close()
 
         # Read stdout of process entirely

branches/mbutscher/work/extensions/PloticusClBridge.py

@@ -128 +128 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
-            
+
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
+
             if u"noerror" in [a.strip() for a in insToken.appendices]:
                 childErr.read()

branches/mbutscher/work/lib/pwiki/DocPages.py

@@ -1 +1 @@
 from __future__ import with_statement
-
-# import profilehooks
-# profile = profilehooks.profile(filename="profile.prf", immediate=False)
+## import profilehooks
+## profile = profilehooks.profile(filename="profile.prf", immediate=False)
 
 
@@ -27 +26 @@
 
 import Serialization
-# from Serialization import SerializeStream
 
 
@@ -1736 +1734 @@
         return valid
 
-    
+
     def putIntoSearchIndex(self, threadstop=DUMBTHREADSTOP):
         """
-        Add or update the reverse index for the given docPage
+        Add or update the index for the given docPage
         """
         with self.textOperationLock:

branches/mbutscher/work/lib/pwiki/PersonalWikiFrame.py

@@ -2 +2 @@
 from __future__ import with_statement
 
-## import hotshot
-## _prof = hotshot.Profile("hotshot.prf")
 ## import profilehooks
 ## profile = profilehooks.profile(filename="profile.prf", immediate=False)

branches/mbutscher/work/lib/pwiki/SearchAndReplace.py

@@ -1697 +1697 @@
         """
         if self.searchStr == u"":
+            self.searchOpTree = AllWikiPagesNode(self)
+            return
+        
+        if self.indexSearch != "no":
+            # Search tree not used, but non-None value needed
             self.searchOpTree = AllWikiPagesNode(self)
             return
@@ -1808 +1813 @@
 
 
+    def getWhooshIndexQuery(self, wikiDocument):
+        from whoosh.qparser import QueryParser
+
+        qp = QueryParser("content", schema=wikiDocument.getWhooshIndexSchema())
+        q = qp.parse(self.searchStr)
+#         print "--getWhooshIndexQuery10", repr((qp, q))
+
+        return q
+
+
+    def hasWhooshHighlighting(self):
+        """
+        Return True iff call to highlightWhooshIndexFound() would work.
+        """
+        return self.indexSearch == "default"
+
+
+    def highlightWhooshIndexFound(self, content, docPage, before, after,
+            formatter=None):
+        """
+        Retrieve formatted output with highlighted search hits for a page.
+        formatter -- whoosh formatter or None (uses SimpleHtmlFormatter then)
+        """
+        if docPage is None:
+            return
+        
+        from whoosh import highlight
+
+        # TODO: Loop invariant, move out?
+        q = self.getWhooshIndexQuery(docPage.getWikiDocument())
+        
+        # Extract the terms the user mentioned
+        terms = [text for fieldname, text in q.all_terms()
+                if fieldname == "content"]
+        
+        analyzer = docPage.getWikiDocument().getWhooshIndexContentAnalyzer()
+        
+        # TODO: Length of before and after from config
+        fragmenter = highlight.ContextFragmenter(terms, (before + after) * 2,
+                before, after)
+
+        if formatter is None:
+            formatter = highlight.SimpleHtmlFormatter()
+        
+        return highlight.highlight(content, terms, analyzer,
+                     fragmenter, formatter, top=1)
+
+
     def hasParticularTextPosition(self):
         if self.indexSearch != "no":
@@ -1855 +1908 @@
         if self.searchOpTree is None:
             self.rebuildSearchOpTree()
-        
+
         if commonCache is None:
             commonCache = {}
-            
+
         self.listWikiPagesOp.beginWikiSearch(wikiDocument,
                 commonCache=commonCache)
-            
+
         return self.searchOpTree.beginWikiSearch(wikiDocument,
                 commonCache=commonCache)

branches/mbutscher/work/lib/pwiki/SearchAndReplaceDialogs.py

@@ -1 @@
-## import hotshot
-## _prof = hotshot.Profile("hotshot.prf")
+# import profilehooks
+# profile = profilehooks.profile(filename="profile.prf", immediate=False)
 
 import sys, traceback, re, threading, time
@@ -80 +80 @@
         self.occNumber = occNumber
         return self
+
+
+    def setHtmlDirectly(self, occHtml):
+        self.occNumber = -1
+        self.occCount = -1
+        self.occHtml = occHtml
+
 
 
@@ -220 +227 @@
                 context = before + after
 
-                if not sarOp.hasParticularTextPosition():
-                    # No specific position to show as context, so show beginning of page
-                    # Also, no occurrence counting possible
-                    if context == 0:
+                if sarOp.hasParticularTextPosition():
+                    if context == 0 and not countOccurrences:
+                        # No context, no occurrence counting
+                        # -> just a list of found pages
                         self.foundinfo = [_SearchResultItemInfo(w) for w in found]
                     else:
-                        for w in found:
-                            text = wikiDocument.getWikiPageNoError(w).\
-                                    getLiveTextNoTemplate()
-                            if text is None:
-                                continue
-                            self.foundinfo.append(
-                                    _SearchResultItemInfo(w).buildOccurrence(
-                                    text, before, after, (-1, -1), -1, 100))
-                    threadstop.testRunning()
-                else:
-                    if context == 0 and not countOccurrences:
-                        # No context, no occurrence counting
-                        self.foundinfo = [_SearchResultItemInfo(w) for w in found]
-                    else:
+                        # "As is" or regex search
                         sarOp.beginWikiSearch(self.pWiki.getWikiDocument())
                         try:
@@ -270 +264 @@
                                 info = _SearchResultItemInfo(w, occPos=pos,
                                         maxOccCount=maxCountOccurrences)
-
+    
                                 if countOccurrences:
                                     occ = 1
@@ -290 +284 @@
                         finally:
                             sarOp.endWikiSearch()
-
+                elif sarOp.hasWhooshHighlighting():
+                    # Index search
+                    if context == 0:
+                        # No context, occurrence counting doesn't matter
+                        # -> just a list of found pages
+                        self.foundinfo = [_SearchResultItemInfo(w) for w in found]
+                    else:
+                        sarOp.beginWikiSearch(self.pWiki.getWikiDocument())
+                        try:
+                            for w in found:
+                                threadstop.testRunning()
+                                docPage = wikiDocument.getWikiPageNoError(w)
+                                text = docPage.getLiveTextNoTemplate()
+                                if text is None:
+                                    continue
+    
+                                html, firstPos = sarOp.highlightWhooshIndexFound(
+                                        text, docPage, before, after)
+                                
+                                info = _SearchResultItemInfo(w, occPos=(firstPos, firstPos))
+                                info.setHtmlDirectly(html)
+    
+                                self.foundinfo.append(info)
+                        finally:
+                            sarOp.endWikiSearch()
+                else:  # not sarOp.hasParticularTextPosition():
+                    # No specific position to show as context, so show beginning of page
+                    # Also, no occurrence counting possible
+                    if context == 0:
+                        self.foundinfo = [_SearchResultItemInfo(w) for w in found]
+                    else:
+                        for w in found:
+                            text = wikiDocument.getWikiPageNoError(w).\
+                                    getLiveTextNoTemplate()
+                            if text is None:
+                                continue
+                            self.foundinfo.append(
+                                    _SearchResultItemInfo(w).buildOccurrence(
+                                    text, before, after, (-1, -1), -1, 100))
+                    threadstop.testRunning()
+                
                 threadstop.testRunning()
                 self.isShowingSearching = False
@@ -1093 +1127 @@
 
 
+#     @profile
     def _refreshPageList(self):
         sarOp = self._buildSearchReplaceOperation()

branches/mbutscher/work/lib/pwiki/wikidata/WikiDataManager.py

@@ -1 +1 @@
 from __future__ import with_statement
+
 
 from weakref import WeakValueDictionary
@@ -1291 +1292 @@
             
             if self.isSearchIndexEnabled():
-                # Step four: update reverse index
+                # Step four: update index
                 for wikiWord in wikiWords:
                     progresshandler.update(step, _(u"Update index of %s") % wikiWord)
@@ -1596 +1597 @@
             return result
         else:
-            # Processing reverse index search
-            from whoosh.qparser import QueryParser
-            
+            # Processing index search
             threadstop.testRunning()
             if not self.isSearchIndexEnabled():
                 return []
 
-            qp = QueryParser("content", schema=self._getSearchIndexSchema())
-            q = qp.parse(sarOp.searchStr)
+            q = sarOp.getWhooshIndexQuery(self)
             s = self.getSearchIndex().searcher()
             threadstop.testRunning()
-            resultList = s.search(q)
+            resultList = s.search(q, limit=None)
+            
+#             docnumList = [(rd.docnum, rd["unifName"]) for rd in resultList]
+#             
+#             docnumList.sort()
+#             docpp = "\n".join(["%3i %s" % rd for rd in docnumList])
+# 
+#             print "--docResults"
+#             print docpp.encode("mbcs", "replace")
+
             result = [rd["unifName"][9:] for rd in resultList
                     if rd["unifName"].startswith(u"wikipage/")]
@@ -1614 +1621 @@
             return result
 
-    
+
+    @staticmethod
+    def getWhooshIndexContentAnalyzer():
+        from whoosh.analysis import StandardAnalyzer        
+        return StandardAnalyzer(stoplist=None)
+
+
+
     _REV_SEARCH_INDEX_SCHEMA = None
     
     @staticmethod
-    def _getSearchIndexSchema():
+    def getWhooshIndexSchema():
         if WikiDataManager._REV_SEARCH_INDEX_SCHEMA is None:
             from whoosh.fields import Schema, ID, NUMERIC, TEXT
-            from whoosh.analysis import StandardAnalyzer
             
             WikiDataManager._REV_SEARCH_INDEX_SCHEMA = Schema(
                     unifName=ID(stored=True, unique=True),
                     modTimestamp=NUMERIC(), content=TEXT(
-                    analyzer=StandardAnalyzer(stoplist=None)))
+                    analyzer=WikiDataManager.getWhooshIndexContentAnalyzer()))
 
         return WikiDataManager._REV_SEARCH_INDEX_SCHEMA
+    
+    
 
 
@@ -1687 +1702 @@
 
             if clear or not whoosh.index.exists_in(indexPath):
-                schema = self._getSearchIndexSchema()
+                schema = self.getWhooshIndexSchema()
                 whoosh.index.create_in(indexPath, schema)
 
@@ -1741 +1756 @@
 
 
-
-    def putIntoSearchIndex(self, wikiPage):
-        """
-        Add or update the reverse index for the given docPage
-        """
-        if not self.isSearchIndexEnabled():
-            return
-
-        if isinstance(wikiPage, AliasWikiPage):
-            wikiPage = WikiPage(self, wikiWord)
-
-        content = wikiPage.getLiveText()
-
-        try:
-            searchIdx = self.getSearchIndex()
-            writer = searchIdx.writer()
-            
-            unifName = wikiPage.getUnifiedPageName()
-            
-            writer.delete_by_term("unifName", unifName)
-            writer.add_document(unifName=unifName,
-                    modTimestamp=wikiPage.getTimestamps()[0],
-                    content=content)
-        except:
-            writer.cancel()
-            raise
-
-        writer.commit()
-        
-    
     def removeFromSearchIndex(self, unifName):
         if not self.isSearchIndexEnabled():

branches/mbutscher/work/lib/pwiki/wikidata/compact_sqlite/DbStructure.py

@@ -11 +11 @@
 from os.path import exists, join
 
+import Consts
 from pwiki.WikiExceptions import *
 from pwiki.StringOps import mbcsDec, mbcsEnc, utf8Enc, utf8Dec, applyBinCompact, \
@@ -1284 +1285 @@
         connwrap.execSql("insert or replace into settings(key, value) "
                 "values ('lastwritever', '"+str(VERSION_DB)+"')")
+                
+        # Write which program version at last wrote to database
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.branchtag', '"+Consts.VERSION_TUPLE[0]+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.major', '"+str(Consts.VERSION_TUPLE[1])+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.minor', '"+str(Consts.VERSION_TUPLE[2])+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.sub', '"+str(Consts.VERSION_TUPLE[3])+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.patch', '"+str(Consts.VERSION_TUPLE[4])+"')")
     except sqlite.ReadOnlyDbError:
         pass

branches/mbutscher/work/lib/pwiki/wikidata/original_gadfly/DbStructure.py

@@ -12 +12 @@
 import glob
 
+import Consts
 from pwiki.WikiExceptions import *
 from pwiki.StringOps import mbcsDec, mbcsEnc, utf8Enc, utf8Dec, \
@@ -1130 +1131 @@
     try:
         setSettingsValue(connwrap, "lastwritever", str(VERSION_DB))
+        
+        # Write which program version at last wrote to database
+        setSettingsValue(connwrap, "lastwriteprogver.branchtag", Consts.VERSION_TUPLE[0])
+        setSettingsValue(connwrap, "lastwriteprogver.major", str(Consts.VERSION_TUPLE[1]))
+        setSettingsValue(connwrap, "lastwriteprogver.minor", str(Consts.VERSION_TUPLE[2]))
+        setSettingsValue(connwrap, "lastwriteprogver.sub", str(Consts.VERSION_TUPLE[3]))
+        setSettingsValue(connwrap, "lastwriteprogver.patch", str(Consts.VERSION_TUPLE[4]))
     except IOError:
         pass

branches/mbutscher/work/lib/pwiki/wikidata/original_sqlite/DbStructure.py

@@ -11 +11 @@
 from os.path import exists, join
 
+import Consts
 from pwiki.WikiExceptions import *
 from pwiki.StringOps import mbcsDec, mbcsEnc, utf8Enc, utf8Dec, applyBinCompact, \
@@ -995 +996 @@
     """
     try:
-        # Write which version at last wrote to database
+        # Write which format version at last wrote to database
         connwrap.execSql("insert or replace into settings(key, value) "
                 "values ('lastwritever', '"+str(VERSION_DB)+"')")
+
+        # Write which program version at last wrote to database
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.branchtag', '"+Consts.VERSION_TUPLE[0]+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.major', '"+str(Consts.VERSION_TUPLE[1])+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.minor', '"+str(Consts.VERSION_TUPLE[2])+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.sub', '"+str(Consts.VERSION_TUPLE[3])+"')")
+        connwrap.execSql("insert or replace into settings(key, value) "
+                "values ('lastwriteprogver.patch', '"+str(Consts.VERSION_TUPLE[4])+"')")
     except sqlite.ReadOnlyDbError:
         pass

branches/mbutscher/work/lib/whoosh/analysis.py

@@ -54 +54 @@
 """
 
+import copy, re
 from array import array
-import copy, re
+from collections import deque
 from itertools import chain
 
@@ -83 +84 @@
 # Token object
 
+
+# Mine:
 class Token(object):
+    __slots__ = (
+        "positions",
+        "chars",
+        "stopped",
+        "boost",
+        "removestops",
+        "mode",
+
+        "startchar",
+        "endchar",
+        "text",
+        "original",
+
+        "__dict__"
+    )
+
+
     """
     Represents a "token" (usually a word) extracted from the source text being
@@ -113 +133 @@
     
     def __init__(self, positions=False, chars=False, removestops=True, mode='',
-                 **kwargs):
+                 stopped=False, boost=1.0, startchar=None, endchar=None,
+                 text=None, original=None, **kwargs):
         """
         :param positions: Whether tokens should have the token position in the
@@ -123 +144 @@
         :param mode: contains a string describing the purpose for which the
             analyzer is being called, i.e. 'index' or 'query'.
-        """
-        
+
+            Do not modify the parameters after mode. They are needed for
+            copying only
+        """
         self.positions = positions
         self.chars = chars
-        self.stopped = False
-        self.boost = 1.0
         self.removestops = removestops
         self.mode = mode
-        self.__dict__.update(kwargs)
-    
-    def __repr__(self):
+
+        self.stopped = stopped
+        self.boost = boost
+        
+        self.startchar = startchar
+        self.endchar = endchar
+        self.text = text
+        self.original = original
+
+        if kwargs:
+            self.__dict__.update(kwargs)
+
+    
+    def __repr__(self):   # TODO!!!
         parms = ", ".join("%s=%r" % (name, value)
                           for name, value in self.__dict__.iteritems())
@@ -139 +171 @@
         
     def copy(self):
-        return copy.copy(self)
+        return Token(positions=self.positions, chars=self.chars,
+                removestops=self.removestops, mode=self.mode,
+                stopped=self.stopped, boost=self.boost, startchar=self.startchar,
+                endchar=self.endchar, text=self.text, original=self.original,
+                **self.__dict__)
+
+
+
+# mchaput:
+# class Token(object):
+#     """
+#     Represents a "token" (usually a word) extracted from the source text being
+#     indexed.
+#     
+#     See "Advanced analysis" in the user guide for more information.
+#     
+#     Because object instantiation in Python is slow, tokenizers should create
+#     ONE SINGLE Token object and YIELD IT OVER AND OVER, changing the attributes
+#     each time.
+#     
+#     This trick means that consumers of tokens (i.e. filters) must never try to
+#     hold onto the token object between loop iterations, or convert the token
+#     generator into a list. Instead, save the attributes between iterations,
+#     not the object::
+#     
+#         def RemoveDuplicatesFilter(self, stream):
+#             # Removes duplicate words.
+#             lasttext = None
+#             for token in stream:
+#                 # Only yield the token if its text doesn't
+#                 # match the previous token.
+#                 if lasttext != token.text:
+#                     yield token
+#                 lasttext = token.text
+# 
+#     ...or, call token.copy() to get a copy of the token object.
+#     """
+#     
+#     def __init__(self, positions=False, chars=False, removestops=True, mode='',
+#                  **kwargs):
+#         """
+#         :param positions: Whether tokens should have the token position in the
+#             'pos' attribute.
+#         :param chars: Whether tokens should have character offsets in the
+#             'startchar' and 'endchar' attributes.
+#         :param removestops: whether to remove stop words from the stream (if
+#             the tokens pass through a stop filter).
+#         :param mode: contains a string describing the purpose for which the
+#             analyzer is being called, i.e. 'index' or 'query'.
+#         """
+#         
+#         self.positions = positions
+#         self.chars = chars
+#         self.stopped = False
+#         self.boost = 1.0
+#         self.removestops = removestops
+#         self.mode = mode
+#         self.__dict__.update(kwargs)
+#     
+#     def __repr__(self):
+#         parms = ", ".join("%s=%r" % (name, value)
+#                           for name, value in self.__dict__.iteritems())
+#         return "%s(%s)" % (self.__class__.__name__, parms)
+#         
+#     def copy(self):
+#         # This is faster than using the copy module
+#         return Token(**self.__dict__.copy())
+
+
+
+# mchaput modif:
+# class Token(object):
+#     """
+#     Represents a "token" (usually a word) extracted from the source text being
+#     indexed.
+#     
+#     See "Advanced analysis" in the user guide for more information.
+#     
+#     Because object instantiation in Python is slow, tokenizers should create
+#     ONE SINGLE Token object and YIELD IT OVER AND OVER, changing the attributes
+#     each time.
+#     
+#     This trick means that consumers of tokens (i.e. filters) must never try to
+#     hold onto the token object between loop iterations, or convert the token
+#     generator into a list. Instead, save the attributes between iterations,
+#     not the object::
+#     
+#         def RemoveDuplicatesFilter(self, stream):
+#             # Removes duplicate words.
+#             lasttext = None
+#             for token in stream:
+#                 # Only yield the token if its text doesn't
+#                 # match the previous token.
+#                 if lasttext != token.text:
+#                     yield token
+#                 lasttext = token.text
+# 
+#     ...or, call token.copy() to get a copy of the token object.
+#     """
+#     
+#     def __init__(self, positions=False, chars=False, removestops=True, mode='',
+#                  **kwargs):
+#         """
+#         :param positions: Whether tokens should have the token position in the
+#             'pos' attribute.
+#         :param chars: Whether tokens should have character offsets in the
+#             'startchar' and 'endchar' attributes.
+#         :param removestops: whether to remove stop words from the stream (if
+#             the tokens pass through a stop filter).
+#         :param mode: contains a string describing the purpose for which the
+#             analyzer is being called, i.e. 'index' or 'query'.
+#         """
+#         
+#         self.positions = positions
+#         self.chars = chars
+#         self.stopped = False
+#         self.boost = 1.0
+#         self.removestops = removestops
+#         self.mode = mode
+#         self.__dict__.update(kwargs)
+#     
+#     def __repr__(self):
+#         parms = ", ".join("%s=%r" % (name, value)
+#                           for name, value in self.__dict__.iteritems())
+#         return "%s(%s)" % (self.__class__.__name__, parms)
+#         
+#     def copy(self):
+#         # This is faster than using the copy module
+#         return Token(**self.__dict__)
+
+
+
 
 
@@ -184 +347 @@
         t = Token(positions, chars, removestops=removestops, mode=mode)
         t.text = value
+        t.boost=1.0
         if keeporiginal:
             t.original = value
@@ -203 +367 @@
     """
     
-    __inittypes__ = dict(expression=unicode, gaps=bool)
-    
-    def __init__(self, expression=r"\w+(\.?\w+)*", gaps=False):
+    __inittypes__ = dict(expression=unicode, gaps=bool, lowercase=bool)
+    
+    def __init__(self, expression=r"\w+(\.?\w+)*", gaps=False, lowercase=False):
         """
         :param expression: A regular expression object or string. Each match
@@ -214 +378 @@
             than matching on the expression.
         """
-        
+
         if isinstance(expression, basestring):
             self.expression = re.compile(expression, re.UNICODE)
@@ -220 +384 @@
             self.expression = expression
         self.gaps = gaps
+        
+        self.lowercase = lowercase
+        
     
     def __eq__(self, other):
@@ -245 +412 @@
         
         assert isinstance(value, unicode), "%r is not unicode" % value
-        
+
+        if self.lowercase:
+            lowervalue = value.lower()
+        else:
+            lowervalue = value
+
         t = Token(positions, chars, removestops=removestops, mode=mode)
         if not tokenize:
-            t.original = t.text = value
+            t.original = value
+            t.text = lowervalue
+            t.boost = 1.0
             if positions: t.pos = start_pos
             if chars:
@@ -256 +430 @@
         elif not self.gaps:
             # The default: expression matches are used as tokens
-            for pos, match in enumerate(self.expression.finditer(value)):
-                t.text = match.group(0)
+            for pos, match in enumerate(self.expression.finditer(lowervalue)):
+                ms = match.start()
+                me = match.end()
+                t.text = lowervalue[ms:me]  # match.group(0)
                 if keeporiginal:
-                    t.original = t.text
+                    t.original = value[ms:me]
                 t.stopped = False
                 if positions:
                     t.pos = start_pos + pos
                 if chars:
-                    t.startchar = start_char + match.start()
-                    t.endchar = start_char + match.end()
+                    t.startchar = start_char + ms
+                    t.endchar = start_char + me
                 yield t
         else:
@@ -272 +448 @@
             prevend = 0
             pos = start_pos
-            for match in self.expression.finditer(value):
+            for match in self.expression.finditer(lowervalue):
                 start = prevend
                 end = match.start()
-                text = value[start:end]
+                text = lowervalue[start:end]
                 if text:
                     t.text = text
+                    t.boost = 1.0
                     if keeporiginal:
-                        t.original = t.text
+                        t.original = value[start:end]
                     t.stopped = False
                     if positions:
@@ -296 +473 @@
             if prevend < len(value):
                 t.text = value[prevend:]
+                t.boost = 1.0
                 if keeporiginal:
                     t.original = t.text
@@ -365 +543 @@
         if not tokenize:
             t.original = t.text = value
+            t.boost = 1.0
             if positions: t.pos = start_pos
             if chars:
@@ -382 +561 @@
                     if currentchar > startchar:
                         t.text = text
+                        t.boost = 1.0
                         if keeporiginal:
                             t.original = t.text
@@ -398 +578 @@
             if currentchar > startchar:
                 t.text = value[startchar:currentchar]
+                t.boost = 1.0
                 if keeporiginal:
                     t.original = t.text
@@ -1169 +1350 @@
         "the-sign", "sign-of", "of-four"
         
-    This can be used in fields dedicated to phrase searching. In the example
-    above,  the three "bi-word" tokens will be faster to find than the four
-    original words since there are fewer of them and they will be much less
-    frequent (especially compared to words like "the" and "of").
+    This can be used to create fields for pseudo-phrase searching, where if
+    all the terms match the document probably contains the phrase, but the
+    searching is faster than actually doing a phrase search on individual word
+    terms.
+    
+    The ``BiWordFilter`` is much faster than using the otherwise equivalent
+    ``ShingleFilter(2)``.
     """
     
@@ -1213 +1397 @@
             if positions: prev_pos = ps
         
-        # If at no bi-words were emitted, that is, the token stream only had
+        # If no bi-words were emitted, that is, the token stream only had
         # a single token, then emit that single token.
         if not atleastone:
             yield token
         
+
+class ShingleFilter(Filter):
+    """Merges a certain number of adjacent tokens into multi-word tokens, so
+    that for example::
+    
+        "better", "a", "witty", "fool", "than", "a", "foolish", "wit"
+        
+    with ``ShingleFilter(3, ' ')`` becomes::
+    
+        'better a witty', 'a witty fool', 'witty fool than', 'fool than a',
+        'than a foolish', 'a foolish wit'
+    
+    This can be used to create fields for pseudo-phrase searching, where if
+    all the terms match the document probably contains the phrase, but the
+    searching is faster than actually doing a phrase search on individual word
+    terms.
+    
+    If you're using two-word shingles, you should use the functionally
+    equivalent ``BiWordFilter`` instead because it's faster than
+    ``ShingleFilter``.
+    """
+    
+    def __init__(self, size=2, sep="-"):
+        self.size = size
+        self.sep = sep
+        
+    def __call__(self, tokens):
+        size = self.size
+        sep = self.sep
+        buf = deque()
+        atleastone = False
+        
+        def make_token():
+            tk = buf[0]
+            tk.text = sep.join([t.text for t in buf])
+            if tk.chars:
+                tk.endchar = buf[-1].endchar
+            return tk
+        
+        for token in tokens:
+            buf.append(token.copy())
+            if len(buf) == size:
+                atleastone = True
+                yield make_token()
+                buf.popleft()
+        
+        # If no shingles were emitted, that is, the token stream had fewer than
+        # 'size' tokens, then emit a single token with whatever tokens there
+        # were
+        if not atleastone:
+            yield make_token()
+
 
 class BoostTextFilter(Filter):
@@ -1260 +1496 @@
 
 
-class DeliminatedAttributeFilter(Filter):
+class DelimitedAttributeFilter(Filter):
     """Looks for delimiter characters in the text of each token and stores the
     data after the delimiter in a named attribute on the token.
@@ -1323 +1559 @@
 
 class DoubleMetaphoneFilter(Filter):
-    def __init__(self, primary_boost=3.0):
+    """Transforms the text of the tokens using Lawrence Philips's Double
+    Metaphone algorithm. This algorithm attempts to encode words in such a way
+    that similar-sounding words reduce to the same code. This may be useful for
+    fields containing the names of people and places, and other uses where
+    tolerance of spelling differences is desireable.
+    """
+    
+    def __init__(self, primary_boost=1.0, secondary_boost=0.5, combine=False):
+        """
+        :param primary_boost: the boost to apply to the token containing the
+            primary code.
+        :param secondary_boost: the boost to apply to the token containing the
+            secondary code, if any.
+        :param combine: if True, the original unencoded tokens are kept in the
+            stream, preceding the encoded tokens.
+        """
+        
         self.primary_boost = primary_boost
+        self.secondary_boost = secondary_boost
+        self.combine = combine
         
     def __eq__(self, other):
@@ -1333 +1587 @@
     def __call__(self, tokens):
         primary_boost = self.primary_boost
+        secondary_boost = self.secondary_boost
+        combine = self.combine
         
         for t in tokens:
+            if combine:
+                yield t
+            
             primary, secondary = double_metaphone(t.text)
+            b = t.boost
+            # Overwrite the token's text and boost and yield it
             if primary:
-                # Save the original boost
-                b = t.boost
-                # Overwrite the token's text and boost and yield it
                 t.text = primary
                 t.boost = b * primary_boost
                 yield t
-                # Restored the original boost
-                t.boost = b 
             if secondary:
                 t.text = secondary
+                t.boost = b * secondary_boost
                 yield t
+                
+
+class SubstitutionFilter(Filter):
+    """Performas a regular expression substitution on the token text.
+    
+    This is especially useful for removing text from tokens, for example
+    hyphens::
+    
+        ana = RegexTokenizer(r"\\S+") | SubstitutionFilter("-", "")
+        
+    Because it has the full power of the re.sub() method behind it, this filter
+    can perform some fairly complex transformations. For example, to take tokens
+    like ``'a=b', 'c=d', 'e=f'`` and change them to ``'b=a', 'd=c', 'f=e'``::
+    
+        # Analyzer that swaps the text on either side of an equal sign
+        ana = RegexTokenizer(r"\\S+") | SubstitutionFilter("([^/]*)/(./*)", r"\\2/\\1")
+    """
+    
+    def __init__(self, pattern, replacement):
+        """
+        :param pattern: a pattern string or compiled regular expression object
+            describing the text to replace.
+        :param replacement: the substitution text.
+        """
+        
+        if isinstance(pattern, basestring):
+            pattern = re.compile(pattern, re.UNICODE)
+        self.pattern = pattern
+        self.replacement = replacement
+    
+    def __eq__(self, other):
+        return (other and self.__class__ is other.__class__
+                and self.pattern == other.pattern
+                and self.replacement == other.replacement)
+    
+    def __call__(self, tokens):
+        pattern = self.pattern
+        replacement = self.replacement
+        
+        for t in tokens:
+            t.text = pattern.sub(replacement, t.text)
+            yield t
+
 
 # Analyzers
@@ -1483 +1783 @@
     """
     
-    ret = RegexTokenizer(expression=expression, gaps=gaps)
-    chain = ret | LowercaseFilter()
+#     ret = RegexTokenizer(expression=expression, gaps=gaps)
+#     chain = ret | LowercaseFilter()
+    chain = RegexTokenizer(expression=expression, gaps=gaps, lowercase=True)
     if stoplist is not None:
         chain = chain | StopFilter(stoplist=stoplist, minsize=minsize,

branches/mbutscher/work/lib/whoosh/fields.py

@@ -65 +65 @@
       being updated.
       
+    * multitoken_query is a string indicating what kind of query to use when
+      a "word" in a user query parses into multiple tokens. The string is
+      interpreted by the query parser. The strings understood by the default
+      query parser are "first" (use first token only), "and" (join the tokens
+      with an AND query), "or" (join the tokens with OR), and "phrase" (join
+      the tokens with a phrase query).
+     
     The constructor for the base field type simply lets you supply your own
     configured field format, vector format, and scorable and stored values.
     Subclasses may configure some or all of this for you.
-    
     """
     
     format = vector = scorable = stored = unique = None
     indexed = True
+    multitoken_query = "first"
     __inittypes__ = dict(format=Format, vector=Format,
                          scorable=bool, stored=bool, unique=bool)
     
     def __init__(self, format, vector=None, scorable=False, stored=False,
-                 unique=False):
+                 unique=False, multitoken_query="first"):
         self.format = format
         self.vector = vector
@@ -83 +90 @@
         self.stored = stored
         self.unique = unique
+        self.multitoken_query = multitoken_query
     
     def __repr__(self):

branches/mbutscher/work/lib/whoosh/filedb/fileindex.py

@@ -144 +144 @@
     
     # Generation
-    stream.read_int()
+    index_gen = stream.read_int()
+    assert gen == index_gen
     
     segment_counter = stream.read_int()
@@ -156 +157 @@
 
 def _next_segment_name(self):
-        #Returns the name of the next segment in sequence.
-        if self.segment_num_lock is None:
-            self.segment_num_lock = Lock()
-        
-        if self.segment_num_lock.acquire():
-            try:
-                self.segment_counter += 1
-                return 
-            finally:
-                self.segment_num_lock.release()
-        else:
-            raise LockError
+    #Returns the name of the next segment in sequence.
+    if self.segment_num_lock is None:
+        self.segment_num_lock = Lock()
+    
+    if self.segment_num_lock.acquire():
+        try:
+            self.segment_counter += 1
+            return 
+        finally:
+            self.segment_num_lock.release()
+    else:
+        raise LockError
 
 
 def _clean_files(storage, indexname, gen, segments):
-        # Attempts to remove unused index files (called when a new generation
-        # is created). If existing Index and/or reader objects have the files
-        # open, they may not be deleted immediately (i.e. on Windows) but will
-        # probably be deleted eventually by a later call to clean_files.
-
-        current_segment_names = set(s.name for s in segments)
-
-        tocpattern = _toc_pattern(indexname)
-        segpattern = _segment_pattern(indexname)
-
-        todelete = set()
-        for filename in storage:
-            tocm = tocpattern.match(filename)
-            segm = segpattern.match(filename)
-            if tocm:
-                if int(tocm.group(1)) != gen:
-                    todelete.add(filename)
-            elif segm:
-                name = segm.group(1)
-                if name not in current_segment_names:
-                    todelete.add(filename)
-        
-        for filename in todelete:
-            try:
-                storage.delete_file(filename)
-            except OSError:
-                # Another process still has this file open
-                pass
+    # Attempts to remove unused index files (called when a new generation
+    # is created). If existing Index and/or reader objects have the files
+    # open, they may not be deleted immediately (i.e. on Windows) but will
+    # probably be deleted eventually by a later call to clean_files.
+
+    current_segment_names = set(s.name for s in segments)
+
+    tocpattern = _toc_pattern(indexname)
+    segpattern = _segment_pattern(indexname)
+
+    todelete = set()
+    for filename in storage:
+        tocm = tocpattern.match(filename)
+        segm = segpattern.match(filename)
+        if tocm:
+            if int(tocm.group(1)) != gen:
+                todelete.add(filename)
+        elif segm:
+            name = segm.group(1)
+            if name not in current_segment_names:
+                todelete.add(filename)
+    
+    for filename in todelete:
+        try:
+            storage.delete_file(filename)
+        except OSError:
+            # Another process still has this file open
+            pass
 
 

branches/mbutscher/work/lib/whoosh/filedb/filepostings.py

@@ -334 +334 @@
         self.stringids = stringids
         
-        assert postfile.get_int(offset) == -48626
+        magic = postfile.get_int(offset)
+        assert magic == -48626
+        
         self.blockcount = postfile.get_uint(offset + _INT_SIZE)
         self.baseoffset = offset + _INT_SIZE * 2

branches/mbutscher/work/lib/whoosh/formats.py

@@ -187 +187 @@
         for t in unstopped(self.analyzer(value, boosts=True, **kwargs)):
             freqs[t.text] += 1
-            weights[t.text] += int(t.boost)
+            weights[t.text] += t.boost
         
         encode = self.encode

branches/mbutscher/work/lib/whoosh/highlight.py

@@ -22 +22 @@
 from __future__ import division
 from heapq import nlargest
-from cgi import escape as htmlescape
+from collections import deque
+
+# from cgi import escape as htmlescape
+from pwiki.StringOps import escapeHtml as htmlescape
 
 # Fragment object
@@ -187 +190 @@
     """
     
-    def __init__(self, termset, maxchars=200, surround=20):
+    def __init__(self, termset, maxchars=200, charsbefore=20, charsafter=20):
         """
         :param termset: A collection (probably a set or frozenset) containing
@@ -193 +196 @@
         :param maxchars: The maximum number of characters allowed in a
             fragment.
-        :param surround: The number of extra characters of context to add both
-            before the first matched term and after the last matched term.
-        """
-        
+        :param charsbefore: The number of extra characters of context to add
+            before the first matched term .
+        :param charsafter: The number of extra characters of context to add
+            after the last matched term.
+        """
         self.maxchars = maxchars
-        self.charsbefore = self.charsafter = surround
+        self.charsbefore = charsbefore
+        self.charsafter = charsafter
+
     
     def __call__(self, text, tokens):
@@ -205 +211 @@
         charsafter = self.charsafter
         
-        current = []
+        current = deque()
         currentlen = 0
         countdown = -1
@@ -222 +228 @@
                 if countdown < 0 or currentlen >= maxchars:
                     yield Fragment(current)
-                    current = []
+                    current.clear()
                     currentlen = 0
-            
+
             else:
-                while current and currentlen > charsbefore:
-                    t = current.pop(0)
+#                 while current and currentlen > charsbefore: len(current) can't be 0 if currentlen > 0 and
+#                         charsbefore should be >= 0
+                while currentlen > charsbefore:
+                    t = current.popleft()
                     currentlen -= t.endchar - t.startchar
 
@@ -340 +348 @@
 
 
+class SimpleHtmlFormatter(object):
+    """Returns a string in which the matched terms are enclosed in <b></b>.
+    """
+    
+    def __init__(self, between=u"... "):
+        """
+        :param between: the text to add between fragments.
+        """
+        self.between = between
+        self.firstPos = -1
+        
+    def _format_fragment(self, text, fragment):
+        output = []
+        index = fragment.startchar
+        
+        for t in fragment.matches:
+            if t.startchar > index:
+                output.append(htmlescape(text[index:t.startchar]))
+
+            ttxt = htmlescape(text[t.startchar:t.endchar])
+            if t.matched:
+                ttxt = "<b>%s</b>" % ttxt
+                if self.firstPos == -1:
+                    self.firstPos = t.startchar
+                else:
+                    self.firstPos = min(self.firstPos, t.startchar)
+
+            output.append(ttxt)
+            index = t.endchar
+        
+        output.append(htmlescape(text[index:fragment.endchar]))
+        return u"".join(output)
+
+    def __call__(self, text, fragments):
+        return self.between.join([self._format_fragment(text, fragment)
+                                  for fragment in fragments]), self.firstPos
+
+
+
+
 class HtmlFormatter(object):
     """Returns a string containing HTML formatting around the matched terms.
@@ -428 +476 @@
 
 
-class GenshiFormatter(object):
-    """Returns a Genshi event stream containing HTML formatting around the
-    matched terms.
-    """
-    
-    def __init__(self, qname="strong", between="..."):
-        """
-        :param qname: the QName for the tag to wrap around matched terms.
-        :param between: the text to add between fragments.
-        """
-        
-        self.qname = qname
-        self.between = between
-        
-        from genshi.core import START, END, TEXT, Attrs, Stream #@UnresolvedImport
-        self.START, self.END, self.TEXT = START, END, TEXT
-        self.Attrs, self.Stream = Attrs, Stream
-
-    def _add_text(self, text, output):
-        if output and output[-1][0] == self.TEXT:
-            output[-1] = (self.TEXT, output[-1][1] + text, output[-1][2])
-        else:
-            output.append((self.TEXT, text, (None, -1, -1)))
-
-    def _format_fragment(self, text, fragment):
-        START, TEXT, END, Attrs = self.START, self.TEXT, self.END, self.Attrs
-        qname = self.qname
-        output = []
-        
-        index = fragment.startchar
-        lastmatched = False
-        for t in fragment.matches:
-            if t.startchar > index:
-                if lastmatched:
-                    output.append((END, qname, (None, -1, -1)))
-                    lastmatched = False
-                self._add_text(text[index:t.startchar], output)
-            
-            ttxt = text[t.startchar:t.endchar]
-            if not lastmatched:
-                output.append((START, (qname, Attrs()), (None, -1, -1)))
-                lastmatched = True
-            output.append((TEXT, ttxt, (None, -1, -1)))
-                                    
-            index = t.endchar
-        
-        if lastmatched:
-            output.append((END, qname, (None, -1, -1)))
-        
-        return output
-
-    def __call__(self, text, fragments):
-        output = []
-        first = True
-        for fragment in fragments:
-            if not first:
-                self._add_text(self.between, output)
-            first = False
-            output += self._format_fragment(text, fragment)
-        
-        return self.Stream(output)
+# class GenshiFormatter(object):
+#     """Returns a Genshi event stream containing HTML formatting around the
+#     matched terms.
+#     """
+#     
+#     def __init__(self, qname="strong", between="..."):
+#         """
+#         :param qname: the QName for the tag to wrap around matched terms.
+#         :param between: the text to add between fragments.
+#         """
+#         
+#         self.qname = qname
+#         self.between = between
+#         
+#         from genshi.core import START, END, TEXT, Attrs, Stream #@UnresolvedImport
+#         self.START, self.END, self.TEXT = START, END, TEXT
+#         self.Attrs, self.Stream = Attrs, Stream
+# 
+#     def _add_text(self, text, output):
+#         if output and output[-1][0] == self.TEXT:
+#             output[-1] = (self.TEXT, output[-1][1] + text, output[-1][2])
+#         else:
+#             output.append((self.TEXT, text, (None, -1, -1)))
+# 
+#     def _format_fragment(self, text, fragment):
+#         START, TEXT, END, Attrs = self.START, self.TEXT, self.END, self.Attrs
+#         qname = self.qname
+#         output = []
+#         
+#         index = fragment.startchar
+#         lastmatched = False
+#         for t in fragment.matches:
+#             if t.startchar > index:
+#                 if lastmatched:
+#                     output.append((END, qname, (None, -1, -1)))
+#                     lastmatched = False
+#                 self._add_text(text[index:t.startchar], output)
+#             
+#             ttxt = text[t.startchar:t.endchar]
+#             if not lastmatched:
+#                 output.append((START, (qname, Attrs()), (None, -1, -1)))
+#                 lastmatched = True
+#             output.append((TEXT, ttxt, (None, -1, -1)))
+#                                     
+#             index = t.endchar
+#         
+#         if lastmatched:
+#             output.append((END, qname, (None, -1, -1)))
+#         
+#         return output
+# 
+#     def __call__(self, text, fragments):
+#         output = []
+#         first = True
+#         for fragment in fragments:
+#             if not first:
+#                 self._add_text(self.between, output)
+#             first = False
+#             output += self._format_fragment(text, fragment)
+#         
+#         return self.Stream(output)
 
 

branches/mbutscher/work/lib/whoosh/lang/porter2.py

@@ -1 @@
-# Copyright (c) 2008 Michael Dirolf (mike at dirolf dot com)
- 
-# Permission is hereby granted, free of charge, to any person
-# obtaining a copy of this software and associated documentation
-# files (the "Software"), to deal in the Software without
-# restriction, including without limitation the rights to use,
-# copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following
-# conditions:
- 
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
- 
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
- 
 """An implementation of the Porter2 stemming algorithm.
 See http://snowball.tartarus.org/algorithms/english/stemmer.html
@@ -29 +6 @@
 This algorithm is more correct but (at least in this implementation)
 several times slower than the original porter algorithm as implemented
-in whoosh.lang.porter.
+in stemming.porter.
 """
 
@@ -146 +123 @@
  
 def step_1c(word):
-    if word.endswith('y') or word.endswith('Y'):
+    if word.endswith('y') or word.endswith('Y') and len(word) > 1:
         if word[-2] not in 'aeiouy':
             if len(word) > 2:
@@ -305 +282 @@
     return word
 
- 
+    
+ 

branches/mbutscher/work/lib/whoosh/matching.py

@@ -252 +252 @@
     
     def __init__(self, ids, weights=None, values=None, format=None,
-                 scorer=None, position=0):
+                 scorer=None, position=0, all_weights=None):
         """
         :param ids: a list of doc IDs.
@@ -267 +267 @@
         self._ids = ids
         self._weights = weights
+        self._all_weights = all_weights
         self._values = values
         self._i = position
@@ -319 +320 @@
     
     def weight(self):
-        if self._weights:
+        if self._all_weights:
+            return self._all_weights
+        elif self._weights:
             return self._weights[self._i]
         else:
@@ -1023 +1026 @@
             child.skip_to(self._id)
         
-        while (self._id < self.limit
-               and ((child.is_active() and self._id == child.id())
-                    or missing(self._id))):
-            self._id += 1
-            if child.is_active():
+        # While self._id is missing or is in the child matcher, increase it
+        while child.is_active() and self._id < self.limit:
+            if missing(self._id):
+                self._id += 1
+                continue
+
+            if self._id == child.id():
+                self._id += 1
                 child.next()
-    
+                continue
+            
+            break
+        
     def id(self):
         return self._id

branches/mbutscher/work/lib/whoosh/qparser/default.py

@@ -20 +20 @@
 """
 
-import re
-
 from whoosh import query
-
-
-class QueryParserError(Exception):
-    def __init__(self, cause, msg=None):
-        super(QueryParserError, self).__init__(str(cause))
-        self.cause = cause
-
-
-def rcompile(pattern, flags=0):
-    if not isinstance(pattern, basestring):
-        # If it's not a string, assume it's already a compiled pattern
-        return pattern
-    return re.compile(pattern, re.UNICODE | flags)
-
-
-def get_single_text(field, text, **kwargs):
-    # Just take the first token
-    for t in field.process_text(text, mode="query", **kwargs):
-        return t
+from whoosh.qparser.syntax import *
+from whoosh.qparser.plugins import *
 
 
@@ -48 +29 @@
 
 
-class SyntaxObject(object):
-    """An object representing parsed text. These objects generally correspond
-    to a query object type, and are intermediate objects used to represent
-    the syntax tree parsed from a query string, and then generate a query
-    tree from the syntax tree. There will be syntax objects that do not have
-    a corresponding query type, such as the object representing whitespace.
-    """
-    
-    def query(self, parser):
-        """Returns a query object tree representing this parser object.
-        """
-        
-        raise NotImplementedError
-
-
-# Grouping objects
-
-class Group(SyntaxObject):
-    """An object representing a group of objects. These generally correspond
-    to compound query objects such as ``query.And`` and ``query.Or``.
-    """
-    
-    def __init__(self, tokens=None, boost=1.0):
-        if tokens:
-            self.tokens = tokens
-        else:
-            self.tokens = []
-        self.boost = boost
-    
-    def __repr__(self):
-        r = "%s(%r)" % (self.__class__.__name__, self.tokens)
-        if self.boost != 1.0:
-            r += "^%s" % self.boost
-        return r
-    
-    def __nonzero__(self):
-        return bool(self.tokens)
-    
-    def __iter__(self):
-        return iter(self.tokens)
-    
-    def __len__(self):
-        return len(self.tokens)
-    
-    def __getitem__(self, n):
-        return self.tokens.__getitem__(n)
-    
-    def __setitem__(self, n, v):
-        self.tokens.__setitem__(n, v)
-    
-    def set_boost(self, b):
-        return self.__class__(self.tokens[:], boost=b)
-    
-    def set_fieldname(self, name):
-        return self.__class__([t.set_fieldname(name) for t in self.tokens])
-    
-    def append(self, item):
-        self.tokens.append(item)
-        
-    def extend(self, items):
-        self.tokens.extend(items)
-    
-    def pop(self):
-        return self.tokens.pop()
-    
-    def query(self, parser):
-        return self.qclass([t.query(parser) for t in self.tokens],
-                           boost=self.boost)
-        
-    def empty(self):
-        return self.__class__(boost=self.boost)
-
-
-class AndGroup(Group):
-    """Syntax group corresponding to an And query.
-    """
-    
-    qclass = query.And
-
-
-class OrGroup(Group):
-    """Syntax group corresponding to an Or query.
-    """
-    
-    qclass = query.Or
-
-
-class AndNotGroup(Group):
-    """Syntax group corresponding to an AndNot query.
-    """
-    
-    def query(self, parser):
-        assert len(self.tokens) == 2
-        return query.AndNot(self.tokens[0].query(parser),
-                            self.tokens[1].query(parser), boost=self.boost)
-    
-class AndMaybeGroup(Group):
-    """Syntax group corresponding to an AndMaybe query.
-    """
-    
-    def query(self, parser):
-        assert len(self.tokens) == 2
-        return query.AndMaybe(self.tokens[0].query(parser),
-                              self.tokens[1].query(parser), boost=self.boost)
-
-
-class DisMaxGroup(Group):
-    """Syntax group corresponding to a DisjunctionMax query.
-    """
-    
-    def __init__(self, tokens=None, tiebreak=0.0, boost=None):
-        super(DisMaxGroup, self).__init__(tokens)
-        self.tiebreak = tiebreak
-    
-    def __repr__(self):
-        r = "dismax(%r" % self.tokens
-        if self.tiebreak != 0:
-            r += " tb=%s" % self.tiebreak
-        r += ")"
-        return r
-    
-    def query(self, parser):
-        return query.DisjunctionMax([t.query(parser) for t in self.tokens],
-                                    tiebreak=self.tiebreak)
-        
-    def empty(self):
-        return self.__class__(tiebreak=self.tiebreak)
-
-
-class NotGroup(Group):
-    """Syntax group corresponding to a Not query.
-    """
-    
-    def __repr__(self):
-        return "NOT(%r)" % self.tokens
-    
-    def query(self, parser):
-        assert len(self.tokens) == 1
-        return query.Not(self.tokens[0].query(parser))
-    
-
-# Parse-able tokens
-
-class Token(SyntaxObject):
-    """A parse-able token object. Each token class has an ``expr`` attribute
-    containing a regular expression that matches the token text. When this
-    expression is found, the class's ``create()`` class method is called and
-    returns a token object to represent the match in the syntax tree. When the
-    syntax tree is finished, the
-    """
-    
-    fieldname = None
-    endpos = None
-    
-    def set_boost(self, b):
-        return self
-    
-    def set_fieldname(self, name):
-        return self
-    
-    @classmethod
-    def match(cls, text, pos):
-        return cls.expr.match(text, pos)
-    
-    @classmethod
-    def create(cls, parser, match):
-        return cls()
-    
-    def query(self, parser):
-        raise NotImplementedError
-
-
-class Singleton(Token):
-    """Base class for tokens that don't carry any information specific to
-    each instance (e.g. "open paranthesis" token), so they can all share the
-    same instance.
-    """
-    
-    me = None
-    
-    def __repr__(self):
-        return self.__class__.__name__
-    
-    @classmethod
-    def create(cls, parser, match):
-        if not cls.me:
-            cls.me = cls()
-        return cls.me
-
-
-class White(Singleton):
-    expr = rcompile("\\s+")
-    
-
-class ErrorToken(Token):
-    """A token representing an unavoidable parsing error. The ``query()``
-    method always returns NullQuery.
-    
-    The default parser usually does not produce "errors" (text that doesn't
-    match the syntax is simply treated as part of the query), so this is mostly
-    for use by plugins that may add more restrictive parsing, for example
-    :class:`DateParserPlugin`.
-    
-    Since the corresponding NullQuery will be filtered out when the query is
-    normalized, this is really only useful for debugging and possibly for
-    plugin filters.
-    
-    The ``token`` attribute may contain the token that produced the error.
-    """
-    
-    def __init__(self, token):
-        self.token = token
-        
-    def __repr__(self):
-        return "<%s (%r)>" % (self.__class__.__name__, self.token)
-    
-    def query(self, parser):
-        return query.NullQuery
-
-
-class BasicSyntax(Token):
-    """Base class for "basic" (atomic) syntax -- term, prefix, wildcard,
-    phrase, range.
-    """
-    
-    expr = None
-    qclass = None
-    tokenize = False
-    removestops = False
-    
-    def __init__(self, text, fieldname=None, boost=1.0):
-        self.fieldname = fieldname
-        self.text = text
-        self.boost = boost
-    
-    def set_boost(self, b):
-        return self.__class__(self.text, fieldname=self.fieldname, boost=b)
-    
-    def set_fieldname(self, name):
-        if self.fieldname is None:
-            return self.__class__(self.text, fieldname=name, boost=self.boost)
-        else:
-            return self
-    
-    def __repr__(self):
-        r = "%s:%r" % (self.fieldname, self.text)
-        if self.boost != 1.0:
-            r += "^%s" % self.boost
-        return r
-    
-    @classmethod
-    def create(cls, parser, match):
-        return cls(match.group(0))
-    
-    def query(self, parser):
-        texts = (self.text, )
-        fieldname = self.fieldname or parser.fieldname
-        cls = self.qclass or parser.termclass
-        
-        if parser.schema and fieldname in parser.schema:
-            field = parser.schema[fieldname]
-            
-            if field.self_parsing():
-                try:
-                    return field.parse_query(fieldname, self.text,
-                                             boost=self.boost)
-                except QueryParserError:
-                    return query.NullQuery
-            
-            texts = list(field.process_text(self.text, mode="query",
-                                            tokenize=self.tokenize,
-                                            removestops=self.removestops))
-        
-        if len(texts) > 1:
-            compound = parser.group.qclass
-            return compound([cls(fieldname, t, boost=self.boost)
-                             for t in texts])
-        elif texts and texts[0] is not None:
-            return cls(fieldname, texts[0], boost=self.boost)
-        else:
-            return query.NullQuery
-
-
-class Word(BasicSyntax):
-    """Syntax object representing a term.
-    """
-    
-    expr = rcompile("[^ \t\r\n)]+")
-    tokenize = True
-    removestops = True
-    
-
-# Parser plugins
-
-class Plugin(object):
-    """Base class for parser plugins.
-    """
-            
-    def tokens(self, parser):
-        """Returns a list of ``(token_class, priority)`` tuples to add to the
-        syntax the parser understands.
-        """
-        
-        return ()
-    
-    def filters(self, parser):
-        """Returns a list of ``(filter_function, priority)`` tuples to add to
-        parser.
-        """
-        
-        return ()
-    
-
-class RangePlugin(Plugin):
-    """Adds the ability to specify term ranges.
-    
-    This plugin has no configuration.
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def tokens(self, parser):
-        return ((RangePlugin.Range, 1), )
-    
-    class Range(Token):
-        expr = rcompile(r"""
-        (?P<open>\{|\[)               # Open paren
-        
-        (                             # Begin optional "start"
-          (                           # Begin choice between start1 and start2
-            ('(?P<start2>[^']+)')     # Quoted start
-            | (?P<start1>[^ ]+)       # ...or regular start
-          )                           # End choice
-        [ ]+)?                        # Space at end of optional "start"
-        
-        [Tt][Oo]                      # "to" between start and end
-        
-        ([ ]+                         # Space at start of optional "end"
-          (                           # Begin choice between end1 and end2
-            ('(?P<end2>[^']+)')       # Quoted end
-            | (?P<end1>[^\]\}]*)      # ...or normal end
-          )                           # End choice
-        )?                            # End of optional "end
-        
-        (?P<close>\}|\])              # Close paren
-        """, re.VERBOSE)
-        
-        def __init__(self, start, end, startexcl, endexcl, fieldname=None, boost=1.0):
-            self.fieldname = fieldname
-            self.start = start
-            self.end = end
-            self.startexcl = startexcl
-            self.endexcl = endexcl
-            self.boost = boost
-        
-        def set_boost(self, b):
-            return self.__class__(self.start, self.end, self.startexcl,
-                                  self.endexcl, fieldname=self.fieldname,
-                                  boost=b)
-        
-        def set_fieldname(self, name):
-            return self.__class__(self.start, self.end, self.startexcl,
-                                  self.endexcl, fieldname=name,
-                                  boost=self.boost)
-        
-        def __repr__(self):
-            r = "%s:(%r, %r, %s, %s)" % (self.fieldname, self.start, self.end,
-                                         self.startexcl, self.endexcl)
-            if self.boost != 1.0:
-                r += "^%s" % self.boost
-            return r
-        
-        @classmethod
-        def create(cls, parser, match):
-            start = match.group("start2") or match.group("start1")
-            end = match.group("end2") or match.group("end1")
-            return cls(start, end, startexcl=match.group("open") == "{",
-                       endexcl=match.group("close") == "}")
-            
-        def query(self, parser):
-            fieldname = self.fieldname or parser.fieldname
-            start, end = self.start, self.end
-            if parser.schema and fieldname in parser.schema:
-                field = parser.schema[fieldname]
-                
-                if field.self_parsing():
-                    try:
-                        rangeq = field.parse_range(fieldname, start, end,
-                                                   self.startexcl, self.endexcl,
-                                                   boost=self.boost)
-                        if rangeq is not None:
-                            return rangeq
-                    except QueryParserError, e:
-                        return query.NullQuery
-                
-                if start:
-                    start = get_single_text(field, start, tokenize=False,
-                                            removestops=False)
-                if end:
-                    end = get_single_text(field, end, tokenize=False,
-                                          removestops=False)
-            
-            if start is None:
-                start = u''
-            if end is None:
-                end = u'\uFFFF'
-            
-            return query.TermRange(fieldname, start, end, self.startexcl,
-                                   self.endexcl, boost=self.boost)
-            
-
-class PhrasePlugin(Plugin):
-    """Adds the ability to specify phrase queries inside double quotes.
-    
-    This plugin has no configuration.
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def tokens(self, parser):
-        return ((PhrasePlugin.Quotes, 0), )
-    
-    class Quotes(BasicSyntax):
-        expr = rcompile('"(.*?)"')
-        
-        def __init__(self, text, fieldname=None, boost=1.0, slop=1):
-            super(PhrasePlugin.Quotes, self).__init__(text, fieldname=fieldname,
-                                                      boost=boost)
-            self.slop = slop
-        
-        def __repr__(self):
-            r = "%s:q(%r)" % (self.fieldname, self.text)
-            if self.boost != 1.0:
-                r += "^%s" % self.boost
-            return r
-        
-        @classmethod
-        def create(cls, parser, match):
-            slop = 1
-            #if match.group(5):
-            #    try:
-            #        slop = int(match.group(5))
-            #    except ValueError:
-            #        pass
-            return cls(match.group(1), slop=slop)
-        
-        def query(self, parser):
-            fieldname = self.fieldname or parser.fieldname
-            if parser.schema and fieldname in parser.schema:
-                field = parser.schema[fieldname]
-                #if field.self_parsing():
-                #    return field.parse_query(fieldname, self.text, boost=self.boost)
-                #else:
-                words = list(field.process_text(self.text, mode="query")) 
-            else:
-                words = self.text.split(" ")
-            
-            return parser.phraseclass(fieldname, words, boost=self.boost,
-                                      slop=self.slop)
-
-
-class SingleQuotesPlugin(Plugin):
-    """Adds the ability to specify single "terms" containing spaces by
-    enclosing them in single quotes.
-    
-    This plugin has no configuration.
-    
-    This plugin is included in the default parser configuration.
-    """
-     
-    def tokens(self, parser):
-        return ((SingleQuotesPlugin.SingleQuotes, 0), )
-    
-    class SingleQuotes(Token):
-        expr = rcompile(r"(^|(?<=\W))'(.*?)'(?=\s|\]|[)}]|$)")
-        
-        @classmethod
-        def create(cls, parser, match):
-            return Word(match.group(2))
-
-
-class PrefixPlugin(Plugin):
-    """Adds the ability to specify prefix queries by ending a term with an
-    asterisk. This plugin is useful if you want the user to be able to create
-    prefix but not wildcard queries (for performance reasons). If you are
-    including the wildcard plugin, you should not include this plugin as well.
-    """
-    
-    def tokens(self, parser):
-        return ((PrefixPlugin.Prefix, 0), )
-    
-    class Prefix(BasicSyntax):
-        expr = rcompile("[^ \t\r\n*]+\\*(?= |$|\\))")
-        qclass = query.Prefix
-        
-        def __repr__(self):
-            r = "%s:pre(%r)" % (self.fieldname, self.text)
-            if self.boost != 1.0:
-                r += "^%s" % self.boost
-            return r
-        
-        @classmethod
-        def create(cls, parser, match):
-            return cls(match.group(0)[:-1])
-        
-
-class WildcardPlugin(Plugin):
-    """Adds the ability to specify wildcard queries by using asterisk and
-    question mark characters in terms. Note that these types can be very
-    performance and memory intensive. You may consider not including this
-    type of query.
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def tokens(self, parser):
-        return ((WildcardPlugin.Wild, 1), )
-    
-    class Wild(BasicSyntax):
-        # \u055E = Armenian question mark
-        # \u061F = Arabic question mark
-        # \u1367 = Ethiopic question mark
-        expr = rcompile(u"[^ \t\r\n*?\u055E\u061F\u1367]*[*?\u055E\u061F\u1367]\\S*")
-        qclass = query.Wildcard
-        
-        def __repr__(self):
-            r = "%s:wild(%r)" % (self.fieldname, self.text)
-            if self.boost != 1.0:
-                r += "^%s" % self.boost
-            return r
-        
-        @classmethod
-        def create(cls, parser, match):
-            return cls(match.group(0))
-        
-
-class WhitespacePlugin(Plugin):
-    """Parses whitespace between words in the query string. You should always
-    include this plugin.
-    
-    This plugin is always automatically included by the QueryParser.
-    """
-    
-    def __init__(self, tokenclass=White):
-        self.tokenclass = tokenclass
-    
-    def tokens(self, parser):
-        return ((self.tokenclass, 100), )
-    
-    def filters(self, parser):
-        return ((self.do_whitespace, 500), )
-    
-    def do_whitespace(self, parser, stream):
-        newstream = stream.empty()
-        for t in stream:
-            if isinstance(t, Group):
-                newstream.append(self.do_whitespace(parser, t))
-            elif not isinstance(t, self.tokenclass):
-                newstream.append(t)
-        return newstream
-
-
-class GroupPlugin(Plugin):
-    """Adds the ability to group clauses using parentheses.
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def tokens(self, parser):
-        return ((GroupPlugin.Open, 0), (GroupPlugin.Close, 0))
-    
-    def filters(self, parser):
-        return ((GroupPlugin.do_groups, 0), )
-    
-    @staticmethod
-    def do_groups(parser, stream):
-        stack = [parser.group()]
-        for t in stream:
-            if isinstance(t, GroupPlugin.Open):
-                stack.append(parser.group())
-            elif isinstance(t, GroupPlugin.Close):
-                if len(stack) > 1:
-                    last = stack.pop()
-                    stack[-1].append(last)
-            else:
-                stack[-1].append(t)
-        
-        top = stack[0]
-        if len(stack) > 1:
-            for ls in stack[1:]:
-                top.extend(ls)
-        
-        if len(top) == 1 and isinstance(top[0], Group):
-            top = top[0].set_boost(top.boost)
-        
-        return top
-    
-    class Open(Singleton):
-        expr = rcompile("\\(")
-        
-    class Close(Singleton):
-        expr = rcompile("\\)")
-
-
-class FieldsPlugin(Plugin):
-    """Adds the ability to specify the field of a clause using a colon.
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def tokens(self, parser):
-        return ((FieldsPlugin.Field, 0), )
-    
-    def filters(self, parser):
-        return ((FieldsPlugin.do_fieldnames, 100), )
-
-    @staticmethod
-    def do_fieldnames(parser, stream):
-        newstream = stream.empty()
-        newname = None
-        for i, t in enumerate(stream):
-            if isinstance(t, FieldsPlugin.Field):
-                valid = False
-                if i < len(stream) - 1:
-                    next = stream[i+1]
-                    if not isinstance(next, (White, FieldsPlugin.Field)):
-                        newname = t.fieldname
-                        valid = True
-                if not valid:
-                    newstream.append(Word(t.fieldname, fieldname=parser.fieldname))
-                continue
-            
-            if isinstance(t, Group):
-                t = FieldsPlugin.do_fieldnames(parser, t)
-                
-            if newname is not None:
-                t = t.set_fieldname(newname)
-            newstream.append(t)
-            newname = None
-        
-        return newstream
-    
-    class Field(Token):
-        expr = rcompile(u"(\w[\w\d]*):")
-        
-        def __init__(self, fieldname):
-            self.fieldname = fieldname
-        
-        def __repr__(self):
-            return "<%s:>" % self.fieldname
-        
-        def set_fieldname(self, fieldname):
-            return self.__class__(fieldname)
-        
-        @classmethod
-        def create(cls, parser, match):
-            fieldname = match.group(1)
-            if not parser.schema or (fieldname in parser.schema):
-                return cls(fieldname)
-    
-
-class CompoundsPlugin(Plugin):
-    """Adds the ability to use AND, OR, ANDMAYBE, and ANDNOT to specify
-    query constraints.
-    
-    You can customize the tokens by passing regular expressions to the ``And``,
-    ``Or``, ``AndNot``, and/or ``AndMaybe`` keywords to the class initializer::
-    
-        qp = qparser.QueryParser("content")
-        
-        cp = qparser.CompoundsPlugin(And="&", Or="\\|", AndNot="&!", AndMaybe="&~")
-        qp.replace_plugin(cp)
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def __init__(self, And=r"\sAND\s", Or=r"\sOR\s", AndNot=r"\sANDNOT\s",
-                 AndMaybe=r"\sANDMAYBE\s"):
-        # Create one-off token classes using the keyword arguments
-        class AndTokenClass(Singleton):
-            expr = rcompile(And)
-        class OrTokenClass(Singleton):
-            expr = rcompile(Or)
-        class AndNotTokenClass(Singleton):
-            expr = rcompile(AndNot)
-        class AndMaybeTokenClass(Singleton):
-            expr = rcompile(AndMaybe)
-            
-        # Store these classes as attributes
-        self.And = AndTokenClass
-        self.Or = OrTokenClass
-        self.AndNot = AndNotTokenClass
-        self.AndMaybe = AndMaybeTokenClass
-    
-    def tokens(self, parser):
-        return ((self.AndNot, -10), (self.AndMaybe, -5), (self.And, 0),
-                (self.Or, 0))
-    
-    def filters(self, parser):
-        return ((self.do_compounds, 600), )
-
-    def do_compounds(self, parser, stream):
-        newstream = stream.empty()
-        i = 0
-        while i < len(stream):
-            # The current token
-            t = stream[i]
-            
-            # Whether this token has other tokens in front and behind; that is,
-            # if ismiddle is True, this is not the first or last token
-            ismiddle = newstream and i < len(stream) - 1
-            
-            if isinstance(t, Group):
-                # The current token is a group: recursively apply this plugin
-                # to the group
-                newstream.append(self.do_compounds(parser, t))
-                
-            elif isinstance(t, (self.And, self.Or)):
-                # This is either an And or Or token. Create a new Group class
-                # of the appropriate type
-                if isinstance(t, self.And):
-                    cls = AndGroup
-                else:
-                    cls = OrGroup
-                
-                if cls != type(newstream) and ismiddle:
-                    last = newstream.pop()
-                    rest = self.do_compounds(parser, cls(stream[i+1:]))
-                    newstream.append(cls([last, rest]))
-                    break
-            
-            elif isinstance(t, (self.AndNot, self.AndMaybe)) and ismiddle:
-                # This is either an AndNot or AndMaybe token. Create a new
-                # Group class of the appropriate type
-                if isinstance(t, self.AndNot):
-                    cls = AndNotGroup
-                else:
-                    cls = AndMaybeGroup
-                
-                last = newstream.pop()
-                i += 1
-                next = stream[i]
-                if isinstance(next, Group):
-                    next = self.do_compounds(parser, next)
-                newstream.append(cls([last, next]))
-            
-            else:
-                newstream.append(t)
-            
-            i += 1
-        
-        return newstream
-
-
-class BoostPlugin(Plugin):
-    """Adds the ability to boost clauses of the query using the circumflex.
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def tokens(self, parser):
-        return ((BoostPlugin.Boost, 0), )
-    
-    def filters(self, parser):
-        return ((BoostPlugin.clean_boost, 0), (BoostPlugin.do_boost, 700))
-
-    @staticmethod
-    def clean_boost(parser, stream):
-        newstream = stream.empty()
-        for i, t in enumerate(stream):
-            if isinstance(t, BoostPlugin.Boost):
-                if i == 0 or isinstance(stream[i-1], (BoostPlugin.Boost, White)):
-                    t = Word(t.original)
-            newstream.append(t)
-        return newstream
-
-    @staticmethod
-    def do_boost(parser, stream):
-        newstream = stream.empty()
-        
-        for t in stream:
-            if isinstance(t, Group):
-                newstream.append(BoostPlugin.do_boost(parser, t))
-                
-            elif isinstance(t, BoostPlugin.Boost):
-                if newstream:
-                    newstream.append(newstream.pop().set_boost(t.boost))
-                
-            else:
-                newstream.append(t)
-        
-        return newstream
-    
-    class Boost(Token):
-        expr = rcompile("\\^([0-9]+(.[0-9]+)?)($|(?=[ \t\r\n]))")
-        
-        def __init__(self, original, boost):
-            self.original = original
-            self.boost = boost
-        
-        def __repr__(self):
-            return "<^%s>" % self.boost
-        
-        @classmethod
-        def create(cls, parser, match):
-            try:
-                return cls(match.group(0), float(match.group(1)))
-            except ValueError:
-                return Word(match.group(0))
-    
-
-class NotPlugin(Plugin):
-    """Adds the ability to negate a clause by preceding it with NOT.
-    
-    You can customize the token by passing a regular expression to the class
-    initializer::
-    
-        qp = qparser.QueryParser("content")
-        
-        # Use - as the not token
-        qp.replace_plugin(qparser.NotPlugin("(^|(?<= ))-"))
-        
-        # Use ! as the not token
-        qp.replace_plugin(qparser.NotPlugin("(^|(?<= ))!"))
-    
-    This plugin is included in the default parser configuration.
-    """
-    
-    def __init__(self, token="(^|(?<= ))NOT "):
-        class Not(Singleton):
-            expr = rcompile(token)
-        
-        self.Not = Not
-    
-    def tokens(self, parser):
-        return ((self.Not, 0), )
-    
-    def filters(self, parser):
-        return ((self.do_not, 800), )
-    
-    def do_not(self, parser, stream):
-        newstream = stream.empty()
-        notnext = False
-        for t in stream:
-            if isinstance(t, self.Not):
-                notnext = True
-                continue
-            
-            if isinstance(t, Group):
-                t = self.do_not(parser, t)
-            
-            if notnext:
-                t = NotGroup([t])
-            
-            newstream.append(t)
-            notnext = False
-            
-        return newstream
-    
-
-class PlusMinusPlugin(Plugin):
-    """Adds the ability to use + and - in a flat OR query to specify required
-    and prohibited terms.
-    
-    This is the basis for the parser configuration returned by
-    ``SimpleParser()``.
-    """
-    
-    def tokens(self, parser):
-        return ((PlusMinusPlugin.Plus, 0), (PlusMinusPlugin.Minus, 0))
-    
-    def filters(self, parser):
-        return ((PlusMinusPlugin.do_plusminus, 510), )
-    
-    @staticmethod
-    def do_plusminus(parser, stream):
-        required = AndGroup()
-        optional = OrGroup()
-        prohibited = OrGroup()
-        
-        nextlist = optional
-        for t in stream:
-            if isinstance(t, PlusMinusPlugin.Plus):
-                nextlist = required
-            elif isinstance(t, PlusMinusPlugin.Minus):
-                nextlist = prohibited
-            else:
-                nextlist.append(t)
-                nextlist = optional
-        
-        r = optional
-        if required:
-            r = AndMaybeGroup([required, optional])
-        if prohibited:
-            r = AndNotGroup([r, prohibited])
-        return r
-    
-    class Plus(Singleton):
-        expr = rcompile("\\+")
-        
-    class Minus(Singleton):
-        expr = rcompile("-")
-
-
-class MultifieldPlugin(Plugin):
-    """Converts any unfielded terms into OR clauses that search for the
-    term in a specified list of fields.
-    """
-    
-    def __init__(self, fieldnames, fieldboosts=None):
-        """
-        :param fieldnames: a list of fields to search.
-        :param fieldboosts: an optional dictionary mapping field names to
-            a boost to use for that field.
-        """
-        
-        self.fieldnames = fieldnames
-        self.boosts = fieldboosts or {}
-    
-    def filters(self, parser):
-        return ((self.do_multifield, 110), )
-    
-    def do_multifield(self, parser, stream):
-        newstream = stream.empty()
-        for t in stream:
-            if isinstance(t, BasicSyntax) and t.fieldname is None:
-                t = OrGroup([t.set_fieldname(fn).set_boost(self.boosts.get(fn, 1.0))
-                             for fn in self.fieldnames])
-            newstream.append(t)
-        return newstream
-        
-
-class DisMaxPlugin(Plugin):
-    """Converts any unfielded terms into DisjunctionMax clauses that search
-    for the term in a specified list of fields.
-    """
-    
-    def __init__(self, fieldboosts, tiebreak=0.0):
-        """
-        :param fieldboosts: a dictionary mapping field names to a boost to use
-            for that in the DisjuctionMax query.
-        """
-        
-        self.fieldboosts = fieldboosts.items()
-        self.tiebreak = tiebreak
-    
-    def filters(self, parser):
-        return ((self.do_dismax, 110), )
-    
-    def do_dismax(self, parser, stream):
-        newstream = stream.empty()
-        for t in stream:
-            if isinstance(t, BasicSyntax) and t.fieldname is None:
-                t = DisMaxGroup([t.set_fieldname(fn).set_boost(b)
-                                 for fn, b in self.fieldboosts],
-                                 tiebreak=self.tiebreak)
-            newstream.append(t)
-        return newstream
-
-
-class FieldAliasPlugin(Plugin):
-    """Adds the ability to use "aliases" of fields in the query string.
-    
-    >>> # Allow users to use 'body' or 'text' to refer to the 'content' field
-    >>> parser.add_plugin(FieldAliasPlugin({"content": ("body", "text")}))
-    >>> parser.parse("text:hello")
-    Term("content", "hello")
-    """
-    
-    def __init__(self, fieldmap):
-        """
-        :param fieldmap: a dictionary mapping fieldnames to a list of
-            aliases for the field.
-        """
-        
-        self.fieldmap = fieldmap
-        self.reverse = {}
-        for key, values in fieldmap.iteritems():
-            for value in values:
-                self.reverse[value] = key
-        
-    def filters(self, parser):
-        return ((self.do_aliases, 90), )
-    
-    def do_aliases(self, parser, stream):
-        newstream = stream.empty()
-        for t in stream:
-            if (not isinstance(t, Group)
-                  and t.fieldname is not None
-                  and t.fieldname in self.reverse):
-                    t = t.set_fieldname(self.reverse[t.fieldname])
-            newstream.append(t)
-        return newstream
-
-
-# Parser object
-
-full_profile = (BoostPlugin, CompoundsPlugin, FieldsPlugin, GroupPlugin,
-                NotPlugin, PhrasePlugin, RangePlugin, SingleQuotesPlugin,
-                WildcardPlugin)
+full_profile = (BoostPlugin, OperatorsPlugin, FieldsPlugin, GroupPlugin,
+                PhrasePlugin, RangePlugin, SingleQuotesPlugin, WildcardPlugin)
 
 
@@ -1065 +48 @@
     And([Term("content", u"hello"), Term("content", u"there")])
     """
+    
+    _multitoken_query_map = {"and": query.And, "or": query.Or,
+                             "phrase": query.Phrase}
     
     def __init__(self, fieldname, schema=None, termclass=query.Term,
@@ -1152 +138 @@
         return [item for item, pri in items_and_priorities]
     
+    def multitoken_query(self, name, texts, fieldname, termclass, boost):
+        qclass = self._multitoken_query_map.get(name.lower())
+        if qclass:
+            return qclass([termclass(fieldname, t, boost=boost)
+                           for t in texts])
+    
+    def term_query(self, fieldname, text, termclass, boost=1.0, tokenize=True,
+                   removestops=True):
+        """Returns the appropriate query object for a single term in the query
+        string.
+        """
+        
+        if self.schema and fieldname in self.schema:
+            field = self.schema[fieldname]
+            
+            # If this field type wants to parse queries itself, let it do so
+            # and return early
+            if field.self_parsing():
+                try:
+                    return field.parse_query(fieldname, text, boost=boost)
+                except QueryParserError:
+                    return query.NullQuery
+            
+            # Otherwise, ask the field to process the text into a list of
+            # tokenized strings
+            texts = list(field.process_text(text, mode="query",
+                                            tokenize=tokenize,
+                                            removestops=removestops))
+            
+            # If the analyzer returned more than one token, use the field's
+            # multitoken_query attribute to decide what query class, if any, to
+            # use to put the tokens together
+            if len(texts) > 1:
+                mtq = self.multitoken_query(field.multitoken_query, texts,
+                                            fieldname, termclass, boost)
+                if mtq:
+                    return mtq
+                
+            # It's possible field.process_text() will return an empty list (for
+            # example, on a stop word)
+            if not texts:
+                return query.NullQuery
+            
+            text = texts[0]
+        
+        return termclass(fieldname, text, boost=boost)
+        
     def tokens(self):
         """Returns a priorized list of tokens from the included plugins.

branches/mbutscher/work/lib/whoosh/query.py

@@ -752 +752 @@
 
 
-_wildcard_exp = re.compile("(.*?)([?*]|$)");
 class Wildcard(MultiTerm):
     """Matches documents that contain any terms that match a wildcard
@@ -862 +861 @@
         """
 
-        if not text:
-            raise QueryError("Fuzzy term is empty")
-
         self.fieldname = fieldname
         self.text = text
@@ -881 +877 @@
 
     def __repr__(self):
-        return "%s(%r, %r, ratio=%f)" % (self.__class__.__name__,
-                                         self.fieldname, self.text,
-                                         self.ratio)
+        r = "%s(%r, %r, boost=%f, minsimilarity=%f, prefixlength=%d)"
+        return r % (self.__class__.__name__, self.fieldname, self.text,
+                    self.boost, self.minsimilarity, self.prefixlength)
 
     def __unicode__(self):
-        return u"~" + self.text
+        r = u"~" + self.text
+        if self.boost != 1.0:
+            r += "^%f" % self.boost
+        return r
 
     def copy(self):
@@ -982 +981 @@
 
     def replace(self, oldtext, newtext):
-        if self.start == oldtext:
-            return TermRange(self.fieldname, newtext, self.end,
-                             self.startexcl, self.endexcl, boost=self.boost)
-        elif self.end == oldtext:
-            return TermRange(self.fieldname, self.start, newtext,
-                             self.startexcl, self.endexcl, boost=self.boost)
-        else:
-            return self
-
+        start = newtext if self.start == oldtext else self.start
+        end = newtext if self.end == oldtext else self.end
+        return self.__class__(self.fieldname, start, end, self.startexcl,
+                              self.endexcl, boost=self.boost)
+        
     def _words(self, ixreader):
         fieldname = self.fieldname
@@ -1206 +1201 @@
 
 
-class Phrase(MultiTerm):
+class Phrase(Query):
     """Matches documents containing a given phrase."""
 
@@ -1309 +1304 @@
 
 
+class Ordered(And):
+    """Matches documents containing a list of sub-queries in the given order.
+    """
+
+    JOINT = " BEFORE "
+    
+    def matcher(self, searcher, exclude_docs=None):
+        from spans import SpanBefore
+        return self._matcher(SpanBefore.SpanBeforeMatcher, searcher,
+                             exclude_docs=exclude_docs)
+
+
 class Every(Query):
     """A query that matches every document containing any word in a given
@@ -1340 +1347 @@
     def matcher(self, searcher, exclude_docs=None):
         fieldname = self.fieldname
-        s = set()
         
         # This is a hacky hack, but just create an in-memory set of all the
         # document numbers of every term in the field
-        for text in searcher.lexicon(fieldname):
-            pr = searcher.postings(fieldname, text)
-            s.update(pr.all_ids())
+        s = set()
+        
+        if fieldname == "*":
+            s.update(xrange(searcher.doc_count_all()))
+        else:
+            for text in searcher.lexicon(fieldname):
+                pr = searcher.postings(fieldname, text)
+                s.update(pr.all_ids())
+        
         if exclude_docs:
             s.difference_update(exclude_docs)
         
-        return ListMatcher(sorted(s), weights=[self.boost] * len(s))
+        return ListMatcher(sorted(s), all_weights=self.boost)
 
             
@@ -1394 +1406 @@
         else:
             ids = array("I", m.all_ids())
-            return ListMatcher(ids, weights=[self.score] * len(ids))
+            return ListMatcher(ids, all_weights=self.score)
         
     def replace(self, oldtext, newtext):
@@ -1451 +1463 @@
 
 
-class Require(CompoundQuery):
+class BinaryQuery(CompoundQuery):
+    """Base class for binary queries (queries which are composed of two
+    sub-queries). Subclasses should set the ``matcherclass`` attribute or
+    override ``matcher()``, and may also need to override ``normalize()``,
+    ``estimate_size()``, and/or ``estimate_min_size()``.
+    """
+    
+    def __init__(self, a, b, boost=1.0):
+        self.a = a
+        self.b = b
+        self.subqueries = (a, b)
+        self.boost = boost
+        
+    def copy(self):
+        return self.__class__(self.a, self.b, boost=self.boost)
+    
+    def normalize(self):
+        a = self.a.normalize()
+        b = self.b.normalize()
+        if a is NullQuery and b is NullQuery:
+            return NullQuery
+        elif a is NullQuery:
+            return b
+        elif b is NullQuery:
+            return a
+    
+        return self.__class__(a, b, boost=self.boost)
+    
+    def matcher(self, searcher, exclude_docs=None):
+        return self.matcherclass(self.a.matcher(searcher, exclude_docs=exclude_docs),
+                                 self.b.matcher(searcher, exclude_docs=exclude_docs))
+
+
+class Require(BinaryQuery):
     """Binary query returns results from the first query that also appear in
     the second query, but only uses the scores from the first query. This lets
@@ -1458 +1503 @@
 
     JOINT = " REQUIRE "
-
-    def __init__(self, scoredquery, requiredquery, boost=1.0):
-        """
-        :param scoredquery: The query that is scored. Only documents that also
-            appear in the second query ('requiredquery') are scored.
-        :param requiredquery: Only documents that match both 'scoredquery' and
-            'requiredquery' are returned, but this query does not
-            contribute to the scoring.
-        """
-
-        # The superclass CompoundQuery expects the subqueries to be in a
-        # sequence in self.subqueries
-        self.subqueries = (scoredquery, requiredquery)
-        self.boost = boost
-
-    def copy(self):
-        return self.__class__(self.subqueries[0], self.subqueries[1],
+    matcherclass = RequireMatcher
+
+    def estimate_size(self, ixreader):
+        return self.b.estimate_size(ixreader)
+    
+    def estimate_min_size(self, ixreader):
+        return self.b.estimate_min_size(ixreader)
+
+    def normalize(self):
+        a = self.a.normalize()
+        b = self.b.normalize()
+        if a is NullQuery or b is NullQuery:
+            return NullQuery
+        return self.__class__(a, b, boost=self.boost)
+    
+    def replace(self, oldtext, newtext):
+        return self.__class__(self.a.replace(oldtext, newtext),
+                              self.b.replace(oldtext, newtext),
                               boost=self.boost)
-
-    def estimate_size(self, ixreader):
-        return self.subqueries[1].estimate_size(ixreader)
-    
-    def estimate_min_size(self, ixreader):
-        return self.subqueries[1].estimate_min_size(ixreader)
-
-    def normalize(self):
-        subqueries = [q.normalize() for q in self.subqueries]
-        if NullQuery in subqueries:
-            return NullQuery
-        return Require(subqueries[0], subqueries[1], boost=self.boost)
-
+    
     def docs(self, searcher, exclude_docs=None):
         return And(self.subqueries).docs(searcher, exclude_docs=exclude_docs)
     
-    def matcher(self, searcher, exclude_docs=None):
-        scored, required = self.subqueries
-        return RequireMatcher(scored.matcher(searcher, exclude_docs=exclude_docs),
-                              required.matcher(searcher, exclude_docs=exclude_docs))
-
-
-class AndMaybe(CompoundQuery):
+
+class AndMaybe(BinaryQuery):
     """Binary query takes results from the first query. If and only if the
     same document also appears in the results from the second query, the score
@@ -1505 +1534 @@
 
     JOINT = " ANDMAYBE "
-
-    def __init__(self, requiredquery, optionalquery, boost=1.0):
-        """
-        :param requiredquery: Documents matching this query are returned.
-        :param optionalquery: If a document matches this query as well as
-            'requiredquery', the score from this query is added to the
-            document score from 'requiredquery'.
-        """
-
-        # The superclass CompoundQuery expects the subqueries to be
-        # in a sequence in self.subqueries
-        self.subqueries = (requiredquery, optionalquery)
-        self.boost = boost
-
-    def copy(self):
-        return self.__class__(self.subqueries[0], self.subqueries[1],
-                              boost=self.boost)
+    matcherclass = AndMaybeMatcher
 
     def normalize(self):
-        required, optional = (q.normalize() for q in self.subqueries)
-        if required is NullQuery:
+        a = self.a.normalize()
+        b = self.b.normalize()
+        if a is NullQuery:
             return NullQuery
-        if optional is NullQuery:
-            return required
-        return AndMaybe(required, optional, boost=self.boost)
+        if b is NullQuery:
+            return a
+        return self.__class__(a, b, boost=self.boost)
 
     def estimate_min_size(self, ixreader):
@@ -1536 +1550 @@
     def docs(self, searcher, exclude_docs=None):
         return self.subqueries[0].docs(searcher, exclude_docs=exclude_docs)
-    
-    def matcher(self, searcher, exclude_docs=None):
-        required, optional = self.subqueries
-        return AndMaybeMatcher(required.matcher(searcher, exclude_docs=exclude_docs),
-                                optional.matcher(searcher, exclude_docs=exclude_docs))
-
-
-class AndNot(Query):
+
+
+class AndNot(BinaryQuery):
     """Binary boolean query of the form 'a ANDNOT b', where documents that
     match b are removed from the matches for a.
     """
 
-    def __init__(self, positive, negative, boost=1.0):
-        """
-        :param positive: query to INCLUDE.
-        :param negative: query whose matches should be EXCLUDED.
-        :param boost: boost factor that should be applied to the raw score of
-            results matched by this query.
-        """
-
-        self.positive = positive
-        self.negative = negative
-        self.boost = boost
-
-    def __eq__(self, other):
-        return (other
-                and self.__class__ is other.__class__
-                and self.positive == other.positive
-                and self.negative == other.negative
-                and self.boost == other.boost)
-
-    def __repr__(self):
-        return "%s(%r, %r)" % (self.__class__.__name__,
-                               self.positive, self.negative)
-
-    def __unicode__(self):
-        return u"%s ANDNOT %s" % (self.positive, self.negative)
-
-    def copy(self):
-        return self.__class__(self.positive, self.negative, boost=self.boost)
+    JOINT = " ANDNOT "
 
     def normalize(self):
-        pos = self.positive.normalize()
-        neg = self.negative.normalize()
-
-        if pos is NullQuery:
+        a = self.a.normalize()
+        b = self.b.normalize()
+
+        if a is NullQuery:
             return NullQuery
-        elif neg is NullQuery:
-            return pos
-
-        return AndNot(pos, neg, boost=self.boost)
-
-    def replace(self, oldtext, newtext):
-        return AndNot(self.positive.replace(oldtext, newtext),
-                      self.negative.replace(oldtext, newtext),
-                      boost=self.boost)
+        elif b is NullQuery:
+            return a
+
+        return self.__class__(a, b, boost=self.boost)
 
     def _all_terms(self, termset, phrases=True):
-        self.positive.all_terms(termset, phrases=phrases)
+        self.a.all_terms(termset, phrases=phrases)
 
     def _existing_terms(self, ixreader, termset, reverse=False, phrases=True):
-        self.positive.existing_terms(ixreader, termset, reverse=reverse,
-                                     phrases=phrases)
-
-    def matcher(self, searcher, exclude_docs=None):
-        notvector = _not_vector(searcher, [self.negative], exclude_docs)
-        return self.positive.matcher(searcher, exclude_docs=notvector)
+        self.a.existing_terms(ixreader, termset, reverse=reverse,
+                              phrases=phrases)
+
+    def matcher(self, searcher, exclude_docs=None):
+        # This is faster than actually using an AndNotMatcher, but could use
+        # a lot of memory on a very large index.
+        # TODO: Switch based on size of index?
+        notvector = _not_vector(searcher, [self.b], exclude_docs)
+        return self.a.matcher(searcher, exclude_docs=notvector)
+
+
+class Otherwise(BinaryQuery):
+    """A binary query that only matches the second clause if the first clause
+    doesn't match any documents.
+    """
+    
+    JOINT = " OTHERWISE "
+    
+    def matcher(self, searcher, exclude_docs=None):
+        m = self.a.matcher(searcher, exclude_docs=exclude_docs)
+        if not m.is_active():
+            m = self.b.matcher(searcher, exclude_docs=exclude_docs)
+        return m
 
 
@@ -1609 +1603 @@
 
 
-
-
-
-
-
-
-
-
-
-
+            
+
+
+
+
+
+
+
+
+

branches/mbutscher/work/lib/whoosh/reading.py

@@ -591 +591 @@
         postreaders = []
         docoffsets = []
+        excl = exclude_docs
+        term = (fieldname, text)
+        
         for i, r in enumerate(self.readers):
-            try:
+            if term in r:
+                offset = self.doc_offsets[i]
+                
+                # If an exclude_docs set was passed in, we need to pull out
+                # the document numbers that apply to this reader and subtract
+                # the offset from them
+                if exclude_docs and i > 0:
+                    limit = offset + r.doc_count_all()
+                    # Create a subset of the exclude_docs set with the offset
+                    # subtracted
+                    excl = set(docnum - offset for docnum in exclude_docs
+                               if docnum >= offset and docnum < limit)
+                
+                # Get a posting reader for the term and add it to the list
                 pr = r.postings(fieldname, text, scorer=scorer,
-                                exclude_docs=exclude_docs)
+                                exclude_docs=excl)
                 postreaders.append(pr)
-                docoffsets.append(self.doc_offsets[i])
-            except TermNotFound:
-                pass
+                docoffsets.append(offset)
         
         if not postreaders:

branches/mbutscher/work/lib/whoosh/scoring.py

@@ -479 +479 @@
                 cache[docid] = i
 
-        self.limit = i
         self._fieldcache = cache
         return cache

branches/mbutscher/work/lib/whoosh/searching.py

@@ -872 +872 @@
     """Represents a single search result ("hit") in a Results object.
     
+    This object acts like a dictionary of the matching document's stored
+    fields. If for some reason you need an actual ``dict`` object, use
+    ``Hit.fields()`` to get one.
+    
     >>> r = searcher.search(query.Term("content", "render"))
     >>> r[0]
     <Hit {title=u"Rendering the scene"}>
+    >>> r[0].rank
+    0
     >>> r[0].docnum
     4592L
     >>> r[0].score
-    2.52045682 
+    2.52045682
+    >>> r[0]["title"]
+    "Rendering the scene"
+    >>> r[0].keys()
+    ["title"]
     """
     
@@ -891 +901 @@
         
         self.searcher = searcher
-        self.pos = pos
+        self.pos = self.rank = pos
         self.docnum = docnum
         self.score = score
         self._fields = None
+    
+    def fields(self):
+        """Returns a dictionary of the stored fields of the document this
+        object represents.
+        """
+        
+        if self._fields is None:
+            self._fields = self.searcher.stored_fields(self.docnum)
+        return self._fields
     
     def __repr__(self):
@@ -907 +926 @@
             return False
     
-    def __iter__(self):
-        return self.fields().iterkeys()
-    
-    def __getitem__(self, key):
-        return self.fields().__getitem__(key)
-    
-    def __len__(self):
-        return len(self.fields())
-    
-    def fields(self):
-        if self._fields is None:
-            self._fields = self.searcher.stored_fields(self.docnum)
-        return self._fields
-    
-    def get(self, key, default=None):
-        return self.fields().get(key, default)
+    def __len__(self): return len(self.fields())
+    def __iter__(self): return self.fields().iterkeys()
+    def __getitem__(self, key): return self.fields().__getitem__(key)
+    def __contains__(self, key): return key in self.fields()
+    def items(self): return self.fields().items()
+    def keys(self): return self.fields().keys()
+    def values(self): return self.fields().values()
+    def iteritems(self): return self.fields().iteritems()
+    def iterkeys(self): return self.fields().iterkeys()
+    def itervalues(self): return self.fields().itervalues()
+    def get(self, key, default=None): return self.fields().get(key, default)
+    
+    def __setitem__(self, key, value):
+        raise NotImplementedError("You cannot modify a search result")
+    def __delitem__(self, key, value):
+        raise NotImplementedError("You cannot modify a search result")
+    def clear(self):
+        raise NotImplementedError("You cannot modify a search result")
+    def update(self, dict=None, **kwargs):
+        raise NotImplementedError("You cannot modify a search result")
     
 

branches/mbutscher/work/lib/whoosh/support/bench.py

@@ -18 +18 @@
 import os.path, random, sys
 from optparse import OptionParser
+from shutil import rmtree
 from zlib import compress, decompress
 
@@ -36 +37 @@
     pass
 
+try:
+    from persistent import Persistent
+    class ZDoc(Persistent):
+        def __init__(self, d):
+            self.__dict__.update(d)
+except ImportError:
+    pass
+
+
+class Module(object):
+    def __init__(self, bench, options, args):
+        self.bench = bench
+        self.options = options
+        self.args = args
+    
+    def __repr__(self):
+        return self.__class__.__name__
+    
+    def indexer(self):
+        pass
+    
+    def index_document(self, d):
+        raise NotImplementedError
+    
+    def finish(self):
+        pass
+    
+    def searcher(self):
+        pass
+    
+    def query(self):
+        raise NotImplementedError
+    
+    def find(self, q):
+        raise NotImplementedError
+    
+    def findterms(self, terms):
+        raise NotImplementedError
+    
+    def results(self, r):
+        return r
+
+
+class Spec(object):
+    headline_field = "title"
+    main_field = "body"
+    whoosh_compress_main = False
+    
+    def __init__(self, options, args):
+        self.options = options
+        self.args = args
+        
+    def documents(self):
+        raise NotImplementedError
+    
+    def setup(self):
+        pass
+    
+    def print_results(self, ls):
+        showbody = self.options.showbody
+        limit = self.options.limit
+        for i, hit in enumerate(ls):
+            if i >= limit:
+                break
+            
+            print "%d. %s" % (i+1, hit.get(self.headline_field))
+            if showbody:
+                print hit.get(self.main_field)
+            
+class WhooshModule(Module):
+    def indexer(self):
+        schema = self.bench.spec.whoosh_schema()
+        path = os.path.join(self.options.dir, "%s_whoosh" % self.options.indexname)
+        if not os.path.exists(path):
+            os.mkdir(path)
+        ix = index.create_in(path, schema)
+        self.writer = ix.writer(procs=int(self.options.procs),
+                                limitmb=int(self.options.limitmb))
+
+    def index_document(self, d):
+        if hasattr(self.bench, "process_document_whoosh"):
+            self.bench.process_document_whoosh(d)
+        if self.bench.spec.whoosh_compress_main:
+            mf = self.bench.spec.main_field
+            d["_stored_%s" % mf] = compress(d[mf], 9)
+        self.writer.add_document(**d)
+
+    def finish(self):
+        self.writer.commit()
+        
+    def searcher(self):
+        path = os.path.join(self.options.dir, "%s_whoosh" % self.options.indexname)
+        ix = index.open_dir(path)
+        self.srch = ix.searcher()
+        self.parser = qparser.QueryParser(self.bench.spec.main_field, schema=ix.schema)
+        
+    def query(self):
+        qstring = " ".join(self.args).decode("utf8")
+        return self.parser.parse(qstring)
+    
+    def find(self, q):
+        return self.srch.search(q, limit=int(self.options.limit))
+    
+    def results(self, r):
+        mf = self.bench.spec.main_field
+        for hit in r:
+            fs = hit.fields()
+            if self.bench.spec.whoosh_compress_main:
+                fs[mf] = decompress(fs[mf])
+            yield fs
+    
+    def findterms(self, terms):
+        limit = int(self.options.limit)
+        s = self.srch
+        q = query.Term(self.main_field, None)
+        for term in terms:
+            q.text = term
+            yield s.search(q, limit=limit)
+    
+
+class XappyModule(Module):
+    def indexer(self):
+        path = os.path.join(self.options.dir, "%s_xappy" % self.options.indexname)
+        conn = self.bench.spec.xappy_connection(path)
+        return conn
+    
+    def index_document(self, conn, d):
+        if hasattr(self.bench, "process_document_xappy"):
+            self.bench.process_document_xappy(d)
+        doc = xappy.UnprocessedDocument()
+        for key, values in d:
+            if not isinstance(values, list):
+                values = [values]
+            for value in values:
+                doc.fields.append(xappy.Field(key, value))
+        conn.add(doc)
+
+    def finish(self, conn):
+        conn.flush()
+        
+    def searcher(self):
+        path = os.path.join(self.options.dir, "%s_xappy" % self.options.indexname)
+        return xappy.SearchConnection(path)
+        
+    def query(self, conn):
+        return conn.query_parse(" ".join(self.args))
+    
+    def find(self, conn, q):
+        return conn.search(q, 0, int(self.options.limit))
+    
+    def findterms(self, conn, terms):
+        limit = int(self.options.limit)
+        for term in terms:
+            q = conn.query_field(self.main_field, term)
+            yield conn.search(q, 0, limit)
+    
+    def results(self, r):
+        hf = self.bench.spec.headline_field
+        mf = self.bench.spec.main_field
+        for hit in r:
+            yield {hf: hit.data[hf], mf: hit.data[mf]}
+        
+
+class XapianModule(Module):
+    def indexer(self):
+        path = os.path.join(self.options.dir, "%s_xapian" % self.options.indexname)
+        self.database = xapian.WritableDatabase(path, xapian.DB_CREATE_OR_OPEN)
+        self.ixer = xapian.TermGenerator()
+        
+    def index_document(self, d):
+        if hasattr(self.bench, "process_document_xapian"):
+            self.bench.process_document_xapian(d)
+        doc = xapian.Document()
+        doc.add_value(0, d.get(self.bench.spec.headline_field, "-"))
+        doc.set_data(d[self.main_field])
+        self.ixer.set_document(doc)
+        self.ixer.index_text(d[self.main_field])
+        self.database.add_document(doc)
+        
+    def finish(self):
+        self.database.flush()
+        
+    def searcher(self):
+        path = os.path.join(self.options.dir, "%s_xappy" % self.options.indexname)
+        self.db = xapian.Database(path)
+        self.enq = xapian.Enquire(self.db)
+        self.qp = xapian.QueryParser()
+        self.qp.set_database(self.db)
+        
+    def query(self):
+        return self.qp.parse_query(" ".join(self.args))
+    
+    def find(self, q):
+        self.enq.set_query(q)
+        return self.enq.get_mset(0, int(self.options.limit))
+    
+    def findterms(self, terms):
+        limit = int(self.options.limit)
+        for term in terms:
+            q = self.qp.parse_query(term)
+            self.enq.set_query(q)
+            yield self.enq.get_mset(0, limit)
+    
+    def results(self, matches):
+        hf = self.bench.spec.headline_field
+        mf = self.bench.spec.main_field
+        for m in matches:
+            yield {hf: m.document.get_value(0), mf: m.document.get_data()}
+
+
+class SolrModule(Module):
+    def indexer(self):
+        self.solr_doclist = []
+        self.conn = pysolr.Solr(self.options.url)
+        self.conn.delete("*:*")
+        self.conn.commit()
+    
+    def index_document(self, d):
+        self.solr_doclist.append(d)
+        if len(self.solr_doclist) >= int(self.options.batch):
+            self.conn.add(self.solr_doclist, commit=False)
+            self.solr_doclist = []
+        
+    def finish(self):
+        if self.solr_doclist:
+            self.conn.add(self.solr_doclist)
+        del self.solr_doclist
+        self.conn.optimize(block=True)
+        
+    def searcher(self):
+        self.solr = pysolr.Solr(self.options.url)
+    
+    def query(self):
+        return " ".join(self.args)
+    
+    def find(self, q):
+        return self.solr.search(q, limit=int(self.options.limit))
+    
+    def findterms(self, terms):
+        limit = int(self.options.limit)
+        for term in terms:
+            yield self.solr.search("body:" + term, limit=limit)
+    
+
+class ZcatalogModule(Module):
+    def indexer(self):
+        from ZODB.FileStorage import FileStorage
+        from ZODB.DB import DB
+        from zcatalog import catalog
+        from zcatalog import indexes
+        import transaction
+        
+        dir = os.path.join(self.options.dir, "%s_zcatalog" % self.options.indexname)
+        if os.path.exists(dir):
+            rmtree(dir)
+        os.mkdir(dir)
+        
+        storage = FileStorage(os.path.join(dir, "index"))
+        db = DB(storage)
+        conn = db.open()
+        
+        self.cat = catalog.Catalog()
+        self.bench.spec.zcatalog_setup(self.cat)
+        conn.root()["cat"] = self.cat
+        transaction.commit()
+        
+        self.zcatalog_count = 0
+    
+    def index_document(self, d):
+        if hasattr(self.bench, "process_document_zcatalog"):
+            self.bench.process_document_zcatalog(d)
+        doc = ZDoc(d)
+        self.cat.index_doc(doc)
+        self.zcatalog_count += 1
+        if self.zcatalog_count >= 100:
+            import transaction
+            transaction.commit()
+            self.zcatalog_count = 0
+        
+    def finish(self):
+        import transaction
+        transaction.commit()
+        del self.zcatalog_count
+        
+    def searcher(self):
+        from ZODB.FileStorage import FileStorage
+        from ZODB.DB import DB
+        from zcatalog import catalog
+        from zcatalog import indexes
+        import transaction
+        
+        path = os.path.join(self.options.dir, "%s_zcatalog" % self.options.indexname, "index")
+        storage = FileStorage(path)
+        db = DB(storage)
+        conn = db.open()
+        
+        self.cat = conn.root()["cat"]
+    
+    def query(self):
+        return " ".join(self.args)
+    
+    def find(self, q):
+        return self.cat.searchResults(body=q)
+    
+    def findterms(self, terms):
+        for term in terms:
+            yield self.cat.searchResults(body=term)
+    
+    def results(self, r):
+        hf = self.bench.spec.headline_field
+        mf = self.bench.spec.main_field
+        for hit in r:
+            # Have to access the attributes for them to be retrieved
+            yield {hf: getattr(hit, hf), mf: getattr(hit, mf)}
 
 
 class Bench(object):
-    solr_url = "http://localhost:8983/solr"
-    main_field = "text"
-    headline_field = "title"
-    
-    libs = ("whoosh", "xappy", "xapian", "solr")
-    
-    _name = "unknown"
-    
-    def name(self):
-        return self._name
-    
-    def process_document_whoosh(self, d):
-        pass
-    
-    def process_document_xappy(self, d):
-        pass
-    
-    def process_document_xapian(self, d):
-        pass
-    
-    def process_document_solr(self, d):
-        pass
+    libs = {"whoosh": WhooshModule, "xappy": XappyModule,
+            "xapian": XapianModule, "solr": SolrModule,
+            "zcatalog": ZcatalogModule}
     
     def index(self, lib):
@@ -73 +369 @@
         
         starttime = chunkstarttime = now()
-        ix = getattr(self, "%s_indexer" % lib)()
-        index_document = getattr(self, "index_document_%s" % lib)
-        for d in self.documents():
+        lib.indexer()
+        for d in self.spec.documents():
             skipc -= 1
             if not skipc:
-                index_document(ix, d)
+                lib.index_document(d)
                 count += 1
                 skipc = skip
@@ -91 +386 @@
         spooltime = now()
         print "Spool time:", spooltime - starttime
-        getattr(self, "finish_%s" % lib)(ix)
+        lib.finish()
         committime = now()
         print "Commit time:", committime - spooltime
         print "Total time to index", count, "documents:",  committime - starttime
     
-    def whoosh_indexer(self):
-        schema = self.whoosh_schema()
-        path = os.path.join(self.options.dir, "%s_whoosh" % self.options.indexname)
-        if not os.path.exists(path):
-            os.mkdir(path)
-        ix = index.create_in(path, schema)
-        w = ix.writer(procs=int(self.options.procs),
-                      limitmb=int(self.options.limitmb))
-        return w
-    
-    def index_document_whoosh(self, writer, d):
-        self.process_document_whoosh(d)
-        writer.add_document(**d)
-        
-    def finish_whoosh(self, writer):
-        writer.commit()
-        
-    def xappy_indexer(self):
-        path = os.path.join(self.options.dir, "%s_xappy" % self.options.indexname)
-        conn = self.xappy_connection(path)
-        return conn
-    
-    def index_document_xappy(self, conn, d):
-        self.process_document_xappy(d)
-        doc = xappy.UnprocessedDocument()
-        for key, values in d:
-            if not isinstance(values, list):
-                values = [values]
-            for value in values:
-                doc.fields.append(xappy.Field(key, value))
-        conn.add(doc)
-        
-    def finish_xappy(self, conn):
-        conn.flush()
-                    
-    def xapian_indexer(self):
-        path = os.path.join(self.options.dir, "%s_xapian" % self.options.indexname)
-        database = xapian.WritableDatabase(path, xapian.DB_CREATE_OR_OPEN)
-        indexer = xapian.TermGenerator()
-        
-        return (database, indexer)
-    
-    def index_document_xapian(self, dix, d):
-        self.process_document_xapian(d)
-        database, indexer = dix
-        doc = xapian.Document()
-        doc.add_value(0, d.get(self.headline_field, "-"))
-        doc.set_data(d[self.main_field])
-        indexer.set_document(doc)
-        indexer.index_text(d[self.main_field])
-        database.add_document(doc)
-        
-    def finish_xapian(self, dix):
-        dix[0].flush()
-        
-    def solr_indexer(self):
-        self.solr_doclist = []
-        conn = pysolr.Solr(self.options.url)
-        conn.delete("*:*")
-        conn.commit()
-        return conn
-    
-    def index_document_solr(self, conn, d):
-        self.solr_doclist.append(d)
-        if len(self.solr_doclist) >= int(self.options.batch):
-            conn.add(self.solr_doclist, commit=False)
-            self.solr_doclist = []
-        
-    def finish_solr(self, conn):
-        if self.solr_doclist:
-            conn.add(self.solr_doclist)
-        del self.solr_doclist
-        conn.optimize(block=True)
-    
-    def whoosh_searcher(self):
-        path = os.path.join(self.options.dir, "%s_whoosh" % self.options.indexname)
-        ix = index.open_dir(path)
-        searcher = ix.searcher()
-        parser = qparser.QueryParser(self.main_field, schema=ix.schema)
-        
-        return (searcher, parser)
-    
-    def whoosh_query(self, s):
-        qstring = " ".join(self.args).decode("utf8")
-        return s[1].parse(qstring)
-    
-    def whoosh_find(self, s, q):
-        return s[0].search(q, limit=int(self.options.limit))
-    
-    def whoosh_findterms(self, s, terms):
-        limit = int(self.options.limit)
-        searcher = s[0]
-        q = query.Term(self.main_field, None)
-        for term in terms:
-            q.text = term
-            yield searcher.search(q, limit=limit)
-    
-    def whoosh_results(self, s, r):
-        showbody = self.options.showbody
-        
-        print "Runtime:", r.runtime
-        for hit in r:
-            print hit.get(self.headline_field)
-            if showbody:
-                print decompress(hit[self.main_field])
-    
-    def xappy_searcher(self):
-        path = os.path.join(self.options.dir, "%s_xappy" % self.options.indexname)
-        return xappy.SearchConnection(path)
-        
-    def xappy_query(self, conn):
-        return conn.query_parse(" ".join(self.args))
-    
-    def xappy_find(self, conn, q):
-        return conn.search(q, 0, int(self.options.limit))
-    
-    def xappy_findterms(self, conn, terms):
-        limit = int(self.options.limit)
-        for term in terms:
-            q = conn.query_field(self.main_field, term)
-            yield conn.search(q, 0, limit)
-    
-    def xappy_results(self, conn, r):
-        showbody = self.options.showbody
-        for hit in r:
-            print hit.rank, hit.data[self.headline_field]
-            if showbody:
-                print hit.data[self.main_field]
-    
-    def xapian_searcher(self):
-        path = os.path.join(self.options.dir, "%s_xappy" % self.options.indexname)
-        db = xapian.Database(path)
-        enq = xapian.Enquire(db)
-        qp = xapian.QueryParser()
-        qp.set_database(db)
-        return db, enq, qp
-    
-    def xapian_query(self, s):
-        return s[2].parse_query(" ".join(self.args))
-    
-    def xapian_find(self, s, q):
-        enq = s[1]
-        enq.set_query(q)
-        return enq.get_mset(0, int(self.options.limit))
-    
-    def xapian_findterms(self, s, terms):
-        limit = int(self.options.limit)
-        db, enq, qp = s
-        for term in terms:
-            q = qp.parse_query(term)
-            enq.set_query(q)
-            yield enq.get_mset(0, limit)
-    
-    def xapian_results(self, s, matches):
-        showbody = self.options.showbody
-        for m in matches:
-            print m.rank, repr(m.document.get_value(0))
-            if showbody:
-                print m.document.get_data()
-    
-    def solr_searcher(self):
-        return pysolr.Solr(self.solr_url)
-    
-    def solr_query(self, solr):
-        return " ".join(self.args)
-    
-    def solr_find(self, solr, q):
-        return solr.search(q, limit=int(self.options.limit))
-    
-    def solr_findterms(self, solr, terms):
-        limit = int(self.options.limit)
-        for term in terms:
-            yield solr.search("body:" + term, limit=limit)
-    
-    def solr_results(self, solr, r):
-        showbody = self.options.showbody
-        print len(r), "results"
-        for hit in r:
-            print hit.get(self.headline_field)
-            if showbody:
-                print hit[self.main_field]
-    
     def search(self, lib):
-        s = getattr(self, "%s_searcher" % lib)()
+        lib.searcher()
+        
         t = now()
-        q = getattr(self, "%s_query" % lib)(s)
+        q = lib.query()
         print "Query:", q
-        r = getattr(self, "%s_find" % lib)(s, q)
+        r = lib.find(q)
         print "Search time:", now() - t
         
         t = now()
-        getattr(self, "%s_results" % lib)(s, r)
+        self.spec.print_results(lib.results(r))
         print "Print time:", now() - t
     
@@ -296 +410 @@
         
         print "Searching %d terms with %s" % (len(terms), lib)
-        s = getattr(self, "%s_searcher" % lib)()
+        lib.searcher()
         starttime = now()
-        for r in getattr(self, "%s_findterms" % lib)(s, terms):
+        for r in lib.findterms(terms):
             pass
         searchtime = now() - starttime
         print "Search time:", searchtime, "searches/s:", float(len(terms))/searchtime
     
-    def generate_search_file(self, lib):
-        if self.args:
-            f = open(self.args[0], "wb")
-        else:
-            f = sys.stdout
-        count = int(self.options.generate)
-        
-        t = now()
-        s = self.whoosh_searcher()[0]
-        terms = list(s.lexicon(self.main_field))
-        sample = random.sample(terms, count)
-        for term in sample:
-            if term.isalnum():
-                f.write(term + "\n")
-        print now() - t
-    
-    def _parser(self):
+    def _parser(self, name):
         p = OptionParser()
         p.add_option("-x", "--lib", dest="lib",
@@ -331 +429 @@
                      help="Index the documents.", default=False)
         p.add_option("-n", "--name", dest="indexname", metavar="PREFIX",
-                     help="Index name prefix.", default="%s_index" % self.name())
+                     help="Index name prefix.", default="%s_index" % name)
         p.add_option("-U", "--url", dest="url", metavar="URL",
                      help="Solr URL", default="http://localhost:8983/solr")
@@ -363 +461 @@
         return p
     
-    def run(self):
-        parser = self._parser()
+    def run(self, specclass):
+        parser = self._parser(specclass.name)
         options, args = parser.parse_args()
         self.options = options
         self.args = args
         
-        lib = options.lib
-        if lib not in self.libs:
-            raise Exception("Unknown library: %r" % lib)
+        if options.lib not in self.libs:
+            raise Exception("Unknown library: %r" % options.lib)
+        lib = self.libs[options.lib](self, options, args)
+        
+        self.spec = specclass(options, args)
         
         if options.setup:
-            self.setup()
+            self.spec.setup()
         
         action = self.search

branches/mbutscher/work/lib/whoosh/support/unicode.py

@@ -249 +249 @@
     return _names[i]
 
+
 def blocknum(ch):
     """Returns the unicode block number for ch, or None if ch has no block.
@@ -268 +269 @@
 
 
-if __name__ == "__main__":
-    pass
-    
-    
-    
-    
-    
+    
+    
+    
+    
+    

branches/stable-2.0/extensions/GnuplotClBridge.py

@@ -117 +117 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
             
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
+
             if u"noerror" in [a.strip() for a in insToken.appendices]:
                 childErr.read()

branches/stable-2.0/extensions/GraphvizClBridge.py

@@ -124 +124 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
+
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
 
             if u"noerror" in [a.strip() for a in insToken.appendices]:

branches/stable-2.0/extensions/GraphvizStructureView.py

@@ -342 +342 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
+
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
 
             if u"noerror" in [a.strip() for a in insParams]:

branches/stable-2.0/extensions/MimeTexCGIBridge.py

@@ -103 +103 @@
         # Run MimeTeX process
         popenObject = subprocess.Popen(cmdline, shell=True,
-                stdout=subprocess.PIPE)
+                 stdout=subprocess.PIPE, stdin=subprocess.PIPE,
+                 stderr=subprocess.PIPE)
+
         childOut = popenObject.stdout
+        
+        # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+        # why this is necessary
+        popenObject.stdin.close()
+        popenObject.stderr.close()
 
         # Read stdout of process entirely

branches/stable-2.0/extensions/PloticusClBridge.py

@@ -128 +128 @@
 #             childIn, childOut, childErr = os.popen3(cmdline, "b")
             popenObject = subprocess.Popen(cmdline, shell=True,
-                    stderr=subprocess.PIPE)
+                    stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stdin=subprocess.PIPE)
             childErr = popenObject.stderr
-            
+
+            # See http://bytes.com/topic/python/answers/634409-subprocess-handle-invalid-error
+            # why this is necessary
+            popenObject.stdin.close()
+            popenObject.stdout.close()
+
             if u"noerror" in [a.strip() for a in insToken.appendices]:
                 childErr.read()