root/branches/mbutscher/next/lib/pwiki/wikidata/compact_sqlite/WikiData.py @ 178

Revision 178, 68.5 kB (checked in by mbutscher, 4 years ago)

2.0preAlpha (internal)

Works so far, now editing help wiki.

Line 
1"""
2
3Used terms:
4   
5    wikiword -- a string matching one of the wiki word regexes
6    wiki page -- real existing content stored and associated with a wikiword
7            (which is the page name). Sometimes page is synonymous for page name
8    alias -- wikiword without content but associated to a page name.
9            For the user it looks as if the content of the alias is the content
10            of the page for the associated page name
11    defined wiki word -- either a page name or an alias
12"""
13
14
15
16from os import mkdir, unlink, listdir, rename, stat, utime
17from os.path import exists, join, basename
18import os.path
19
20from time import time, localtime
21import datetime
22import string, glob, types, traceback
23import re # import pwiki.srePersistent as re
24
25from pwiki.WikiExceptions import *   # TODO make normal import
26from pwiki import SearchAndReplace
27
28try:
29    import pwiki.sqlite3api as sqlite
30    import DbStructure
31    from DbStructure import createWikiDB, WikiDBExistsException
32except:
33    traceback.print_exc()
34    sqlite = None
35# finally:
36#     pass
37
38
39from pwiki.StringOps import getBinCompactForDiff, applyBinCompact, longPathEnc, \
40        longPathDec, binCompactToCompact, fileContentToUnicode, utf8Enc, utf8Dec, \
41        uniWithNone, loadEntireTxtFile, Conjunction, lineendToInternal
42
43
44import Consts
45
46class WikiData:
47    "Interface to wiki data."
48    def __init__(self, wikiDocument, dataDir, tempDir):
49        self.wikiDocument = wikiDocument
50        self.dataDir = dataDir
51        self.cachedContentNames = None
52#         tempDir = uniWithNone(tempDir)
53
54        dbfile = join(dataDir, "wiki.sli")
55
56        try:
57            if (not exists(longPathEnc(dbfile))):
58                DbStructure.createWikiDB(None, dataDir)  # , True
59        except (IOError, OSError, sqlite.Error), e:
60            traceback.print_exc()
61            raise DbWriteAccessError(e)
62
63        dbfile = longPathDec(dbfile)
64        try:
65            self.connWrap = DbStructure.ConnectWrapSyncCommit(
66                    sqlite.connect(dbfile))
67        except (IOError, OSError, sqlite.Error), e:
68            traceback.print_exc()
69            raise DbReadAccessError(e)
70       
71#         self.connWrap.execSql("pragma temp_store_directory = '%s'" %
72#                 utf8Enc(tempDir)[0])
73
74        DbStructure.registerSqliteFunctions(self.connWrap)
75
76
77    def checkDatabaseFormat(self):
78        return DbStructure.checkDatabaseFormat(self.connWrap)
79
80
81    def connect(self):
82        formatcheck, formatmsg = self.checkDatabaseFormat()
83
84        if formatcheck == 2:
85            # Unknown format
86            raise WikiDataException, formatmsg
87
88        # Update database from previous versions if necessary
89        if formatcheck == 1:
90            try:
91                DbStructure.updateDatabase(self.connWrap, self.dataDir)
92            except Exception, e:
93                traceback.print_exc()
94                try:
95                    self.connWrap.rollback()
96                except Exception, e2:
97                    traceback.print_exc()
98                    raise DbWriteAccessError(e2)
99                raise DbWriteAccessError(e)
100
101        lastException = None
102        try:
103            # Further possible updates
104            DbStructure.updateDatabase2(self.connWrap)
105        except sqlite.Error, e:
106            # Remember but continue
107            lastException = DbWriteAccessError(e)
108
109        # Activate UTF8 support for text in database (content is blob!)
110        DbStructure.registerUtf8Support(self.connWrap)
111
112        # Function to convert from content in database to
113        # return value, used by getContent()
114        self.contentDbToOutput = lambda c: utf8Dec(c, "replace")[0]
115       
116        try:
117            # Set marker for database type
118            self.wikiDocument.getWikiConfig().set("main", "wiki_database_type",
119                    "compact_sqlite")
120        except (IOError, OSError, sqlite.Error), e:
121            # Remember but continue
122            lastException = DbWriteAccessError(e)
123
124        # Function to convert unicode strings from input to content in database
125        # used by setContent
126
127        def contentUniInputToDb(unidata):
128            return utf8Enc(unidata, "replace")[0]
129
130        self.contentUniInputToDb = contentUniInputToDb
131
132        try:
133            self._createTempTables()
134
135            # reset cache
136            self.cachedContentNames = None
137   
138            self.cachedGlobalProps = None
139            self.getGlobalProperties()
140        except (IOError, OSError, sqlite.Error), e:
141            traceback.print_exc()
142            try:
143                self.connWrap.rollback()
144            except (IOError, OSError, sqlite.Error), e2:
145                traceback.print_exc()
146                raise DbReadAccessError(e2)
147            raise DbReadAccessError(e)
148           
149        if lastException:
150            raise lastException
151
152
153    def _reinit(self):
154        """
155        Actual initialization or reinitialization after rebuildWiki()
156        """
157        pass       
158
159
160    def _createTempTables(self):
161        # Temporary table for findBestPathFromWordToWord
162        # TODO: Possible for read-only dbs?
163
164        # These schema changes are only on a temporary table so they are not
165        # in DbStructure.py
166        self.connWrap.execSql("create temp table temppathfindparents "
167                "(word text primary key, child text, steps integer)")
168
169        self.connWrap.execSql("create index temppathfindparents_steps "
170                "on temppathfindparents(steps)")
171
172
173    # ---------- Direct handling of page data ----------
174   
175    def getContent(self, word):
176        try:
177            result = self.connWrap.execSqlQuerySingleItem("select content from "+\
178                "wikiwordcontent where word = ?", (word,), None)
179
180            if result is None:
181                raise WikiFileNotFoundException(_(u"Wiki page not found: %s") % word)
182   
183            return self.contentDbToOutput(result)
184        except (IOError, OSError, sqlite.Error), e:
185            traceback.print_exc()
186            raise DbReadAccessError(e)
187
188
189    def _getContentAndInfo(self, word):
190        """
191        Get content and further information about a word
192       
193        Not part of public API!
194        """
195        try:
196            result = self.connWrap.execSqlQuery("select content, modified from "+\
197                "wikiwordcontent where word = ?", (word,))
198            if len(result) == 0:
199                raise WikiFileNotFoundException, "wiki page not found: %s" % word
200   
201            content = self.contentDbToOutput(result[0][0])
202            return (content, result[0][1])
203        except (IOError, OSError, sqlite.Error), e:
204            traceback.print_exc()
205            raise DbReadAccessError(e)
206
207
208    def setContent(self, word, content, moddate = None, creadate = None):
209        """
210        Sets the content, does not modify the cache information
211        except self.cachedContentNames
212        """
213        if not content: content = u""  # ?
214       
215        assert type(content) is unicode
216
217        content = self.contentUniInputToDb(content)
218        self.setContentRaw(word, content, moddate, creadate)
219
220        self.cachedContentNames = None
221
222
223    def setContentRaw(self, word, content, moddate = None, creadate = None):
224        """
225        Sets the content without applying any encoding, used by versioning,
226        does not modify the cache information
227       
228        moddate -- Modification date to store or None for current
229        creadate -- Creation date to store or None for current
230       
231        Not part of public API!
232        """
233        ti = time()
234        if moddate is None:
235            moddate = ti
236
237        # if not content: content = ""
238       
239        assert type(content) is str
240
241        try:
242            if self.connWrap.execSqlQuerySingleItem("select word from "+\
243                    "wikiwordcontent where word=?", (word,), None) is not None:
244   
245                # Word exists already
246    #             self.connWrap.execSql("insert or replace into wikiwordcontent"+\
247    #                 "(word, content, modified) values (?,?,?)",
248    #                 (word, sqlite.Binary(content), moddate))
249                self.connWrap.execSql("update wikiwordcontent set "
250                    "content=?, modified=? where word=?",
251                    (sqlite.Binary(content), moddate, word))
252            else:
253                if creadate is None:
254                    creadate = ti
255   
256                # Word does not exist -> record creation date
257                self.connWrap.execSql("insert or replace into wikiwordcontent"
258                    "(word, content, modified, created) "
259                    "values (?,?,?,?)",
260                    (word, sqlite.Binary(content), moddate, creadate))
261        except (IOError, OSError, sqlite.Error), e:
262            traceback.print_exc()
263            raise DbWriteAccessError(e)
264
265
266    def renameContent(self, oldWord, newWord):
267        """
268        The content which was stored under oldWord is stored
269        after the call under newWord. The self.cachedContentNames
270        dictionary is updated, other caches won't be updated.
271        """
272        try:
273            self.connWrap.execSql("update wikiwordcontent set word = ? "
274                    "where word = ?", (newWord, oldWord))
275   
276            self.cachedContentNames = None
277        except (IOError, OSError, sqlite.Error), e:
278            traceback.print_exc()
279            raise DbWriteAccessError(e)
280
281
282    def deleteContent(self, word):
283        try:
284            self.connWrap.execSql("delete from wikiwordcontent where word = ?", (word,))
285            self.cachedContentNames = None
286        except (IOError, OSError, sqlite.Error), e:
287            traceback.print_exc()
288            raise DbWriteAccessError(e)
289
290
291    def getTimestamps(self, word):
292        """
293        Returns a tuple with modification, creation and visit date of
294        a word or (None, None, None) if word is not in the database
295        """
296        try:
297            dates = self.connWrap.execSqlQuery(
298                    "select modified, created from wikiwordcontent where word = ?",
299                    (word,))
300
301            if len(dates) > 0:
302                return (float(dates[0][0]), float(dates[0][1]), 0.0)
303            else:
304                return (None, None, None)  # ?
305        except (IOError, OSError, sqlite.Error), e:
306            traceback.print_exc()
307            raise DbReadAccessError(e)
308
309
310    def setTimestamps(self, word, timestamps):
311        """
312        Set timestamps for an existing wiki page.
313        Aliases must be resolved beforehand.
314        """
315        moddate, creadate = timestamps[:2]
316
317        try:
318            data = self.connWrap.execSqlQuery("select word from wikiwordcontent "
319                    "where word = ?", (word,))
320        except (IOError, OSError, sqlite.Error), e:
321            traceback.print_exc()
322            raise DbReadAccessError(e)
323
324        try:
325            if len(data) < 1:
326                raise WikiFileNotFoundException
327            else:
328                self.connWrap.execSql("update wikiwordcontent set modified = ?, "
329                        "created = ? where word = ?", (moddate, creadate, word))
330        except (IOError, OSError, sqlite.Error), e:
331            traceback.print_exc()
332            raise DbWriteAccessError(e)
333
334
335    def getExistingWikiWordInfo(self, wikiWord, withFields=()):
336        """
337        Get information about an existing wiki word
338        Aliases must be resolved beforehand.
339        Function must work for read-only wiki.
340        withFields -- Seq. of names of fields which should be included in
341            the output. If this is not empty, a tuple is returned
342            (relation, ...) with ... as further fields in the order mentioned
343            in withfields.
344
345            Possible field names:
346                "modified": Modification date of page
347                "created": Creation date of page
348                "visited": Last visit date of page (currently always returns 0)
349                "firstcharpos": Dummy returning very high value
350        """
351        if withFields is None:
352            withFields = ()
353
354        addFields = ""
355        converters = [lambda s: s]
356
357        for field in withFields:
358            if field == "modified":
359                addFields += ", modified"
360                converters.append(float)
361            elif field == "created":
362                addFields += ", created"
363                converters.append(float)
364            elif field == "visited":
365                # Fake "visited" field
366                addFields += ", visited"
367#                 converters.append(lambda s: 0.0)
368                converters.append(float)
369            elif field == "firstcharpos":
370                # Fake character position. TODO More elegantly
371                addFields += ", 0"
372                converters.append(lambda s: 2000000000L)
373
374
375        sql = "select word%s from wikiwordcontent where word = ?" % addFields
376
377        try:
378            if len(withFields) > 0:
379                dbresult = [tuple(c(item) for c, item in zip(converters, row))
380                        for row in self.connWrap.execSqlQuery(sql, (wikiWord,))]
381            else:
382                dbresult = self.connWrap.execSqlQuerySingleColumn(sql, (wikiWord,))
383           
384            if len(dbresult) == 0:
385                raise WikiWordNotFoundException(wikiWord)
386           
387            return dbresult[0]
388        except (IOError, OSError, sqlite.Error), e:
389            traceback.print_exc()
390            raise DbReadAccessError(e)
391
392
393    # ---------- Renaming/deleting pages with cache update or invalidation ----------
394
395    def renameWord(self, word, toWord):
396        try:
397            # commit anything pending so we can rollback on error
398            self.connWrap.syncCommit()
399
400            try:
401                self.connWrap.execSql("update wikirelations set word = ? where word = ?", (toWord, word))
402                self.connWrap.execSql("update wikiwordprops set word = ? where word = ?", (toWord, word))
403                self.connWrap.execSql("update todos set word = ? where word = ?", (toWord, word))
404                self.connWrap.execSql("update wikiwordmatchterms set word = ? where word = ?", (toWord, word))
405                self.renameContent(word, toWord)
406   
407                self.connWrap.commit()
408            except:
409                self.connWrap.rollback()
410                raise
411        except (IOError, OSError, sqlite.Error), e:
412            traceback.print_exc()
413            raise DbWriteAccessError(e)
414
415
416    def deleteWord(self, word):
417        """
418        delete everything about the wikiword passed in. an exception is raised
419        if you try and delete the wiki root node.
420        """
421        if word != self.wikiDocument.getWikiName():
422            try:
423                self.connWrap.syncCommit()
424                try:
425                    # don't delete the relations to the word since other
426                    # pages still have valid outward links to this page.
427                    # just delete the content
428   
429                    self.deleteChildRelationships(word)
430                    self.deleteProperties(word)
431                    self.deleteTodos(word)
432                    # self.connWrap.execSql("delete from wikiwordcontent where word = ?", (word,))
433                    self.deleteContent(word)
434                    self.deleteWikiWordMatchTerms(word, syncUpdate=False)
435                    self.deleteWikiWordMatchTerms(word, syncUpdate=True)
436                    self.connWrap.commit()
437                except:
438                    self.connWrap.rollback()
439                    raise
440            except (IOError, OSError, sqlite.Error), e:
441                traceback.print_exc()
442                raise DbWriteAccessError(e)
443        else:
444            raise WikiDataException(_(u"You cannot delete the root wiki node"))
445
446
447    def setMetaDataState(self, word, state):
448        """
449        Set the state of meta-data processing for a particular word.
450        See Consts.WIKIWORDMETADATA_STATE_*
451        """
452        try:
453            self.connWrap.execSql("update wikiwordcontent set metadataprocessed = ? "
454                    "where word = ?", (state, word))
455        except (IOError, OSError, sqlite.Error), e:
456            traceback.print_exc()
457            raise DbWriteAccessError(e)
458
459
460    def getMetaDataState(self, word):
461        """
462        Retrieve meta-data processing state of a particular wiki word.
463        """
464        try:
465            return self.connWrap.execSqlQuerySingleItem("select metadataprocessed "
466                    "from wikiwordcontent where word = ?", (word,))
467        except (IOError, OSError, sqlite.Error), e:
468            traceback.print_exc()
469            raise DbReadAccessError(e)
470
471
472    def fullyResetMetaDataState(self, state=0):
473        """
474        Reset state of all wikiwords.
475        """
476        self.connWrap.execSql("update wikiwordcontent set metadataprocessed = ?",
477                (state,))
478
479
480    def getWikiWordsForMetaDataState(self, state):
481        """
482        Retrieve a list of all words with a particular meta-data processing
483        state.
484        """
485        try:
486            return self.connWrap.execSqlQuerySingleColumn("select word "
487                    "from wikiwordcontent where metadataprocessed = ?", (state,))
488        except (IOError, OSError, sqlite.Error), e:
489            traceback.print_exc()
490            raise DbReadAccessError(e)
491
492
493    def validateFileSignatureForWord(self, word, setMetaDataDirty=False,
494            refresh=False):
495        """
496        Returns True if file signature stored in DB matches the file
497        containing the content, False otherwise.
498        For compact_sqlite it always returns True.
499        """
500        return True
501
502
503    def refreshFileSignatureForWord(self, word):
504        """
505        Sets file signature to match current file.
506        For compact_sqlite it does nothing.
507        """
508        pass
509
510
511
512    # ---------- Handling of relationships cache ----------
513
514    def getChildRelationships(self, wikiWord, existingonly=False,
515            selfreference=True, withFields=()):
516        """
517        get the child relations of this word
518        Function must work for read-only wiki.
519        existingonly -- List only existing wiki words
520        selfreference -- List also wikiWord if it references itself
521        withFields -- Seq. of names of fields which should be included in
522            the output. If this is not empty, tuples are returned
523            (relation, ...) with ... as further fields in the order mentioned
524            in withfields.
525
526            Possible field names:
527                "firstcharpos": position of link in page (may be -1 to represent
528                    unknown)
529                "modified": Modification date of child
530        """
531        if withFields is None:
532            withFields = ()
533
534        addFields = ""
535        converters = [lambda s: s]
536        for field in withFields:
537            if field == "firstcharpos":
538                addFields += ", firstcharpos"
539                converters.append(lambda s: s)
540            elif field == "modified":
541                # "modified" isn't a field of wikirelations. We need
542                # some SQL magic to retrieve the modification date
543                addFields += (", ifnull((select modified from wikiwordcontent "
544                        "where wikiwordcontent.word = relation or "
545                        "wikiwordcontent.word = (select word from wikiwordmatchterms "
546                        "where wikiwordmatchterms.matchterm = relation and "
547                        "(wikiwordmatchterms.type & 2) != 0 limit 1)), 0.0)")
548
549                converters.append(float)
550
551
552        sql = "select relation%s from wikirelations where word = ?" % addFields
553
554        if not selfreference:
555            sql += " and relation != word"
556
557        if existingonly:
558            # filter to only words in wikiwords or aliases
559#             sql += " and (exists (select word from wikiwordcontent "+\
560#                     "where word = relation) or exists "+\
561#                     "(select value from wikiwordprops "+\
562#                     "where value = relation and key = 'alias'))"
563            sql += (" and (exists (select 1 from wikiwordcontent "
564                    "where word = relation) or exists "
565                    "(select 1 from wikiwordmatchterms "
566                    "where wikiwordmatchterms.matchterm = relation and "
567                    "(wikiwordmatchterms.type & 2) != 0))")
568            # Consts.WIKIWORDMATCHTERMS_TYPE_ASLINK == 2
569
570
571        try:
572            if len(withFields) > 0:
573                return self.connWrap.execSqlQuery(sql, (wikiWord,))
574            else:
575                return self.connWrap.execSqlQuerySingleColumn(sql, (wikiWord,))
576        except (IOError, OSError, sqlite.Error), e:
577            traceback.print_exc()
578            raise DbReadAccessError(e)
579
580
581    def getParentRelationships(self, wikiWord):
582        """
583        get the parent relations to this word
584        Function must work for read-only wiki.
585        """
586        # Parents of the real word
587        wikiWord = self.getUnAliasedWikiWord(wikiWord)
588        try:
589            return self.connWrap.execSqlQuerySingleColumn(
590                    "select word from wikirelations where relation = ? or "
591                    "relation in (select matchterm from wikiwordmatchterms "
592                    "where word = ? and "
593                    "(wikiwordmatchterms.type & 2) != 0)", (wikiWord, wikiWord))
594            # Consts.WIKIWORDMATCHTERMS_TYPE_ASLINK == 2
595
596        except (IOError, OSError, sqlite.Error), e:
597            traceback.print_exc()
598            raise DbReadAccessError(e)
599
600
601
602    # TODO Optimize!
603    def getParentlessWikiWords(self):
604        """
605        get the words that have no parents.
606        Function must work for read-only wiki.
607
608        NO LONGER VALID: (((also returns nodes that have files but
609        no entries in the wikiwords table.)))
610        """
611        try:
612            return self.connWrap.execSqlQuerySingleColumn(
613                    "select word from wikiwordcontent except "
614                    "select word as mtword from wikiwordmatchterms "
615                    "where matchterm in (select relation from wikirelations "
616                    "where wikirelations.word != mtword)")
617        except (IOError, OSError, sqlite.Error), e:
618            traceback.print_exc()
619            raise DbReadAccessError(e)
620
621
622    def getUndefinedWords(self):
623        """
624        List words which are childs of a word but are not defined, neither
625        directly nor as alias.
626        Function must work for read-only wiki.
627        """
628        try:
629            return self.connWrap.execSqlQuerySingleColumn(
630                    "select relation from wikirelations "
631                    "except select word from wikiwordcontent "
632                    "except select matchterm from wikiwordmatchterms "
633                    "where (type & 2) != 0")
634            # Consts.WIKIWORDMATCHTERMS_TYPE_ASLINK == 2
635
636        except (IOError, OSError, sqlite.Error), e:
637            traceback.print_exc()
638            raise DbReadAccessError(e)
639
640
641    def _addRelationship(self, word, rel):
642        """
643        Add a relationship from word to rel. rel is a tuple (toWord, pos).
644        A relation from one word to another is unique and can't be added twice.
645        """
646        try:
647            self.connWrap.execSql(
648                    "insert or replace into wikirelations(word, relation, firstcharpos) "
649                    "values (?, ?, ?)", (word, rel[0], rel[1]))
650        except (IOError, OSError, sqlite.Error), e:
651            traceback.print_exc()
652            raise DbWriteAccessError(e)
653
654    def updateChildRelations(self, word, childRelations):
655        self.deleteChildRelationships(word)
656        self.getExistingWikiWordInfo(word)
657        for r in childRelations:
658            self._addRelationship(word, r)
659
660    def deleteChildRelationships(self, fromWord):
661        try:
662            self.connWrap.execSql("delete from wikirelations where word = ?",
663                    (fromWord,))
664        except (IOError, OSError, sqlite.Error), e:
665            traceback.print_exc()
666            raise DbWriteAccessError(e)
667
668
669    # TODO Maybe optimize
670    def getAllSubWords(self, words, level=-1):
671        """
672        Return all words which are children, grandchildren, etc.
673        of words and the words itself. Used by the "export/print Sub-Tree"
674        functions. All returned words are real existing words, no aliases.
675        Function must work for read-only wiki.
676        """
677        checkList = [(w, 0)
678                for w in (self.getUnAliasedWikiWord(w) for w in words)
679                if w is not None]
680        checkList.reverse()
681       
682        resultSet = {}
683        result = []
684
685        while len(checkList) > 0:
686            toCheck, chLevel = checkList.pop()
687            if resultSet.has_key(toCheck):
688                continue
689
690            result.append(toCheck)
691            resultSet[toCheck] = None
692           
693            if level > -1 and chLevel >= level:
694                continue  # Don't go deeper
695           
696            children = self.getChildRelationships(toCheck, existingonly=True,
697                    selfreference=False)
698                   
699            children = [(self.getUnAliasedWikiWord(c), chLevel + 1)
700                    for c in children]
701            children.reverse()
702            checkList += children
703
704        return result
705
706
707    def findBestPathFromWordToWord(self, word, toWord):
708        """
709        finds the shortest path from word to toWord going through the parents.
710        word and toWord are included as first/last element. If word == toWord,
711        it is included only once as the single element of the list.
712        If there is no path from word to toWord, [] is returned
713        Function must work for read-only wiki (should hold although function
714        writes to temporary table).
715        """
716        # TODO Aliases supported?
717       
718        if word == toWord:
719            return [word]
720        try:
721            # Clear temporary table
722            self.connWrap.execSql("delete from temppathfindparents")
723   
724            self.connWrap.execSql("insert into temppathfindparents "+
725                    "(word, child, steps) select word, relation, 1 from wikirelations "+
726                    "where relation = ?", (word,))
727
728            step = 1
729            while True:
730                changes = self.connWrap.rowcount
731   
732                if changes == 0:
733                    # No more (grand-)parents
734                    return []
735
736                if self.connWrap.execSqlQuerySingleItem("select word from "+
737                        "temppathfindparents where word=?", (toWord,)) is not None:
738                    # Path found
739                    result = [toWord]
740                    crumb = toWord
741   
742                    while crumb != word:
743                        crumb = self.connWrap.execSqlQuerySingleItem(
744                                "select child from temppathfindparents where "+
745                                "word=?", (crumb,))
746                        result.append(crumb)
747
748                    # print "findBestPathFromWordToWord result", word, toWord, repr(result)
749   
750                    # Clear temporary table
751                    self.connWrap.execSql("delete from temppathfindparents")
752
753                    return result
754   
755                self.connWrap.execSql("""
756                    insert or ignore into temppathfindparents (word, child, steps)
757                    select wikirelations.word, temppathfindparents.word, ? from
758                        temppathfindparents inner join wikirelations on
759                        temppathfindparents.word == wikirelations.relation where
760                        temppathfindparents.steps == ?
761                    """, (step+1, step))
762   
763                step += 1
764       
765        except (IOError, OSError, sqlite.Error), e:
766            traceback.print_exc()
767            raise DbReadAccessError(e)
768
769
770    # ---------- Listing/Searching wiki words (see also "alias handling", "searching pages")----------
771
772    def getAllDefinedContentNames(self):
773        """
774        get the names of all the content elements in the db, no aliases
775        Function must work for read-only wiki.
776        """
777        try:
778            return self.connWrap.execSqlQuerySingleColumn(
779                    "select word from wikiwordcontent")
780        except (IOError, OSError, sqlite.Error), e:
781            traceback.print_exc()
782            raise DbReadAccessError(e)
783
784
785    getAllDefinedWikiPageNames = getAllDefinedContentNames
786
787
788    def refreshDefinedContentNames(self):
789        """
790        Refreshes the internal list of defined pages which
791        may be different from the list of pages for which
792        content is available (not possible for compact database).
793        The function tries to conserve additional informations
794        (creation/modif. date) if possible.
795       
796        It is mainly called during rebuilding of the wiki
797        so it must not rely on the presence of other cache
798        information (e.g. relations).
799
800        The self.cachedContentNames is invalidated.
801        """
802        self.cachedContentNames = None
803
804
805    def _getCachedContentNames(self):
806        """
807        Function works for read-only wiki.
808        """
809        try:
810            if self.cachedContentNames is None:
811                self.cachedContentNames = dict(self.connWrap.execSqlQuery(
812                        "select word, word from wikiwordcontent union "
813                        "select matchterm, word from wikiwordmatchterms "
814                        "where (type & 2) != 0 and not matchterm in "
815                        "(select word from wikiwordcontent)"))
816                # Consts.WIKIWORDMATCHTERMS_TYPE_ASLINK == 2
817
818            return self.cachedContentNames
819        except (IOError, OSError, sqlite.Error), e:
820            traceback.print_exc()
821            raise DbReadAccessError(e)
822
823
824    def isDefinedWikiPage(self, word):
825        try:
826            return bool(self.connWrap.execSqlQuerySingleItem(
827                    "select 1 from wikiwordcontent where word = ?", (word,)))
828        except (IOError, OSError, sqlite.Error), e:
829            traceback.print_exc()
830            raise DbReadAccessError(e)
831
832
833    def isDefinedWikiLink(self, word):
834        "check if a word is a valid wikiword (page name or alias)"
835        return bool(self.getUnAliasedWikiWord(word))
836
837
838#     # TODO More reliably esp. for aliases
839#     def isDefinedWikiWord(self, word):
840#         "check if a word is a valid wikiword (page name or alias)"
841#         return self._getCachedContentNames().has_key(word)
842
843
844    def getAllProducedWikiLinks(self):
845        """
846        Return all links stored by production (in contrast to resolution)
847        Function must work for read-only wiki.
848        """
849        return self._getCachedContentNames().keys()
850
851
852
853    def getWikiLinksStartingWith(self, thisStr, includeAliases=False,
854            caseNormed=False):
855        "get the list of words starting with thisStr. used for autocompletion."
856        if caseNormed:
857            thisStr = sqlite.escapeForGlob(thisStr.lower())   # TODO More general normcase function
858
859            try:
860                return self.connWrap.execSqlQuerySingleColumn(
861                        "select matchterm from wikiwordmatchterms "
862                        "where matchtermnormcase glob (? || '*') and "
863                        "(type & 2) != 0",
864                        (thisStr,))
865                # Consts.WIKIWORDMATCHTERMS_TYPE_ASLINK == 2
866
867            except (IOError, OSError, sqlite.Error), e:
868                traceback.print_exc()
869                raise DbReadAccessError(e)
870
871        else:
872            try:
873                thisStr = sqlite.escapeForGlob(thisStr)
874
875                # To ensure that at least all real wikiwords are found,
876                # the wikiwords table is also read
877                return self.connWrap.execSqlQuerySingleColumn(
878                        "select matchterm from wikiwordmatchterms "
879                        "where matchterm glob (? || '*') and "
880                        "(type & 2) != 0 union "
881                        "select word from wikiwordcontent where word glob (? || '*')",
882                        (thisStr,thisStr))
883                # Consts.WIKIWORDMATCHTERMS_TYPE_ASLINK == 2
884
885            except (IOError, OSError, sqlite.Error), e:
886                traceback.print_exc()
887                raise DbReadAccessError(e)
888
889
890    def getWikiWordsModifiedWithin(self, startTime, endTime):
891        """
892        Function must work for read-only wiki.
893        startTime and endTime are floating values as returned by time.time()
894        startTime is inclusive, endTime is exclusive
895        """
896        try:
897            return self.connWrap.execSqlQuerySingleColumn(
898                    "select word from wikiwordcontent where modified >= ? and "
899                    "modified < ?",
900                    (startTime, endTime))
901        except (IOError, OSError, sqlite.Error), e:
902            traceback.print_exc()
903            raise DbReadAccessError(e)
904
905
906    _STAMP_TYPE_TO_FIELD = {
907            0: "modified",
908            1: "created"
909        }
910
911    def getTimeMinMax(self, stampType):
912        """
913        Return the minimal and maximal timestamp values over all wiki words
914        as tuple (minT, maxT) of float time values.
915        A time value of 0.0 is not taken into account.
916        If there are no wikiwords with time value != 0.0, (None, None) is
917        returned.
918       
919        stampType -- 0: Modification time, 1: Creation, 2: Last visit
920        """
921        field = self._STAMP_TYPE_TO_FIELD.get(stampType)
922        if field is None:
923            # Visited not supported yet
924            return (None, None)
925
926        try:
927            result = self.connWrap.execSqlQuery(
928                    ("select min(%s), max(%s) from wikiwordcontent where %s > 0") %
929                    (field, field, field))
930        except (IOError, OSError, sqlite.Error), e:
931            traceback.print_exc()
932            raise DbReadAccessError(e)
933
934        if len(result) == 0:
935            # No matching wiki words found
936            return (None, None)
937        else:
938            return tuple(result[0])
939
940
941    def getWikiWordsBefore(self, stampType, stamp, limit=None):
942        """
943        Get a list of tuples of wiki words and dates related to a particular
944        time before stamp.
945       
946        stampType -- 0: Modification time, 1: Creation, 2: Last visit
947        limit -- How much words to return or None for all
948        """
949        field = self._STAMP_TYPE_TO_FIELD.get(stampType)
950        if field is None:
951            # Visited not supported yet
952            return []
953           
954        if limit is None:
955            limit = -1
956           
957        try:
958            return self.connWrap.execSqlQuery(
959                    ("select word, %s from wikiwordcontent where %s > 0 and %s < ? "
960                    "order by %s desc limit ?") %
961                    (field, field, field, field), (stamp, limit))
962        except (IOError, OSError, sqlite.Error), e:
963            traceback.print_exc()
964            raise DbReadAccessError(e)
965
966
967    def getWikiWordsAfter(self, stampType, stamp, limit=None):
968        """
969        Get a list of of tuples of wiki words and dates related to a particular
970        time after OR AT stamp.
971       
972        stampType -- 0: Modification time, 1: Creation, 2: Last visit
973        limit -- How much words to return or None for all
974        """
975        field = self._STAMP_TYPE_TO_FIELD.get(stampType)
976        if field is None:
977            # Visited not supported yet
978            return []
979           
980        if limit is None:
981            limit = -1
982
983        try:
984            return self.connWrap.execSqlQuery(
985                    ("select word, %s from wikiwordcontent where %s > ? "
986                    "order by %s asc limit ?") %
987                    (field, field, field), (stamp, limit))
988        except (IOError, OSError, sqlite.Error), e:
989            traceback.print_exc()
990            raise DbReadAccessError(e)
991
992
993    def getFirstWikiWord(self):
994        """
995        Returns the name of the "first" wiki word. See getNextWikiWord()
996        for details. Returns either an existing wiki word or None if no
997        wiki words in database.
998        """
999        try:
1000            return self.connWrap.execSqlQuerySingleItem(
1001                    "select word from wikiwordcontent "
1002                    "order by word limit 1")
1003        except (IOError, OSError, sqlite.Error), e:
1004            traceback.print_exc()
1005            raise DbReadAccessError(e)
1006
1007
1008    def getNextWikiWord(self, currWord):
1009        """
1010        Returns the "next" wiki word after currWord or None if no
1011        next word exists. If you begin with the first word returned
1012        by getFirstWikiWord() and then use getNextWikiWord() to
1013        go to the next word until no more words are available
1014        and if the list of existing wiki words is not modified during
1015        iteration, it is guaranteed that you have visited all real
1016        wiki words (no aliases) then.
1017        currWord  doesn't have to be an existing word itself.
1018        Function must work for read-only wiki.
1019        """
1020        try:
1021            return self.connWrap.execSqlQuerySingleItem(
1022                    "select word from wikiwordcontent where "
1023                    "word > ? order by word limit 1", (currWord,))
1024        except (IOError, OSError, sqlite.Error), e:
1025            traceback.print_exc()
1026            raise DbReadAccessError(e)
1027
1028
1029
1030    # ---------- Property cache handling ----------
1031
1032    def getPropertyNames(self):
1033        """
1034        Return all property names not beginning with "global."
1035        Function must work for read-only wiki.
1036        """
1037        try:
1038            return self.connWrap.execSqlQuerySingleColumn(
1039                    "select distinct(key) from wikiwordprops "
1040                    "where key not glob 'global.*'")
1041        except (IOError, OSError, sqlite.Error), e:
1042            traceback.print_exc()
1043            raise DbReadAccessError(e)
1044
1045
1046    # TODO More efficient? (used by autocompletion)
1047    def getPropertyNamesStartingWith(self, startingWith):
1048        """
1049        Function must work for read-only wiki.
1050        """
1051        try:
1052            return self.connWrap.execSqlQuerySingleColumn(
1053                    "select distinct(key) from wikiwordprops "
1054                    "where key glob (? || '*')",
1055                    (sqlite.escapeForGlob(startingWith),))   #  order by key")
1056#             names = self.connWrap.execSqlQuerySingleColumn(
1057#                     "select distinct(key) from wikiwordprops")   #  order by key")
1058        except (IOError, OSError, sqlite.Error), e:
1059            traceback.print_exc()
1060            raise DbReadAccessError(e)
1061
1062#         return [name for name in names if name.startswith(startingWith)]
1063
1064    def getGlobalProperties(self):
1065        """
1066        Function must work for read-only wiki.
1067        """
1068        if not self.cachedGlobalProps:
1069            return self.updateCachedGlobalProps()
1070
1071        return self.cachedGlobalProps
1072
1073    def getDistinctPropertyValues(self, key):
1074        """
1075        Function must work for read-only wiki.
1076        """
1077        try:
1078            return self.connWrap.execSqlQuerySingleColumn(
1079                    "select distinct(value) from wikiwordprops where key = ? ",
1080                    (key,))
1081        except (IOError, OSError, sqlite.Error), e:
1082            traceback.print_exc()
1083            raise DbReadAccessError(e)
1084
1085
1086    def getPropertyTriples(self, word, key, value):
1087        """
1088        Function must work for read-only wiki.
1089        word, key and value can either be unistrings or None.
1090        """
1091        conjunction = Conjunction("where ", "and ")
1092
1093        query = "select distinct word, key, value from wikiwordprops "
1094        parameters = []
1095       
1096        if word is not None:
1097            parameters.append(word)
1098            query += conjunction() + "word = ? "
1099       
1100        if key is not None:
1101            parameters.append(key)
1102            query += conjunction() + "key = ? "
1103
1104        if value is not None:
1105            parameters.append(value)
1106            query += conjunction() + "value = ? "
1107
1108        try:
1109            return self.connWrap.execSqlQuery(query, tuple(parameters))
1110        except (IOError, OSError, sqlite.Error), e:
1111            traceback.print_exc()
1112            raise DbReadAccessError(e)
1113
1114
1115    def getWordsForPropertyName(self, key):
1116        """
1117        Function must work for read-only wiki.
1118        """
1119        try:
1120            return self.connWrap.execSqlQuerySingleColumn(
1121                    "select distinct(word) from wikiwordprops where key = ? ",
1122                    (key,))
1123        except (IOError, OSError, sqlite.Error), e:
1124            traceback.print_exc()
1125            raise DbReadAccessError(e)
1126
1127
1128    def getPropertiesForWord(self, word):
1129        """
1130        Returns list of tuples (key, value) of key and value
1131        of all properties for word.
1132        Function must work for read-only wiki.
1133        """
1134        try:
1135            return self.connWrap.execSqlQuery("select key, value "+
1136                        "from wikiwordprops where word = ?", (word,))
1137        except (IOError, OSError, sqlite.Error), e:
1138            traceback.print_exc()
1139            raise DbReadAccessError(e)
1140
1141
1142    def _setProperty(self, word, key, value):
1143        try:
1144            self.connWrap.execSql(
1145                    "insert into wikiwordprops(word, key, value) "
1146                    "values (?, ?, ?)", (word, key, value))
1147        except (IOError, OSError, sqlite.Error), e:
1148            traceback.print_exc()
1149            raise DbWriteAccessError(e)
1150
1151
1152    def updateProperties(self, word, props):
1153        self.deleteProperties(word)
1154        self.getExistingWikiWordInfo(word)
1155        for k in props.keys():
1156            values = props[k]
1157            for v in values:
1158                self._setProperty(word, k, v)
1159
1160        self.cachedGlobalProps = None   # reset global properties cache
1161
1162
1163    def updateCachedGlobalProps(self):
1164        """
1165        TODO: Should become part of public API!
1166        Function must work for read-only wiki.
1167        """
1168        try:
1169            data = self.connWrap.execSqlQuery("select key, value from wikiwordprops "
1170                    "where key glob 'global.*'")
1171        except (IOError, OSError, sqlite.Error), e:
1172            traceback.print_exc()
1173            raise DbReadAccessError(e)
1174
1175        globalMap = {}
1176        for (key, val) in data:
1177            globalMap[key] = val
1178
1179        self.cachedGlobalProps = globalMap
1180
1181        return globalMap
1182
1183
1184    def deleteProperties(self, word):
1185        try:
1186            self.connWrap.execSql("delete from wikiwordprops where word = ?",
1187                    (word,))
1188        except (IOError, OSError, sqlite.Error), e:
1189            traceback.print_exc()
1190            raise DbWriteAccessError(e)
1191
1192
1193
1194    # ---------- Alias handling ----------
1195
1196    def getUnAliasedWikiWord(self, alias):
1197        """
1198        If alias is an alias for another word, return that,
1199        otherwise return None.
1200        Function should only be called by WikiDocument as some methods
1201        of unaliasing must be performed in WikiDocument.
1202        Function must work for read-only wiki.
1203        """
1204        return self._getCachedContentNames().get(alias, None)
1205
1206
1207    # ---------- Todo cache handling ----------
1208
1209    def getTodos(self):
1210        """
1211        Function must work for read-only wiki.
1212        """
1213        try:
1214            return self.connWrap.execSqlQuery("select word, todo from todos")
1215        except (IOError, OSError, sqlite.Error), e:
1216            traceback.print_exc()
1217            raise DbReadAccessError(e)
1218
1219    def getTodosForWord(self, word):
1220        """
1221        Returns list of all todo items of word.
1222        Function must work for read-only wiki.
1223        """
1224        try:
1225            return self.connWrap.execSqlQuerySingleColumn("select todo from todos "
1226                    "where word = ?", (word,))
1227        except (IOError, OSError, sqlite.Error), e:
1228            traceback.print_exc()
1229            raise DbReadAccessError(e)
1230
1231
1232    def updateTodos(self, word, todos):
1233        self.deleteTodos(word)
1234        self.getExistingWikiWordInfo(word)
1235        for t in todos:
1236            self._addTodo(word, t)
1237
1238
1239    def _addTodo(self, word, todo):
1240        try:
1241            self.connWrap.execSql("insert into todos(word, todo) values (?, ?)",
1242                    (word, todo))
1243        except (IOError, OSError, sqlite.Error), e:
1244            traceback.print_exc()
1245            raise DbWriteAccessError(e)
1246
1247
1248    def deleteTodos(self, word):
1249        try:
1250            self.connWrap.execSql("delete from todos where word = ?", (word,))
1251        except (IOError, OSError, sqlite.Error), e:
1252            traceback.print_exc()
1253            raise DbWriteAccessError(e)
1254
1255
1256    # ---------- Wikiword matchterm cache handling ----------
1257
1258    def getWikiWordMatchTermsWith(self, thisStr):
1259        "get the list of match terms with thisStr in them."
1260        thisStr = sqlite.escapeForGlob(thisStr.lower())   # TODO More general normcase function
1261
1262        try:
1263            result1 = self.connWrap.execSqlQuery(
1264                    "select matchterm, type, word, firstcharpos "
1265                    "from wikiwordmatchterms where "
1266                    "matchtermnormcase glob (? || '*')", (thisStr,))
1267
1268            result2 = self.connWrap.execSqlQuery(
1269                    "select matchterm, type, word, firstcharpos "
1270                    "from wikiwordmatchterms where "
1271                    "not matchtermnormcase glob (? || '*') "
1272                    "and matchtermnormcase glob ('*' || ? || '*')",
1273                    (thisStr, thisStr))
1274        except (IOError, OSError, sqlite.Error), e:
1275            traceback.print_exc()
1276            raise DbReadAccessError(e)
1277
1278        coll = self.wikiDocument.getCollator()
1279
1280        coll.sortByFirst(result1)
1281        coll.sortByFirst(result2)
1282
1283        return result1 + result2
1284
1285
1286    def updateWikiWordMatchTerms(self, word, wwmTerms, syncUpdate=False):
1287        self.deleteWikiWordMatchTerms(word, syncUpdate=syncUpdate)
1288        self.getExistingWikiWordInfo(word)
1289        for t in wwmTerms:
1290            assert t[2] == word
1291            self._addWikiWordMatchTerm(t)
1292
1293
1294    def _addWikiWordMatchTerm(self, wwmTerm):
1295        matchterm, typ, word, firstcharpos = wwmTerm
1296        try:
1297            # TODO Check for name collisions
1298            self.connWrap.execSql("insert into wikiwordmatchterms(matchterm, "
1299                    "type, word, firstcharpos, matchtermnormcase) values "
1300                    "(?, ?, ?, ?, ?)",
1301                    (matchterm, typ, word, firstcharpos, matchterm.lower()))
1302        except (IOError, OSError, sqlite.Error), e:
1303            traceback.print_exc()
1304            raise DbWriteAccessError(e)
1305
1306
1307    def deleteWikiWordMatchTerms(self, word, syncUpdate=False):
1308        if syncUpdate:
1309            addSql = " and (type & 16) != 0"
1310        else:
1311            addSql = " and (type & 16) == 0"
1312            # Consts.WIKIWORDMATCHTERMS_TYPE_SYNCUPDATE == 16
1313
1314        try:
1315            self.connWrap.execSql("delete from wikiwordmatchterms where "
1316                    "word = ?" + addSql, (word,))
1317            self.cachedContentNames = None
1318        except (IOError, OSError, sqlite.Error), e:
1319            traceback.print_exc()
1320            raise DbWriteAccessError(e)
1321
1322
1323    # ---------- Data block handling ----------
1324
1325    def getDataBlockUnifNamesStartingWith(self, startingWith):
1326        """
1327        Return all unified names starting with startingWith (case sensitive)
1328        """
1329        try:
1330            return self.connWrap.execSqlQuerySingleColumn(
1331                    "select distinct(unifiedname) from datablocks where "
1332                    "unifiedname glob (? || '*')",
1333                    (sqlite.escapeForGlob(startingWith),))
1334        except (IOError, OSError, sqlite.Error), e:
1335            traceback.print_exc()
1336            raise DbReadAccessError(e)
1337
1338
1339    def retrieveDataBlock(self, unifName):
1340        """
1341        Retrieve data block as binary string.
1342        """
1343        try:
1344            # TODO exception if not present?
1345            return self.connWrap.execSqlQuerySingleItem(
1346                    "select data from datablocks where unifiedname = ?",
1347                    (unifName,))
1348
1349        except (IOError, OSError, sqlite.Error), e:
1350            traceback.print_exc()
1351            raise DbReadAccessError(e)
1352
1353
1354    def retrieveDataBlockAsText(self, unifName):
1355        """
1356        Retrieve data block as unicode string (assuming it was encoded properly)
1357        and with normalized line-ending (Un*x-style).
1358        """
1359        datablock = self.retrieveDataBlock(unifName)
1360        if datablock is None:
1361            return None
1362
1363        return fileContentToUnicode(lineendToInternal(datablock))
1364
1365
1366    def storeDataBlock(self, unifName, newdata, storeHint=None):
1367        """
1368        Store newdata under unified name. If previously data was stored under the
1369        same name, it is deleted.
1370       
1371        unifName -- unistring. Unified name to store data under
1372        newdata -- Data to store, either bytestring or unistring. The latter one
1373            will be converted using utf-8 before storing and the file gets
1374            the appropriate line-ending of the OS for external data blocks .
1375        storeHint -- Hint if data should be stored intern in table or extern
1376            in a file (using DATABLOCK_STOREHINT_* constants from Consts.py).
1377            storeHint is ignored in compact_sqlite
1378        """
1379        try:
1380            self.connWrap.execSql("insert or replace into "
1381                    "datablocks(unifiedname, data) values (?, ?)",
1382                    (unifName, sqlite.Binary(newdata)))
1383        except (IOError, OSError, sqlite.Error), e:
1384            traceback.print_exc()
1385            raise DbWriteAccessError(e)
1386
1387
1388
1389    def deleteDataBlock(self, unifName):
1390        """
1391        Delete data block with the associated unified name. If the unified name
1392        is not in database, nothing happens.
1393        """
1394        try:
1395            self.connWrap.execSql(
1396                    "delete from datablocks where unifiedname = ?", (unifName,))
1397        except (IOError, OSError, sqlite.Error), e:
1398            traceback.print_exc()
1399            raise DbWriteAccessError(e)
1400
1401
1402    # ---------- Searching pages ----------
1403
1404    # TODO Other searchmodes
1405    def search_fallback(self, forPattern, processAnds=True, caseSensitive=False,
1406            searchmode=0):
1407        """
1408        Backup method for non sqlite (without user-defined functions).
1409        Currently unused
1410        """
1411        if caseSensitive:
1412            reFlags = re.MULTILINE | re.UNICODE
1413        else:
1414            reFlags = re.IGNORECASE | re.MULTILINE | re.UNICODE
1415       
1416        if processAnds:
1417            andPatterns = [re.compile(pattern, reFlags)
1418                           for pattern in forPattern.split(u' and ')]
1419#                            for pattern in forPattern.lower().split(u' and ')]
1420        else:
1421            andPatterns = [re.compile(forPattern, reFlags)]
1422
1423
1424        # execSqlQueryIter is insecure, don't use
1425        itr = self.connWrap.execSqlQueryIter(
1426                "select word, content from wikiwordcontent")
1427
1428        results = []
1429
1430        for word, content in itr:
1431            for pattern in andPatterns:
1432                if not pattern.search(content):
1433                    word = None
1434                    break
1435
1436            if word:
1437                results.append(word)
1438
1439        return results
1440
1441
1442    def search(self, sarOp, exclusionSet):
1443        """
1444        Search all wiki pages using the SearchAndReplaceOperation sarOp and
1445        return set of all page names that match the search criteria.
1446        sarOp.beginWikiSearch() must be called before calling this function,
1447        sarOp.endWikiSearch() must be called after calling this function.
1448        This version uses sqlite user-defined functions.
1449       
1450        exclusionSet -- set of wiki words for which their pages shouldn't be
1451        searched here and which must not be part of the result set
1452        """
1453        try:
1454            result = self.connWrap.execSqlQuerySingleColumn(
1455                    "select word from wikiwordcontent where "
1456                    "testMatch(word, content, ?)",
1457                    (sqlite.addTransObject(sarOp),))
1458        except (IOError, OSError, sqlite.Error), e:
1459            traceback.print_exc()
1460            raise DbReadAccessError(e)
1461        finally:
1462            sqlite.delTransObject(sarOp)
1463
1464        result = set(result)
1465        result -= exclusionSet
1466
1467        return result
1468
1469
1470# explain select distinct type from wikiwordmatchterms where type & 2
1471# explain select type from (select distinct type from wikiwordmatchterms) where type & 2
1472# explain select type, type & 2 from (select distinct type from wikiwordmatchterms where type > 1)
1473
1474
1475    # ---------- Miscellaneous ----------
1476
1477    _CAPABILITIES = {
1478        "rebuild": 1,
1479        "compactify": 1,     # = sqlite vacuum
1480#         "versioning": 1,     # TODO (old versioning)
1481        "plain text import":1,
1482#         "asynchronous commit":1  # Commit can be done in separate thread, but
1483#                 # calling any other function during running commit is not allowed
1484        }
1485
1486
1487    def checkCapability(self, capkey):
1488        """
1489        Check the capabilities of this WikiData implementation.
1490        The capkey names the capability, the function returns normally
1491        a version number or None if not supported
1492        """
1493        return WikiData._CAPABILITIES.get(capkey, None)
1494
1495
1496        # TODO drop and recreate tables and indices!
1497    def clearCacheTables(self):
1498        """
1499        Clear all tables in the database which contain non-essential
1500        (cache) information as well as other cache information.
1501        Needed before rebuilding the whole wiki
1502        """
1503        DbStructure.recreateCacheTables(self.connWrap)
1504        self.connWrap.syncCommit()
1505
1506        self.cachedContentNames = None
1507        self.cachedGlobalProps = None
1508
1509
1510    def setDbSettingsValue(self, key, value):
1511        assert isinstance(value, basestring)
1512        self.connWrap.execSql("insert or replace into settings(key, value) "
1513                "values (?, ?)", (key, value))
1514
1515    def getDbSettingsValue(self, key, default=None):
1516        return DbStructure.getSettingsValue(self.connWrap, key, default)
1517
1518
1519    def setPresentationBlock(self, word, datablock):
1520        """
1521        Save the presentation datablock (a byte string) for a word to
1522        the database.
1523        """
1524        try:
1525            self.connWrap.execSql(
1526                    "update wikiwordcontent set presentationdatablock = ? where "
1527                    "word = ?", (sqlite.Binary(datablock), word))
1528        except (IOError, OSError, sqlite.Error), e:
1529            traceback.print_exc()
1530            raise DbWriteAccessError(e)
1531
1532
1533    def getPresentationBlock(self, word):
1534        """
1535        Returns the presentation datablock (a byte string).
1536        The function may return either an empty string or a valid datablock
1537        """
1538        try:
1539            return self.connWrap.execSqlQuerySingleItem(
1540                    "select presentationdatablock from wikiwordcontent where word = ?",
1541                    (word,))
1542        except (IOError, OSError, sqlite.Error), e:
1543            traceback.print_exc()
1544            raise DbReadAccessError(e)
1545
1546
1547    def testWrite(self):
1548        """
1549        Test if writing to database is possible. Throws a DbWriteAccessError
1550        if writing failed.
1551        TODO !
1552        """
1553        pass
1554
1555
1556    def close(self):
1557        self.connWrap.syncCommit()
1558        self.connWrap.close()
1559
1560        self.connWrap = None
1561
1562
1563    # ---------- Versioning (optional) ----------
1564    # Must be implemented if checkCapability returns a version number
1565    #     for "versioning".
1566       
1567    def storeModification(self, word):
1568        """ Store the modification for a single word (wikicontent and headversion for the word must exist)
1569        between wikicontents and headversion in the changelog.
1570        Does not modify headversion. It is recommended to not call this directly
1571
1572        Values for the op-column in the changelog:
1573        0 set content: set content as it is in content column
1574        1 modify: content is a binary compact diff as defined in StringOps,
1575            apply it to new revision to get the old one.
1576        2 create page: content contains data of the page
1577        3 delete page: content is undefined
1578        """
1579
1580        content, moddate = self._getContentAndInfo(word)[:2]
1581
1582        headcontent, headmoddate = self.connWrap.execSqlQuery("select content, modified from headversion "+\
1583                "where word=?", (word,))[0]
1584
1585        bindiff = getBinCompactForDiff(content, headcontent)
1586        self.connWrap.execSql("insert into changelog (word, op, content, moddate) values (?, ?, ?, ?)",
1587                (word, 1, sqlite.Binary(bindiff), headmoddate))  # Modify  # TODO: Support overwrite
1588        return self.connWrap.lastrowid
1589
1590
1591    def hasVersioningData(self):
1592        """
1593        Returns true iff any version information is stored in the database
1594        """
1595        return DbStructure.hasVersioningData(self.connWrap)
1596
1597
1598    def storeVersion(self, description):
1599        """
1600        Store the current version of a wiki in the changelog
1601
1602        Values for the op-column in the changelog:
1603        0 set content: set content as it is in content column
1604        1 modify: content is a binary compact diff as defined in StringOps,
1605            apply it to new revision to get the old one.
1606        2 create page: content contains data of the page
1607        3 delete page: content is undefined
1608
1609        Renaming is not supported directly.
1610        """
1611        # Test if tables were created already
1612
1613        if not DbStructure.hasVersioningData(self.connWrap):
1614            # Create the tables
1615            self.connWrap.syncCommit()
1616            try:
1617                DbStructure.createVersioningTables(self.connWrap)
1618                # self.connWrap.commit()
1619            except:
1620                self.connWrap.rollback()
1621                raise
1622
1623        self.connWrap.syncCommit()
1624        try:
1625            # First move head version to normal versions
1626            headversion = self.connWrap.execSqlQuery("select description, "+\
1627                    "created from versions where id=0") # id 0 is the special head version
1628            if len(headversion) == 1:
1629                firstchangeid = self.connWrap.execSqlQuerySingleItem("select id from changelog order by id desc limit 1 ",
1630                        default = -1) + 1
1631
1632                # Find modified words
1633                modwords = self.connWrap.execSqlQuerySingleColumn("select headversion.word from headversion inner join "+\
1634                        "wikiwordcontent on headversion.word = wikiwordcontent.word where "+\
1635                        "headversion.modified != wikiwordcontent.modified")
1636
1637                for w in modwords:
1638                    self.storeModification(w)
1639
1640
1641                # Store changes for deleted words
1642                self.connWrap.execSql("insert into changelog (word, op, content, moddate) "+\
1643                        "select word, 2, content, modified from headversion where "+\
1644                        "word not in (select word from wikiwordcontent)")
1645
1646                # Store changes for inserted words
1647                self.connWrap.execSql("insert into changelog (word, op, content, moddate) "+\
1648                        "select word, 3, x'', modified from wikiwordcontent where "+\
1649                        "word not in (select word from headversion)")
1650
1651                if firstchangeid == (self.connWrap.execSqlQuerySingleItem("select id from changelog order by id desc limit 1 ",
1652                        default = -1) + 1):
1653
1654                    firstchangeid = -1 # No changes recorded in changelog
1655
1656                headversion = headversion[0]
1657                self.connWrap.execSql("insert into versions(description, firstchangeid, created) "+\
1658                        "values(?, ?, ?)", (headversion[0], firstchangeid, headversion[1]))
1659
1660            self.connWrap.execSql("insert or replace into versions(id, description, firstchangeid, created) "+\
1661                    "values(?, ?, ?, ?)", (0, description, -1, time()))
1662
1663            # Copy from wikiwordcontent everything to headversion
1664            self.connWrap.execSql("delete from headversion")
1665            self.connWrap.execSql("insert into headversion select * from wikiwordcontent")
1666
1667            self.connWrap.commit()
1668        except:
1669            self.connWrap.rollback()
1670            raise
1671
1672
1673    def getStoredVersions(self):
1674        """
1675        Return a list of tuples for each stored version with (<id>, <description>, <creation date>).
1676        Newest versions at first
1677        """
1678        # Head version first
1679        result = self.connWrap.execSqlQuery("select id, description, created "+\
1680                    "from versions where id == 0")
1681
1682        result += self.connWrap.execSqlQuery("select id, description, created "+\
1683                    "from versions where id != 0 order by id desc")
1684        return result
1685
1686
1687    # TODO: Wrong moddate?
1688    def applyChange(self, word, op, content, moddate):
1689        """
1690        Apply a single change to wikiwordcontent. word, op, content and modified have the
1691        same meaning as in the changelog table
1692        """
1693        if op == 0:
1694            self.setContentRaw(word, content, moddate)
1695        elif op == 1:
1696            self.setContentRaw(word, applyBinCompact(self.getContent(word), content), moddate)
1697        elif op == 2:
1698            self.setContentRaw(word, content, moddate)
1699        elif op == 3:
1700            self.deleteContent(word)
1701
1702
1703    # TODO: Wrong date?, more efficient
1704    def applyStoredVersion(self, id):
1705        """
1706        Set the content back to the version identified by id (retrieved by getStoredVersions).
1707        Only wikiwordcontent is modified, the cache information must be updated separately
1708        """
1709
1710        self.connWrap.syncCommit()
1711        try:
1712            # Start with head version
1713            self.connWrap.execSql("delete from wikiwordcontent") #delete all rows
1714            self.connWrap.execSql("insert into wikiwordcontent select * from headversion") # copy from headversion
1715
1716            if id != 0:
1717                lowestchangeid = self.connWrap.execSqlQuerySingleColumn("select firstchangeid from versions where id == ?",
1718                        (id,))
1719                if len(lowestchangeid) == 0:
1720                    raise WikiFileNotFoundException()  # TODO: Better exception
1721
1722                lowestchangeid = lowestchangeid[0]
1723
1724                changes = self.connWrap.execSqlQuery("select word, op, content, moddate from changelog "+\
1725                        "where id >= ? order by id desc", (lowestchangeid,))
1726
1727                for c in changes:
1728                    self.applyChange(*c)
1729
1730
1731            self.connWrap.commit()
1732        except:
1733            self.connWrap.rollback()
1734            raise
1735
1736
1737    def deleteVersioningData(self):
1738        """
1739        Completely delete all versioning information
1740        """
1741        DbStructure.deleteVersioningTables(self.connWrap)
1742
1743
1744    # ---------- Other optional functionality ----------
1745
1746    def cleanupAfterRebuild(self, progresshandler):
1747        """
1748        Rebuild cached structures, try to repair database inconsistencies.
1749
1750        Must be implemented if checkCapability returns a version number
1751        for "rebuild".
1752       
1753        progresshandler -- Object, fulfilling the GuiProgressHandler
1754            protocol
1755        """
1756        try:
1757            self.connWrap.execSql("update wikiwordmatchterms "
1758                    "set matchtermnormcase=utf8Normcase(matchterm)")
1759            DbStructure.rebuildIndices(self.connWrap)
1760        except (IOError, OSError, sqlite.Error), e:
1761            traceback.print_exc()
1762            raise DbWriteAccessError(e)
1763
1764
1765            # TODO
1766            # Check the presence of important indexes
1767
1768            indexes = self.connWrap.execSqlQuerySingleColumn(
1769                    "select name from sqlite_master where type='index'")
1770            indexes = map(string.upper, indexes)
1771
1772            if not "WIKIWORDCONTENT_PKEY" in indexes:
1773                # Maybe we have multiple pages with the same name in the database
1774               
1775                # Copy valid creation date to all pages
1776                self.connWrap.execSql("update wikiwordcontent set "
1777                        "created=(select max(created) from wikiwordcontent as "
1778                        "inner where inner.word=wikiwordcontent.word)")
1779   
1780                # Delete all but the newest page
1781                self.connWrap.execSql("delete from wikiwordcontent where "
1782                        "ROWID not in (select max(ROWID) from wikiwordcontent as "
1783                        "outer where modified=(select max(modified) from "
1784                        "wikiwordcontent as inner where inner.word=outer.word) "
1785                        "group by outer.word)")
1786   
1787                DbStructure.rebuildIndices(self.connWrap)
1788        except (IOError, OSError, sqlite.Error), e:
1789            traceback.print_exc()
1790            raise DbWriteAccessError(e)
1791
1792
1793       # TODO: More repair operations
1794
1795
1796#         # recreate word caches
1797#         self.cachedContentNames = {}
1798#         for word in self.getAllDefinedContentNames():
1799#             self.cachedContentNames[word] = 1
1800#
1801#         # cache aliases
1802#         aliases = self.getAllAliases()
1803#         for alias in aliases:
1804#             self.cachedContentNames[alias] = 2
1805
1806
1807#         finally:           
1808#             progresshandler.close()
1809
1810
1811    def commit(self):
1812        """
1813        Do not call from this class, only from outside to handle errors.
1814        """
1815        try:
1816            self.connWrap.commit()
1817        except (IOError, OSError, sqlite.Error), e:
1818            traceback.print_exc()
1819            raise DbWriteAccessError(e)
1820
1821
1822    def rollback(self):
1823        """
1824        Do not call from this class, only from outside to handle errors.
1825        """
1826        try:
1827            self.connWrap.rollback()
1828        except (IOError, OSError, sqlite.Error), e:
1829            traceback.print_exc()
1830            raise DbWriteAccessError(e)
1831
1832
1833    def vacuum(self):
1834        """
1835        Reorganize the database, free unused space.
1836       
1837        Must be implemented if checkCapability returns a version number
1838        for "compactify".       
1839        """
1840        try:
1841            self.connWrap.syncCommit()
1842            self.connWrap.execSql("vacuum")
1843        except (IOError, OSError, sqlite.Error), e:
1844            traceback.print_exc()
1845            raise DbWriteAccessError(e)
1846
1847
1848
1849    # TODO: Better error checking
1850    # TODO: Process 2.0-named files
1851    def copyWikiFilesToDatabase(self):
1852        """
1853        Helper to transfer wiki files into database for migrating from
1854        original WikidPad to specialized databases.
1855
1856        Must be implemented if checkCapability returns a version number
1857        for "plain text import".
1858        """
1859        self.connWrap.syncCommit()
1860
1861        fnames = glob.glob(longPathEnc(join(self.dataDir, '*.wiki')))
1862        for fn in fnames:
1863            word = longPathDec(basename(fn)).replace('.wiki', '')
1864
1865            content = fileContentToUnicode(loadEntireTxtFile(fn))
1866            langHelper = wx.GetApp().createWikiLanguageHelper(
1867                    self.wikiDocument.getWikiDefaultWikiLanguage())
1868
1869            if not langHelper.checkForInvalidWikiWord(word, self.wikiDocument):
1870                self.setContent(word, content, moddate=stat(fn).st_mtime)
1871
1872        self.connWrap.commit()
1873
1874
1875def listAvailableWikiDataHandlers():
1876    """
1877    Returns a list with the names of available handlers from this module.
1878    Each item is a tuple (<internal name>, <descriptive name>)
1879    """
1880    if sqlite is not None:
1881        return [("compact_sqlite", "Compact Sqlite")]
1882    else:
1883        return []
1884
1885
1886def getWikiDataHandler(name):
1887    """
1888    Returns a factory function (or class) for an appropriate
1889    WikiData object and a createWikiDB function or (None, None)
1890    if name is unknown
1891    """
1892    if name == "compact_sqlite":
1893        return WikiData, createWikiDB
1894   
1895    return (None, None)
Note: See TracBrowser for help on using the browser.