First version of search

ChangeLog

@@ -1,4 +1,5 @@
-v0.2 (08/11/2015)
+v0.2 (17/01/2016)
 
 **User**
 	Add autofocus to login page
+	Add search

denote/models.py

@@ -24,8 +24,12 @@ import re
 from django.db import models
 from django.contrib.auth.models import AbstractUser
 from django.http import HttpResponse
+from django.db.models.signals import pre_init, post_init, pre_delete, post_delete
+from django.db.models.signals import pre_save, post_save
+from django.dispatch import receiver
 
 import markdown2
+import search
 
 class User(AbstractUser):
     hidden_categories = models.TextField(blank=True)
@@ -113,8 +117,20 @@ class Note(models.Model):
         self.modified_date = datetime.now()
         self.transformed_text = markdown2.markdown(self.text, extras=['fenced-code-blocks'])
         self._summarize()
+        s = Search()
+        
         super(Note, self).save()
 
+@receiver(post_save, sender=Note)
+def post_save_note_signal(sender, **kwargs):
+    s = Search()
+    s.edit_note(kwargs['instance'].id)
+
+@receiver(pre_delete, sender=Note)
+def pre_delete_note_signal(sender, **kwargs):
+    s = Search()
+    s.remove_note(kwargs['instance'].id)
+
 def manage_category(user, cat_name):
     category = None
     if cat_name:

denote/search.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+"""
+  Copyright 2016 Grégory Soutadé
+
+  This file is part of Dénote.
+
+  Dynastie is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Dynastie is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with Dynastie.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import re
+import unicodedata
+import os
+import operator
+import pickle
+from django.db import models
+
+import models
+#from models import Note
+
+class Search:
+    MINIMUM_LETTERS = 3
+
+    def __init__(self):
+        self.report = ''
+
+        self.tagreg = re.compile('<[^>]+>')
+        self.htmlreg = re.compile('&[^;]+;')
+        self.numreg = re.compile('[0-9]+')
+        self.pat = re.compile(r'\s+')
+
+        self.replace_by_space = (u'(', u')', u'#', u'\'', u'{', u'}', u'[', u']',
+                                 u'-', u'|', u'\t', u'\\', u'_', u'^' '=', u'+', u'$',
+                                 u'£', u'%', u'µ', u'*', u',', u'?', u';', u'.', u'/',
+                                 u':', u'!', u'§', u'€', u'²')
+
+    # Imported from generator.py
+    def _addReport(self, string, color=''):
+        if color != '':
+            self.report = self.report + '<span style="color:' + color + '">'
+        self.report = self.report + '<b>' + self.__class__.__name__ + '</b> : '
+        self.report = self.report + string
+        if color != '':
+            self.report = self.report + '</span>'
+        self.report = self.report + '<br/>\n'
+
+    def _addWarning(self, string):
+        self.addReport(string, 'yellow')
+
+    def _addError(self, string):
+        self.addReport(string, 'red')
+
+
+    def _saveDatabase(self, hashtable):
+        d = pickle.dumps(hashtable)
+
+        f = open(os.environ['DENOTE_ROOT'] + '/_search.db', 'w')
+        f.write(d)
+        f.close()
+
+    def _loadDatabase(self):
+        filename = os.environ['DENOTE_ROOT'] + '/_search.db'
+
+        if not os.path.exists(filename):
+            print 'No search index !'
+            return {}
+
+        f = open(filename, 'rb')
+        hashtable = pickle.load(f)
+        f.close()
+
+        return hashtable
+
+    def _strip_accents(self, s):
+        return ''.join((c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn'))
+
+    def _remove_tag(self, content):
+        content = self.htmlreg.sub('', content)
+        content = self.numreg.sub('', content)
+
+        content = content.replace('\n', '')
+        content = content.replace('\r', '')
+        content = content.replace('"', '')
+
+        for c in self.replace_by_space:
+            content = content.replace(c, ' ')
+        
+        content = self.tagreg.sub('', content)
+
+        content = self.pat.sub(' ', content)
+
+        return content
+
+    def _prepare_string(self, content):
+        content = self._remove_tag(content)
+        content = self._strip_accents(content)
+
+        return content
+
+    def _indexContent(self, hashtable, index, content, word_weight):
+        content = self._prepare_string(content)
+
+        wordlist = content.split(' ')
+
+        for word in wordlist:
+            if len(word) < self.MINIMUM_LETTERS:
+                continue
+            word = word.lower()
+            if not word in hashtable:
+                hashtable[word] = []
+            if not index in hashtable[word]:
+                hashtable[word].insert(0, [index, word_weight])
+            else:
+                weight = hashtable[word][1]
+                hashtable[word][1] = weight + word_weight
+
+    def _index(self, hashtable, index):
+        try:
+            note = Note.objects.get(pk=index)
+        except:
+            return
+
+        self._indexContent(hashtable, index, note.text, 1)
+        self._indexContent(hashtable, index, note.title.encode('utf-8'), 5)
+        
+    def _index_note(self, note, saveDatabase=True):
+        hashtable = self._loadDatabase()
+
+        self._index(hashtable, int(note))
+
+        if saveDatabase:
+            self._saveDatabase(hashtable)
+
+    def _remove_note(self, note, saveDatabase=True):
+        hashtable = self._loadDatabase()
+
+        if hashtable is None: return
+
+        for k, v in hashtable.items():
+            # For tuples in values
+            for t in v:
+                if note == v[0]:
+                    v.remove(t)
+
+        if saveDatabase:
+            self._saveDatabase(hashtable)
+
+    def generate_index(self, notes):
+        hashtable = self._loadDatabase()
+
+        for note in notes:
+            self._indexContent(hashtable, note.id, note.text, 1)
+            self._indexContent(hashtable, note.id, note.title, 5)
+
+        self._saveDatabase(hashtable)
+    
+    def index_note(self, note):
+        return self._index_note(note, True)
+
+    def delete_note(self, note):
+        return self._remove_note(note, True)
+
+    def edit_note(self, note, saveDatabase=True):
+        self._remove_note(note, False)
+        self._index_note(note, True)
+
+    def search(self, string):
+        hashtable = self._loadDatabase()
+        
+        string = self._prepare_string(string.encode('utf-8'))
+
+        wordlist = string.split(' ')
+
+        res = {}
+        for word in wordlist:
+            if len(word) < Search.MINIMUM_LETTERS:
+                continue
+            word = word.lower()
+            reg = re.compile('.*' + word + '.*')
+            for key in hashtable.keys():
+                if reg.match(key):
+                    for note in hashtable[key]:
+                        res[note[0]] = res.get(note[0],0) + note[1]
+
+        sorted_res = sorted(res.iteritems(), key=operator.itemgetter(1))
+        sorted_res.reverse()
+
+        res = [sorted_res[i][0] for i in range(len(sorted_res))]
+
+        return res

denote/templates/base.html

@@ -8,7 +8,8 @@
   </head>
   <body>
     <!-- Header -->
-    <div class="settings"><a href="/user/edit">Settings</a> <a href="/disconnect">Disconnect</a></div>
+    <div class="settings"><a href="/user/edit">Settings</a> <a href="/disconnect">Disconnect</a><br/>
+    <form action="/search">{% csrf_token %}<input name="text"/><input type="button" value="Search"/></form></div>
     <!-- Left panel -->
     <div id="left_panel">
       <a id="home_icon" href="/" alt="Home"><img src="{{ STATIC_URL }}images/home.png"/></a><br/><br/>

denote/templates/base_user.html

@@ -14,7 +14,8 @@
   </head>
   <body onload="startup();">
     <!-- Header -->
-    <div class="settings"><a href="/user/edit">Settings</a> <a href="/disconnect">Disconnect</a></div>
+    <div class="settings"><a href="/user/edit">Settings</a> <a href="/disconnect">Disconnect</a><br/><br/>
+    <form method="post" action="/search">{% csrf_token %}<input name="text"/><input type="submit" value="Search"/></form></div>
     <!-- Left panel -->
     <div id="left_panel">
       <a id="home_icon" href="/" alt="Home"><img src="{{ STATIC_URL }}images/home.png"/></a><br/><br/>

denote/urls.py

@@ -30,4 +30,6 @@ urlpatterns = patterns('',
     url(r'^note/(\d+)$', 'denote.views.note', name='note'),
     url(r'^category/edit/(\d)$','denote.views.edit_category',   name='edit_category'),
     url(r'^preferences$', 'denote.views.preferences', name='preferences'),
+    url(r'^search$', 'denote.views.search', name='search'),
+    url(r'^generate_search_index$', 'denote.views.generate_search_index', name='generate_search_index'),
 )

denote/views.py

@@ -18,6 +18,7 @@
   along with Dénote.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+import os
 from datetime import datetime
 
 from django.http import HttpResponseRedirect, HttpResponse, Http404
@@ -27,6 +28,7 @@ from django.shortcuts import render
 
 from denote.models import *
 from denote.forms import *
+from denote.search import *
 
 def index(request):
     if request.user.is_authenticated():
@@ -238,3 +240,33 @@ def preferences(request):
     else:
         raise Http404
 
+@login_required
+def search(request):
+    context = _prepare_note_context(request.user)
+
+    ref = request.META['HTTP_REFERER']
+
+    if 'text' in request.POST:
+        text = request.POST['text']
+    else:
+        return HttpResponseRedirect(ref)
+
+    s = Search()
+    note_list = s.search(text)
+
+    notes = Note.objects.filter(pk__in=note_list, author=request.user)
+    context['notes'] = notes
+    context['note_form'] = NoteForm()
+
+    return render(request, 'user_index.html', context)
+
+@login_required
+def generate_search_index(request):
+
+    if os.path.exists('_search.db'):
+        os.path.remove('_search.db')
+
+    s = Search()
+    s.generate_index(Note.objects.all())
+
+    return HttpResponseRedirect('/')