Dynastie/dynastie/generators/index.py

# -*- coding: utf-8 -*-
"""
  Copyright 2012-2014 Grégory Soutadé

  This file is part of Dynastie.

  Dynastie is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  Dynastie is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with Dynastie.  If not, see <http://www.gnu.org/licenses/>.
"""
import os
import re
import datetime
import hashlib
import xml
from xml.parsers.expat import *
import xml.parsers.expat
from xml.dom.minidom import parse, parseString
import codecs
from dynastie.generators.generator import DynastieGenerator, StrictUTF8Writer
from django.db import models
from dynastie.generators import markdown2

class Index(DynastieGenerator):

    def __init__(self, request=None, hash_posts={}, hash_posts_content={}):
        DynastieGenerator.__init__(self, request, hash_posts, hash_posts_content)

        self.hooks = {'posts' : self.createPosts,
                      'title' : self.createTitle,
                      'date'  : self.createDate,
                      'navigation' : self.createNavigation,
                      'recents'    : self.createRecents,
                      'tags' : self.createTags,
                      'replace'    : self.createReplace,
                      'first_page_only'    : self.createFirstPageOnly,
                      'ljdc_last'  : self.createLJDCLast,
                      'comments_count' : self.createCommentsCount,
                      'category_name' : self.createCategoryName,
                      }

        self.first_try = True

        self.posts_per_page = 5
        self.filename = 'index'
        self.dirname = ''
        self.blog = None
        self.parent_posts = []

        self.resetCounters()

    def resetCounters(self):
        self.nb_pages = 0
        self.cur_page = 0
        self.cur_post = 0
        self.cur_post_obj = None

    def createReplace(self, posts, dom, root, replace_elem):
        if not replace_elem.hasAttribute('div_name'):
            self.addError('No attribute div_name for a replace tag')
            return

        div_element = replace_elem.cloneNode(True)
        div_element.tagName = replace_elem.getAttribute('div_name')
        div_element.removeAttribute('div_name')
        for key,value in replace_elem.attributes.items():
            if key == 'div_name': continue

            value = value.replace('dyn:blog_id', str(self.blog.id))
                
            if self.cur_post_obj:
                url = self.cur_post_obj.getPath()
                full_url = self.cur_post_obj.blog.name + url
                value = value.replace('dyn:post_url', url)
                value = value.replace('dyn:post_full_url', full_url)

            div_element.setAttribute(key, value)

        root.replaceChild(div_element, replace_elem)
        return div_element

    def createCommentsCount(self, posts, dom, root, node):
        from dynastie.models import Comment
        if self.cur_post_obj is None:
            count = 0
        else:
            count = Comment.objects.filter(post=self.cur_post_obj.id).count()
        self.replaceByText(dom, root, node, str(count))

    def createCategoryName(self, posts, dom, root, node):
        from dynastie.models import Category
        if self.cur_post_obj is None:
            category = ''
        else:
            category = Category.objects.filter(id=self.cur_post_obj.category.id)[0].name
        self.replaceByText(dom, root, node, category)
            
    def createNavigation(self, posts, dom, root, node):
        if 0 <= self.nb_pages <= 1:
            return None
        
        if self.dirname:
            if self.dirname.startswith('/'):
                href = '<a href="' + self.dirname + '/' + self.filename
            else:
                href = '<a href="/' + self.dirname + '/' + self.filename
        else:
            href = '<a href="/' + self.filename

        nav = ''
        if self.cur_page != 0:
            nav = nav + href + '.html">&lt;&lt; First</a>  '
            if self.cur_page == 1:
                nav = nav + href + '.html">&lt; Prev</a> '
            else:
                nav = nav + href + str(self.cur_page-1) + '.html">&lt; Prev</a> '

        start = self.cur_page-5
        if start < 0:
            start = 0
        end = start + 10
        
        if end > self.nb_pages:
            end = self.nb_pages

        for i in range(start, end):
            if i == self.cur_page:
                nav = nav + str(i+1) + ' '
            else:
                if i == 0:
                    nav = nav + href + '.html">1</a> '
                else:
                    nav = nav + href + str(i) + '.html">' + str(i+1) + '</a> '

        if self.cur_page < self.nb_pages-1:
            nav = nav + href + str(self.cur_page+1) + '.html">Next &gt;</a>  '
            nav = nav + href + str(self.nb_pages-1) + '.html">Last &gt;&gt;</a>'

        s = '<div class="navigation">' + nav + '</div>'
        new_dom = parseString(s.encode('utf-8'))
        new_node = new_dom.getElementsByTagName('div')[0]
        res = new_node.cloneNode(True)
        root.replaceChild(res, node)
        
        return res

    def createTitle(self, posts, dom, root, title_elem):
        create_link = (title_elem.getAttribute('link') == '1')
        post = self.cur_post_obj
        if create_link == True:
            node = self.createElement(dom, 'title')
            node.appendChild(self.createLinkElem(dom, post.getPath(), post.title))
        else:
            node = self.createElement(dom, 'title', post.title)

        root.replaceChild(node, title_elem)

    def createFirstPageOnly(self, posts, dom, root, node):
        if self.cur_page == 0:
            for n in node.childNodes:
                root.insertBefore(n.cloneNode(True), node)
        root.removeChild(node)

    def createLJDCLast(self, posts, dom, root, node):
        from dynastie.generators.ljdc import LJDC

        l = LJDC()
        img = l.getLast(dom, self.blog.src_path)
        
        if not img is None:
            root.replaceChild(img, node)
        else:
            root.removeChild(node)

    def createDate(self, posts, dom, root, date_elem):
        date_format = date_elem.getAttribute('format')
        if date_format == '':
            date_format = '%A, %d %B %Y %H:%m'

        post = self.cur_post_obj
        node = self.createElement(dom, 'date', post.creation_date.strftime(date_format))

        root.replaceChild(node, date_elem)

    def pygmentCode(self, code):
        while True:
            start = code.find('<dyn:code')

            if start == -1: break
            end = code.find('</dyn:code>')

            if end < start:
                self.addError('Invalid <dyn:code> tags in ' + self.filename)
                break

            try:
                try:
                    dom = parseString(code[start:end+11])
                except UnicodeEncodeError:
                    dom = parseString(code[start:end+11].encode('utf-8'))
            except xml.dom.DOMException as e:
                self.addError('Error parsing ' + self.filename)
                break

            res = self.createCode(dom, dom.firstChild)
            if res:
                code = code.replace(code[start:end+11], res)
        return code

    def _have_I_right(self, user, post_id):
        from dynastie.models import Post, Blog

        p = Post.objects.get(pk=post_id)
        
        if p is None: return None

        blog_id = p.blog.id

        if not user.is_superuser:
            b = Blog.objects.filter(pk=blog_id, writers=user.id)
            if not b: return None
            b = b[0]
        else:
            b = Blog.objects.get(pk=blog_id)
            if b is None: return None

        return (b, p)

    def _manageInternalPosts(self, post, text, user=None):
        from dynastie.models import Post
        if not user: user = post.author
        # Markdown replace
        if not post or (post and post.content_format == Post.CONTENT_TEXT):
            internal_posts = re.finditer('\[\[([0-9]+)\]\]', text)
            if internal_posts:
                for post in internal_posts:
                    post_id = post.groups()
                    post_id = int(post_id[0])
                    if post_id in self.parent_posts: continue
                    _,post = self._have_I_right(user, post_id)
                    if not post: continue
                    new_content = self._loadPostContent(post)
                    if new_content:
                        p = '[[' + str(post_id) + ']]'
                        text = text.replace(p, new_content)
            if internal_posts: return text
        # HTML replace
        if not post or (post and post.content_format == Post.CONTENT_HTML):
            while True:
                start = text.find('<dyn:postinclude')

                if start == -1: break
                end = text.find('</dyn:postinclude>')

                if end < start:
                    self.addError('Invalid <dyn:postinclude> tags in ' + self.filename)
                    break

                internal_posts = re.search('post_id="([0-9]+)"', text[start:])
                if not internal_posts: break
                for post_id in internal_posts.groups():
                    _,post = self._have_I_right(user, post_id)
                    if not post: break
                    new_content = self._loadPostContent(post)
                    if new_content:
                        text = text.replace(text[start:end+18], new_content)
        return text

    def _loadPostContent(self, post):
        from dynastie.models import Post

        blog = post.blog
        blog.create_paths()

        filename = blog.src_path + '/_post/' + str(post.id)

        if not os.path.exists(filename):
            filename2 = blog.src_path + '/_draft/' + str(post.id)
            if not os.path.exists(filename2):
                self.addError('File does not exists ' + filename)
                return None
            else:
                filename = filename2

        if not filename in self.hash_posts_content:
            f = codecs.open(filename, 'rb', 'utf-8')
            post_content = f.read()
            f.close()
            self.parent_posts.append(post.id)
            post_content = self._manageInternalPosts(post, post_content)
            if post.content_format == Post.CONTENT_TEXT:
                post_content = markdown2.markdown(post_content, extras=['fenced-code-blocks'])
            self.hash_posts_content[filename] = post_content
        else:
            post_content = self.hash_posts_content[filename]

        return post_content

    def createPost(self, posts, dom, post_elem, root):
        from dynastie.models import Post
        post = self.cur_post_obj

        if post.id > 0 and post.id in self.hash_posts.keys() and not self.first_try:
            node,_ = self.hash_posts[post.id]
            return node.cloneNode(0)

        values = {'post_content': '', 'author': 'Unknown'}
        try:
            values['author'] = post.author.first_name + ' ' + post.author.last_name
        except:
            pass

        self.parent_posts = []
        post_content = self._loadPostContent(post)
        if not post_content: return None

        post_content = self.pygmentCode(post_content)

        self.simpleTransform(values, dom, post_elem, root)
        
        content_nodes = post_elem.getElementsByTagName('div')
        post_transform = ['post_content']
        for content_node in content_nodes:
            the_class = content_node.getAttribute('class')
            if not the_class in post_transform:
                continue
            new_node = dom.createTextNode(post_content)
            content_node.appendChild(new_node)

        # Disable this cache
        # writer = StrictUTF8Writer()
        # post_elem.writexml(writer)
        # content = writer.getvalue().encode('utf-8')
            
        # md5 = hashlib.md5()
        # md5.update(content)

        # if post.id in self.hash_posts:
        #     # Here, we are in first_try, check that computed
        #     # post has the same result than the one in cache
        #     self.first_try = False

        #     _,md5_2 = self.hash_posts[post.id]
            
        #     # If not, clear cache
        #     if md5.digest() != md5_2:
        #         self.hash_posts = {}
        #         self.hash_posts[post.id] = (post_elem.cloneNode(0), md5.digest())
        # else:
        #     self.hash_posts[post.id] = (post_elem.cloneNode(0), md5.digest())

        return post_elem

    def createPosts(self, posts, dom, root, node):
        posts_elem = self.createElement(dom, 'posts')
        create_link = (node.getAttribute('link') == '1')
        for i in range(0, self.posts_per_page):
            post_elem = self.createElement(dom, 'post')
            if self.cur_post < len(posts):
                self.cur_post_obj = posts[self.cur_post]
                post_elem = self.createPost(posts, dom, post_elem, node)
            else:
                post_elem = self.createElement(dom, '', '<b>No posts yet</b>')
                self.cur_post_obj = None
            if post_elem:
                posts_elem.appendChild(post_elem)

                # Parse inner HTML
                self._parse(self.hooks, posts, dom, post_elem)

            self.cur_post = self.cur_post + 1
            if self.cur_post == len(posts):
                break
        root.replaceChild(posts_elem, node)
        return posts_elem

    def createRecents(self, posts, dom, root, node):
        if self.cur_post == len(posts):
            root.removeChild(node)
            return

        if node.hasAttribute("limit"):
            nb_recents = int(node.getAttribute("limit"))
        else:
            nb_recents = 5
        recents_elem = self.createElement(dom, 'recents')
        for child in node.childNodes:
            recents_elem.appendChild(child.cloneNode(True))
        list_elem = dom.createElement('ul')
        for i in range(0, nb_recents):
            post_elem = dom.createElement('li')
            if self.cur_post+i < len(posts):
                post = posts[self.cur_post+i]
                link_elem = self.createLinkElem(dom, post.getPath(), post.title)
                post_elem.appendChild(link_elem)
            else:
                break
            list_elem.appendChild(post_elem)

        recents_elem.appendChild(list_elem)
        root.replaceChild(recents_elem, node)
        
        return recents_elem

    def createTags(self, posts, dom, root, node):
        from dynastie.models import Post
        tags_elem = self.createElement(dom, 'tags')
        create_link = (node.getAttribute('link') == '1')
        if type(posts) == models.query.QuerySet or type(posts) == list:
            if len(posts) > self.cur_post:
                cur_post = posts[self.cur_post]
            else:
                cur_post = None
        elif type(posts) == Post:
            cur_post = posts
        else:
            cur_post = None

        if cur_post:
            for tag in cur_post.tags.all():
                if create_link:
                    tag_elem = self.createElement(dom, 'tag')
                    link_elem = self.createLinkElem(dom, '/tag/' + tag.name_slug, '#' + tag.name)
                    tag_elem.appendChild(link_elem)
                else:
                    tag_elem = self.createElement(dom, 'tag', '#' + tag.name)
                tags_elem.appendChild(tag_elem)

            if len(cur_post.tags.all()) == 0:
                root.removeChild(node)
                return None
            else:
                root.replaceChild(tags_elem, node)
        else:
            root.removeChild(node)
            return None

        return tags_elem

    def createCode(self, dom, node):
        try:
            from pygments import highlight
            from pygments.util import ClassNotFound
            from pygments.lexers import get_lexer_by_name
            from pygments.formatters import get_formatter_by_name
        except ImportError:
            self.addError('Pygments package is missing, please install it in order to use <dyn:code>')
            return None

        language = node.getAttribute('language')
        
        if language is None:
            self.addWarning('No language defined for <dyn:code> assuming C language')
            language = "c"
        else:
            language = language.lower()

        lexer_options = {}
        try:
            lexer = get_lexer_by_name(language, **lexer_options)
        except ClassNotFound:
            self.addWarning('Language ' + language + ' not supported by current version of pygments')
            lexer = get_lexer_by_name('c', **lexer_options)
        
        formatter_options = {'classprefix' : 'color_emacs_', 'style' : 'emacs'}

        for k in node.attributes.keys():
            attr = node.attributes[k]
            if attr.prefix != '': continue
            if attr.name == 'language': continue
            name = attr.name
            value = attr.value
            if name == 'colouring': 
                name = 'style'
                formatter_options['classprefix'] = 'color_' + value + '_' 
            formatter_options[name] = value

        formatter = get_formatter_by_name('html', **formatter_options)

        lexer.encoding = 'utf-8'
        formatter.encoding = 'utf-8'
        
        writer = StrictUTF8Writer()
        node.firstChild.writexml(writer)
        code = writer.getvalue().encode('utf-8')

        r,w = os.pipe()
        r,w=os.fdopen(r,'r',0), os.fdopen(w,'w',0)
        highlight(code, lexer, formatter, w)
        w.close()
        
        code = r.read()
        r.close()

        # Remove <pre> after <div class="highlight">
        code = code[28:-13]
        code = u'<div class="highlight">' + unicode(code, 'utf-8') + u'</div>'

        return code

    def parseTemplate(self, blog, src, output, name, directory=None, parsePostsTag=True):
        self.blog = blog

        if not os.path.exists(src + '/_%s.html' % name):
            self.addError('No _%s.html found, exiting' % name)
            return None

        try:
            dom = parse(src + '/_%s.html' % name)
        except xml.dom.DOMException as e:
            self.addError('Error parsing _%s.html : ' + e)
            return None
            
        if directory and not os.path.exists(output + '/' + directory):
            os.makedirs(output + '/' + directory)

        if not parsePostsTag: return dom

        post_nodes = dom.getElementsByTagNameNS(self.URI, "posts")

        if post_nodes:
            if post_nodes[0].hasAttribute("limit"):
                self.posts_per_page = int(post_nodes[0].getAttribute("limit"))
        else:
            self.addWarning('No tag dyn:posts found')

        return dom

    def generatePages(self, dom, posts, src, output, name):
        if len(posts) > self.posts_per_page:
            self.nb_pages = self.computeNbPages(len(posts), self.posts_per_page)

        if not os.path.exists(output + self.dirname):
            os.mkdir(output + self.dirname)

        filename = self.dirname + '/' + self.filename + '.html'
        
        impl = xml.dom.getDOMImplementation()

        while self.cur_page <= self.nb_pages:
            #print 'Generate ' + filename
            dom_ = impl.createDocument('', 'xml', None)
            dom_.replaceChild(dom.firstChild.cloneNode(True), dom_.firstChild)
            nodes = self.parse(src, self.hooks, posts, dom_, dom_.firstChild)
            self.writeIfNotTheSame(output + filename, nodes)
            self.cur_page = self.cur_page + 1
            filename = self.dirname + '/' + self.filename + str(self.cur_page) + '.html'

        while os.path.exists(filename):
            self.addReport('Removing unused ' + filename)
            os.unlink(filename)
            filename = filename + '.gz'
            if os.path.exists(filename):
                self.addReport('Removing unused ' + filename)
                os.unlink(filename)
            self.cur_page = self.cur_page + 1
            filename = self.dirname + '/' + self.filename + str(self.cur_page) + '.html'

    def generate(self, blog, src, output):
        from dynastie.models import Post, Blog

        self.blog = blog
        self.parent_posts = []

        dom = self.parseTemplate(blog, src, output, 'index')
        if dom is None: return self.report

        now = datetime.datetime.now()
        cur_year = now.year
        posts = Post.objects.filter(creation_date__year=cur_year, published=True, front_page=True).order_by('-creation_date')

        if posts.count() < self.posts_per_page:
            posts = Post.objects.filter(published=True, front_page=True).order_by('-creation_date')[:self.posts_per_page]

        self.dirname = ''
        self.generatePages(dom, posts, src, output, 'index')

        if not self.somethingWrote:
            self.addReport('Nothing changed')

        return self.report