source: enpraxis.staticsite/trunk/enpraxis/staticsite/utilities/staticsiteutility.py @ 291

import os
import re
import string
from BeautifulSoup import BeautifulSoup
from Globals import BobobaseName
from os import makedirs as os_makedirs
from os.path import join as os_join
from os.path import lexists as os_lexists
from os.path import split as os_split

from urllib2 import urlopen
from urllib2 import HTTPError

from OFS.SimpleItem import SimpleItem
from zope.interface import implements
from interfaces import IStaticSiteUtility
from zope.component import getUtility, getMultiAdapter

class StaticSiteUtility(SimpleItem):
    """ Deploy a static site """

    implements(IStaticSiteUtility)

    def deploySite(self, context):
        """ Deploy the site """

        ssprops = context.portal_url.portal_properties.staticsite_properties
        domain = ssprops.getProperty('domain')
        dpath = self._getDeploymentPath(ssprops.getProperty('deployment_path'))
        pstates = ssprops.getProperty('published_states')
        
        self._deploySiteStructure(context, ssprops, domain, dpath)
        self._deploySiteActions(context, dpath)

        brains = context.portal_catalog.searchResults(
            path={'query':'/'.join(context.getPhysicalPath()),
                  'depth':1,},
            review_state=pstates)
        for x in brains:
            url = x.getURL()
            self.deployObject(url, x, dpath, domain)
            self.traverse(x, dpath, domain, pstates)

    def traverse(self, brain, dpath, domain, pstates):
        """ Traverse the site. """
        brains = brain.portal_catalog.searchResults(
            path={'query':brain.getPath(), 'depth':1},
                  review_state=pstates)
        for x in brains:
            url = x.getURL()
            self.deployObject(url, x, dpath, domain)
            if x.is_folderish:
                self.traverse(x, dpath, domain, pstates)
            
    def deployObject(self, url, brain, dpath, domain):
        """ Deploy an object """
        portal_url = brain.portal_url()
        soup = ''
        if brain.is_folderish:
            #create dir before processing html         
            objpath = self._getObjPath(url, portal_url, dpath)
            self._createDirectory(objpath)
            path = '%s/index.html' % objpath
            raw = self._httpget(url)
            soup = BeautifulSoup(raw)
            #process document actions
            self._deployDocumentActions(brain, portal_url, path, soup)
            self.runFilters(brain, soup, portal_url, dpath)
            self._writeFile(path, soup.prettify())
        elif brain.Type in ['Page']:
            path = self._getObjPath(url, portal_url, dpath)
            if '.htm' not in path:
                path += '.html'
            raw = self._httpget(url)        
            soup = BeautifulSoup(raw)
            #process document actions
            self._deployDocumentActions(brain, portal_url, path, soup)            
            self.runFilters(brain, soup, portal_url, dpath)
            self._writeFile(path, soup.prettify())
        else:
            #get file object
            path = self._getObjPath(url, portal_url, dpath)
            self.linkTable[url] = {'dpath':path, 'type' : 'file'}  
            raw = self._httpget(url) 
            soup = BeautifulSoup(raw)
            self.runFilters(brain, soup, portal_url, dpath)       
            self._writeFile(path, raw)
        #process rdf for the object
        self._deployRDF(brain, url, path)

    def _deployRDF(self, brain, url, path):
        """ Deploy the RDF for an object """
        rdfurl = '%s/rdf' %url
        rdfpath = '%s-rdf' %path
        if brain.Type in ['Page', 'Course', 'Division', 'Folder', 'File', 'Image']:
            rdfraw = self._httpget(rdfurl)
            self._writeFile(rdfpath, rdfraw)     

    def runFilters(self, brain, soup, portal_url, dpath):
        """ Filter content """
        rel_path = './'
        #overwrite file
        soup.base.extract()
        self._rewriteHREF(brain, soup, dpath)

    def _deploySiteStructure(self, context, ssprops, domain, dpath):
        """ Get the base framework and resources needed to build the chrome locally  """
        base_files = ssprops.base_files
        css_images = ssprops.css_images
        extra_views = ssprops.extra_views
        
        portal_url = context.portal_url()
        
        for x in base_files:
            #Download files/images that are used sitewide in the chrome
            url = '%s/%s' % (portal_url, x)
            path = self._getObjPath(url, portal_url, dpath)
            raw = self._httpget(url)
            self._writeFile(path, raw)
            
        for x in extra_views:
            #Download zope3 views and/or additional views that are not included in the site catalog search
            url = '%s/%s' % (portal_url, x)
            path = self._getObjPath(url, portal_url, dpath)
            path += '.html'
            raw = self._httpget(url)
            self._writeFile(path, raw)

        #get soup of front page
        fp = self._httpget(portal_url)
        soup = BeautifulSoup(fp)
        types = ['css', 'js']
            
        for x in types:
            if x == 'css':
                for tag in soup.head.findAll('style'):
                    #analyze content and extract url to CSS
                    contents = tag.contents[0]
                    url = re.match(r"""[^(]*\((.*)\)[^)]*$""", contents, re.X)
                    url = url.group(1)
                    path = self._deployCompressedResources(url, portal_url, dpath)
                    css_path = os_split(path)[0]
                for img in css_images:
                    #download css_images
                    url = '%s/%s' % (portal_url, img)
                    path = '%s/%s' % (css_path, img)
                    try:
                        raw = self._httpget(url)
                    except HTTPError:
                        continue
                    self._writeFile(path, raw)                    
                    
            elif x == 'js':
                for tag in soup.head.findAll('script'):
                    if tag.has_key('src'):
                        url = tag['src']
                        self._deployCompressedResources(url, portal_url, dpath)


    def _deploySiteActions(self, context, dpath):
        """  deploys portal_actions site_actions content """
        excluded = ['plone_setup', 'deploy']        
        site_actions = context.portal_actions.site_actions.listActions()
        for x in site_actions:
            if x.id not in excluded and x.visible == True:
                portal_url = context.portal_url()
                action_url = x.url_expr.split('/')[-1]
                url = '%s/%s' % (portal_url, action_url)
                path = self._getObjPath(url, portal_url, dpath)
                path += '.html'
                raw = self._httpget(url)        
                self._writeFile(path, raw)
        
    def _deployDocumentActions(self, brain, portal_url, path, soup):
        """ Deploys the document actions for the site """
        doc_actions = brain.portal_actions.document_actions.listActions()
        for doc_action in doc_actions:
            if doc_action.id in ['skinless', 's5']:
                action_url = self._findDocActionUrl(soup, doc_action.id)
                if action_url:
                    path = '%s-%s.html' %(path, doc_action.id)
                    if action_url.find(portal_url) == -1:
                        action_url = '%s/%s' %(portal_url, action_url)
                    path = '%s-%s.html' %(path, doc_action.id)
                    raw = self._httpget(action_url)
                    self._writeFile(path, raw)
            elif doc_action.id in ['rss', 'rss_front']:
                action_url = self._findDocActionUrl(soup, doc_action.id)
                if action_url:
                    path = '%s-%s' %(path, doc_action.id)
                    if action_url.find(portal_url) == -1:
                        action_url = '%s/%s' %(portal_url, action_url)
                    path = '%s-%s' %(path, doc_action.id)
                    raw = self._httpget(action_url)
                    self._writeFile(path, raw)
                    

    def _findDocActionUrl(self, soup, id):
        """ Parse document soup to find a document action """
        docdivs = soup.body.findAll('div')
        action_url = None
        for x in docdivs:
            if x.has_key('class') and x['class'] == 'documentActions':
                action = x.find('li', id='document-action-%s' %id)
                if action:
                    anchor = action.find('a')
                    if anchor and anchor.has_key('href'):
                        action_url = anchor['href']
                break
        return action_url

    def _getURLPath(self, brain, url):
        """ Retrieve the URL path on the filesystem. """

        rel_path = self._getRelativePath('/'.join(brain.getURL().split('/')[:-1]), url)
        rel_path = rel_path.split('#')[0]
        urlbrains = brain.portal_catalog.searchResults(query={'path':rel_path,},id=rel_path.split('/')[0])
        for x in urlbrains:
            if brains.is_folderish:
                pass
            elif brain.Type == 'Page':
                pass
            else:
                pass

    def _deployCompressedResources(self, url, portal_url, dpath):
        """ extracts resources from soup and writes to static path location """
        path = self._getObjPath(url, portal_url, dpath)
        path = path.replace('%20','_')
        raw = self._httpget(url)
        self._writeFile(path, raw)
        return path
                    
    def _getDeploymentPath(self, sp):
        """ Get the default static path location. """
        dpath = os_split(BobobaseName)[0]
        return os_join(dpath, sp)
        
    def _getObjPath(self, url, portal_url, dpath):
        """ Get the object path based on the deployment path. """
        objpath = url.replace(portal_url + '/', '')
        path = dpath
        for x in objpath.split('/'):
            path = os_join(path, x)
        return path
    
    def _createDirectory(self, path):
        """ Create a directory on the filesystem """      
        if not os_lexists(path):
            os_makedirs(path)
            
    def _httpget(self, url):
        """ Get html for the url """
        f = urlopen(url)
        data = f.read()
        f.close()
        return data

    def _writeFile(self, fn, data):
        self._createDirectory(os_split(fn)[0])
        f = open(fn, 'w')
        f.write(data)
        f.close()
    
    def _getRelativePath(self, path1, path2):
        common = os.path.commonprefix ([path1, path2])
        p1 = path1[len(common):]
        p2 = path2[len(common):]
        if os.path.isfile (path1):
            p1 = os.path.dirname (p1)
        dirs = string.split (p1, os.sep)
        dotdot = map (lambda x: '..', dirs)
        dotpath = string.join (dotdot, os.sep)
        rel_path = os.path.join (dotpath, p2)        
        if rel_path.startswith(os.sep):
            rel_path = '.%s' % rel_path
        return rel_path
            
    def _rewriteHREF(self, brain, soup, dpath):
        tags = soup.findAll('a')
        for tag in tags:
            if tag.has_key('href'):
                self._getURLPath(brain, tag['href'])
        return soup