Changeset 378

Timestamp:

07/13/09 14:26:21 (4 years ago)

Author:

jon

Message:

Changing blackboard import to accomodate tufts blackboard packages

Location:

collective.imstransport/trunk/collective/imstransport/utilities

Files:

• bb/bbreader.py (modified) (5 diffs)
• bb/configbb.py (modified) (1 diff)
• bb/imsbbreader.py (modified) (3 diffs)
• imsutility.py (modified) (2 diffs)
• moodle/imsmoodlewriter.py (modified) (1 diff)
• packagingio.py (modified) (2 diffs)
• webct/imswebctreader.py (modified) (1 diff)
• webct/webctreader.py (modified) (1 diff)

collective.imstransport/trunk/collective/imstransport/utilities/bb/bbreader.py

@@ -1 +1 @@
 from xml.dom import minidom
+from urlparse import urlparse
 from collective.imstransport.IMS_exceptions import ManifestError
-from configbb import LOM_BB_namespace, XML_namespace
+from configbb import LOM_BB_namespace, XML_namespace, EMBEDDED_STRING
+from Products.PageTemplates.PageTemplateFile import PageTemplateFile
+from collective.imstransport.utilities.interfaces import IIMSTransportUtility
 
 class BBReader(object):
@@ -53 +56 @@
                 resource.getAttributeNS(XML_namespace, 'base'))
 
-    def readFiles(self, resource):
+    def readFiles(self, resource, bbase):
         files = []
         flns = resource.getElementsByTagName('file')
@@ -59 +62 @@
             for fln in flns:
                 file = fln.getAttribute('href')
-                files.append(file)
+                if bbase:
+                    files.append('%s/%s' %(bbase, file))
+                else:
+                    files.append(file)
         return files
 
@@ -93 +99 @@
                     value = isfolder_nodes[0].getAttribute('value')
                     if value ==  'true':
-                        md['type'] = 'Folder'
+                        md['bbtype'] = 'Folder'
+            handler_nodes = content_node.getElementsByTagName('CONTENTHANDLER')
+            if handler_nodes:
+                value = handler_nodes[0].getAttribute('value')
+                if value == 'resource/x-bb-externallink':
+                    url_nodes = content_node.getElementsByTagName('URL')
+                    if url_nodes:
+                        url = url_nodes[0].getAttribute('value') 
+                        if url:
+                            md['bbtype'] = 'Link'
+                            md['remoteUrl'] = url        
 
     def readTocItem(self, manifest, resid):
@@ -107 +123 @@
                     idref = item.getAttribute('identifierref')
                     if idref == resid:
-                        childitems = item.getElementsByTagName('item')
+                        childitems = item.childNodes
                         for x in childitems:
-                            itemid = x.getAttribute('identifierref')
-                            tocitems.append(itemid)
+                            if x.nodeName == 'item':
+                                itemid = x.getAttribute('identifierref')
+                                tocitems.append(itemid)
         return tocitems
+
+    def readEmbeddedTags(self, soup):
+        """ Read embedded tags from a text file """
+        prevlink = None
+        links = self.getDocumentHrefLinks(soup)
+        for link in links:
+            if 'embedded' in link['href']:
+                if prevlink and prevlink != link['href']:
+                    return None
+                prevlink = link['href']
+        links = self.getDocumentSrcLinks(soup)
+        for link in links:
+            if 'embedded' in link['src']:
+                if prevlink  and prevlink != link['src']:
+                    return None
+        return prevlink
+
+    def createTocPage(self, entries):
+        """ Create a table of links """
+        text = '<table>'
+        for z in entries:
+            text += "<tr><td><a href='%s'>%s</a></td></tr>" %(z[0], z[1])
+        text += '</table>'
+        return text
+
+    def runDocumentFilters(self, utils, soup, vars, base):
+        """ Run a filter over the links """
+        links = []
+        links = self.getDocumentHrefLinks(soup)
+        for link in links:
+            orig = link['href']
+            link['href'] = self.filterDocumentLink(link['href'], utils, vars, base)
+        links = []
+        links = self.getDocumentSrcLinks(soup)
+        for link in links:
+            orig = link['src']
+            link['src'] = self.filterDocumentLink(link['src'], utils, vars, base)
+        return soup.prettify()
+
+    def getDocumentHrefLinks(self, soup):
+        links = []
+        tags = soup.findAll(href=True)
+        from urlparse import urlparse
+        for tag in tags:
+            if tag.has_key('href'):
+                url = urlparse('href')
+                if not url[1] or 'localhost' in url[1]:
+                    links.append(tag)
+        return links
+
+    def getDocumentSrcLinks(self, soup):
+        links = []
+        tags = soup.findAll(src=True)
+        from urlparse import urlparse
+        for tag in tags:
+            if tag.has_key('src'):
+                url = urlparse('src')
+                if not url[1] or 'localhost' in url[1]:
+                    links.append(tag)
+        return links
+
+    def _convertBBVariables(self, link, vars, base):
+        """ Convert BB variables to their counterparts """
+        lnk = link
+        for var in vars:
+            if base:
+                replace = '%s/%s/' %(base, var[1])
+            else:
+                replace = var[1]
+            lnk = link.replace(var[0], replace)
+        return lnk
+
+    def _convertURLEntities(self, link):
+        """ Convert BB variables to their counterparts """
+        lnk = link
+        import urllib
+        lnk = urllib.unquote(lnk)
+        return lnk
+
+    def _convertToNormalizedLink(self, link, utils):
+        """ Normalize the link so it can be imported without errors """
+        lnk = link
+        url = urlparse(lnk)
+        urlfile = url[2].split('/')
+        urlfile[-1] = utils.normalizeString(urlfile[-1])
+        lnk = '/'.join(urlfile)
+        return lnk
+        
+    def filterDocumentLink(self, link, utils, vars, base):
+        lnk = link
+        from urlparse import urlparse
+        url = urlparse(lnk)
+        if url[2] and not url[0]:
+            lnk = self._convertBBVariables(lnk, vars, base)
+            lnk = self._convertURLEntities(lnk)
+            lnk = self._convertToNormalizedLink(lnk, utils)
+        return lnk
+
+    def runFilters(self, text, filters, **kw):
+        rettext = text
+        for filter in filters:
+            if 'embed' == filter:
+                rettext = self.replaceEmbedVariables(rettext, kw['base'])
+            elif 'reflinks' == filter:
+                rettext = self.rewriteReferenceLinks(rettext)
+        return rettext
+
+    def replaceEmbedVariables(self, text, base):
+        """ Remove the proprietary embed variables and replace with proper path """
+        if base:
+            rpath = '%s/embed/' %base
+        return text.replace(EMBEDDED_STRING, rpath)
+
+    def rewriteReferenceLinks(self, text):
+        """ Rewrite all reference links """
+
+    def removeUrlEntities(self, href):
+        """ Remove the URL entities from the string """
+        
+        
+
+
+
+        
+
+
+
+
+        
+        
 
         

collective.imstransport/trunk/collective/imstransport/utilities/bb/configbb.py

@@ -4 +4 @@
 LOM_BB_namespace = 'http://www.blackboard.com/content-packaging/'
 XML_namespace = 'http://www.w3.org/XML/1998/namespace'
+EMBEDDED_STRING = '@X@EmbeddedFile.location@X@'
 
 namespaces = [('xmlns', 'http://www.imsglobal.org/xsd/imscp_v1p1'),

collective.imstransport/trunk/collective/imstransport/utilities/bb/imsbbreader.py

@@ -6 +6 @@
 from xml.parsers.expat import ExpatError
 from collective.imstransport.IMS_exceptions import ManifestError
+from BeautifulSoup import BeautifulSoup
+from Products.CMFCore.interfaces import ISiteRoot
+import htmlentitydefs
+from configbb import EMBEDDED_STRING
 import re
 
@@ -30 +34 @@
         for x in resources:
             resid, restype, bbfile, bbtitle, bbase = bbreader.readResourceAttributes(x)
+            doctext = filetoc = restoc = ''
+            metadata = {}
             if restype == 'resource/x-bb-document':
                 metadata = {}
@@ -37 +43 @@
                     resnode = bbreader.parseDataFile(dataxml)
                     metadata = bbreader.readMetadata(resnode)
-                files = bbreader.readFiles(x)
-                # If the resource is a file
-                if files:
-                    hash = resid
-                    objDict[hash] = metadata
-                    excludeFromNav = True
-                    file = '%s/%s' %(bbase, files[0])
-                    type = self.determineType(objDict[hash], files[0])
-                    id = self.createIdFromFile(files[0])
-                    path = '%s' %bbase
-                    if orgs.has_key(resid):
-                        title = orgs[resid]
+                if metadata.has_key('text') and metadata['text']:
+                    mtext = metadata['text']                    
+                    if type(mtext) == type(u''):
+                        mtext = mtext.encode('utf-8')
+                    ptext = unquotehtml(mtext)
+                    utils = getUtility(ISiteRoot).plone_utils
+                    soup = BeautifulSoup(ptext)
+                    doctext = bbreader.runDocumentFilters(utils, soup, [(EMBEDDED_STRING, 'embedded'),], bbase)
+                # Handle Files
+                files = bbreader.readFiles(x, bbase)
+                entries = []
+                for y in files:
+                    dhash = resid + y
+                    objDict[dhash] = {}
+                    dexcludeFromNav = True
+                    dfile = y
+                    dfileparts = y.split('/')
+                    # File is embedded
+                    if 'embedded' in y:
+                        # Link is encrypted
+                        if len(dfileparts) > 0 and dfileparts[-1][0] == '!' and doctext:
+                            soup = BeautifulSoup(doctext)
+                            embeddedpath = bbreader.readEmbeddedTags(soup)
+                            if embeddedpath:
+                                dfile = embeddedpath
+                    dfilepath = self.createPathFromFile(dfile)
+                    did = self.createIdFromFile(dfile)
+                    dtype = self.determineType(objDict[dhash], dfile)
+                    portal = getUtility(ISiteRoot)
+                    utils = portal.plone_utils
+                    did = utils.normalizeString(did)
+                    dtitle = did
+                    if dfilepath:
+                        linkpath = '%s/%s' %(dfilepath, did)
                     else:
-                        title = id
-                    self.applyCoreMetadata(objDict, hash, id, path, excludeFromNav, type, title, file=file)
-                # If the resource is a document
-                else:
-                    hash = resid 
-                    objDict[hash] = metadata
-                    excludeFromNav = True
-                    type = 'Document'
-                    id = resid
-                    path = ''
-                    if orgs.has_key(resid):
-                        title = orgs[resid]
-                    else:
-                        title = id
-                    self.applyCoreMetadata(objDict, hash, id, path, excludeFromNav, type, title)
-            # The resource is a table of contents page.
-            elif restype == 'course/x-bb-coursetoc':
+                        linkpath = did
+                    binfile = y
+                    folder_files = source.getFolderFiles(dfilepath)
+                    if len(folder_files) == 1:
+                        binfile = folder_files[0]
+                    entries.append((linkpath, dtitle))
+                    self.applyCoreMetadata(objDict[dhash], did, dfilepath, dexcludeFromNav, dtype, dtitle, file=binfile)
+                if entries:
+                    filetoc = bbreader.createTocPage(entries)
+            # Handle links
+            if metadata.has_key('bbtype') and metadata['bbtype'] == 'Link':
                 hash = resid
-                objDict[hash] = {}
-                excludeFromNav = False
-                tocpages.append(resid)
-                type = 'Document'
-                path = ''
-                id = resid
+                objDict[hash] = metadata
+                filepath = ''
+                id = resid + '.link'
+                excludeFromNav = True
+                ptype = metadata['bbtype']
                 if orgs.has_key(resid):
-                    title = re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', orgs[resid].split('.')[1])
+                    title = orgs[resid]
                 else:
                     title = id
-                self.applyCoreMetadata(objDict, hash, id, path, excludeFromNav, type, title)
-
-            # Build table of contents pages
+                self.applyCoreMetadata(objDict[hash], id, filepath, excludeFromNav, ptype, title)
+            elif restype in ['resource/x-bb-document', 'course/x-bb-coursetoc']:
+                # Handle normal bb-documents
+                hash = resid
+                objDict[hash] = metadata
+                filepath = ''
+                excludeFromNav = True
+                ptype = 'Document'
+                id = resid + '.html'
+                if orgs.has_key(resid):
+                    title = orgs[resid]
+                else:
+                    title = id
+                # It's a folder object:
+                isFolder = metadata.has_key('bbtype') and metadata['bbtype'] == 'Folder'
+                if isFolder or restype == 'course/x-bb-coursetoc':
+                    tocpages.append(resid)
+                # It's a table of contents object
+                if restype == 'course/x-bb-coursetoc':
+                    if orgs.has_key(resid):
+                        excludeFromNav = False
+                        orgstitle = orgs[resid].split('.')
+                        # Rewrite label tag
+                        if len(orgstitle) > 1 and orgstitle[-1] == 'label':
+                            title = re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', orgs[resid].split('.')[1])
+                text = doctext
+                if type(filetoc) == type(u''):
+                    text += filetoc.encode('utf-8')
+                self.applyCoreMetadata(objDict[hash], id, filepath, excludeFromNav, ptype, title, text=text)
+        # Build table of contents pages
         for z in tocpages:
-            text = '<table>'
             tocitems = bbreader.readTocItem(doc, z)
+            entries = []
             for titem in tocitems:
-                met = objDict[titem]
-                path = met['path']
-                if path:
-                    linkpath = '%s/%s' %(path, objDict[titem]['id'])
+                if objDict.has_key(titem):
+                    met = objDict[titem]
+                    path = met['path']
+                    if path:
+                        linkpath = '%s/%s' %(path, objDict[titem]['id'])
+                    else:
+                        linkpath = objDict[titem]['id']
+                    entries.append((linkpath, met['title']))
+            if entries:
+                if objDict[z].has_key('text') and objDict[z]['text']:
+                    gtext = objDict[z]['text']
+                    if type(gtext) == type(u''):
+                        gtext = gtext.encode('utf-8')
+                    objDict[z]['text'] = bbreader.createTocPage(entries).encode('utf-8') + gtext
                 else:
-                    linkpath = objDict[titem]['id']
-                text += "<tr><td><a href='%s/view'>%s</a></td></tr>" %(linkpath, objDict[titem]['title'])
-            text += '</table>'
-            objDict[z]['text'] = text
-                
+                    objDict[z]['text'] = bbreader.createTocPage(entries).encode('utf-8')
+            else:
+                objDict[z]['excludeFromNav'] = True
         objcreator = getUtility(IIMSObjectCreator)
         objcreator.createObjects(objDict, context, source)
 
-    def applyCoreMetadata(self, objDict, hash, id, path, excludeFromNav, type, title, file=None, text=None):
+    def applyCoreMetadata(self, metadata, id, path, excludeFromNav, type, title, file=None, text=None):
         """ Helper function for applying metadata """
-        objDict[hash]['id'] = id
-        objDict[hash]['path'] = path
-        objDict[hash]['excludeFromNav'] = excludeFromNav
-        if not (objDict[hash].has_key('type') and objDict[hash]['type']):
-            objDict[hash]['type'] = type
-        if not (objDict[hash].has_key('title') and objDict[hash]['title']):
-            objDict[hash]['title'] = title
+        portal = getUtility(ISiteRoot)
+        utils = portal.plone_utils
+        metadata['id'] = utils.normalizeString(id)
+        metadata['path'] = path
+        metadata['excludeFromNav'] = excludeFromNav
+        metadata['type'] = type
+        if not (metadata.has_key('title') and metadata['title']):
+            metadata['title'] = title
         if file:
-            objDict[hash]['file'] =  file
+            metadata['file'] =  file
         if text:
-            objDict[hash]['text'] = text
+            metadata['text'] = text
 
+def convertentity(m):
+    """Convert a HTML entity into normal string (ISO-8859-1)"""
+    if m.group(1)=='#':
+        try:
+            return chr(int(m.group(2)))
+        except ValueError:
+            return '&#%s;' % m.group(2)
+    try:
+        return htmlentitydefs.entitydefs[m.group(2)]
+    except KeyError:
+        return '&%s;' % m.group(2)
+
+def unquotehtml(s):
+    """Convert a HTML quoted string into normal string (ISO-8859-1).
+
+    Works with &#XX; and with &nbsp; &gt; etc."""
+    return re.sub(r'&(#?)(.+?);',convertentity,s) 

collective.imstransport/trunk/collective/imstransport/utilities/imsutility.py

@@ -3 +3 @@
 from collective.imstransport.utilities.interfaces import IIMSTransportUtility, IIMSManifestWriter, IIMSManifestReader
 from OFS.SimpleItem import SimpleItem
-
 
 class IMSTransportUtility(SimpleItem):
@@ -30 +29 @@
             values.append((x[1].getPackageName(), x[0]))
         return values
-

collective.imstransport/trunk/collective/imstransport/utilities/moodle/imsmoodlewriter.py

@@ -120 +120 @@
                     path += '.html'
 
-        return path
+        return path 
 
 

collective.imstransport/trunk/collective/imstransport/utilities/packagingio.py

@@ -52 +52 @@
         return self.files.read(fn)
 
+    def getFolderFiles(self, path):
+        """ Returns the files within a specific folder """
+        retfiles = []
+        for fn in self.files.namelist():
+            if path +'/' in fn:
+                retfiles.append(fn)
+        return retfiles
+
 
 class ZipfileWriter:
@@ -62 +70 @@
         self.archive = StringIO()
         self.zipfile = ZipFile(self.archive, 'w', ZIP_DEFLATED)
-
 
     def writeFile(self, path, data):

collective.imstransport/trunk/collective/imstransport/utilities/webct/imswebctreader.py

@@ -27 +27 @@
         for manifest in manifests:
             orgs =[]
-            manifestmetadata = webctreader.readMetadata(manifest)
+            manifestmetadata = webctreader.readPackageMetadata(manifest)
             if manifestmetadata.has_key('webcttype') and manifestmetadata['webcttype'] == 'Course':
                 objDict['package'] = manifestmetadata

collective.imstransport/trunk/collective/imstransport/utilities/webct/webctreader.py

@@ -24 +24 @@
         if organizations:
             organization_nodes = organizations[0].getElementsByTagName('organization')
-            if organization_nodes:
-                organization_node = organization_nodes[0]
-                item_nodes = organization_nodes[0].getElementsByTagName('item')
+            for organization_node in organization_nodes:
+                item_nodes = organization_node.getElementsByTagName('item')
                 for item in item_nodes:
                     idref = item.getAttribute('identifierref')