source: Products.ecmigration/trunk/Products/ecmigration/migrate.py @ 756

Revision 756, 14.3 KB checked in by brent, 3 years ago (diff)

Split out data from metadata.

Line 
1# -*- coding: us-ascii -*-
2# _______________________________________________________________________
3#              __________                      .__       
4#   ____   ____\______   \____________  ___  __|__| ______
5# _/ __ \ /    \|     ___/\_  __ \__  \ \  \/  /  |/  ___/
6# \  ___/|   |  \    |     |  | \// __ \_>    <|  |\___ \
7#  \___  >___|  /____|     |__|  (____  /__/\_ \__/____  >
8#      \/     \/                      \/      \/       \/
9# _______________________________________________________________________
10#
11#    This file is part of the eduCommons software package.
12#
13#    Copyright (c) 2011 enPraxis, LLC
14#    http://enpraxis.net
15#
16#    This program is free software; you can redistribute it and/or modify
17#    it under the terms of the GNU General Public License as published by
18#    the Free Software Foundation, version 2.8 
19#
20#    This program is distributed in the hope that it will be useful,
21#    but WITHOUT ANY WARRANTY; without even the implied warranty of
22#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23#    GNU General Public License for more details.
24#
25#    You should have received a copy of the GNU General Public License
26#    along with this program; if not, write to the Free Software
27#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
28# _______________________________________________________________________
29
30__author__ = 'Brent Lambert'
31__version__ = '$ Revision 0.0 $'[11:-2]
32
33
34from Products.CMFPlone.utils import _createObjectByType
35from cStringIO import StringIO
36from DateTime import DateTime
37import cPickle as Pickle
38import Globals
39import os
40import tarfile
41import tempfile
42
43
44# Try to import annotation interfaces
45
46try:
47    from collective.contentlicensing.DublinCoreExtensions.interfaces import ILicensable, ILicense
48except ImportError:
49    from Products.ContentLicensing.DublinCoreExtensions.interfaces import ILicensable, ILicense
50
51try:
52    from enpraxis.educommons.annotations.interfaces import IClearCopyrightable, IClearCopyright
53    has_clearcopyrightable = True
54except ImportError:
55    has_clearcopyrightable = False
56   
57
58try:
59    from enpraxis.educommons.annotations.interfaces import IAccessibilityCompliantable, IAccessibilityCompliant
60    has_accessibilitycompliance = True
61except ImportError:
62    has_accessibilitycompliance = False
63
64try:
65    from enpraxis.educommons.annotations.interfaces import ICourseOrderable, ICourseOrder
66    has_courseorderable = True
67except ImportError:
68    has_courseorderable = False
69
70
71class TarArchiveManager(object):
72    """ Archive data using tarfile """
73
74    def __init__(self, archname, mode='w:bz2'):
75        """ Open the tar file """
76        self.tar_archive = tarfile.open(archname, mode)
77
78    def addFileFromString(self, fn, data, lmod):
79        """ Write a file from a string to the archive """
80        tinfo = tarfile.TarInfo(fn)
81        tinfo.size = len(data)
82        tinfo.mtime = lmod
83        self.tar_archive.addfile(tinfo, StringIO(data))
84        # Circumvent caching for large file handling
85        self.tar_archive.members = []
86        print fn
87
88    def addFileFromDisk(self, fn, arcname):
89        """ Write a file from the filesystemto the archive """
90        self.tar_archive.add(fn, arcname)
91        print arcname
92
93    def getNextFileInfo(self):
94        """ Return file information or None if no files left """
95        return self.tar_archive.next()
96
97    def readFile(self, info):
98        """ Read a file from the archive """
99        f = self.tar_archive.extractfile(info)
100        data = Pickle.load(f)
101        f.close()
102        # Circumvent caching for large file handling
103        self.tar_archive.members = []
104        return data
105
106    def close(self):
107        """ Close the tar file """
108        self.tar_archive.close()
109       
110
111class ECMigration:
112    """ Migrate an eduCommons site """
113
114    transforms = {
115        'ECDepartment':'Division',
116        'ECCourse':'Course',
117        'ECDocument':'Document',
118        'ECFile':'File',
119        'ECFolder':'Folder',
120        'ECImage':'Image',
121        'ECLink':'Link',
122        'GFolder':'Folder',
123        'GDocument':'Document',
124        'GFile':'File',
125        'GImage':'Image',
126        'GLink':'Link',
127        }
128
129    id_transforms = {
130        'About':'about',
131        'Help':'help',
132        'terms_of_use':'terms-of-use',
133        'privacy_policy':'privacy-policy'
134        }
135
136       
137
138    def __init__(self, context):
139        self.context = context
140
141    def importContent(self):
142        """ Import a migration """
143        fn = os.path.split(Globals.BobobaseName)[0]
144        fn = os.path.join(fn, 'migration')
145        fn = os.path.join(fn, 'eduCommons-20110216.tar.bz2')
146        tf = TarArchiveManager(fn, 'r:bz2')
147        while (1):
148            info = tf.getNextFileInfo()
149            if info:
150                data = tf.readFile(info)
151                if 'UserInfo.ecmigration' == data['type']:
152                    self.importUsers(data)
153                else:
154                    self.importObject(data)
155            else:
156                break
157
158    def importUsers(self, data):
159        """ Import Users """
160        users = data['users']
161        portal = self.portal_url.getPortalObject()
162        pr = portal.portal_registration
163        for x in users:
164            pr.addMember(id=x,
165                         password=pr.generatePassword(),
166                         roles=users[x]['roles'],
167                         properties = {'fullname':users[x]['fullname'],
168                                       'username':x,
169                                       'email':users[x]['email']})
170        for x in users:
171            portal.acl_users.source_users._user_passwords[x] = users[x]['password']
172
173    def importObject(self, data):
174        """ Import an object from data out of the tar archive """
175        oid = '/'.join(data['filename'].split(os.sep))
176        print oid
177        try:
178            obj = self.getObjectByPath(oid)
179        except TypeError:
180            pass
181        else:
182            if not obj:
183                # Object does not exist, create it
184                parent = self.getObjectByPath('/'.join(oid.split('/')[:-1]))
185                if parent:
186                    nid = oid.split('/')[-1]
187                    _createObjectByType(data['type'], parent, id=nid)
188                    obj = getattr(parent, nid)
189            if obj:
190                if getattr(obj, 'getId', None):
191                    self.updateObject(obj, data)
192
193    def updateObject(self, obj, data):
194        """ Update the object with the new settings """
195        for x in data['fields']:
196            try:
197                field = obj.getField(x)
198            except AttributeError:
199                import pdb; pdb.set_trace()
200            if field:
201                field.set(obj, data['fields'][x])
202        obj.workflow_history = data['workflowhistory']
203        if data.has_key('rightsholder'):
204            if ILicensable.providedBy(obj):
205                lic = ILicense(obj)
206                lic.setRightsHolder(data['rightsholder'])
207                lic.setRightsLicense(data['rightslicense'])
208        if data.has_key('clearedcopyright'):
209            if IClearCopyrightable.providedBy(obj):
210                cc = IClearCopyright(obj)
211                cc.setClearedCopyright(data['clearedcopyright'])
212        if data.has_key('accessibilitycompliant'):
213            if IAccessibilityCompliantable.providedBy(obj):
214                acc = IAccessibilityCompliant(obj)
215                acc.setAccessible(data['accessibilitycompliant'])
216        if data.has_key('courseorder'):
217            if ICourseOrderable.providedBy(obj):
218                order = ICourseOrder(obj)
219                order.setPositionInCourse(data['courseorder'])
220        try:
221            obj.reindexObject()
222        except AttributeError:
223            import pdb; pdb.set_trace()
224
225    def getObjectByPath(self, path):
226        """ Return an object via its path """
227        try:
228            obj = self.context.restrictedTraverse(path)
229        except KeyError:
230            obj = None
231        except AttributeError:
232            obj = None
233        return obj
234
235    def exportContent(self, filename=None):
236        """ Export eduCommons content out of site ready for migration. """
237        fn = os.path.split(Globals.BobobaseName)[0]
238        fn = os.path.join(fn, 'migration')
239        if not os.path.exists(fn):
240            os.mkdir(fn)
241        fn = os.path.join(fn, 'eduCommons-20110216.tar.bz2')
242        brains = self.context.portal_catalog(path={'query':'/', 'depth':2,},)
243        tf = TarArchiveManager(fn, 'w:bz2')
244        self.exportUsers(tf)
245        self.exportObjects(brains, tf)
246        tf.close()
247
248    def exportUsers(self, archive):
249        fn = '%s/userinfo.ecmigration' %self.context.getId()
250        objstore = {'filename':fn, 'type':'UserInfo.ecmigration', 'users':{},}
251        for user in self.context.acl_users.getUsers():
252            username = user.getName()
253            password = self.context.acl_users.source_users._user_passwords[username]
254            objstore['users'][username] = {
255                'password':password,
256                'fullname':user.getProperty('fullname'),
257                'email':user.getProperty('email'),
258                'roles':user.getRoles(),
259                }
260            data = Pickle.dumps(objstore)
261            archive.addFileFromString(fn, data, float(DateTime()))
262
263    def exportObjects(self, brains, tf):
264        for x in brains:
265            self.exportObject(x, tf)
266            if x.is_folderish:
267                brains = self.context.portal_catalog(path={'query':x.getPath(), 'depth':1,},)
268                self.exportObjects(brains, tf)
269
270    def exportObject(self, brain, archive):
271        """ Export an object """
272        if 'Download this Course' == brain.Title:
273            return
274        fn = self._getPath(brain)
275        obj = brain.getObject()
276        # Get the metadata and store it
277        if obj.isPrincipiaFolderish:
278            self.exportMetadata(obj, fn + '.ecmigration.directory', archive)
279        else:
280            self.exportMetadata(obj, fn + '.ecmigration.metadata', archive)
281            self.exportFile(obj, fn, archive)
282       
283    def exportMetadata(self, obj, fn, archive):
284        """ Export an object's metadata """
285        objstore = {'filename':fn}
286        self._storeMetaData(obj, objstore)
287        lmod = obj.getRawModification_date()
288        data = Pickle.dumps(objstore)
289        archive.addFileFromString(fn, data, float(lmod))
290
291    def exportFile(self, obj, fn, archive):
292        """ Export the primary field of an object as a file in the archive """
293        pfield = obj.getPrimaryField()
294        fd = pfield.get(obj)
295        data = None
296        if type(fd) == type(''):
297            data = fd #StringIO(fd)
298        if type(data) != type(''):
299            data = fd.data
300            if type(data) != type(''):
301                data = fd.data.data
302#        elif fd.meta_type in ['Image', 'File']:
303#            try:
304#                data = fd.data #StringIO(fd.data)
305#            except TypeError:
306#                data = fd.data.data #StringIO(fd.data.data)
307        tf = tempfile.NamedTemporaryFile()
308        tf.write(data)
309        #while 1:
310        #    buf = data.read()
311        #    if buf:
312        #        tf.write(buf)
313        #    else:
314        #        break
315        tf.seek(0)
316        archive.addFileFromDisk(tf.name, fn)
317        tf.close()
318
319    def _getPath(self, brain):
320        opath = brain.getPath()
321        fn = opath.split('/')
322        self._transformId(fn)
323        return os.sep.join(fn[1:])
324
325    def _storeMetaData(self, obj, objstore):
326        """ Export Object Metadata """
327        objstore['type'] = self._transformType(obj.portal_type)
328        # Get field data
329        pfield = obj.getPrimaryField()
330        objstore['fields'] = {}
331        for x in obj.Schema().fields():
332            if x != pfield:
333                fid = x.getName()
334                if 'id' != fid:
335                    pass # Do not store the ID, get it from the filename instead
336                if 'clearedCopyright' == fid:
337                    # if clearedcopyright flag is stored as a field
338                    # move it to annotations instead
339                    objstore['clearedcopyright'] = x.get(obj)
340                else:
341                    objstore['fields'][fid] = x.get(obj)
342#                    # Use the usual method to get the field data
343#                    data = x.get(obj)
344#                    if getattr(data, 'data', None):
345#                        pass
346#                    else:
347#                        objstore['fields'][fid] = data
348#                    #objstore['fields'][fid] = x.get(obj)
349#                    #if getattr(objstore['fields'][fid], 'data', None):
350#                    #    objstore['fields'][fid] = x.get(obj).data
351#                    #    if getattr(objstore['fields'][fid], 'data', None):
352#                    #        objstore['fields'][fid] = objstore['fields'][fid].data
353        # Get non field related data
354        objstore['owner'] = obj.getOwner().getId()
355        objstore['workflowhistory'] = obj.workflow_history
356        # Get annotation related data
357        if ILicensable.providedBy(obj):
358            lic = ILicense(obj)
359            objstore['rightsholder'] = lic.getRightsHolder()
360            objstore['rightslicense'] = lic.getRightsLicense()
361        if has_clearcopyrightable:
362            if IClearCopyrightable.providedBy(obj):
363                cc = IClearCopyright(obj)
364                objstore['clearedcopyright'] = cc.getClearedCopyright()
365        if has_accessibilitycompliance:
366            if IAccessibilityCompliantable.providedBy(obj):
367                acc = IAccessibilityCompliant(obj)
368                objstore['accessibilitycompliant'] = acc.getAccessible()
369        if has_courseorderable:
370            if ICourseOrderable.providedBy(obj):
371                order = ICourseOrder(obj)
372                objstore['courseorder'] = order.getPositionInCourse()
373
374    def _transformType(self, ptype):
375        """ Change type """
376        if self.transforms.has_key(ptype):
377            return self.transforms[ptype]
378        else:
379            return ptype
380
381    def _transformId(self, fn):
382        """ Change IDs """
383        if len(fn) > 2 and self.id_transforms.has_key(fn[2]):
384            fn[2] = self.id_transforms[fn[2]]
385            if len(fn) > 3 and 'about' == fn[2] and 'index_html' == fn[3]:
386                fn[3] = 'abouttext_text'
387            if len(fn) > 3 and 'help' == fn[2] and 'index_html' == fn[3]:
388                fn[3] = 'help_text'
389        if len(fn) > 3 and self.id_transforms.has_key(fn[3]):
390            fn[3] = self.id_transforms[fn[3]]
Note: See TracBrowser for help on using the repository browser.