Changeset 756

Timestamp:

03/03/11 16:58:21 (2 years ago)

Author:

brent

Message:

Split out data from metadata.

File:

• Products.ecmigration/trunk/Products/ecmigration/migrate.py (modified) (9 diffs)

Products.ecmigration/trunk/Products/ecmigration/migrate.py

@@ -39 +39 @@
 import os
 import tarfile
+import tempfile
 
 
@@ -75 +76 @@
         self.tar_archive = tarfile.open(archname, mode)
 
-    def addFile(self, fn, finfo, lmod):
-        """ Write a file to the archive """
-        data = Pickle.dumps(finfo, -1)
+    def addFileFromString(self, fn, data, lmod):
+        """ Write a file from a string to the archive """
         tinfo = tarfile.TarInfo(fn)
         tinfo.size = len(data)
@@ -84 +84 @@
         # Circumvent caching for large file handling
         self.tar_archive.members = []
+        print fn
+
+    def addFileFromDisk(self, fn, arcname):
+        """ Write a file from the filesystemto the archive """
+        self.tar_archive.add(fn, arcname)
+        print arcname
 
     def getNextFileInfo(self):
@@ -124 +130 @@
         'About':'about',
         'Help':'help',
+        'terms_of_use':'terms-of-use',
+        'privacy_policy':'privacy-policy'
         }
 
@@ -234 +242 @@
         brains = self.context.portal_catalog(path={'query':'/', 'depth':2,},)
         tf = TarArchiveManager(fn, 'w:bz2')
-        self.exportUsers(context, tf)
+        self.exportUsers(tf)
         self.exportObjects(brains, tf)
         tf.close()
 
-    def exportUsers(self, portal, archive):
-        fn = '%s/userinfo.ecmigration' %portal.getId()
+    def exportUsers(self, archive):
+        fn = '%s/userinfo.ecmigration' %self.context.getId()
         objstore = {'filename':fn, 'type':'UserInfo.ecmigration', 'users':{},}
-        for user in portal.acl_users.getUsers():
+        for user in self.context.acl_users.getUsers():
             username = user.getName()
-            password = portal.acl_users.source_users._user_passwords[username]
+            password = self.context.acl_users.source_users._user_passwords[username]
             objstore['users'][username] = {
                 'password':password,
@@ -250 +258 @@
                 'roles':user.getRoles(),
                 }
-            archive.addFile(fn, objstore, float(DateTime()))
+            data = Pickle.dumps(objstore)
+            archive.addFileFromString(fn, data, float(DateTime()))
 
     def exportObjects(self, brains, tf):
@@ -261 +270 @@
     def exportObject(self, brain, archive):
         """ Export an object """
+        if 'Download this Course' == brain.Title:
+            return
         fn = self._getPath(brain)
-        print fn
-        objstore = { 'filename':fn }
         obj = brain.getObject()
-        self._storeData(obj, objstore)
+        # Get the metadata and store it
+        if obj.isPrincipiaFolderish:
+            self.exportMetadata(obj, fn + '.ecmigration.directory', archive)
+        else:
+            self.exportMetadata(obj, fn + '.ecmigration.metadata', archive)
+            self.exportFile(obj, fn, archive)
+        
+    def exportMetadata(self, obj, fn, archive):
+        """ Export an object's metadata """
+        objstore = {'filename':fn}
+        self._storeMetaData(obj, objstore)
         lmod = obj.getRawModification_date()
-        if obj.isPrincipiaFolderish:
-            fn += '.ecmigration.directory'
-        try:
-            archive.addFile(fn, objstore, float(lmod))
-        except TypeError:
-            import pdb; pdb.set_trace()
+        data = Pickle.dumps(objstore)
+        archive.addFileFromString(fn, data, float(lmod))
+
+    def exportFile(self, obj, fn, archive):
+        """ Export the primary field of an object as a file in the archive """
+        pfield = obj.getPrimaryField()
+        fd = pfield.get(obj)
+        data = None
+        if type(fd) == type(''):
+            data = fd #StringIO(fd) 
+        if type(data) != type(''):
+            data = fd.data
+            if type(data) != type(''):
+                data = fd.data.data
+#        elif fd.meta_type in ['Image', 'File']:
+#            try:
+#                data = fd.data #StringIO(fd.data)
+#            except TypeError:
+#                data = fd.data.data #StringIO(fd.data.data)
+        tf = tempfile.NamedTemporaryFile()
+        tf.write(data)
+        #while 1:
+        #    buf = data.read()
+        #    if buf:
+        #        tf.write(buf)
+        #    else:
+        #        break
+        tf.seek(0)
+        archive.addFileFromDisk(tf.name, fn)
+        tf.close()
 
     def _getPath(self, brain):
@@ -280 +323 @@
         return os.sep.join(fn[1:])
 
-    def _storeData(self, obj, objstore):
+    def _storeMetaData(self, obj, objstore):
         """ Export Object Metadata """
         objstore['type'] = self._transformType(obj.portal_type)
         # Get field data
+        pfield = obj.getPrimaryField()
         objstore['fields'] = {}
         for x in obj.Schema().fields():
-            fid = x.getName()
-            if 'id' != fid:
-                if fid in ['file', 'image']: # and 'ATBlob' == obj.meta_type:
-                    # Object is a file, must get data from the file
-                    # In order to pickle it
-                    objstore['fields'][fid] = x.get(obj).data
+            if x != pfield:
+                fid = x.getName()
+                if 'id' != fid:
+                    pass # Do not store the ID, get it from the filename instead
+                if 'clearedCopyright' == fid:
+                    # if clearedcopyright flag is stored as a field
+                    # move it to annotations instead
+                    objstore['clearedcopyright'] = x.get(obj)
                 else:
-                    if 'clearedCopyright' == fid:
-                        # if clearedcopyright flag is stored as a field
-                        # move it to annotations instead
-                        objstore['clearedcopyright'] = x.get(obj)
-                    else:
-                        # Use the usual method to get the field data
-                        objstore['fields'][fid] = x.get(obj)
+                    objstore['fields'][fid] = x.get(obj)
+#                    # Use the usual method to get the field data
+#                    data = x.get(obj)
+#                    if getattr(data, 'data', None):
+#                        pass
+#                    else:
+#                        objstore['fields'][fid] = data
+#                    #objstore['fields'][fid] = x.get(obj)
+#                    #if getattr(objstore['fields'][fid], 'data', None):
+#                    #    objstore['fields'][fid] = x.get(obj).data
+#                    #    if getattr(objstore['fields'][fid], 'data', None):
+#                    #        objstore['fields'][fid] = objstore['fields'][fid].data
         # Get non field related data
         objstore['owner'] = obj.getOwner().getId()
@@ -330 +381 @@
     def _transformId(self, fn):
         """ Change IDs """
-        if len(fn) > 1 and self.id_transforms.has_key(fn[1]):
-            import pdb; pdb.set_trace()
-            fn[1] = self.id_transforms[fn[1]]
+        if len(fn) > 2 and self.id_transforms.has_key(fn[2]):
+            fn[2] = self.id_transforms[fn[2]]
+            if len(fn) > 3 and 'about' == fn[2] and 'index_html' == fn[3]:
+                fn[3] = 'abouttext_text'
+            if len(fn) > 3 and 'help' == fn[2] and 'index_html' == fn[3]:
+                fn[3] = 'help_text'
+        if len(fn) > 3 and self.id_transforms.has_key(fn[3]):
+            fn[3] = self.id_transforms[fn[3]]