Changeset 3266

Timestamp:

02/24/08 03:20:27 (9 months ago)

Author:

hvr

Message:

GitPlugin: significantly reduced memory usage of revision cache by cutting down usage of set objects in favor of tuples in cache data structures, and
aggressively reusing sha strings

Files:

• gitplugin/0.11/tracext/git/PyGIT.py (modified) (10 diffs)

gitplugin/0.11/tracext/git/PyGIT.py

@@ -114 +114 @@
-
-class Storage:
-6
+4
-
-    @staticmethod
@@ -185 +185 @@
-                    new_tags.add(revs.strip())
-
+                # helper for reusing strings
+                __rev_seen = {}
+                def __rev_reuse(rev):
+                    rev = str(rev)
+                    return __rev_seen.setdefault(rev, rev)
+
-                for revs in self.repo.rev_list("--parents", "--all").readlines():
-                    revs = revs.strip().split()
-
+                    revs = map(__rev_reuse, revs)
+
-                    rev = revs[0]
-
-                    # shortrev "hash" map
-
-
-
+
+
+
+
+
-
-                    ord_rev += 1
@@ -206 +216 @@
-                        assert not _parents
-                        assert _ord_rev == 0
-                        new_db[rev] = (_children, parents, ord_rev)
-                    else:
-
+
+
+
+
-
-                    # update all parents(rev)'s children
-                    for parent in parents:
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                self._commit_db = new_db, parent, new_tags, new_sdb
-                self.last_youngest_rev = youngest
-                self.logger.debug("rebuilt commit tree db for %d with %d entries" % (id(self),len(new_db)))
-
-
-
+
-
-            return self._commit_db[0]
@@ -316 +354 @@
-
-        srev = rev[:self.__SREV_MIN]
-
+
+
-
-        if len(srevs) == 1:
@@ -429 +468 @@
-            yield p
-
-db[p][0]
+set(db[p][0])
-
-            seen.update(_children)
@@ -536 +575 @@
-            yield __chg_tuple()
-
+############################################################################
+############################################################################
+############################################################################
+
-if __name__ == '__main__':
-    import sys, logging, timeit
@@ -541 +584 @@
-    print "git version [%s]" % str(Storage.git_version())
-
+    # custom linux hack reading `/proc/<PID>/statm`
+    if sys.platform == "linux2":
+        __pagesize = os.sysconf('SC_PAGESIZE')
+
+        def proc_statm(pid = os.getpid()):
+            __proc_statm = '/proc/%d/statm' % pid
+            try:
+                t = open(__proc_statm)
+                result = t.read().split()
+                t.close()
+                assert len(result) == 7
+                return tuple([ __pagesize*int(p) for p in result ])
+            except:
+                raise RuntimeError("failed to get memory stats")
+
+    else: # not linux2
+        print "WARNING - meminfo.proc_statm() not available"
+        def proc_statm():
+            return (0,)*7
+
+    print "statm =", proc_statm()
+    __data_size = proc_statm()[5]
+
+    def print_data_usage():
+        print "DATA:", proc_statm()[5] - __data_size
+
+    print_data_usage()
+
-    g = Storage(sys.argv[1], logging)
+
+    print_data_usage()
-
-    print "[%s]" % g.head()
-    print g.ls_tree(g.head())
-    print "--------------"
+    print_data_usage()
-    print g.read_commit(g.head())
-    print "--------------"
+    print_data_usage()
-    p = g.parents(g.head())
-    print list(p)
@@ -559 +634 @@
-    print "--------------"
-    print g.hist_prev_revision(g.oldest_rev()), g.oldest_rev(), g.hist_next_revision(g.oldest_rev())
-
+
-    print "--------------"
-    p = g.youngest_rev()
-    print g.hist_prev_revision(p), p, g.hist_next_revision(p)
-    print "--------------"
+
-    p = g.head()
-    for i in range(-5,5):
@@ -584 +660 @@
-    #p = g.head()
-    #revs = [ g.history_relative_rev(p, i) for i in range(0,10) ]
+    print_data_usage()
-    revs = g.get_commits().keys()
+    print_data_usage()
-
-    def shortrev_test():
@@ -600 +678 @@
-
-    #print len(list(g.children_recursive(g.oldest_rev())))
+
+    print_data_usage()
+
+    # perform typical trac operations:
+
+    if 0:
+        print "--------------"
+        rev = g.head()
+        for mode,type,sha,name in g.ls_tree(rev):
+            [last_rev] = g.history(rev, name, limit=1)
+            s = g.get_obj_size(sha) if type == "blob" else 0
+            msg = g.read_commit(last_rev)
+
+            print "%s %s %10d [%s]" % (type, last_rev, s, name)