1
=== modified file 'lib/canonical/librarian/ftests/test_gc.py'
2
--- lib/canonical/librarian/ftests/test_gc.py	2010-03-16 10:08:38 +0000
3
+++ lib/canonical/librarian/ftests/test_gc.py	2010-04-23 22:13:24 +0000
4
@@ -7,6 +7,7 @@
5
7
7
6
8
import shutil
8
import shutil
7
9
import sys
9
import sys
8
10
import tempfile
9
10
import os
11
import os
10
11
from subprocess import Popen, PIPE, STDOUT
12
from subprocess import Popen, PIPE, STDOUT
11
12
from cStringIO import StringIO
13
from cStringIO import StringIO
12
@@ -559,6 +560,36 @@
13
559
        # This should cope.
560
        # This should cope.
14
560
        librariangc.delete_unwanted_files(self.con)
561
        librariangc.delete_unwanted_files(self.con)
15
561
562
16
563
    def test_delete_unwanted_files_follows_symlinks(self):
17
564
        # In production, our tree has symlinks in it now.  We need to be able
18
565
        # to cope.
19
566
        # First, let's make sure we have some trash.
20
567
        self.layer.switchDbUser(dbuser='testadmin')
21
568
        content = 'foo'
22
569
        self.client.addFile(
23
570
            'foo.txt', len(content), StringIO(content), 'text/plain')
24
571
        # Roll back the database changes, leaving the file on disk.
25
572
        transaction.abort()
26
573
27
574
        self.layer.switchDbUser(config.librarian_gc.dbuser)
28
575
29
576
        # Now, we will move the directory containing the trash somewhere else
30
577
        # and make a symlink to it.
31
578
        original = os.path.join(config.librarian_server.root, '00', '00')
32
579
        newdir = tempfile.mkdtemp()
33
580
        alt = os.path.join(newdir, '00')
34
581
        shutil.move(original, alt)
35
582
        os.symlink(alt, original)
36
583
37
584
        # Now we will do our thing.  This is the actual test.  It used to
38
585
        # fail.
39
586
        librariangc.delete_unwanted_files(self.con)
40
587
41
588
        # Clean up.
42
589
        os.remove(original)
43
590
        shutil.move(alt, original)
44
591
        shutil.rmtree(newdir)
45
592
46
562
    def test_cronscript(self):
593
    def test_cronscript(self):
47
563
        script_path = os.path.join(
594
        script_path = os.path.join(
48
564
                config.root, 'cronscripts', 'librarian-gc.py'
595
                config.root, 'cronscripts', 'librarian-gc.py'
49
565
596
50
=== modified file 'lib/canonical/librarian/librariangc.py'
51
--- lib/canonical/librarian/librariangc.py	2010-03-16 10:08:38 +0000
52
+++ lib/canonical/librarian/librariangc.py	2010-04-23 22:13:24 +0000
53
@@ -495,6 +495,99 @@
54
495
    loop_tuner = DBLoopTuner(UnreferencedContentPruner(con), 5, log=log)
495
    loop_tuner = DBLoopTuner(UnreferencedContentPruner(con), 5, log=log)
55
496
    loop_tuner.run()
496
    loop_tuner.run()
56
497
497
57
498
# XXX gary 2010-04-22 bug=569217
58
499
# We should remove this and use Python 2.6's os.walk once we switch to
59
500
# Python 2.6.
60
501
def _walk(top, topdown=True, onerror=None, followlinks=False):
61
502
    """Directory tree generator.
62
503
63
504
    For each directory in the directory tree rooted at top (including top
64
505
    itself, but excluding '.' and '..'), yields a 3-tuple
65
506
66
507
        dirpath, dirnames, filenames
67
508
68
509
    dirpath is a string, the path to the directory.  dirnames is a list of
69
510
    the names of the subdirectories in dirpath (excluding '.' and '..').
70
511
    filenames is a list of the names of the non-directory files in dirpath.
71
512
    Note that the names in the lists are just names, with no path components.
72
513
    To get a full path (which begins with top) to a file or directory in
73
514
    dirpath, do os.path.join(dirpath, name).
74
515
75
516
    If optional arg 'topdown' is true or not specified, the triple for a
76
517
    directory is generated before the triples for any of its subdirectories
77
518
    (directories are generated top down).  If topdown is false, the triple
78
519
    for a directory is generated after the triples for all of its
79
520
    subdirectories (directories are generated bottom up).
80
521
81
522
    When topdown is true, the caller can modify the dirnames list in-place
82
523
    (e.g., via del or slice assignment), and walk will only recurse into the
83
524
    subdirectories whose names remain in dirnames; this can be used to prune
84
525
    the search, or to impose a specific order of visiting.  Modifying
85
526
    dirnames when topdown is false is ineffective, since the directories in
86
527
    dirnames have already been generated by the time dirnames itself is
87
528
    generated.
88
529
89
530
    By default errors from the os.listdir() call are ignored.  If
90
531
    optional arg 'onerror' is specified, it should be a function; it
91
532
    will be called with one argument, an os.error instance.  It can
92
533
    report the error to continue with the walk, or raise the exception
93
534
    to abort the walk.  Note that the filename is available as the
94
535
    filename attribute of the exception object.
95
536
96
537
    By default, os.walk does not follow symbolic links to subdirectories on
97
538
    systems that support them.  In order to get this functionality, set the
98
539
    optional argument 'followlinks' to true.
99
540
100
541
    Caution:  if you pass a relative pathname for top, don't change the
101
542
    current working directory between resumptions of walk.  walk never
102
543
    changes the current directory, and assumes that the client doesn't
103
544
    either.
104
545
105
546
    Example:
106
547
107
548
    import os
108
549
    from os.path import join, getsize
109
550
    for root, dirs, files in os.walk('python/Lib/email'):
110
551
        print root, "consumes",
111
552
        print sum([getsize(join(root, name)) for name in files]),
112
553
        print "bytes in", len(files), "non-directory files"
113
554
        if 'CVS' in dirs:
114
555
            dirs.remove('CVS')  # don't visit CVS directories
115
556
    """
116
557
117
558
    from os.path import join, isdir, islink
118
559
119
560
    # We may not have read permission for top, in which case we can't
120
561
    # get a list of the files the directory contains.  os.path.walk
121
562
    # always suppressed the exception then, rather than blow up for a
122
563
    # minor reason when (say) a thousand readable directories are still
123
564
    # left to visit.  That logic is copied here.
124
565
    try:
125
566
        # Note that listdir and error are globals in this module due
126
567
        # to earlier import-*.
127
568
        names = os.listdir(top)
128
569
    except error, err:
129
570
        if onerror is not None:
130
571
            onerror(err)
131
572
        return
132
573
133
574
    dirs, nondirs = [], []
134
575
    for name in names:
135
576
        if isdir(join(top, name)):
136
577
            dirs.append(name)
137
578
        else:
138
579
            nondirs.append(name)
139
580
140
581
    if topdown:
141
582
        yield top, dirs, nondirs
142
583
    for name in dirs:
143
584
        path = join(top, name)
144
585
        if followlinks or not islink(path):
145
586
            for x in _walk(path, topdown, onerror, followlinks):
146
587
                yield x
147
588
    if not topdown:
148
589
        yield top, dirs, nondirs
149
590
150
498
591
151
499
def delete_unwanted_files(con):
592
def delete_unwanted_files(con):
152
500
    """Delete files found on disk that have no corresponding record in the
593
    """Delete files found on disk that have no corresponding record in the
153
@@ -524,12 +617,15 @@
154
524
            return result[0]
617
            return result[0]
155
525
618
156
526
    removed_count = 0
619
    removed_count = 0
158
527
    next_wanted_content_id = -1
620
    content_id = next_wanted_content_id = -1
159
528
621
160
529
    hex_content_id_re = re.compile('^[0-9a-f]{8}$')
622
    hex_content_id_re = re.compile('^[0-9a-f]{8}$')
161
530
    ONE_DAY = 24 * 60 * 60
623
    ONE_DAY = 24 * 60 * 60
162
531
624
164
532
    for dirpath, dirnames, filenames in os.walk(get_storage_root()):
625
    # XXX gary 2010-04-22 bug=569217
165
626
    # We should switch back to os.walk once we switch to Python 2.6.
166
627
    for dirpath, dirnames, filenames in _walk(
167
628
        get_storage_root(), followlinks=True):
168
533
629
169
534
        # Ignore known and harmless noise in the Librarian storage area.
630
        # Ignore known and harmless noise in the Librarian storage area.
170
535
        if 'incoming' in dirnames:
631
        if 'incoming' in dirnames:
Reviewer	Review Type	Date Requested	Status
Aaron Bentley (community)		2010-04-23	Approve on 2010-04-26
Review via email: mp+24044@code.launchpad.net