Ubuntu One Client

Merge lp:~verterok/ubuntuone-client/tritcask-win-compat into lp:ubuntuone-client

  1. tritcask-win-compat
  2. Merge into trunk
Proposed by Guillermo Gonzalez
Status: Merged
Approved by: Natalia Bidart
Approved revision: 1077
Merged at revision: 1083
Proposed branch: lp:~verterok/ubuntuone-client/tritcask-win-compat
Merge into: lp:ubuntuone-client
Diff against target: 167 lines (+26/-17)
2 files modified
tests/syncdaemon/test_tritcask.py (+17/-15)
ubuntuone/syncdaemon/tritcask.py (+9/-2)
To merge this branch: bzr merge lp:~verterok/ubuntuone-client/tritcask-win-compat
Reviewer Review Type Date Requested Status
Natalia Bidart (community) Approve
Manuel de la Peña (community) Approve
Review via email: mp+69725@code.launchpad.net

Commit message

Fix tritcask to use time.clock() on windows and time.time() on others.

Description of the change

Fix tritcask to use time.clock() on windows and time.time() on others.

To post a comment you must log in.
Revision history for this message
Manuel de la Peña (mandel) wrote :

The expected tests pass and the failing ones are due to a fd being open when we try to do a rename.

review: Approve
Revision history for this message
Natalia Bidart (nataliabidart) wrote :

I have 2 tests failing in windows:

===============================================================================
[ERROR]
Traceback (most recent call last):
  File "E:\u1\client\review_tritcask-win-compat\tests\syncdaemon\test_tritcask.p
y", line 1406, in test_merge_mixed_dead_entries
    db = Tritcask(self.base_dir, auto_merge=False)
  File "E:\u1\client\review_tritcask-win-compat\ubuntuone\syncdaemon\tritcask.py
", line 590, in __init__
    self._rotate_and_merge()
  File "E:\u1\client\review_tritcask-win-compat\ubuntuone\syncdaemon\tritcask.py
", line 646, in _rotate_and_merge
    self.rotate(create_file=False)
  File "E:\u1\client\review_tritcask-win-compat\ubuntuone\syncdaemon\tritcask.py
", line 694, in rotate
    data_file = self.live_file.make_immutable()
  File "E:\u1\client\review_tritcask-win-compat\ubuntuone\syncdaemon\tritcask.py
", line 187, in make_immutable
    os.rename(self.filename, new_name)
exceptions.WindowsError: [Error 32] The process cannot access the file because i
t is being used by another process

tests.syncdaemon.test_tritcask.MergeTests.test_merge_mixed_dead_entries
===============================================================================
[ERROR]
Traceback (most recent call last):
  File "E:\u1\client\review_tritcask-win-compat\tests\syncdaemon\test_tritcask.p
y", line 323, in test_delete
    new_file.delete()
  File "E:\u1\client\review_tritcask-win-compat\ubuntuone\syncdaemon\tritcask.py
", line 398, in delete
    os.unlink(self.filename)
exceptions.WindowsError: [Error 32] The process cannot access the file because i
t is being used by another process: 'C:\\Temp\\5\\_trial_temp\\tmp\\tests.syncda
emon.test_tritcask\\TempDataFileTest\\test_delete\\data_dir\\1258630.tmp7u7m6a.t
ritcask-v1.data'

tests.syncdaemon.test_tritcask.TempDataFileTest.test_delete
-------------------------------------------------------------------------------

Can you please replace os.rename by the ubuntuone.platform.rename?

review: Needs Fixing
Revision history for this message
Guillermo Gonzalez (verterok) wrote :

> I have 2 tests failing in windows:

Those tests area already failing in trunk, and will be fixed in a different branch

> Can you please replace os.rename by the ubuntuone.platform.rename?

tritcask isn't using any of the platform stuff as it will be moved out of syncdaemon.

Revision history for this message
Natalia Bidart (nataliabidart) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'tests/syncdaemon/test_tritcask.py'
--- tests/syncdaemon/test_tritcask.py 2011-07-27 17:31:25 +0000
+++ tests/syncdaemon/test_tritcask.py 2011-07-28 22:31:19 +0000
@@ -22,7 +22,6 @@
22import mmap22import mmap
23import marshal23import marshal
24import os24import os
25import time
26import types25import types
27import uuid26import uuid
28import zlib27import zlib
@@ -59,6 +58,7 @@
59 is_live,58 is_live,
60 is_immutable,59 is_immutable,
61 is_hint,60 is_hint,
61 timestamp,
62 logger,62 logger,
63)63)
6464
@@ -89,9 +89,10 @@
8989
90 def test__get_next_file_id(self):90 def test__get_next_file_id(self):
91 """Test for _get_next_file_id method."""91 """Test for _get_next_file_id method."""
92 curr = int(time.time() * 100000)92 curr = int(timestamp() * 100000)
93
93 self.assertTrue(curr <= int(self.file_class._get_next_file_id()))94 self.assertTrue(curr <= int(self.file_class._get_next_file_id()))
94 curr = int(time.time() * 100000)95 curr = int(timestamp() * 100000)
95 self.assertTrue(curr <= int(self.file_class._get_next_file_id()))96 self.assertTrue(curr <= int(self.file_class._get_next_file_id()))
9697
97 def test_exists(self):98 def test_exists(self):
@@ -125,6 +126,7 @@
125 """Test for make_immutable method."""126 """Test for make_immutable method."""
126 new_file = self.file_class(self.base_dir)127 new_file = self.file_class(self.base_dir)
127 new_file.fd.write('foo')128 new_file.fd.write('foo')
129 new_file.fd.flush()
128 immutable_file = new_file.make_immutable()130 immutable_file = new_file.make_immutable()
129 # the DataFile should be closed131 # the DataFile should be closed
130 self.assertEqual(None, new_file.fd)132 self.assertEqual(None, new_file.fd)
@@ -630,10 +632,10 @@
630 def test_write(self):632 def test_write(self):
631 """Test the write method."""633 """Test the write method."""
632 hint_file = HintFile(os.path.join(self.base_dir, 'hint_file'))634 hint_file = HintFile(os.path.join(self.base_dir, 'hint_file'))
633 tstamp1 = time.time()635 tstamp1 = timestamp()
634 entry = HintEntry(tstamp1, len('foo'), 0, len('bar'), 100, 'foo')636 entry = HintEntry(tstamp1, len('foo'), 0, len('bar'), 100, 'foo')
635 hint_file.write(entry)637 hint_file.write(entry)
636 tstamp2 = time.time()638 tstamp2 = timestamp()
637 entry2 = HintEntry(tstamp2, len('foo1'), 1, len('bar1'), 100, 'foo1')639 entry2 = HintEntry(tstamp2, len('foo1'), 1, len('bar1'), 100, 'foo1')
638 hint_file.write(entry2)640 hint_file.write(entry2)
639 hint_file.close()641 hint_file.close()
@@ -665,7 +667,7 @@
665667
666 def test_header_property(self):668 def test_header_property(self):
667 """Test the header property."""669 """Test the header property."""
668 tstamp = time.time()670 tstamp = timestamp()
669 entry = HintEntry(tstamp, len('foo'), 0, len('bar'), 100, 'foo')671 entry = HintEntry(tstamp, len('foo'), 0, len('bar'), 100, 'foo')
670 self.assertEqual((entry.tstamp, entry.key_sz, entry.row_type,672 self.assertEqual((entry.tstamp, entry.key_sz, entry.row_type,
671 entry.value_sz, entry.value_pos), entry.header)673 entry.value_sz, entry.value_pos), entry.header)
@@ -1494,11 +1496,11 @@
1494 keydir = Keydir()1496 keydir = Keydir()
1495 file_id = DataFile._get_next_file_id()1497 file_id = DataFile._get_next_file_id()
1496 for i in range(10):1498 for i in range(10):
1497 keydir[(0, str(uuid.uuid4()))] = KeydirEntry(file_id, time.time(),1499 keydir[(0, str(uuid.uuid4()))] = KeydirEntry(file_id, timestamp(),
1498 len(str(uuid.uuid4())), i+10)1500 len(str(uuid.uuid4())), i+10)
1499 file_id_1 = DataFile._get_next_file_id()1501 file_id_1 = DataFile._get_next_file_id()
1500 for i in range(20):1502 for i in range(20):
1501 keydir[(0, str(uuid.uuid4()))] = KeydirEntry(file_id_1, time.time(),1503 keydir[(0, str(uuid.uuid4()))] = KeydirEntry(file_id_1, timestamp(),
1502 len(str(uuid.uuid4())), i+10)1504 len(str(uuid.uuid4())), i+10)
1503 entry_size = len(str(uuid.uuid4()))*2 + header_size + crc32_size1505 entry_size = len(str(uuid.uuid4()))*2 + header_size + crc32_size
1504 self.assertEqual(entry_size*10, keydir._stats[file_id]['live_bytes'])1506 self.assertEqual(entry_size*10, keydir._stats[file_id]['live_bytes'])
@@ -1509,11 +1511,11 @@
1509 keydir = Keydir()1511 keydir = Keydir()
1510 file_id = DataFile._get_next_file_id()1512 file_id = DataFile._get_next_file_id()
1511 key = str(uuid.uuid4())1513 key = str(uuid.uuid4())
1512 entry = KeydirEntry(file_id, time.time(), 1, 1)1514 entry = KeydirEntry(file_id, timestamp(), 1, 1)
1513 keydir[(0, key)] = entry1515 keydir[(0, key)] = entry
1514 base_size = len(key) + header_size + crc32_size1516 base_size = len(key) + header_size + crc32_size
1515 self.assertEqual(keydir._stats[file_id]['live_bytes'], base_size + 1)1517 self.assertEqual(keydir._stats[file_id]['live_bytes'], base_size + 1)
1516 new_entry = KeydirEntry(file_id, time.time(), 2, 2)1518 new_entry = KeydirEntry(file_id, timestamp(), 2, 2)
1517 keydir[(0, key)] = new_entry1519 keydir[(0, key)] = new_entry
1518 self.assertEqual(keydir._stats[file_id]['live_bytes'], base_size + 2)1520 self.assertEqual(keydir._stats[file_id]['live_bytes'], base_size + 2)
15191521
@@ -1525,12 +1527,12 @@
1525 keydir = Keydir()1527 keydir = Keydir()
1526 file_id = DataFile._get_next_file_id()1528 file_id = DataFile._get_next_file_id()
1527 key = str(uuid.uuid4())1529 key = str(uuid.uuid4())
1528 entry = KeydirEntry(file_id, time.time(), 10, 1)1530 entry = KeydirEntry(file_id, timestamp(), 10, 1)
1529 keydir[(0, key)] = entry1531 keydir[(0, key)] = entry
1530 base_size = len(key) + header_size + crc32_size1532 base_size = len(key) + header_size + crc32_size
1531 self.assertEqual(keydir._stats[file_id]['live_bytes'], base_size + 10)1533 self.assertEqual(keydir._stats[file_id]['live_bytes'], base_size + 10)
1532 new_file_id = DataFile._get_next_file_id()1534 new_file_id = DataFile._get_next_file_id()
1533 new_entry = KeydirEntry(new_file_id, time.time(), 5, 10)1535 new_entry = KeydirEntry(new_file_id, timestamp(), 5, 10)
1534 keydir[(0, key)] = new_entry1536 keydir[(0, key)] = new_entry
1535 self.assertEqual(keydir._stats[new_file_id]['live_bytes'],1537 self.assertEqual(keydir._stats[new_file_id]['live_bytes'],
1536 base_size + 5)1538 base_size + 5)
@@ -1542,14 +1544,14 @@
1542 file_id = DataFile._get_next_file_id()1544 file_id = DataFile._get_next_file_id()
1543 for i in range(10):1545 for i in range(10):
1544 key = str(uuid.uuid4())1546 key = str(uuid.uuid4())
1545 keydir[(0, key)] = KeydirEntry(file_id, time.time(),1547 keydir[(0, key)] = KeydirEntry(file_id, timestamp(),
1546 len(str(uuid.uuid4())), i+10)1548 len(str(uuid.uuid4())), i+10)
1547 if i % 2:1549 if i % 2:
1548 keydir.remove((0, key))1550 keydir.remove((0, key))
1549 file_id_1 = DataFile._get_next_file_id()1551 file_id_1 = DataFile._get_next_file_id()
1550 for i in range(20):1552 for i in range(20):
1551 key = str(uuid.uuid4())1553 key = str(uuid.uuid4())
1552 keydir[(0, key)] = KeydirEntry(file_id_1, time.time(),1554 keydir[(0, key)] = KeydirEntry(file_id_1, timestamp(),
1553 len(str(uuid.uuid4())), i+10)1555 len(str(uuid.uuid4())), i+10)
1554 if i % 2:1556 if i % 2:
1555 keydir.remove((0, key))1557 keydir.remove((0, key))
@@ -1573,7 +1575,7 @@
1573 keydir = Keydir()1575 keydir = Keydir()
1574 file_id = DataFile._get_next_file_id()1576 file_id = DataFile._get_next_file_id()
1575 for i in range(10):1577 for i in range(10):
1576 keydir[(0, str(uuid.uuid4()))] = KeydirEntry(file_id, time.time(),1578 keydir[(0, str(uuid.uuid4()))] = KeydirEntry(file_id, timestamp(),
1577 len(str(uuid.uuid4())), i+10)1579 len(str(uuid.uuid4())), i+10)
1578 self.assertEqual(keydir._stats[file_id], keydir.get_stats(file_id))1580 self.assertEqual(keydir._stats[file_id], keydir.get_stats(file_id))
1579 self.assertTrue(keydir._stats[file_id] is not \1581 self.assertTrue(keydir._stats[file_id] is not \
15801582
=== modified file 'ubuntuone/syncdaemon/tritcask.py'
--- ubuntuone/syncdaemon/tritcask.py 2011-06-29 14:50:05 +0000
+++ ubuntuone/syncdaemon/tritcask.py 2011-07-28 22:31:19 +0000
@@ -137,6 +137,13 @@
137 """Return True if it's a dead data file."""137 """Return True if it's a dead data file."""
138 return DEAD in filename138 return DEAD in filename
139139
140def timestamp():
141 """Return a timestamp (float)"""
142 if sys.platform == "win32":
143 return time.clock()
144 else:
145 return time.time()
146
140147
141class DataFile(object):148class DataFile(object):
142 """Class that encapsulates data file handling."""149 """Class that encapsulates data file handling."""
@@ -163,7 +170,7 @@
163 @staticmethod170 @staticmethod
164 def _get_next_file_id():171 def _get_next_file_id():
165 """Return the next file id."""172 """Return the next file id."""
166 return str(int(time.time() * 100000))173 return str(int(timestamp() * 100000))
167174
168 @property175 @property
169 def has_hint(self):176 def has_hint(self):
@@ -236,7 +243,7 @@
236 """Write an entry to the file."""243 """Write an entry to the file."""
237 key_sz = len(key)244 key_sz = len(key)
238 value_sz = len(value)245 value_sz = len(value)
239 tstamp = time.time()246 tstamp = timestamp()
240 header = header_struct.pack(tstamp, key_sz, value_sz, row_type)247 header = header_struct.pack(tstamp, key_sz, value_sz, row_type)
241 crc32 = crc32_struct.pack(zlib.crc32(header+key+value))248 crc32 = crc32_struct.pack(zlib.crc32(header+key+value))
242 if EXTRA_SEEK:249 if EXTRA_SEEK:

Subscribers

People subscribed via source and target branches