Ubuntu system image

Merge lp:~barry/ubuntu-system-image/citrain30 into lp:~ubuntu-managed-branches/ubuntu-system-image/system-image

  1. citrain30
  2. Merge into system-image
Proposed by Barry Warsaw
Status: Merged
Approved by: Barry Warsaw
Approved revision: 252
Merged at revision: 241
Proposed branch: lp:~barry/ubuntu-system-image/citrain30
Merge into: lp:~ubuntu-managed-branches/ubuntu-system-image/system-image
Diff against target: 15774 lines (+8086/-2651)
138 files modified
MANIFEST.in (+1/-1)
NEWS.rst (+72/-1)
PKG-INFO (+1/-1)
cli-manpage.rst (+31/-18)
coverage-curl.ini (+21/-0)
coverage-udm.ini (+21/-0)
coverage.ini (+0/-15)
dbus-manpage.rst (+50/-28)
debian/changelog (+77/-0)
debian/control (+3/-2)
debian/rules (+15/-10)
debian/tests/control (+7/-7)
debian/tests/dryrun (+1/-1)
debian/tests/prep.py (+9/-2)
debian/tests/smoketest (+2/-2)
debian/tests/smoketest-noreboot (+2/-2)
debian/tests/unittests (+1/-1)
ini-manpage.rst (+33/-30)
setup.cfg (+1/-1)
setup.py (+1/-1)
system_image.egg-info/PKG-INFO (+1/-1)
system_image.egg-info/SOURCES.txt (+105/-59)
systemimage/api.py (+20/-10)
systemimage/apply.py (+32/-9)
systemimage/bag.py (+7/-1)
systemimage/candidates.py (+1/-1)
systemimage/channel.py (+1/-1)
systemimage/config.py (+174/-108)
systemimage/curl.py (+275/-0)
systemimage/data/client.ini (+0/-35)
systemimage/dbus.py (+100/-95)
systemimage/device.py (+1/-1)
systemimage/docs/conf.py (+1/-1)
systemimage/download.py (+119/-208)
systemimage/gpg.py (+6/-1)
systemimage/helpers.py (+53/-43)
systemimage/image.py (+5/-1)
systemimage/index.py (+2/-5)
systemimage/keyring.py (+5/-5)
systemimage/logging.py (+4/-2)
systemimage/main.py (+117/-47)
systemimage/reactor.py (+7/-4)
systemimage/scores.py (+32/-15)
systemimage/service.py (+15/-18)
systemimage/settings.py (+1/-1)
systemimage/state.py (+32/-23)
systemimage/testing/controller.py (+95/-45)
systemimage/testing/dbus.py (+22/-10)
systemimage/testing/demo.py (+6/-6)
systemimage/testing/helpers.py (+174/-62)
systemimage/testing/nose.py (+7/-2)
systemimage/testing/service.py (+0/-50)
systemimage/tests/data/00.ini (+1/-2)
systemimage/tests/data/01.ini (+2/-3)
systemimage/tests/data/api.channels_01.json (+13/-0)
systemimage/tests/data/api.index_01.json (+36/-0)
systemimage/tests/data/api.index_02.json (+251/-0)
systemimage/tests/data/api.index_03.json (+37/-0)
systemimage/tests/data/candidates.index_01.json (+6/-0)
systemimage/tests/data/candidates.index_02.json (+23/-0)
systemimage/tests/data/candidates.index_08.json (+244/-0)
systemimage/tests/data/candidates.index_10.json (+36/-0)
systemimage/tests/data/candidates.index_11.json (+37/-0)
systemimage/tests/data/candidates.index_13.json (+244/-0)
systemimage/tests/data/channel.channels_01.json (+23/-0)
systemimage/tests/data/channel.channels_02.json (+38/-0)
systemimage/tests/data/channel.channels_03.json (+70/-0)
systemimage/tests/data/channel.channels_04.json (+56/-0)
systemimage/tests/data/channel.channels_05.json (+23/-0)
systemimage/tests/data/channel_06.ini (+0/-8)
systemimage/tests/data/channel_07.ini (+0/-8)
systemimage/tests/data/config.config_01.ini (+34/-0)
systemimage/tests/data/config.config_02.ini (+34/-0)
systemimage/tests/data/config.config_03.ini (+3/-4)
systemimage/tests/data/config.config_04.ini (+36/-0)
systemimage/tests/data/config.config_05.ini (+3/-4)
systemimage/tests/data/config.config_06.ini (+3/-4)
systemimage/tests/data/config.config_07.ini (+1/-2)
systemimage/tests/data/config.config_08.ini (+1/-2)
systemimage/tests/data/config.config_09.ini (+0/-3)
systemimage/tests/data/config.config_10.ini (+27/-0)
systemimage/tests/data/config.config_11.ini (+2/-0)
systemimage/tests/data/config_04.ini (+0/-36)
systemimage/tests/data/config_09.ini (+0/-27)
systemimage/tests/data/config_10.ini (+0/-35)
systemimage/tests/data/dbus.index_03.json (+36/-0)
systemimage/tests/data/dbus.index_06.json (+37/-0)
systemimage/tests/data/download.index_01.json (+6/-0)
systemimage/tests/data/gpg.channels_01.json (+23/-0)
systemimage/tests/data/helpers.config_01.ini (+7/-0)
systemimage/tests/data/helpers.config_02.ini (+6/-0)
systemimage/tests/data/index.channels_01.json (+9/-0)
systemimage/tests/data/index.channels_02.json (+13/-0)
systemimage/tests/data/index.channels_05.json (+9/-0)
systemimage/tests/data/index.index_01.json (+251/-0)
systemimage/tests/data/index.index_04.json (+244/-0)
systemimage/tests/data/main.channels_01.json (+13/-0)
systemimage/tests/data/main.channels_03.json (+13/-0)
systemimage/tests/data/main.config_01.ini (+5/-6)
systemimage/tests/data/main.config_05.ini (+7/-0)
systemimage/tests/data/main.config_07.ini (+1/-1)
systemimage/tests/data/main.index_04.json (+36/-0)
systemimage/tests/data/main.index_05.json (+36/-0)
systemimage/tests/data/scores.index_01.json (+245/-0)
systemimage/tests/data/scores.index_05.json (+245/-0)
systemimage/tests/data/scores.index_06.json (+253/-0)
systemimage/tests/data/scores.index_07.json (+252/-0)
systemimage/tests/data/state.channels_01.json (+64/-0)
systemimage/tests/data/state.channels_02.json (+13/-0)
systemimage/tests/data/state.config_01.ini (+2/-0)
systemimage/tests/data/state.config_02.ini (+6/-0)
systemimage/tests/data/state.index_01.json (+244/-0)
systemimage/tests/data/state.index_02.json (+245/-0)
systemimage/tests/data/state.index_03.json (+36/-0)
systemimage/tests/data/state.index_04.json (+37/-0)
systemimage/tests/data/state.index_07.json (+2/-3)
systemimage/tests/test_api.py (+71/-93)
systemimage/tests/test_bag.py (+1/-1)
systemimage/tests/test_candidates.py (+37/-45)
systemimage/tests/test_channel.py (+19/-19)
systemimage/tests/test_config.py (+202/-178)
systemimage/tests/test_dbus.py (+365/-167)
systemimage/tests/test_download.py (+229/-115)
systemimage/tests/test_gpg.py (+129/-19)
systemimage/tests/test_helpers.py (+183/-171)
systemimage/tests/test_image.py (+1/-1)
systemimage/tests/test_index.py (+32/-68)
systemimage/tests/test_keyring.py (+1/-1)
systemimage/tests/test_main.py (+725/-432)
systemimage/tests/test_scores.py (+127/-15)
systemimage/tests/test_settings.py (+4/-6)
systemimage/tests/test_state.py (+244/-142)
systemimage/tests/test_winner.py (+23/-19)
systemimage/udm.py (+212/-0)
systemimage/version.txt (+1/-1)
tools/demo.ini (+0/-1)
tools/runme.sh (+10/-0)
tox.ini (+21/-16)
To merge this branch: bzr merge lp:~barry/ubuntu-system-image/citrain30
Reviewer Review Type Date Requested Status
Ubuntu CI managed package branches Pending
Review via email: mp+259635@code.launchpad.net

Commit message

system-image 3.0

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'MANIFEST.in'
2--- MANIFEST.in 2014-01-30 16:56:57 +0000
3+++ MANIFEST.in 2015-05-20 14:55:53 +0000
4@@ -1,5 +1,5 @@
5 include *.py MANIFEST.in
6-global-include *.txt *.rst *.json *.ini *.gpg *.pem *.service *.in *.conf *.cfg
7+global-include *.txt *.rst *.json *.ini *.gpg *.pem *.service *.in *.conf *.cfg *.sh
8 prune build
9 prune dist
10 prune .tox
11
12=== modified file 'NEWS.rst'
13--- NEWS.rst 2014-09-26 14:36:34 +0000
14+++ NEWS.rst 2015-05-20 14:55:53 +0000
15@@ -2,7 +2,78 @@
16 NEWS for system-image updater
17 =============================
18
19-2.5 (2014-XX-XX)
20+3.0 (2015-05-08)
21+================
22+ * Support a built-in PyCURL-based downloader in addition to the traditional
23+ ubuntu-download-manager (over D-BUS) downloader. Auto-detects which
24+ downloader to use based on whether udm is available on the system bus,
25+ pycurl is importable, and the setting of the SYSTEMIMAGE_PYCURL environment
26+ variable. Initial contribution by Michael Vogt. (LP: #1374459)
27+ * Support alternative machine-id files as fall backs if the D-Bus file does
28+ not exist. Specifically, add systemd's /etc/machine-id to the list.
29+ Initial contribution by Michael Vogt. (LP: #1384859)
30+ * Support multiple configuration files, as in a `config.d` directory. Now,
31+ configuration files are named `NN_whatever.ini` where "NN" must be a
32+ numeric prefix. Files are loaded in sorted numeric order, with later files
33+ overriding newer files. Support for both the `client.ini` and
34+ `channel.ini` files has been removed. (LP: #1373467)
35+ * The `[system]build_file` variable has been removed. Build number
36+ information now must come from the `.ini` files, and last update date
37+ comes from the newest `.ini` file loaded.
38+ * The `-C` command line option now takes a path to the configuration
39+ directory.
40+ * Reworked the checking and downloading locks/flags to so that they will work
41+ better with configuration reloading. (LP: #1412698)
42+ * Support for the `/etc/ubuntu-build` file has been removed. The build
43+ number now comes from the configuration files. (LP: #1377312)
44+ * Move the `archive-master.tar.xz` file to `/usr/share/system-image` for
45+ better FHS compliance. (LP: #1377184)
46+ * Since devices do not always reboot to apply changes, the `[hooks]update`
47+ variable has been renamed to `[hooks]apply`. (LP: #1381538)
48+ * For testing purposes only, `system-image-cli` now supports an
49+ undocumented command line switch `--skip-gpg-verification`. Originally
50+ given by Jani Monoses. (LP: #1333414)
51+ * A new D-Bus signal `Applied(bool)` is added, which is returned in
52+ response to the `ApplyUpdate()` asynchronous method call. For devices
53+ which do not need to reboot in order to apply the update, this is the only
54+ signal you will get. If your device needs to reboot you will also receive
55+ the `Rebooting(bool)` command as with earlier versions. The semantics of
56+ the flag argument are the same in both cases, as are the race timing issues
57+ inherent in these signals. See the `system-image-dbus(8)` manpage for
58+ details. (LP: #1417176)
59+ * As part of LP: #1417176, the `--no-reboot` switch for
60+ `system-image-cli(1)` has been deprecated. Use `--no-apply` instead
61+ (`-g` is still the shortcut).
62+ * Support production factory resets. `system-image-cli --production-reset`
63+ and a new D-Bus API method `ProductionReset()` are added. Given by Ricardo
64+ Salveti. (LP: #1419027)
65+ * A new key, `target_version_detail` has been added to the dictionary
66+ returned by the `.Information()` D-Bus method. (LP: #1399687)
67+ * The `User-Agent` HTTP header now also includes device and channel names.
68+ (LP: #1387719)
69+ * Added `--progress` flag to `system-image-cli` for specifying methods for
70+ reporting progress. Current available values are: `dots` (compatible with
71+ system-image 2.5), `logfile` (compatible with system-image 2.5's
72+ `--verbose` flag), and `json` for JSON records on stdout. (LP: #1423622)
73+ * Support for the `SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS` environment
74+ variable has been removed.
75+ * Fix `system-image-cli --list-channels`. (LP: #1448153)
76+
77+2.5.1 (2014-10-21)
78+==================
79+ * Make phased upgrade percentage calculation idempotent for each tuple of
80+ (channel, target-build-number, machine-id). Also, modify the candidate
81+ upgrade path selection process such that if the lowest scored candidate
82+ path has a phased percentage greater than the device's percentage, the
83+ candidate will be ignored, and the next lowest scored candidate will be
84+ checked until either a winner is found or no candidates are left, in which
85+ case the device is deemed to be up-to-date. (LP: #1383539)
86+ * `system-image-cli -p/--percentage` is added to allow command line override
87+ of the device's phased percentage.
88+ * `system-image-cli --dry-run` now also displays the phase percentage of the
89+ winning candidate upgrade path.
90+
91+2.5 (2014-09-29)
92 ================
93 * Remove the previously deprecated `system-image-cli --dbus` command line
94 switch. (LP: #1369717)
95
96=== modified file 'PKG-INFO'
97--- PKG-INFO 2014-09-26 14:36:34 +0000
98+++ PKG-INFO 2015-05-20 14:55:53 +0000
99@@ -1,6 +1,6 @@
100 Metadata-Version: 1.0
101 Name: system-image
102-Version: 2.5
103+Version: 3.0
104 Summary: Ubuntu System Image Based Upgrades
105 Home-page: UNKNOWN
106 Author: Barry Warsaw
107
108=== modified file 'cli-manpage.rst'
109--- cli-manpage.rst 2014-09-17 13:41:31 +0000
110+++ cli-manpage.rst 2015-05-20 14:55:53 +0000
111@@ -7,9 +7,9 @@
112 ------------------------------------------------
113
114 :Author: Barry Warsaw <barry@ubuntu.com>
115-:Date: 2014-09-16
116-:Copyright: 2013-2014 Canonical Ltd.
117-:Version: 2.4
118+:Date: 2015-01-15
119+:Copyright: 2013-2015 Canonical Ltd.
120+:Version: 3.0
121 :Manual section: 1
122
123
124@@ -68,10 +68,17 @@
125
126 -n, --dry-run
127 Calculate and print the upgrade path, but do not download or apply it.
128-
129---no-reboot
130- Downloads all files and prepares for a reboot into recovery, but doesn't
131- actually issue the reboot.
132+ *New in system-image 2.5.1: output displays the target phase percentage*
133+
134+-p VALUE, --percentage VALUE
135+ For testing purposes, force a device specific phase percentage. The value
136+ must be an integer between 0 and 100. *New in system-image 2.5.1*
137+
138+-g, --no-apply
139+ Downloads all files and prepares for, but does not actually apply the
140+ update. On devices which require a reboot to apply the update, no reboot
141+ is performed. *New in system-image 3.0: --no-reboot is renamed to
142+ --no-apply*
143
144 -v, --verbose
145 Increase the logging verbosity. With one ``-v``, logging goes to the
146@@ -79,10 +86,13 @@
147 enabled. With two ``-v`` (or ``-vv``), logging both to the console and to
148 the log file are output at ``DEBUG`` level.
149
150--C FILE, --config FILE
151- Use the given configuration file, otherwise use the default. The program
152- will optionally also read a ``channel.ini`` file in the same directory as
153- ``FILE``.
154+-C DIR, --config DIR
155+ Use the given configuration directory, otherwise use the system default.
156+ The program will read all the files in this directory that begin with a
157+ number, followed by an underscore, and ending in ``.ini``
158+ (e.g. ``03_myconfig.ini``). The files are read in sorted numerical order
159+ from lowest prefix number to highest, with later configuration files able
160+ to override any variable in any section.
161
162 --factory-reset
163 Wipes the data partition and issues a reboot into recovery. This
164@@ -91,6 +101,13 @@
165 --show-settings
166 Show all the key/value pairs in the settings database.
167
168+--progress [dots|logfile|json]
169+ Report progress in various ways. `dots` prints some dots every once in a
170+ while to stderr; this mimic what was available in system-image 2.5.
171+ `logfile` prints messages at debug level to the system-image log file, and
172+ is also available in 2.5 (via the `--verbose` flag). `json` prints JSON
173+ records to stdout. *New in system-image 3.0*
174+
175 --get KEY
176 Print the value for the given key in the settings database. If the key is
177 missing, a default value is printed. May be given multiple times.
178@@ -107,15 +124,11 @@
179 FILES
180 =====
181
182-/etc/system-image/client.ini
183- Default configuration file.
184-
185-/etc/system-image/channel.ini
186- Optional configuration file overrides (for the ``[service]`` section
187- only).
188+/etc/system-image/[0-9]+*.ini
189+ Default configuration files.
190
191
192 SEE ALSO
193 ========
194
195-client.ini(5), system-image-dbus(8)
196+system-image.ini(5), system-image-dbus(8)
197
198=== added file 'coverage-curl.ini'
199--- coverage-curl.ini 1970-01-01 00:00:00 +0000
200+++ coverage-curl.ini 2015-05-20 14:55:53 +0000
201@@ -0,0 +1,21 @@
202+[run]
203+branch = true
204+parallel = true
205+omit =
206+ setup*
207+ systemimage/data/*
208+ systemimage/docs/*
209+ systemimage/testing/*
210+ systemimage/tests/*
211+ systemimage/udm.py
212+ /usr/lib/*
213+ .tox/coverage-curl/lib/python3.4/distutils/*
214+ .tox/coverage-curl/lib/python3.4/site-packages/pkg_resources*
215+ .tox/coverage-udm/lib/python3.4/distutils/*
216+ .tox/coverage-udm/lib/python3.4/site-packages/pkg_resources*
217+
218+[paths]
219+source =
220+ systemimage
221+ .tox/coverage-curl/lib/python*/site-packages/systemimage
222+ .tox/coverage-udm/lib/python*/site-packages/systemimage
223
224=== added file 'coverage-udm.ini'
225--- coverage-udm.ini 1970-01-01 00:00:00 +0000
226+++ coverage-udm.ini 2015-05-20 14:55:53 +0000
227@@ -0,0 +1,21 @@
228+[run]
229+branch = true
230+parallel = true
231+omit =
232+ setup*
233+ systemimage/data/*
234+ systemimage/docs/*
235+ systemimage/testing/*
236+ systemimage/tests/*
237+ systemimage/curl.py
238+ /usr/lib/*
239+ .tox/coverage-curl/lib/python3.4/distutils/*
240+ .tox/coverage-curl/lib/python3.4/site-packages/pkg_resources*
241+ .tox/coverage-udm/lib/python3.4/distutils/*
242+ .tox/coverage-udm/lib/python3.4/site-packages/pkg_resources*
243+
244+[paths]
245+source =
246+ systemimage
247+ .tox/coverage-curl/lib/python*/site-packages/systemimage
248+ .tox/coverage-udm/lib/python*/site-packages/systemimage
249
250=== removed file 'coverage.ini'
251--- coverage.ini 2014-09-17 02:58:58 +0000
252+++ coverage.ini 1970-01-01 00:00:00 +0000
253@@ -1,15 +0,0 @@
254-[run]
255-branch = true
256-parallel = true
257-omit =
258- setup*
259- systemimage/data/*
260- systemimage/docs/*
261- systemimage/testing/*
262- systemimage/tests/*
263- /usr/lib/*
264-
265-[paths]
266-source =
267- systemimage
268- .tox/coverage/lib/python*/site-packages/systemimage
269
270=== modified file 'dbus-manpage.rst'
271--- dbus-manpage.rst 2014-09-26 14:36:34 +0000
272+++ dbus-manpage.rst 2015-05-20 14:55:53 +0000
273@@ -7,9 +7,9 @@
274 -----------------------------------------
275
276 :Author: Barry Warsaw <barry@ubuntu.com>
277-:Date: 2014-07-15
278-:Copyright: 2013-2014 Canonical Ltd.
279-:Version: 2.3
280+:Date: 2015-01-15
281+:Copyright: 2013-2015 Canonical Ltd.
282+:Version: 3.0
283 :Manual section: 8
284
285
286@@ -42,10 +42,13 @@
287 enabled. With two ``-v`` (or ``-vv``), logging both to the console and to
288 the log file are output at ``DEBUG`` level.
289
290--C FILE, --config FILE
291- Use the given configuration file, otherwise use the default. The program
292- will optionally also read a ``channel.ini`` file in the same directory as
293- ``FILE``.
294+-C DIR, --config DIR
295+ Use the given configuration directory, otherwise use the system default.
296+ The program will read all the files in this directory that begin with a
297+ number, followed by an underscore, and ending in ``.ini``
298+ (e.g. ``03_myconfig.ini``). The files are read in sorted numerical order
299+ from lowest prefix number to highest, with later configuration files able
300+ to override any variable in any section.
301
302
303 D-BUS API
304@@ -94,11 +97,12 @@
305
306 ``ApplyUpdate()``
307 This is an **asynchronous** call used to apply a previously downloaded
308- update and initiate a reboot to apply the update. It is a no-op if no new
309- update has been downloaded. Just before the device reboots, a
310- ``Rebooting`` signal is sent, although the timing of this signal both
311- being sent and received depends on how quickly the device is shut down for
312- reboot.
313+ update. After the update has been applied, an ``Applied`` signal is
314+ sent. Some devices require a reboot in order to apply the update, and
315+ such devices may also issue a ``Rebooting`` signal. However, on devices
316+ which require a reboot, the timing and emission of both the ``Applied``
317+ and ``Rebooting`` signals are in a race condition with system shutdown,
318+ and may not occur.
319
320 ``CancelUpdate()``
321 This is a **synchronous** call to cancel any update check or download in
322@@ -143,12 +147,21 @@
323 * *version_detail* - A string containing a comma-separated list of
324 key-value pairs providing additional component version details,
325 e.g. "ubuntu=123,mako=456,custom=789".
326+ * *target_version_detail* - Like *version_detail* but contains the
327+ information from the server. If an update is known to be available,
328+ this will be taken from ``index.json`` file's image specification, for
329+ the image that the upgrade will leave the device at. If no update is
330+ available this will be identical to *version_detail*. If no
331+ `CheckForUpdate()` as been previously performed, then the
332+ *target_version_detail* will be the empty string.
333 * *last_check_date* - The last time a ``CheckForUpdate()`` call was
334 performed.
335
336 *New in system-image 2.3*
337
338- *New in system-image 2.5: target_build_number*
339+ *New in system-image 2.5: target_build_number was added.*
340+
341+ *New in system-image 3.0: target_version_detail was added.*
342
343 ``FactoryReset()``
344 This is a **synchronous** call which wipes the data partition and issue a
345@@ -157,6 +170,13 @@
346
347 *New in system-image 2.3*.
348
349+``ProductionReset()``
350+ This is a **synchronous** call which wipes the data partition, sets a flag
351+ for factory wipe (used in production), and issue a reboot to recovery.
352+ A ``Rebooting`` signal may be sent, depending on timing.
353+
354+ *New in system-image 3.0*.
355+
356 ``SetSetting(key, value)``
357 This is a **synchronous** call to write or update a setting. ``key`` and
358 ``value`` are strings. While any key/value pair may be set, some keys
359@@ -265,15 +285,20 @@
360 * **last_reason** - A string containing the reason for why this updated
361 failed.
362
363+``Applied(status)``
364+ Sent in response to an ``ApplyUpdate()`` call. See the timing caveats for
365+ that method. **New in system-image 3.0**
366+
367+ * **status** - A boolean indicating whether an update has been applied or
368+ not.
369+
370 ``Rebooting(status)``
371- Sent just before the device reboots. Because the system is in the process
372- of being rebooted, clients may or may not receive this signal.
373+ On devices which require a reboot in order to apply an update, this signal
374+ may be sent in response to an ``ApplyUpdate()`` call. See the timing
375+ caveats for that method.
376
377- * **status** - A boolean indicating whether the application of the update
378- is successful or not. Generally, when status is true you won't ever
379- receive the signal because the device will be rebooting. When status is
380- false it means the application of the update or reboot failed for some
381- reason.
382+ * **status** - A boolean indicating whether the device has initiated a
383+ reboot sequence or not.
384
385 ``SettingChanged(key, value)``
386 Sent when a setting is changed. This signal is not sent if the new value
387@@ -286,7 +311,7 @@
388 Additional API details
389 ----------------------
390
391-The ``SetSettings()`` call takes a key string and a value string. The
392+The ``SetSetting()`` call takes a key string and a value string. The
393 following keys are predefined.
394
395 * *min_battery* - The minimum battery strength which will allow downloads
396@@ -311,12 +336,8 @@
397 FILES
398 =====
399
400-/etc/system-image/client.ini
401- Default configuration file.
402-
403-/etc/system-image/channel.ini
404- Optional configuration file overrides (for the ``[service]`` section
405- only).
406+/etc/system-image/[0-9]+*.ini
407+ Default configuration files.
408
409 /etc/dbus-1/system.d/com.canonical.SystemImage.conf
410 DBus service permissions file.
411@@ -328,6 +349,7 @@
412 SEE ALSO
413 ========
414
415-client.ini(5), system-image-cli(1)
416+system-image.ini(5), system-image-cli(1)
417+
418
419 .. _`ISO 8601`: http://en.wikipedia.org/wiki/ISO_8601
420
421=== modified file 'debian/changelog'
422--- debian/changelog 2014-09-29 19:02:48 +0000
423+++ debian/changelog 2015-05-20 14:55:53 +0000
424@@ -1,3 +1,80 @@
425+system-image (3.0-0ubuntu2) UNRELEASED; urgency=medium
426+
427+ * New upstream release.
428+ - LP: #1374459 - Support a built-in PyCURL-based downloader in
429+ addition to the traditional ubuntu-download-manager (over D-BUS)
430+ downloader. Auto-detects which downloader to use based on whether
431+ udm is available on the system bus, pycurl is importable, and the
432+ setting of the SYSTEMIMAGE_PYCURL environment variable. Initial
433+ contribution by Michael Vogt.
434+ - LP: #1384859 - Support alternative machine-id files as fall backs if
435+ the D-Bus file does not exist. Specifically, add systemd's
436+ /etc/machine-id to the list. Initial contribution by Michael Vogt.
437+ - LP: #1373467 - Support multiple configuration files, as in a
438+ `config.d` directory. Now, configuration files are named
439+ `NN_whatever.ini` where "NN" must be a numeric prefix. Files are
440+ loaded in sorted numeric order, with later files overriding newer
441+ files. Support for both the `client.ini` and `channel.ini` files has
442+ been removed.
443+ - The `[system]build_file` variable has been removed. Build number
444+ information now must come from the `.ini` files, and last update
445+ date comes from the newest `.ini` file loaded.
446+ - The `-C` command line option now takes a path to the configuration
447+ directory.
448+ - LP: #1412698 - Reworked the checking and downloading locks/flags to
449+ so that they will work better with configuration reloading.
450+ - LP: #1377312 - Support for the `/etc/ubuntu-build` file has been
451+ removed. The build number now comes from the configuration files.
452+ - LP: #1377184 - Move the `archive-master.tar.xz` file to
453+ `/usr/share/system-image` for better FHS compliance.
454+ - LP: #1381538 - Since devices do not always reboot to apply changes,
455+ the `[hooks]update` variable has been renamed to `[hooks]apply`.
456+ - LP: #1333414 - For testing purposes only, `system-image-cli` now
457+ supports an undocumented command line switch
458+ `--skip-gpg-verification`. Originally given by Jani Monoses.
459+ - LP: #1417176 - A new D-Bus signal `Applied(bool)` is added, which is
460+ returned in response to the `ApplyUpdate()` asynchronous method
461+ call. For devices which do not need to reboot in order to apply the
462+ update, this is the only signal you will get. If your device needs
463+ to reboot you will also receive the `Rebooting(bool)` command as
464+ with earlier versions. The semantics of the flag argument are the
465+ same in both cases, as are the race timing issues inherent in these
466+ signals. See the `system-image-dbus(8)` manpage for details.
467+ - As part of LP: #1417176, the `--no-reboot` switch for
468+ `system-image-cli(1)` has been deprecated. Use `--no-apply` instead
469+ (`-g` is still the shortcut).
470+ - LP: #1419027 - Support production factory resets. `system-image-cli
471+ --production-reset` and a new D-Bus API method `ProductionReset()`
472+ are added. Given by Ricardo Salveti.
473+ - LP: #1399687 - A new key, `target_version_detail` has been added to
474+ the dictionary returned by the `.Information()` D-Bus method.
475+ - LP: #1387719 - The `User-Agent` HTTP header now also includes device
476+ and channel names.
477+ - LP: #1423622 - Added `--progress` flag to `system-image-cli` for
478+ specifying methods for reporting progress. Current available values
479+ are: `dots` (compatible with system-image 2.5), `logfile`
480+ (compatible with system-image 2.5's `--verbose` flag), and `json`
481+ for JSON records on stdout.
482+ - Support for the `SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS` environment
483+ variable has been removed.
484+ - LP: #1448153 - Fix `system-image-cli --list-channels`.
485+ * d/rules:
486+ - Run both the cURL and UDM based tests.
487+ - Run tests with more verbosity.
488+ - Install the archive-master keyring files to /usr/share instead of
489+ /etc for better FHS compliance. (LP: #1377184)
490+ * d/control:
491+ - Add python3-pycurl to Build-Depends.
492+ - Bump Standards-Version to 3.9.6 with no other changes necessary.
493+ - system-image-common now depends on
494+ `ubuntu-download-manager | python3-pycurl` so that UDM doesn't need to
495+ be pulled in for snappy. (LP: #1431696)
496+ * d/tests/control: Disable DEP-8 "smoketests" which try to access
497+ external resources. This is now prohibited by policy for
498+ pocket-promotion tests. (LP: #1457070)
499+
500+ -- Barry Warsaw <barry@ubuntu.com> Wed, 20 May 2015 10:46:17 -0400
501+
502 system-image (2.5-0ubuntu1) utopic; urgency=medium
503
504 [ Barry Warsaw ]
505
506=== modified file 'debian/control'
507--- debian/control 2014-09-09 17:27:17 +0000
508+++ debian/control 2015-05-20 14:55:53 +0000
509@@ -16,10 +16,11 @@
510 python3-nose2,
511 python3-pkg-resources,
512 python3-psutil,
513+ python3-pycurl,
514 python3-setuptools,
515 python3-xdg,
516 ubuntu-download-manager
517-Standards-Version: 3.9.5
518+Standards-Version: 3.9.6
519 XS-Testsuite: autopkgtest
520 Vcs-Bzr: https://code.launchpad.net/~ubuntu-managed-branches/ubuntu-system-image/system-image
521 Vcs-Browser: http://bazaar.launchpad.net/~ubuntu-managed-branches/ubuntu-system-image/system-image/files
522@@ -48,7 +49,7 @@
523 python3-gnupg,
524 python3-pkg-resources,
525 python3-xdg,
526- ubuntu-download-manager,
527+ ubuntu-download-manager | python3-pycurl,
528 ${misc:Depends},
529 ${python3:Depends}
530 Description: Ubuntu system image updater
531
532=== modified file 'debian/rules'
533--- debian/rules 2014-09-09 17:27:17 +0000
534+++ debian/rules 2015-05-20 14:55:53 +0000
535@@ -9,11 +9,18 @@
536 dh $@ --with python3 --buildsystem=pybuild
537
538 override_dh_auto_test:
539- unset http_proxy; unset https_proxy; \
540- export SYSTEMIMAGE_REACTOR_TIMEOUT=1200; \
541- export SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS=2; \
542- PYBUILD_SYSTEM=custom \
543- PYBUILD_TEST_ARGS="{interpreter} -m nose2 -v" dh_auto_test
544+ export http_proxy= ; \
545+ export https_proxy= ; \
546+ export SYSTEMIMAGE_REACTOR_TIMEOUT=1200; \
547+ export SYSTEMIMAGE_PYCURL=1; \
548+ PYBUILD_SYSTEM=custom \
549+ PYBUILD_TEST_ARGS="{interpreter} -m nose2 -vv" dh_auto_test
550+ export http_proxy= ; \
551+ export https_proxy= ; \
552+ export SYSTEMIMAGE_REACTOR_TIMEOUT=1200; \
553+ export SYSTEMIMAGE_PYCURL=0; \
554+ PYBUILD_SYSTEM=custom \
555+ PYBUILD_TEST_ARGS="{interpreter} -m nose2 -vv" dh_auto_test
556
557 # pybuild can't yet handle Python 3 packages that don't start with "python3-".
558 # See bug #751908 - In the meantime, this override isn't perfect, but it gets
559@@ -30,11 +37,9 @@
560 usr/lib/python3.?/dist-packages/systemimage/testing
561 dh_install -p system-image-cli usr/bin/system-image-cli
562 dh_install -p system-image-common \
563- debian/archive-master.tar.xz etc/system-image
564- dh_install -p system-image-common \
565- debian/archive-master.tar.xz.asc etc/system-image
566- dh_install -p system-image-common \
567- systemimage/data/client.ini etc/system-image
568+ debian/archive-master.tar.xz usr/share/system-image
569+ dh_install -p system-image-common \
570+ debian/archive-master.tar.xz.asc usr/share/system-image
571 dh_install -p system-image-dbus usr/bin/system-image-dbus usr/sbin
572 dh_install -p system-image-dbus \
573 systemimage/data/com.canonical.SystemImage.service \
574
575=== renamed file 'debian/tests/client.ini.in' => 'debian/tests/00_default.ini.in'
576=== modified file 'debian/tests/control'
577--- debian/tests/control 2014-07-23 22:51:19 +0000
578+++ debian/tests/control 2015-05-20 14:55:53 +0000
579@@ -1,11 +1,11 @@
580-Tests: smoketest
581-Restrictions: isolation-container
582-Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg
583+#Tests: smoketest
584+#Restrictions: isolation-container
585+#Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-pycurl
586
587-Tests: smoketest-noreboot
588-Restrictions: isolation-container allow-stderr
589-Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg
590+#Tests: smoketest-noreboot
591+#Restrictions: isolation-container allow-stderr
592+#Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-pycurl
593
594 Tests: dryrun
595 Restrictions: allow-stderr
596-Depends: @, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-setuptools, python3-nose2
597+Depends: @, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-setuptools, python3-nose2, python3-pycurl
598
599=== modified file 'debian/tests/dryrun'
600--- debian/tests/dryrun 2014-07-18 16:32:44 +0000
601+++ debian/tests/dryrun 2015-05-20 14:55:53 +0000
602@@ -5,7 +5,7 @@
603 # require network access, so it is compatible with less isolated (but also
604 # lighter weight) containers such as schroot.
605 #
606-# Copyright (C) 2014 Canonical Ltd.
607+# Copyright (C) 2014-2015 Canonical Ltd.
608 # Author: Barry Warsaw <barry@ubuntu.com>
609
610 python3 -m nose2 -vv -P TestCLIMainDryRun
611
612=== modified file 'debian/tests/prep.py'
613--- debian/tests/prep.py 2013-12-13 13:55:51 +0000
614+++ debian/tests/prep.py 2015-05-20 14:55:53 +0000
615@@ -1,5 +1,8 @@
616 #!/usr/bin/python3
617
618+# Copyright (C) 2013-2015 Canonical Ltd.
619+# Author: Barry Warsaw <barry@ubuntu.com>
620+
621 import os
622
623 tmpdir = os.environ['ADTTMP']
624@@ -8,15 +11,19 @@
625 os.makedirs(os.path.join(tmpdir, 'android'), exist_ok=True)
626 os.makedirs(os.path.join(tmpdir, 'ubuntu'), exist_ok=True)
627
628+config_d = os.path.join(tmpdir, 'config.d')
629+os.makedirs(config_d, exist_ok=True)
630+
631 substitutions = dict(
632 TMPDIR=tmpdir,
633 ARTIFACTS=artifacts,
634 )
635
636-with open('debian/tests/client.ini.in', encoding='utf-8') as fp:
637+with open('debian/tests/00_default.ini.in', encoding='utf-8') as fp:
638 ini_template = fp.read()
639
640 ini_contents = ini_template.format(**substitutions)
641
642-with open(os.path.join(tmpdir, 'client.ini'), 'w', encoding='utf-8') as fp:
643+default_ini = os.path.join(config_d, '00_default.ini')
644+with open(default_ini, 'w', encoding='utf-8') as fp:
645 fp.write(ini_contents)
646
647=== modified file 'debian/tests/smoketest'
648--- debian/tests/smoketest 2014-07-18 16:32:44 +0000
649+++ debian/tests/smoketest 2015-05-20 14:55:53 +0000
650@@ -5,9 +5,9 @@
651 # isolation-container restricted, so it requires an isolated test container
652 # like QEMU.
653 #
654-# Copyright (C) 2013-2014 Canonical Ltd.
655+# Copyright (C) 2013-2015 Canonical Ltd.
656 # Author: Barry Warsaw <barry@ubuntu.com>
657
658 set -e
659 python3 debian/tests/prep.py
660-system-image-cli -C $ADTTMP/client.ini -d mako -c devel -b 0 --dry-run
661+system-image-cli -C $ADTTMP/config.d -d mako -c devel -b 0 --dry-run
662
663=== modified file 'debian/tests/smoketest-noreboot'
664--- debian/tests/smoketest-noreboot 2014-07-23 22:51:19 +0000
665+++ debian/tests/smoketest-noreboot 2015-05-20 14:55:53 +0000
666@@ -7,9 +7,9 @@
667 #
668 # This is like smoketest except that it does a full download.
669 #
670-# Copyright (C) 2013-2014 Canonical Ltd.
671+# Copyright (C) 2013-2015 Canonical Ltd.
672 # Author: Barry Warsaw <barry@ubuntu.com>
673
674 set -e
675 python3 debian/tests/prep.py
676-system-image-cli -C $ADTTMP/client.ini -d mako -c devel -b 0 --no-reboot -v
677+system-image-cli -C $ADTTMP/config.d -d mako -c devel -b 0 --no-reboot -v
678
679=== modified file 'debian/tests/unittests'
680--- debian/tests/unittests 2013-12-13 13:55:51 +0000
681+++ debian/tests/unittests 2015-05-20 14:55:53 +0000
682@@ -2,7 +2,7 @@
683 #
684 # autopkgtest check: Run tox against the built package.
685 #
686-# Copyright (C) 2013 Canonical Ltd.
687+# Copyright (C) 2013-2015 Canonical Ltd.
688 # Author: Barry Warsaw <barry@ubuntu.com>
689
690 set -e
691
692=== modified file 'ini-manpage.rst'
693--- ini-manpage.rst 2014-09-17 13:41:31 +0000
694+++ ini-manpage.rst 2015-05-20 14:55:53 +0000
695@@ -1,39 +1,43 @@
696-==========
697-client.ini
698-==========
699-
700-
701------------------------------------------------
702-Ubuntu System Image Upgrader configuration file
703------------------------------------------------
704+================
705+system-image.ini
706+================
707+
708+
709+------------------------------------------------
710+Ubuntu System Image Upgrader configuration files
711+------------------------------------------------
712
713 :Author: Barry Warsaw <barry@ubuntu.com>
714-:Date: 2014-09-11
715-:Copyright: 2013-2014 Canonical Ltd.
716-:Version: 2.4
717+:Date: 2015-01-15
718+:Copyright: 2013-2015 Canonical Ltd.
719+:Version: 3.0
720 :Manual section: 5
721
722
723 DESCRIPTION
724 ===========
725
726-``/etc/system-image/client.ini`` is the configuration file for the system
727-image upgrader. It is an ini-style configuration file with sections that
728-define the service to connect to, as well as local system resources.
729-Generally, the options never need to be changed.
730-
731-The system image upgrader will also optionally read a
732-``/etc/system-image/channel.ini`` file with the same format as ``client.ini``.
733-This file should only contain a ``[service]`` section for overriding in the
734-``client.ini`` file. All other sections are ignored.
735+``/etc/system-image/config.d`` is the default configuration directory for the
736+system image upgrader. It contains ini-style configuration files with
737+sections that define the service to connect to, as well as local system
738+resources. Generally, the options never need to be changed.
739+
740+The system image upgrader will read all files in this directory that start
741+with a numeric prefix, followed by an underscore, and then any alphanumeric
742+suffix, ending in ``.ini``. E.g. ``07_myconfig.ini``.
743+
744+The files are read in sorted numerical order, from lowest prefix number to
745+highest, with later configuration files able to override any variable in any
746+section.
747
748
749 SYNTAX
750 ======
751
752-Sections are delimited by square brackets, e.g. ``[service]``. Variables
753-inside the service separate the variable name and value by a colon. Blank
754-lines and lines that start with a ``#`` are ignored.
755+Sections in the ``.ini`` files are delimited by square brackets,
756+e.g. ``[service]``. Variables inside the service separate the variable name
757+and value by a colon. Blank lines and lines that start with a ``#`` are
758+ignored.
759
760
761 THE SERVICE SECTION
762@@ -82,10 +86,6 @@
763
764 This section contains the following variables:
765
766-build_file
767- The file on the local file system containing the system's current build
768- number.
769-
770 tempdir
771 The base temporary directory on the local file system. When any of the
772 system-image processes run, a secure subdirectory inside `tempdir` will be
773@@ -181,9 +181,11 @@
774 The Python import path to the class implementing the upgrade scoring
775 algorithm.
776
777-reboot
778- The Python import path to the class that implements the system reboot
779- command.
780+apply
781+ The Python import path to the class that implements the mechanism for
782+ applying the update. This often reboots the device.
783+
784+ *New in system-image 3.0: ``reboot`` was renamed to ``apply``*
785
786
787 THE DBUS SECTION
788@@ -204,6 +206,7 @@
789
790 system-image-cli(1)
791
792+
793 [1]: https://wiki.ubuntu.com/ImageBasedUpgrades/Server
794
795 [2]: https://wiki.ubuntu.com/ImageBasedUpgrades/GPG
796
797=== modified file 'setup.cfg'
798--- setup.cfg 2014-09-26 14:36:34 +0000
799+++ setup.cfg 2015-05-20 14:55:53 +0000
800@@ -4,7 +4,7 @@
801 logging-filter = systemimage
802
803 [egg_info]
804-tag_svn_revision = 0
805 tag_build =
806 tag_date = 0
807+tag_svn_revision = 0
808
809
810=== modified file 'setup.py'
811--- setup.py 2014-02-20 23:03:24 +0000
812+++ setup.py 2015-05-20 14:55:53 +0000
813@@ -1,4 +1,4 @@
814-# Copyright (C) 2013-2014 Canonical Ltd.
815+# Copyright (C) 2013-2015 Canonical Ltd.
816 # Author: Barry Warsaw <barry@ubuntu.com>
817
818 # This program is free software: you can redistribute it and/or modify
819
820=== modified file 'system_image.egg-info/PKG-INFO'
821--- system_image.egg-info/PKG-INFO 2014-09-26 14:36:34 +0000
822+++ system_image.egg-info/PKG-INFO 2015-05-20 14:55:53 +0000
823@@ -1,6 +1,6 @@
824 Metadata-Version: 1.0
825 Name: system-image
826-Version: 2.5
827+Version: 3.0
828 Summary: Ubuntu System Image Based Upgrades
829 Home-page: UNKNOWN
830 Author: Barry Warsaw
831
832=== modified file 'system_image.egg-info/SOURCES.txt'
833--- system_image.egg-info/SOURCES.txt 2014-09-26 14:36:34 +0000
834+++ system_image.egg-info/SOURCES.txt 2015-05-20 14:55:53 +0000
835@@ -2,7 +2,8 @@
836 NEWS.rst
837 README.rst
838 cli-manpage.rst
839-coverage.ini
840+coverage-curl.ini
841+coverage-udm.ini
842 dbus-manpage.rst
843 ini-manpage.rst
844 setup.cfg
845@@ -17,10 +18,12 @@
846 system_image.egg-info/top_level.txt
847 systemimage/__init__.py
848 systemimage/api.py
849+systemimage/apply.py
850 systemimage/bag.py
851 systemimage/candidates.py
852 systemimage/channel.py
853 systemimage/config.py
854+systemimage/curl.py
855 systemimage/dbus.py
856 systemimage/device.py
857 systemimage/download.py
858@@ -32,14 +35,13 @@
859 systemimage/logging.py
860 systemimage/main.py
861 systemimage/reactor.py
862-systemimage/reboot.py
863 systemimage/scores.py
864 systemimage/service.py
865 systemimage/settings.py
866 systemimage/state.py
867+systemimage/udm.py
868 systemimage/version.txt
869 systemimage/data/__init__.py
870-systemimage/data/client.ini
871 systemimage/data/com.canonical.SystemImage.conf
872 systemimage/data/com.canonical.SystemImage.service
873 systemimage/docs/__init__.py
874@@ -70,77 +72,121 @@
875 systemimage/tests/test_settings.py
876 systemimage/tests/test_state.py
877 systemimage/tests/test_winner.py
878+systemimage/tests/data/00.ini
879+systemimage/tests/data/01.ini
880 systemimage/tests/data/__init__.py
881+systemimage/tests/data/api.channels_01.json
882+systemimage/tests/data/api.index_01.json
883+systemimage/tests/data/api.index_02.json
884+systemimage/tests/data/api.index_03.json
885 systemimage/tests/data/archive-master.gpg
886 systemimage/tests/data/bad_cert.pem
887 systemimage/tests/data/bad_key.pem
888+systemimage/tests/data/candidates.index_01.json
889+systemimage/tests/data/candidates.index_02.json
890+systemimage/tests/data/candidates.index_03.json
891+systemimage/tests/data/candidates.index_04.json
892+systemimage/tests/data/candidates.index_05.json
893+systemimage/tests/data/candidates.index_06.json
894+systemimage/tests/data/candidates.index_07.json
895+systemimage/tests/data/candidates.index_08.json
896+systemimage/tests/data/candidates.index_09.json
897+systemimage/tests/data/candidates.index_10.json
898+systemimage/tests/data/candidates.index_11.json
899+systemimage/tests/data/candidates.index_12.json
900+systemimage/tests/data/candidates.index_13.json
901 systemimage/tests/data/cert.pem
902-systemimage/tests/data/channel_01.ini
903-systemimage/tests/data/channel_02.ini
904-systemimage/tests/data/channel_03.ini
905-systemimage/tests/data/channel_04.ini
906-systemimage/tests/data/channel_05.ini
907-systemimage/tests/data/channel_06.ini
908-systemimage/tests/data/channel_07.ini
909-systemimage/tests/data/channels_01.json
910-systemimage/tests/data/channels_02.json
911-systemimage/tests/data/channels_03.json
912-systemimage/tests/data/channels_04.json
913-systemimage/tests/data/channels_05.json
914-systemimage/tests/data/channels_06.json
915-systemimage/tests/data/channels_07.json
916-systemimage/tests/data/channels_08.json
917-systemimage/tests/data/channels_09.json
918-systemimage/tests/data/channels_10.json
919-systemimage/tests/data/channels_11.json
920+systemimage/tests/data/channel.channels_01.json
921+systemimage/tests/data/channel.channels_02.json
922+systemimage/tests/data/channel.channels_03.json
923+systemimage/tests/data/channel.channels_04.json
924+systemimage/tests/data/channel.channels_05.json
925 systemimage/tests/data/com.canonical.SystemImage.service.in
926 systemimage/tests/data/com.canonical.applications.Downloader.service.in
927-systemimage/tests/data/config_00.ini
928-systemimage/tests/data/config_01.ini
929-systemimage/tests/data/config_02.ini
930-systemimage/tests/data/config_03.ini
931-systemimage/tests/data/config_04.ini
932-systemimage/tests/data/config_05.ini
933-systemimage/tests/data/config_06.ini
934-systemimage/tests/data/config_07.ini
935-systemimage/tests/data/config_08.ini
936-systemimage/tests/data/config_09.ini
937-systemimage/tests/data/config_10.ini
938+systemimage/tests/data/config.config_01.ini
939+systemimage/tests/data/config.config_02.ini
940+systemimage/tests/data/config.config_03.ini
941+systemimage/tests/data/config.config_04.ini
942+systemimage/tests/data/config.config_05.ini
943+systemimage/tests/data/config.config_06.ini
944+systemimage/tests/data/config.config_07.ini
945+systemimage/tests/data/config.config_08.ini
946+systemimage/tests/data/config.config_09.ini
947+systemimage/tests/data/config.config_10.ini
948+systemimage/tests/data/config.config_11.ini
949 systemimage/tests/data/dbus-system.conf.in
950+systemimage/tests/data/dbus.channels_01.json
951+systemimage/tests/data/dbus.index_01.json
952+systemimage/tests/data/dbus.index_02.json
953+systemimage/tests/data/dbus.index_03.json
954+systemimage/tests/data/dbus.index_04.json
955+systemimage/tests/data/dbus.index_05.json
956+systemimage/tests/data/dbus.index_06.json
957 systemimage/tests/data/device-signing.gpg
958+systemimage/tests/data/download.index_01.json
959 systemimage/tests/data/expired_cert.pem
960 systemimage/tests/data/expired_key.pem
961+systemimage/tests/data/gpg.channels_01.json
962+systemimage/tests/data/helpers.config_01.ini
963+systemimage/tests/data/helpers.config_02.ini
964 systemimage/tests/data/image-master.gpg
965 systemimage/tests/data/image-signing.gpg
966-systemimage/tests/data/index_01.json
967-systemimage/tests/data/index_02.json
968-systemimage/tests/data/index_03.json
969-systemimage/tests/data/index_04.json
970-systemimage/tests/data/index_05.json
971-systemimage/tests/data/index_06.json
972-systemimage/tests/data/index_07.json
973-systemimage/tests/data/index_08.json
974-systemimage/tests/data/index_09.json
975-systemimage/tests/data/index_10.json
976-systemimage/tests/data/index_11.json
977-systemimage/tests/data/index_12.json
978-systemimage/tests/data/index_13.json
979-systemimage/tests/data/index_14.json
980-systemimage/tests/data/index_15.json
981-systemimage/tests/data/index_16.json
982-systemimage/tests/data/index_17.json
983-systemimage/tests/data/index_18.json
984-systemimage/tests/data/index_19.json
985-systemimage/tests/data/index_20.json
986-systemimage/tests/data/index_21.json
987-systemimage/tests/data/index_22.json
988-systemimage/tests/data/index_23.json
989-systemimage/tests/data/index_24.json
990-systemimage/tests/data/index_25.json
991+systemimage/tests/data/index.channels_01.json
992+systemimage/tests/data/index.channels_02.json
993+systemimage/tests/data/index.channels_03.json
994+systemimage/tests/data/index.channels_04.json
995+systemimage/tests/data/index.channels_05.json
996+systemimage/tests/data/index.index_01.json
997+systemimage/tests/data/index.index_02.json
998+systemimage/tests/data/index.index_03.json
999+systemimage/tests/data/index.index_04.json
1000+systemimage/tests/data/index.index_05.json
1001 systemimage/tests/data/key.pem
1002+systemimage/tests/data/main.channels_01.json
1003+systemimage/tests/data/main.channels_02.json
1004+systemimage/tests/data/main.channels_03.json
1005+systemimage/tests/data/main.config_01.ini
1006+systemimage/tests/data/main.config_02.ini
1007+systemimage/tests/data/main.config_03.ini
1008+systemimage/tests/data/main.config_04.ini
1009+systemimage/tests/data/main.config_05.ini
1010+systemimage/tests/data/main.config_07.ini
1011+systemimage/tests/data/main.index_01.json
1012+systemimage/tests/data/main.index_02.json
1013+systemimage/tests/data/main.index_03.json
1014+systemimage/tests/data/main.index_04.json
1015+systemimage/tests/data/main.index_05.json
1016 systemimage/tests/data/master-secring.gpg
1017 systemimage/tests/data/nasty_cert.pem
1018 systemimage/tests/data/nasty_key.pem
1019+systemimage/tests/data/scores.index_01.json
1020+systemimage/tests/data/scores.index_02.json
1021+systemimage/tests/data/scores.index_03.json
1022+systemimage/tests/data/scores.index_04.json
1023+systemimage/tests/data/scores.index_05.json
1024+systemimage/tests/data/scores.index_06.json
1025+systemimage/tests/data/scores.index_07.json
1026 systemimage/tests/data/spare.gpg
1027-systemimage/tests/data/sprint_nexus7_index_01.json
1028-tools/demo.ini
1029\ No newline at end of file
1030+systemimage/tests/data/state.channels_01.json
1031+systemimage/tests/data/state.channels_02.json
1032+systemimage/tests/data/state.channels_03.json
1033+systemimage/tests/data/state.channels_04.json
1034+systemimage/tests/data/state.channels_05.json
1035+systemimage/tests/data/state.channels_06.json
1036+systemimage/tests/data/state.channels_07.json
1037+systemimage/tests/data/state.config_01.ini
1038+systemimage/tests/data/state.config_02.ini
1039+systemimage/tests/data/state.index_01.json
1040+systemimage/tests/data/state.index_02.json
1041+systemimage/tests/data/state.index_03.json
1042+systemimage/tests/data/state.index_04.json
1043+systemimage/tests/data/state.index_05.json
1044+systemimage/tests/data/state.index_06.json
1045+systemimage/tests/data/state.index_07.json
1046+systemimage/tests/data/state.index_08.json
1047+systemimage/tests/data/winner.channels_01.json
1048+systemimage/tests/data/winner.channels_02.json
1049+systemimage/tests/data/winner.index_01.json
1050+systemimage/tests/data/winner.index_02.json
1051+tools/demo.ini
1052+tools/runme.sh
1053\ No newline at end of file
1054
1055=== modified file 'systemimage/api.py'
1056--- systemimage/api.py 2014-09-17 13:41:31 +0000
1057+++ systemimage/api.py 2015-05-20 14:55:53 +0000
1058@@ -1,4 +1,4 @@
1059-# Copyright (C) 2013-2014 Canonical Ltd.
1060+# Copyright (C) 2013-2015 Canonical Ltd.
1061 # Author: Barry Warsaw <barry@ubuntu.com>
1062
1063 # This program is free software: you can redistribute it and/or modify
1064@@ -24,10 +24,9 @@
1065
1066 import logging
1067
1068-from systemimage.helpers import last_update_date
1069-from systemimage.reboot import factory_reset
1070+from systemimage.apply import factory_reset, production_reset
1071 from systemimage.state import State
1072-from unittest.mock import patch
1073+
1074
1075 log = logging.getLogger('systemimage')
1076
1077@@ -63,8 +62,12 @@
1078 return ''
1079
1080 @property
1081- def last_update_date(self):
1082- return last_update_date()
1083+ def version_detail(self):
1084+ try:
1085+ return self._winners[-1].version_detail
1086+ except IndexError:
1087+ # No winners.
1088+ return ''
1089
1090
1091 class Mediator:
1092@@ -115,13 +118,20 @@
1093 def download(self):
1094 """Download the available update."""
1095 # We only want callback progress during the actual download.
1096- with patch.object(self._state.downloader, 'callback', self._callback):
1097- self._state.run_until('reboot')
1098+ old_callbacks = self._state.downloader.callbacks[:]
1099+ try:
1100+ self._state.downloader.callbacks = [self._callback]
1101+ self._state.run_until('apply')
1102+ finally:
1103+ self._state.downloader.callbacks = old_callbacks
1104
1105- def reboot(self):
1106- """Issue the reboot."""
1107+ def apply(self):
1108+ """Apply the update."""
1109 # Transition through all remaining states.
1110 list(self._state)
1111
1112 def factory_reset(self):
1113 factory_reset()
1114+
1115+ def production_reset(self):
1116+ production_reset()
1117
1118=== renamed file 'systemimage/reboot.py' => 'systemimage/apply.py'
1119--- systemimage/reboot.py 2014-09-17 13:41:31 +0000
1120+++ systemimage/apply.py 2015-05-20 14:55:53 +0000
1121@@ -1,4 +1,4 @@
1122-# Copyright (C) 2013-2014 Canonical Ltd.
1123+# Copyright (C) 2013-2015 Canonical Ltd.
1124 # Author: Barry Warsaw <barry@ubuntu.com>
1125
1126 # This program is free software: you can redistribute it and/or modify
1127@@ -16,9 +16,11 @@
1128 """Reboot issuer."""
1129
1130 __all__ = [
1131- 'BaseReboot',
1132+ 'BaseApply',
1133+ 'Noop',
1134 'Reboot',
1135 'factory_reset',
1136+ 'production_reset',
1137 ]
1138
1139
1140@@ -32,24 +34,34 @@
1141 log = logging.getLogger('systemimage')
1142
1143
1144-class BaseReboot:
1145- """Common reboot actions."""
1146+class BaseApply:
1147+ """Common apply-the-update actions."""
1148
1149- def reboot(self): # pragma: no cover
1150+ def apply(self): # pragma: no cover
1151 """Subclasses must override this."""
1152 raise NotImplementedError
1153
1154
1155-class Reboot(BaseReboot):
1156- """Issue a standard reboot."""
1157+class Reboot(BaseApply):
1158+ """Apply the update by rebooting the device."""
1159
1160- def reboot(self):
1161+ def apply(self):
1162 try:
1163 check_call('/sbin/reboot -f recovery'.split(),
1164 universal_newlines=True)
1165 except CalledProcessError as error:
1166 log.exception('reboot exit status: {}'.format(error.returncode))
1167 raise
1168+ # This code may or may not run. We're racing against the system
1169+ # reboot procedure.
1170+ config.dbus_service.Rebooting(True)
1171+
1172+
1173+class Noop(BaseApply):
1174+ """No-op apply, mostly for testing."""
1175+
1176+ def apply(self):
1177+ pass
1178
1179
1180 def factory_reset():
1181@@ -59,4 +71,15 @@
1182 with atomic(command_file) as fp:
1183 print('format data', file=fp)
1184 log.info('Performing a factory reset')
1185- config.hooks.reboot().reboot()
1186+ config.hooks.apply().apply()
1187+
1188+
1189+def production_reset():
1190+ """Perform a production reset."""
1191+ command_file = os.path.join(
1192+ config.updater.cache_partition, 'ubuntu_command')
1193+ with atomic(command_file) as fp:
1194+ print('format data', file=fp)
1195+ print('enable factory_wipe', file=fp)
1196+ log.info('Performing a production factory reset')
1197+ config.hooks.apply().apply()
1198
1199=== modified file 'systemimage/bag.py'
1200--- systemimage/bag.py 2014-09-17 13:41:31 +0000
1201+++ systemimage/bag.py 2015-05-20 14:55:53 +0000
1202@@ -1,4 +1,4 @@
1203-# Copyright (C) 2013-2014 Canonical Ltd.
1204+# Copyright (C) 2013-2015 Canonical Ltd.
1205 # Author: Barry Warsaw <barry@ubuntu.com>
1206
1207 # This program is free software: you can redistribute it and/or modify
1208@@ -42,6 +42,12 @@
1209
1210
1211 class Bag:
1212+ # NOTE: This class's methods share a namespace with the possible
1213+ # configuration variable names in the various sections. Thus no variable
1214+ # in any section can be named `update`, `keys`, or `get`. They also can't
1215+ # be named like any of the non-public methods, but that's usually not a
1216+ # problem. Ideally, we'd name the methods part of the reserved namespace,
1217+ # but it seems like a low tech debt for now.
1218 def __init__(self, *, converters=None, **kws):
1219 self._converters = make_converter(converters)
1220 self.__original__ = {}
1221
1222=== modified file 'systemimage/candidates.py'
1223--- systemimage/candidates.py 2014-09-17 13:41:31 +0000
1224+++ systemimage/candidates.py 2015-05-20 14:55:53 +0000
1225@@ -1,4 +1,4 @@
1226-# Copyright (C) 2013-2014 Canonical Ltd.
1227+# Copyright (C) 2013-2015 Canonical Ltd.
1228 # Author: Barry Warsaw <barry@ubuntu.com>
1229
1230 # This program is free software: you can redistribute it and/or modify
1231
1232=== modified file 'systemimage/channel.py'
1233--- systemimage/channel.py 2014-09-17 13:41:31 +0000
1234+++ systemimage/channel.py 2015-05-20 14:55:53 +0000
1235@@ -1,4 +1,4 @@
1236-# Copyright (C) 2013-2014 Canonical Ltd.
1237+# Copyright (C) 2013-2015 Canonical Ltd.
1238 # Author: Barry Warsaw <barry@ubuntu.com>
1239
1240 # This program is free software: you can redistribute it and/or modify
1241
1242=== modified file 'systemimage/config.py'
1243--- systemimage/config.py 2014-09-17 13:41:31 +0000
1244+++ systemimage/config.py 2015-05-20 14:55:53 +0000
1245@@ -1,4 +1,4 @@
1246-# Copyright (C) 2013-2014 Canonical Ltd.
1247+# Copyright (C) 2013-2015 Canonical Ltd.
1248 # Author: Barry Warsaw <barry@ubuntu.com>
1249
1250 # This program is free software: you can redistribute it and/or modify
1251@@ -17,7 +17,6 @@
1252
1253 __all__ = [
1254 'Configuration',
1255- 'DISABLED',
1256 'config',
1257 ]
1258
1259@@ -27,79 +26,180 @@
1260
1261 from configparser import ConfigParser
1262 from contextlib import ExitStack
1263-from pkg_resources import resource_filename
1264+from pathlib import Path
1265 from systemimage.bag import Bag
1266 from systemimage.helpers import (
1267- as_loglevel, as_object, as_timedelta, makedirs, temporary_directory)
1268-
1269-
1270-DISABLED = object()
1271+ NO_PORT, as_loglevel, as_object, as_port, as_stripped, as_timedelta,
1272+ makedirs, temporary_directory)
1273+
1274+
1275+SECTIONS = ('service', 'system', 'gpg', 'updater', 'hooks', 'dbus')
1276+USER_AGENT = ('Ubuntu System Image Upgrade Client: '
1277+ 'device={0.device};channel={0.channel};build={0.build_number}')
1278
1279
1280 def expand_path(path):
1281 return os.path.abspath(os.path.expanduser(path))
1282
1283
1284-def port_value_converter(value):
1285- if value.lower() in ('disabled', 'disable'):
1286- return DISABLED
1287- result = int(value)
1288- if result < 0:
1289- raise ValueError(value)
1290- return result
1291-
1292-
1293-def device_converter(value):
1294- return value.strip()
1295+class SafeConfigParser(ConfigParser):
1296+ """Like ConfigParser, but with default empty sections.
1297+
1298+ This makes the **style of loading keys/values into the Bag objects a
1299+ little cleaner since it doesn't have to worry about KeyErrors when a
1300+ configuration file doesn't contain a section, which is allowed.
1301+ """
1302+
1303+ def __init__(self, *args, **kws):
1304+ super().__init__(args, **kws)
1305+ for section in SECTIONS:
1306+ self[section] = {}
1307
1308
1309 class Configuration:
1310- def __init__(self, ini_file=None):
1311- # Defaults.
1312- self.config_file = None
1313- self.service = Bag()
1314- self.system = Bag()
1315- if ini_file is None:
1316- ini_file = resource_filename('systemimage.data', 'client.ini')
1317- self.load(ini_file)
1318- self._override = False
1319- # 2013-10-14 BAW This is a placeholder for rendezvous between the
1320- # downloader and the D-Bus service. When running udner D-Bus and we
1321- # get a `paused` signal from the download manager, we need this to
1322- # plumb through an UpdatePaused signal to our clients. It rather
1323- # sucks that we need a global for this, but I can't get the plumbing
1324- # to work otherwise. This seems like the least horrible place to
1325- # stash this global.
1326+ def __init__(self, directory=None):
1327+ self._set_defaults()
1328+ # Because the configuration object is a global singleton, it makes for
1329+ # a convenient place to stash information used by widely separate
1330+ # components. For example, this is a placeholder for rendezvous
1331+ # between the downloader and the D-Bus service. When running under
1332+ # D-Bus and we get a `paused` signal from the download manager, we need
1333+ # this to plumb through an UpdatePaused signal to our clients. It
1334+ # rather sucks that we need a global for this, but I can't get the
1335+ # plumbing to work otherwise. This seems like the least horrible place
1336+ # to stash this global.
1337 self.dbus_service = None
1338- # Cache/overrides.
1339+ # This is used to plumb command line arguments from the main() to
1340+ # other parts of the system.
1341+ self.skip_gpg_verification = False
1342+ # Cache.
1343 self._device = None
1344 self._build_number = None
1345+ self.build_number_override = False
1346 self._channel = None
1347+ # This is used only to override the phased percentage via command line
1348+ # and the property setter.
1349+ self._phase_override = None
1350 self._tempdir = None
1351+ self.config_d = None
1352+ self.ini_files = []
1353+ self.http_base = None
1354+ self.https_base = None
1355+ if directory is not None:
1356+ self.load(directory)
1357+ self._calculate_http_bases()
1358 self._resources = ExitStack()
1359 atexit.register(self._resources.close)
1360
1361- def load(self, path, *, override=False):
1362- parser = ConfigParser()
1363- files_read = parser.read(path)
1364- if files_read != [path]:
1365- raise FileNotFoundError(path)
1366- self.config_file = path
1367- self.service.update(converters=dict(http_port=port_value_converter,
1368- https_port=port_value_converter,
1369+ def _set_defaults(self):
1370+ self.service = Bag(
1371+ base='system-image.ubuntu.com',
1372+ http_port=80,
1373+ https_port=443,
1374+ channel='daily',
1375+ build_number=0,
1376+ )
1377+ self.system = Bag(
1378+ timeout=as_timedelta('1h'),
1379+ tempdir='/tmp',
1380+ logfile='/var/log/system-image/client.log',
1381+ loglevel=as_loglevel('info'),
1382+ settings_db='/var/lib/system-image/settings.db',
1383+ )
1384+ self.gpg = Bag(
1385+ archive_master='/usr/share/system-image/archive-master.tar.xz',
1386+ image_master='/var/lib/system-image/keyrings/image-master.tar.xz',
1387+ image_signing=
1388+ '/var/lib/system-image/keyrings/image-signing.tar.xz',
1389+ device_signing=
1390+ '/var/lib/system-image/keyrings/device-signing.tar.xz',
1391+ )
1392+ self.updater = Bag(
1393+ cache_partition='/android/cache/recovery',
1394+ data_partition='/var/lib/system-image',
1395+ )
1396+ self.hooks = Bag(
1397+ device=as_object('systemimage.device.SystemProperty'),
1398+ scorer=as_object('systemimage.scores.WeightedScorer'),
1399+ apply=as_object('systemimage.apply.Reboot'),
1400+ )
1401+ self.dbus = Bag(
1402+ lifetime=as_timedelta('10m'),
1403+ )
1404+
1405+ def _load_file(self, path):
1406+ parser = SafeConfigParser()
1407+ str_path = str(path)
1408+ parser.read(str_path)
1409+ self.ini_files.append(path)
1410+ self.service.update(converters=dict(http_port=as_port,
1411+ https_port=as_port,
1412 build_number=int,
1413- device=device_converter,
1414+ device=as_stripped,
1415 ),
1416- **parser['service'])
1417- if (self.service.http_port is DISABLED and
1418- self.service.https_port is DISABLED):
1419+ **parser['service'])
1420+ self.system.update(converters=dict(timeout=as_timedelta,
1421+ loglevel=as_loglevel,
1422+ settings_db=expand_path,
1423+ tempdir=expand_path),
1424+ **parser['system'])
1425+ self.gpg.update(**parser['gpg'])
1426+ self.updater.update(**parser['updater'])
1427+ self.hooks.update(converters=dict(device=as_object,
1428+ scorer=as_object,
1429+ apply=as_object),
1430+ **parser['hooks'])
1431+ self.dbus.update(converters=dict(lifetime=as_timedelta),
1432+ **parser['dbus'])
1433+
1434+ def load(self, directory):
1435+ """Load up the configuration from a config.d directory."""
1436+ # Look for all the files in the given directory with .ini or .cfg
1437+ # suffixes. The files must start with a number, and the files are
1438+ # loaded in numeric order.
1439+ if self.config_d is not None:
1440+ raise RuntimeError('Configuration already loaded; use .reload()')
1441+ self.config_d = directory
1442+ if not Path(directory).is_dir():
1443+ raise TypeError(
1444+ '.load() requires a directory: {}'.format(directory))
1445+ candidates = []
1446+ for child in Path(directory).glob('*.ini'):
1447+ order, _, base = child.stem.partition('_')
1448+ # XXX 2014-10-03: The logging system isn't initialized when we get
1449+ # here, so we can't log that these files are being ignored.
1450+ if len(_) == 0:
1451+ continue
1452+ try:
1453+ serial = int(order)
1454+ except ValueError:
1455+ continue
1456+ candidates.append((serial, child))
1457+ for serial, path in sorted(candidates):
1458+ self._load_file(path)
1459+ self._calculate_http_bases()
1460+
1461+ def reload(self):
1462+ """Reload the configuration directory."""
1463+ # Reset some cached attributes.
1464+ directory = self.config_d
1465+ self.ini_files = []
1466+ self.config_d = None
1467+ self._build_number = None
1468+ # Now load the defaults, then reload the previous config.d directory.
1469+ self._set_defaults()
1470+ self.load(directory)
1471+
1472+ def _calculate_http_bases(self):
1473+ if (self.service.http_port is NO_PORT and
1474+ self.service.https_port is NO_PORT):
1475 raise ValueError('Cannot disable both http and https ports')
1476 # Construct the HTTP and HTTPS base urls, which most applications will
1477- # actually use. We do this in two steps, in order to support
1478- # disabling one or the other (but not both) protocols.
1479+ # actually use. We do this in two steps, in order to support disabling
1480+ # one or the other (but not both) protocols.
1481 if self.service.http_port == 80:
1482 http_base = 'http://{}'.format(self.service.base)
1483- elif self.service.http_port is DISABLED:
1484+ elif self.service.http_port is NO_PORT:
1485 http_base = None
1486 else:
1487 http_base = 'http://{}:{}'.format(
1488@@ -107,7 +207,7 @@
1489 # HTTPS.
1490 if self.service.https_port == 443:
1491 https_base = 'https://{}'.format(self.service.base)
1492- elif self.service.https_port is DISABLED:
1493+ elif self.service.https_port is NO_PORT:
1494 https_base = None
1495 else:
1496 https_base = 'https://{}:{}'.format(
1497@@ -119,45 +219,13 @@
1498 if https_base is None:
1499 assert http_base is not None
1500 https_base = http_base
1501- self.service['http_base'] = http_base
1502- self.service['https_base'] = https_base
1503- try:
1504- self.system.update(converters=dict(timeout=as_timedelta,
1505- build_file=expand_path,
1506- loglevel=as_loglevel,
1507- settings_db=expand_path,
1508- tempdir=expand_path),
1509- **parser['system'])
1510- except KeyError:
1511- # If we're overriding via a channel.ini file, it's okay if the
1512- # [system] section is missing. However, the main configuration
1513- # ini file must include all sections.
1514- if not override:
1515- raise
1516- # Short-circuit, since we're loading a channel.ini file.
1517- self._override = override
1518- if override:
1519- return
1520- self.gpg = Bag(**parser['gpg'])
1521- self.updater = Bag(**parser['updater'])
1522- self.hooks = Bag(converters=dict(device=as_object,
1523- scorer=as_object,
1524- reboot=as_object),
1525- **parser['hooks'])
1526- self.dbus = Bag(converters=dict(lifetime=as_timedelta),
1527- **parser['dbus'])
1528+ self.http_base = http_base
1529+ self.https_base = https_base
1530
1531 @property
1532 def build_number(self):
1533 if self._build_number is None:
1534- if self._override:
1535- return self.service.build_number
1536- else:
1537- try:
1538- with open(self.system.build_file, encoding='utf-8') as fp:
1539- return int(fp.read().strip())
1540- except FileNotFoundError:
1541- return 0
1542+ self._build_number = self.service.build_number
1543 return self._build_number
1544
1545 @build_number.setter
1546@@ -166,24 +234,18 @@
1547 raise ValueError(
1548 'integer is required, got: {}'.format(type(value).__name__))
1549 self._build_number = value
1550+ self.build_number_override = True
1551
1552 @build_number.deleter
1553 def build_number(self):
1554 self._build_number = None
1555
1556 @property
1557- def build_number_cli(self):
1558- return self._build_number
1559-
1560- @property
1561 def device(self):
1562 if self._device is None:
1563 # Start by looking for a [service]device setting. Use this if it
1564 # exists, otherwise fall back to calling the hook.
1565 self._device = getattr(self.service, 'device', None)
1566- # The key could exist in the channel.ini file, but its value could
1567- # be empty. That's semantically equivalent to a missing
1568- # [service]device setting.
1569 if not self._device:
1570 self._device = self.hooks.device().get_device()
1571 return self._device
1572@@ -203,6 +265,18 @@
1573 self._channel = value
1574
1575 @property
1576+ def phase_override(self):
1577+ return self._phase_override
1578+
1579+ @phase_override.setter
1580+ def phase_override(self, value):
1581+ self._phase_override = max(0, min(100, int(value)))
1582+
1583+ @phase_override.deleter
1584+ def phase_override(self):
1585+ self._phase_override = None
1586+
1587+ @property
1588 def tempdir(self):
1589 if self._tempdir is None:
1590 makedirs(self.system.tempdir)
1591@@ -211,21 +285,13 @@
1592 dir=self.system.tempdir))
1593 return self._tempdir
1594
1595-
1596-# Define the global configuration object. Normal use can be as simple as:
1597-#
1598-# from systemimage.config import config
1599-# build_file = config.system.build_file
1600-#
1601-# In the test suite though, the actual configuration object can be easily
1602-# patched by doing something like this:
1603-#
1604-# test_config = Configuration(...)
1605-# with unittest.mock.patch('config._config', test_config):
1606-# run_test()
1607-#
1608-# and now every module which does the first code example will get build_file
1609-# from the mocked Configuration instance.
1610+ @property
1611+ def user_agent(self):
1612+ return USER_AGENT.format(self)
1613+
1614+
1615+# Define the global configuration object. We use a proxy here so that
1616+# post-object creation loading will work.
1617
1618 _config = Configuration()
1619
1620
1621=== added file 'systemimage/curl.py'
1622--- systemimage/curl.py 1970-01-01 00:00:00 +0000
1623+++ systemimage/curl.py 2015-05-20 14:55:53 +0000
1624@@ -0,0 +1,275 @@
1625+# Copyright (C) 2014-2015 Canonical Ltd.
1626+# Author: Barry Warsaw <barry@ubuntu.com>
1627+
1628+# This program is free software: you can redistribute it and/or modify
1629+# it under the terms of the GNU General Public License as published by
1630+# the Free Software Foundation; version 3 of the License.
1631+#
1632+# This program is distributed in the hope that it will be useful,
1633+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1634+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1635+# GNU General Public License for more details.
1636+#
1637+# You should have received a copy of the GNU General Public License
1638+# along with this program. If not, see <http://www.gnu.org/licenses/>.
1639+
1640+"""Download files via PyCURL."""
1641+
1642+__all__ = [
1643+ 'CurlDownloadManager',
1644+ ]
1645+
1646+
1647+import pycurl
1648+import hashlib
1649+import logging
1650+
1651+from contextlib import ExitStack
1652+from gi.repository import GLib
1653+from systemimage.config import config
1654+from systemimage.download import Canceled, DownloadManagerBase
1655+
1656+log = logging.getLogger('systemimage')
1657+
1658+
1659+# Some cURL defaults. XXX pull these out of the configuration file.
1660+CONNECTION_TIMEOUT = 120 # seconds
1661+LOW_SPEED_LIMIT = 10
1662+LOW_SPEED_TIME = 120 # seconds
1663+MAX_REDIRECTS = 5
1664+MAX_TOTAL_CONNECTIONS = 4
1665+SELECT_TIMEOUT = 0.05 # 20fps
1666+
1667+
1668+def _curl_debug(debug_type, debug_msg): # pragma: no cover
1669+ from systemimage.testing.helpers import debug
1670+ with debug(end='') as ddlog:
1671+ ddlog('PYCURL:', debug_type, debug_msg)
1672+
1673+
1674+def make_testable(c):
1675+ # The test suite needs to make the PyCURL object accept the testing
1676+ # server's self signed certificate. It will mock this function.
1677+ pass
1678+
1679+
1680+class SingleDownload:
1681+ def __init__(self, record):
1682+ self.url, self.destination, self.expected_checksum = record
1683+ self._checksum = None
1684+ self._fp = None
1685+ self._resources = ExitStack()
1686+
1687+ def make_handle(self, *, HEAD):
1688+ # If we're doing GET, record some more information.
1689+ if not HEAD:
1690+ self._checksum = hashlib.sha256()
1691+ # Create the basic PyCURL object.
1692+ c = pycurl.Curl()
1693+ # Set the common options.
1694+ c.setopt(pycurl.URL, self.url)
1695+ c.setopt(pycurl.USERAGENT, config.user_agent)
1696+ # If we're doing a HEAD, then we don't want the body of the
1697+ # file. Otherwise, set things up to write the body data to the
1698+ # destination file.
1699+ if HEAD:
1700+ c.setopt(pycurl.NOBODY, 1)
1701+ else:
1702+ c.setopt(pycurl.WRITEDATA, self)
1703+ self._fp = self._resources.enter_context(
1704+ open(self.destination, 'wb'))
1705+ # Set some limits. XXX Pull these out of the configuration files.
1706+ c.setopt(pycurl.FOLLOWLOCATION, 1)
1707+ c.setopt(pycurl.MAXREDIRS, MAX_REDIRECTS)
1708+ c.setopt(pycurl.CONNECTTIMEOUT, CONNECTION_TIMEOUT)
1709+ # If the average transfer speed is below 10 bytes per second for 2
1710+ # minutes, libcurl will consider the connection too slow and abort.
1711+ ## c.setopt(pycurl.LOW_SPEED_LIMIT, LOW_SPEED_LIMIT)
1712+ ## c.setopt(pycurl.LOW_SPEED_TIME, LOW_SPEED_TIME)
1713+ # Fail on error codes >= 400.
1714+ c.setopt(pycurl.FAILONERROR, 1)
1715+ # Switch off the libcurl progress meters. The multi that uses
1716+ # this handle will set the transfer info function.
1717+ c.setopt(pycurl.NOPROGRESS, 1)
1718+ # ssl: no need to set SSL_VERIFYPEER, SSL_VERIFYHOST, CAINFO
1719+ # they all use sensible defaults
1720+ #
1721+ # Enable debugging.
1722+ self._make_debuggable(c)
1723+ # For the test suite.
1724+ make_testable(c)
1725+ return c
1726+
1727+ def _make_debuggable(self, c):
1728+ """Add some additional debugging options."""
1729+ ## c.setopt(pycurl.VERBOSE, 1)
1730+ ## c.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
1731+ pass
1732+
1733+ def write(self, data):
1734+ """Update the checksum and write the data out to the file."""
1735+ self._checksum.update(data)
1736+ self._fp.write(data)
1737+ # Returning None implies that all bytes were written
1738+ # successfully, so it's better to be explicit.
1739+ return None
1740+
1741+ def close(self):
1742+ self._resources.close()
1743+
1744+ @property
1745+ def checksum(self):
1746+ # If no checksum was expected, pretend none was gotten. This
1747+ # makes the verification step below a wee bit simpler.
1748+ if self.expected_checksum == '':
1749+ return ''
1750+ return self._checksum.hexdigest()
1751+
1752+
1753+class CurlDownloadManager(DownloadManagerBase):
1754+ """The PyCURL based download manager."""
1755+
1756+ def __init__(self, callback=None):
1757+ super().__init__()
1758+ if callback is not None:
1759+ self.callbacks.append(callback)
1760+ self._pausables = []
1761+ self._paused = False
1762+
1763+ def _get_files(self, records, pausable):
1764+ # Start by doing a HEAD on all the URLs so that we can get the total
1765+ # target download size in bytes, at least as best as is possible.
1766+ with ExitStack() as resources:
1767+ handles = []
1768+ multi = pycurl.CurlMulti()
1769+ multi.setopt(
1770+ pycurl.M_MAX_TOTAL_CONNECTIONS, MAX_TOTAL_CONNECTIONS)
1771+ for record in records:
1772+ download = SingleDownload(record)
1773+ resources.callback(download.close)
1774+ handle = download.make_handle(HEAD=True)
1775+ handles.append(handle)
1776+ multi.add_handle(handle)
1777+ # .add_handle() does not bump the reference count, so we
1778+ # need to keep the PyCURL object alive for the duration
1779+ # of this download.
1780+ resources.callback(multi.remove_handle, handle)
1781+ self._perform(multi, handles)
1782+ self.total = sum(
1783+ handle.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
1784+ for handle in handles)
1785+ # Now do a GET on all the URLs. This will write the data to the
1786+ # destination file and collect the checksums.
1787+ with ExitStack() as resources:
1788+ resources.callback(setattr, self, '_handles', None)
1789+ downloads = []
1790+ multi = pycurl.CurlMulti()
1791+ multi.setopt(
1792+ pycurl.M_MAX_TOTAL_CONNECTIONS, MAX_TOTAL_CONNECTIONS)
1793+ for record in records:
1794+ download = SingleDownload(record)
1795+ downloads.append(download)
1796+ resources.callback(download.close)
1797+ handle = download.make_handle(HEAD=False)
1798+ self._pausables.append(handle)
1799+ multi.add_handle(handle)
1800+ # .add_handle() does not bump the reference count, so we
1801+ # need to keep the PyCURL object alive for the duration
1802+ # of this download.
1803+ resources.callback(multi.remove_handle, handle)
1804+ self._perform(multi, self._pausables)
1805+ # Verify internally calculated checksums. The API requires
1806+ # a FileNotFoundError to be raised when they don't match.
1807+ # Since it doesn't matter which one fails, log them all and
1808+ # raise the first one.
1809+ first_mismatch = None
1810+ for download in downloads:
1811+ if download.checksum != download.expected_checksum:
1812+ log.error('Checksum mismatch. got:{} != exp:{}: {}',
1813+ download.checksum, download.expected_checksum,
1814+ download.destination)
1815+ if first_mismatch is None:
1816+ first_mismatch = download
1817+ if first_mismatch is not None:
1818+ # For backward compatibility with ubuntu-download_manager.
1819+ raise FileNotFoundError('HASH ERROR: {}'.format(
1820+ first_mismatch.destination))
1821+ self._pausables = []
1822+
1823+ def _do_once(self, multi, handles):
1824+ status, active_count = multi.perform()
1825+ if status == pycurl.E_CALL_MULTI_PERFORM:
1826+ # Call .perform() again before calling select.
1827+ return True
1828+ elif status != pycurl.E_OK:
1829+ # An error occurred in the multi, so be done with the
1830+ # whole thing. We can't get a description string out of
1831+ # PyCURL though. Just raise one of the urls.
1832+ log.error('CurlMulti() error: {}', status)
1833+ raise FileNotFoundError(handles[0].getinfo(pycurl.EFFECTIVE_URL))
1834+ # The multi is okay, but it's possible there are errors pending on
1835+ # the individual downloads; check those now.
1836+ queued_count, ok_list, error_list = multi.info_read()
1837+ if len(error_list) > 0:
1838+ # It helps to have at least one URL in the FileNotFoundError.
1839+ first_url = None
1840+ log.error('Curl() errors encountered:')
1841+ for c, code, message in error_list:
1842+ url = c.getinfo(pycurl.EFFECTIVE_URL)
1843+ if first_url is None:
1844+ first_url = url
1845+ log.error(' {} ({}): {}', message, code, url)
1846+ raise FileNotFoundError('{}: {}'.format(message, first_url))
1847+ # For compatibility with .io_add_watch(), we return False if we want
1848+ # to stop the callbacks, and True if we want to call back here again.
1849+ return active_count > 0
1850+
1851+ def _perform(self, multi, handles):
1852+ # While we're performing the cURL downloads, we need to periodically
1853+ # process D-Bus events, otherwise we won't be able to cancel downloads
1854+ # or handle other interruptive events. To do this, we grab the GLib
1855+ # main loop context and then ask it to do an iteration over its events
1856+ # once in a while. It turns out that even if we're not running a D-Bus
1857+ # main loop (i.e. during the in-process tests) periodically dispatching
1858+ # into GLib doesn't hurt, so just do it unconditionally.
1859+ self.received = 0
1860+ context = GLib.main_context_default()
1861+ while True:
1862+ # Do the progress callback, but only if the current received size
1863+ # is different than the last one. Don't worry about in which
1864+ # direction it's different.
1865+ received = int(
1866+ sum(c.getinfo(pycurl.SIZE_DOWNLOAD) for c in handles))
1867+ if received != self.received:
1868+ self._do_callback()
1869+ self.received = received
1870+ if not self._do_once(multi, handles):
1871+ break
1872+ multi.select(SELECT_TIMEOUT)
1873+ # Let D-Bus events get dispatched, but only block if downloads are
1874+ # paused.
1875+ while context.iteration(may_block=self._paused):
1876+ pass
1877+ if self._queued_cancel:
1878+ raise Canceled
1879+ # One last callback, unconditionally.
1880+ self.received = int(
1881+ sum(c.getinfo(pycurl.SIZE_DOWNLOAD) for c in handles))
1882+ self._do_callback()
1883+
1884+ def pause(self):
1885+ for c in self._pausables:
1886+ c.pause(pycurl.PAUSE_ALL)
1887+ self._paused = True
1888+ # 2014-10-20 BAW: We could plumb through the `service` object from
1889+ # service.py (the main entry point for system-image-dbus, but that's
1890+ # actually a bit of a pain, so do the expedient thing and grab the
1891+ # interface here.
1892+ percentage = (int(self.received / self.total * 100.0)
1893+ if self.total > 0 else 0)
1894+ config.dbus_service.UpdatePaused(percentage)
1895+
1896+ def resume(self):
1897+ self._paused = False
1898+ for c in self._pausables:
1899+ c.pause(pycurl.PAUSE_CONT)
1900
1901=== removed file 'systemimage/data/client.ini'
1902--- systemimage/data/client.ini 2014-01-30 15:41:03 +0000
1903+++ systemimage/data/client.ini 1970-01-01 00:00:00 +0000
1904@@ -1,35 +0,0 @@
1905-# Default and example .ini configuration file.
1906-# Edit this and put it in /etc/system-image/client.ini
1907-
1908-[service]
1909-base: system-image.ubuntu.com
1910-http_port: 80
1911-https_port: 443
1912-channel: daily
1913-build_number: 0
1914-
1915-[system]
1916-timeout: 1h
1917-build_file: /etc/ubuntu-build
1918-tempdir: /tmp
1919-logfile: /var/log/system-image/client.log
1920-loglevel: info
1921-settings_db: /var/lib/system-image/settings.db
1922-
1923-[gpg]
1924-archive_master: /etc/system-image/archive-master.tar.xz
1925-image_master: /var/lib/system-image/keyrings/image-master.tar.xz
1926-image_signing: /var/lib/system-image/keyrings/image-signing.tar.xz
1927-device_signing: /var/lib/system-image/keyrings/device-signing.tar.xz
1928-
1929-[updater]
1930-cache_partition: /android/cache/recovery
1931-data_partition: /var/lib/system-image
1932-
1933-[hooks]
1934-device: systemimage.device.SystemProperty
1935-scorer: systemimage.scores.WeightedScorer
1936-reboot: systemimage.reboot.Reboot
1937-
1938-[dbus]
1939-lifetime: 10m
1940
1941=== modified file 'systemimage/dbus.py'
1942--- systemimage/dbus.py 2014-09-26 14:36:34 +0000
1943+++ systemimage/dbus.py 2015-05-20 14:55:53 +0000
1944@@ -1,4 +1,4 @@
1945-# Copyright (C) 2013-2014 Canonical Ltd.
1946+# Copyright (C) 2013-2015 Canonical Ltd.
1947 # Author: Barry Warsaw <barry@ubuntu.com>
1948
1949 # This program is free software: you can redistribute it and/or modify
1950@@ -94,14 +94,14 @@
1951
1952 def __init__(self, bus, object_path, loop):
1953 super().__init__(bus, object_path)
1954- self._loop = loop
1955+ self.loop = loop
1956 self._api = Mediator(self._progress_callback)
1957 log.info('Mediator created {}', self._api)
1958 self._checking = Lock()
1959+ self._downloading = Lock()
1960 self._update = None
1961- self._downloading = False
1962 self._paused = False
1963- self._rebootable = False
1964+ self._applicable = False
1965 self._failure_count = 0
1966 self._last_error = ''
1967
1968@@ -110,8 +110,15 @@
1969 # Asynchronous method call.
1970 log.info('Enter _check_for_update()')
1971 self._update = self._api.check_for_update()
1972+ log.info('_check_for_update(): checking lock releasing')
1973+ try:
1974+ self._checking.release()
1975+ except RuntimeError:
1976+ log.info('_check_for_update(): checking lock already released')
1977+ else:
1978+ log.info('_check_for_update(): checking lock released')
1979 # Do we have an update and can we auto-download it?
1980- downloading = False
1981+ delayed_download = False
1982 if self._update.is_available:
1983 settings = Settings()
1984 auto = settings.get('auto_download')
1985@@ -119,14 +126,19 @@
1986 if auto in ('1', '2'):
1987 # XXX When we have access to the download service, we can
1988 # check if we're on the wifi (auto == '1').
1989+ delayed_download = True
1990 GLib.timeout_add(50, self._download)
1991- downloading = True
1992+ # We have a timing issue. We can't lock the downloading lock here,
1993+ # otherwise when _download() starts running in ~50ms it will think a
1994+ # download is already in progress. But we want to send the UAS signal
1995+ # here and now, *and* indicate whether the download is about to happen.
1996+ # So just lie for now since in ~50ms the download will begin.
1997 self.UpdateAvailableStatus(
1998 self._update.is_available,
1999- downloading,
2000+ delayed_download,
2001 self._update.version,
2002 self._update.size,
2003- self._update.last_update_date,
2004+ last_update_date(),
2005 self._update.error)
2006 # Stop GLib from calling this method again.
2007 return False
2008@@ -148,24 +160,24 @@
2009 completes. The argument to that signal is a boolean indicating
2010 whether the update is available or not.
2011 """
2012- self._loop.keepalive()
2013+ self.loop.keepalive()
2014 # Check-and-acquire the lock.
2015- log.info('test and acquire checking lock')
2016+ log.info('CheckForUpdate(): checking lock test and acquire')
2017 if not self._checking.acquire(blocking=False):
2018+ log.info('CheckForUpdate(): checking lock not acquired')
2019 # Check is already in progress, so there's nothing more to do. If
2020 # there's status available (i.e. we are in the auto-downloading
2021 # phase of the last CFU), then send the status.
2022 if self._update is not None:
2023 self.UpdateAvailableStatus(
2024 self._update.is_available,
2025- self._downloading,
2026+ self._downloading.locked(),
2027 self._update.version,
2028 self._update.size,
2029- self._update.last_update_date,
2030+ last_update_date(),
2031 "")
2032- log.info('checking lock not acquired')
2033 return
2034- log.info('checking lock acquired')
2035+ log.info('CheckForUpdate(): checking lock acquired')
2036 # We've now acquired the lock. Reset any failure or in-progress
2037 # state. Get a new mediator to reset any of its state.
2038 self._api = Mediator(self._progress_callback)
2039@@ -176,7 +188,7 @@
2040 # this method can return immediately.
2041 GLib.timeout_add(50, self._check_for_update)
2042
2043- @log_and_exit
2044+ #@log_and_exit
2045 def _progress_callback(self, received, total):
2046 # Plumb the progress through our own D-Bus API. Our API is defined as
2047 # signalling a percentage and an eta. We can calculate the percentage
2048@@ -187,12 +199,12 @@
2049
2050 @log_and_exit
2051 def _download(self):
2052- if self._downloading and self._paused:
2053+ if self._downloading.locked() and self._paused:
2054 self._api.resume()
2055 self._paused = False
2056 log.info('Download previously paused')
2057 return
2058- if (self._downloading # Already in progress.
2059+ if (self._downloading.locked() # Already in progress.
2060 or self._update is None # Not yet checked.
2061 or not self._update.is_available # No update available.
2062 ):
2063@@ -204,47 +216,33 @@
2064 log.info('Update failures: {}; last error: {}',
2065 self._failure_count, self._last_error)
2066 return
2067- self._downloading = True
2068- log.info('Update is downloading')
2069- try:
2070- # Always start by sending a UpdateProgress(0, 0). This is
2071- # enough to get the u/i's attention.
2072- self.UpdateProgress(0, 0)
2073- self._api.download()
2074- except Exception:
2075- log.exception('Download failed')
2076- self._failure_count += 1
2077- # Set the last error string to the exception's class name.
2078- exception, value = sys.exc_info()[:2]
2079- # if there's no meaningful value, omit it.
2080- value_str = str(value)
2081- name = exception.__name__
2082- self._last_error = ('{}'.format(name)
2083- if len(value_str) == 0
2084- else '{}: {}'.format(name, value))
2085- self.UpdateFailed(self._failure_count, self._last_error)
2086- else:
2087- log.info('Update downloaded')
2088- self.UpdateDownloaded()
2089- self._failure_count = 0
2090- self._last_error = ''
2091- self._rebootable = True
2092- self._downloading = False
2093- log.info('releasing checking lock from _download()')
2094- try:
2095- self._checking.release()
2096- except RuntimeError:
2097- # 2014-09-11 BAW: We don't own the lock. There are several reasons
2098- # why this can happen including: 1) the client canceled the
2099- # download while it was in progress, and CancelUpdate() already
2100- # released the lock; 2) the client called DownloadUpdate() without
2101- # first calling CheckForUpdate(); 3) the client called DU()
2102- # multiple times in a row but the update was already downloaded and
2103- # all the file signatures have been verified. I can't think of
2104- # reason why we shouldn't just ignore the double release, so
2105- # that's what we do. See LP: #1365646.
2106- pass
2107- log.info('released checking lock from _download()')
2108+ log.info('_download(): downloading lock entering critical section')
2109+ with self._downloading:
2110+ log.info('Update is downloading')
2111+ try:
2112+ # Always start by sending a UpdateProgress(0, 0). This is
2113+ # enough to get the u/i's attention.
2114+ self.UpdateProgress(0, 0)
2115+ self._api.download()
2116+ except Exception:
2117+ log.exception('Download failed')
2118+ self._failure_count += 1
2119+ # Set the last error string to the exception's class name.
2120+ exception, value = sys.exc_info()[:2]
2121+ # if there's no meaningful value, omit it.
2122+ value_str = str(value)
2123+ name = exception.__name__
2124+ self._last_error = ('{}'.format(name)
2125+ if len(value_str) == 0
2126+ else '{}: {}'.format(name, value))
2127+ self.UpdateFailed(self._failure_count, self._last_error)
2128+ else:
2129+ log.info('Update downloaded')
2130+ self.UpdateDownloaded()
2131+ self._failure_count = 0
2132+ self._last_error = ''
2133+ self._applicable = True
2134+ log.info('_download(): downloading lock finished critical section')
2135 # Stop GLib from calling this method again.
2136 return False
2137
2138@@ -257,15 +255,15 @@
2139 """
2140 # Arrange for the update to happen in a little while, so that this
2141 # method can return immediately.
2142- self._loop.keepalive()
2143+ self.loop.keepalive()
2144 GLib.timeout_add(50, self._download)
2145
2146 @log_and_exit
2147 @method('com.canonical.SystemImage', out_signature='s')
2148 def PauseDownload(self):
2149 """Pause a downloading update."""
2150- self._loop.keepalive()
2151- if self._downloading:
2152+ self.loop.keepalive()
2153+ if self._downloading.locked():
2154 self._api.pause()
2155 self._paused = True
2156 error_message = ''
2157@@ -277,40 +275,34 @@
2158 @method('com.canonical.SystemImage', out_signature='s')
2159 def CancelUpdate(self):
2160 """Cancel a download."""
2161- self._loop.keepalive()
2162+ self.loop.keepalive()
2163 # During the download, this will cause an UpdateFailed signal to be
2164 # issued, as part of the exception handling in _download(). If we're
2165 # not downloading, then no signal need be sent. There's no need to
2166 # send *another* signal when downloading, because we never will be
2167 # downloading by the time we get past this next call.
2168 self._api.cancel()
2169- # If we're holding the checking lock, release it.
2170- try:
2171- log.info('releasing checking lock from CancelUpdate()')
2172- self._checking.release()
2173- log.info('released checking lock from CancelUpdate()')
2174- except RuntimeError:
2175- # We're not holding the lock.
2176- pass
2177 # XXX 2013-08-22: If we can't cancel the current download, return the
2178 # reason in this string.
2179 return ''
2180
2181 @log_and_exit
2182 def _apply_update(self):
2183- self._loop.keepalive()
2184- if not self._rebootable:
2185+ self.loop.keepalive()
2186+ if not self._applicable:
2187 command_file = os.path.join(
2188 config.updater.cache_partition, 'ubuntu_command')
2189 if not os.path.exists(command_file):
2190- # Not enough has been downloaded to allow for a reboot.
2191- self.Rebooting(False)
2192+ # Not enough has been downloaded to allow for the update to be
2193+ # applied.
2194+ self.Applied(False)
2195 return
2196- self._api.reboot()
2197- # This code may or may not run. We're racing against the system
2198- # reboot procedure.
2199- self._rebootable = False
2200- self.Rebooting(True)
2201+ self._api.apply()
2202+ # This code may or may not run. On devices for which applying the
2203+ # update requires a system reboot, we're racing against that reboot
2204+ # procedure.
2205+ self._applicable = False
2206+ self.Applied(True)
2207
2208 @log_and_exit
2209 @method('com.canonical.SystemImage')
2210@@ -322,7 +314,7 @@
2211 @log_and_exit
2212 @method('com.canonical.SystemImage', out_signature='isssa{ss}')
2213 def Info(self):
2214- self._loop.keepalive()
2215+ self.loop.keepalive()
2216 return (config.build_number,
2217 config.device,
2218 config.channel,
2219@@ -332,23 +324,27 @@
2220 @log_and_exit
2221 @method('com.canonical.SystemImage', out_signature='a{ss}')
2222 def Information(self):
2223- self._loop.keepalive()
2224+ self.loop.keepalive()
2225 settings = Settings()
2226 current_build_number = str(config.build_number)
2227+ version_detail = getattr(config.service, 'version_detail', '')
2228 response = dict(
2229 current_build_number=current_build_number,
2230 device_name=config.device,
2231 channel_name=config.channel,
2232 last_update_date=last_update_date(),
2233- version_detail=getattr(config.service, 'version_detail', ''),
2234+ version_detail=version_detail,
2235 last_check_date=settings.get('last_check_date'),
2236 )
2237 if self._update is None:
2238 response['target_build_number'] = '-1'
2239+ response['target_version_detail'] = ''
2240 elif not self._update.is_available:
2241 response['target_build_number'] = current_build_number
2242+ response['target_version_detail'] = version_detail
2243 else:
2244 response['target_build_number'] = str(self._update.version)
2245+ response['target_version_detail'] = self._update.version_detail
2246 return response
2247
2248 @log_and_exit
2249@@ -359,7 +355,7 @@
2250 Some values are special, e.g. min_battery and auto_downloads.
2251 Implement these special semantics here.
2252 """
2253- self._loop.keepalive()
2254+ self.loop.keepalive()
2255 if key == 'min_battery':
2256 try:
2257 as_int = int(value)
2258@@ -385,22 +381,24 @@
2259 @method('com.canonical.SystemImage', in_signature='s', out_signature='s')
2260 def GetSetting(self, key):
2261 """Get a setting."""
2262- self._loop.keepalive()
2263+ self.loop.keepalive()
2264 return Settings().get(key)
2265
2266 @log_and_exit
2267 @method('com.canonical.SystemImage')
2268 def FactoryReset(self):
2269 self._api.factory_reset()
2270- # This code may or may not run. We're racing against the system
2271- # reboot procedure.
2272- self.Rebooting(True)
2273+
2274+ @log_and_exit
2275+ @method('com.canonical.SystemImage')
2276+ def ProductionReset(self):
2277+ self._api.production_reset()
2278
2279 @log_and_exit
2280 @method('com.canonical.SystemImage')
2281 def Exit(self):
2282 """Quit the daemon immediately."""
2283- self._loop.quit()
2284+ self.loop.quit()
2285
2286 @log_and_exit
2287 @signal('com.canonical.SystemImage', signature='bbsiss')
2288@@ -416,21 +414,21 @@
2289 log.debug('EMIT UpdateAvailableStatus({}, {}, {}, {}, {}, {})',
2290 is_available, downloading, available_version, update_size,
2291 last_update_date, repr(error_reason))
2292- self._loop.keepalive()
2293+ self.loop.keepalive()
2294
2295- @log_and_exit
2296+ #@log_and_exit
2297 @signal('com.canonical.SystemImage', signature='id')
2298 def UpdateProgress(self, percentage, eta):
2299 """Download progress."""
2300 log.debug('EMIT UpdateProgress({}, {})', percentage, eta)
2301- self._loop.keepalive()
2302+ self.loop.keepalive()
2303
2304 @log_and_exit
2305 @signal('com.canonical.SystemImage')
2306 def UpdateDownloaded(self):
2307 """The update has been successfully downloaded."""
2308 log.debug('EMIT UpdateDownloaded()')
2309- self._loop.keepalive()
2310+ self.loop.keepalive()
2311
2312 @log_and_exit
2313 @signal('com.canonical.SystemImage', signature='is')
2314@@ -438,21 +436,28 @@
2315 """The update failed for some reason."""
2316 log.debug('EMIT UpdateFailed({}, {})',
2317 consecutive_failure_count, repr(last_reason))
2318- self._loop.keepalive()
2319+ self.loop.keepalive()
2320
2321 @log_and_exit
2322 @signal('com.canonical.SystemImage', signature='i')
2323 def UpdatePaused(self, percentage):
2324 """The download got paused."""
2325 log.debug('EMIT UpdatePaused({})', percentage)
2326- self._loop.keepalive()
2327+ self.loop.keepalive()
2328
2329 @log_and_exit
2330 @signal('com.canonical.SystemImage', signature='ss')
2331 def SettingChanged(self, key, new_value):
2332 """A setting value has change."""
2333 log.debug('EMIT SettingChanged({}, {})', repr(key), repr(new_value))
2334- self._loop.keepalive()
2335+ self.loop.keepalive()
2336+
2337+ @log_and_exit
2338+ @signal('com.canonical.SystemImage', signature='b')
2339+ def Applied(self, status):
2340+ """The update has been applied."""
2341+ log.debug('EMIT Applied({})', status)
2342+ self.loop.keepalive()
2343
2344 @log_and_exit
2345 @signal('com.canonical.SystemImage', signature='b')
2346
2347=== modified file 'systemimage/device.py'
2348--- systemimage/device.py 2014-09-17 13:41:31 +0000
2349+++ systemimage/device.py 2015-05-20 14:55:53 +0000
2350@@ -1,4 +1,4 @@
2351-# Copyright (C) 2013-2014 Canonical Ltd.
2352+# Copyright (C) 2013-2015 Canonical Ltd.
2353 # Author: Barry Warsaw <barry@ubuntu.com>
2354
2355 # This program is free software: you can redistribute it and/or modify
2356
2357=== modified file 'systemimage/docs/conf.py'
2358--- systemimage/docs/conf.py 2014-02-20 23:03:24 +0000
2359+++ systemimage/docs/conf.py 2015-05-20 14:55:53 +0000
2360@@ -41,7 +41,7 @@
2361
2362 # General information about the project.
2363 project = u'Image Update Resolver'
2364-copyright = u'2013-2014, Canonical Ltd.'
2365+copyright = u'2013-2015, Canonical Ltd.'
2366
2367 # The version info for the project you're documenting, acts as replacement for
2368 # |version| and |release|, also used in various other places throughout the
2369
2370=== modified file 'systemimage/download.py'
2371--- systemimage/download.py 2014-09-17 13:41:31 +0000
2372+++ systemimage/download.py 2015-05-20 14:55:53 +0000
2373@@ -1,4 +1,4 @@
2374-# Copyright (C) 2013-2014 Canonical Ltd.
2375+# Copyright (C) 2013-2015 Canonical Ltd.
2376 # Author: Barry Warsaw <barry@ubuntu.com>
2377
2378 # This program is free software: you can redistribute it and/or modify
2379@@ -17,9 +17,9 @@
2380
2381 __all__ = [
2382 'Canceled',
2383- 'DBusDownloadManager',
2384 'DuplicateDestinationError',
2385 'Record',
2386+ 'get_download_manager',
2387 ]
2388
2389
2390@@ -28,39 +28,18 @@
2391 import logging
2392
2393 from collections import namedtuple
2394-from contextlib import suppress
2395 from io import StringIO
2396 from pprint import pformat
2397-from systemimage.config import config
2398-from systemimage.reactor import Reactor
2399-from systemimage.settings import Settings
2400-
2401-# The systemimage.testing module will not be available on installed systems
2402-# unless the system-image-dev binary package is installed, which is not usually
2403-# the case. Disable _print() debugging in that case.
2404-def _print(*args, **kws):
2405- with suppress(ImportError):
2406- # We must import this here to avoid circular imports.
2407- from systemimage.testing.helpers import debug
2408- with debug(check_flag=True) as ddlog:
2409- ddlog(*args, **kws)
2410-
2411-
2412-# Parameterized for testing purposes.
2413-DOWNLOADER_INTERFACE = 'com.canonical.applications.Downloader'
2414-MANAGER_INTERFACE = 'com.canonical.applications.DownloadManager'
2415-OBJECT_NAME = 'com.canonical.applications.Downloader'
2416-OBJECT_INTERFACE = 'com.canonical.applications.GroupDownload'
2417-USER_AGENT = 'Ubuntu System Image Upgrade Client; Build {}'
2418+
2419+try:
2420+ import pycurl
2421+except ImportError: # pragma: no cover
2422+ pycurl = None
2423
2424
2425 log = logging.getLogger('systemimage')
2426
2427
2428-def _headers():
2429- return {'User-Agent': USER_AGENT.format(config.build_number)}
2430-
2431-
2432 class Canceled(Exception):
2433 """Raised when the download was canceled."""
2434
2435@@ -89,78 +68,10 @@
2436 url=url, destination=destination, checksum=checksum)
2437
2438
2439-class DownloadReactor(Reactor):
2440- def __init__(self, bus, callback=None, pausable=False):
2441- super().__init__(bus)
2442- self._callback = callback
2443- self._pausable = pausable
2444- self.error = None
2445- self.canceled = False
2446- self.received = 0
2447- self.total = 0
2448- self.react_to('canceled')
2449- self.react_to('error')
2450- self.react_to('finished')
2451- self.react_to('paused')
2452- self.react_to('progress')
2453- self.react_to('resumed')
2454- self.react_to('started')
2455-
2456- def _do_started(self, signal, path, started):
2457- _print('STARTED:', started)
2458-
2459- def _do_finished(self, signal, path, local_paths):
2460- _print('FINISHED:', local_paths)
2461- self.quit()
2462-
2463- def _do_error(self, signal, path, error_message):
2464- _print('ERROR:', error_message)
2465- log.error(error_message)
2466- self.error = error_message
2467- self.quit()
2468-
2469- def _do_progress(self, signal, path, received, total):
2470- self.received = received
2471- self.total = total
2472- _print('PROGRESS:', received, total)
2473- if self._callback is not None:
2474- # Be defensive, so yes, use a bare except. If an exception occurs
2475- # in the callback, log it, but continue onward.
2476- try:
2477- self._callback(received, total)
2478- except:
2479- log.exception('Exception in progress callback')
2480-
2481- def _do_canceled(self, signal, path, canceled):
2482- # Why would we get this signal if it *wasn't* canceled? Anyway,
2483- # this'll be a D-Bus data type so converted it to a vanilla Python
2484- # boolean.
2485- _print('CANCELED:', canceled)
2486- self.canceled = bool(canceled)
2487- self.quit()
2488-
2489- def _do_paused(self, signal, path, paused):
2490- _print('PAUSE:', paused, self._pausable)
2491- send_paused = self._pausable and config.dbus_service is not None
2492- if send_paused: # pragma: no branch
2493- # We could plumb through the `service` object from service.py (the
2494- # main entry point for system-image-dbus, but that's actually a
2495- # bit of a pain, so do the expedient thing and grab the interface
2496- # here.
2497- percentage = (int(self.received / self.total * 100.0)
2498- if self.total > 0 else 0)
2499- config.dbus_service.UpdatePaused(percentage)
2500-
2501- def _do_resumed(self, signal, path, resumed):
2502- _print('RESUME:', resumed)
2503- # There currently is no UpdateResumed() signal.
2504-
2505- def _default(self, *args, **kws):
2506- _print('SIGNAL:', args, kws) # pragma: no cover
2507-
2508-
2509-class DBusDownloadManager:
2510- def __init__(self, callback=None):
2511+class DownloadManagerBase:
2512+ """Base class for all download managers."""
2513+
2514+ def __init__(self):
2515 """
2516 :param callback: If given, a function that is called every so often
2517 during downloading.
2518@@ -168,12 +79,79 @@
2519 of bytes received so far, and the total amount of bytes to be
2520 downloaded.
2521 """
2522- self._iface = None
2523+ # This is a list of functions that are called every so often during
2524+ # downloading. Functions in this list take two arguments, the number
2525+ # of bytes received so far, and the total amount of bytes to be
2526+ # downloaded.
2527+ self.callbacks = []
2528+ self.total = 0
2529+ self.received = 0
2530 self._queued_cancel = False
2531- self.callback = callback
2532
2533 def __repr__(self): # pragma: no cover
2534- return '<DBusDownloadManager at 0x{:x}>'.format(id(self))
2535+ return '<{} at 0x{:x}>'.format(self.__class__.__name__, id(self))
2536+
2537+ def _get_download_records(self, downloads):
2538+ """Convert the downloads items to download records."""
2539+ records = [item if isinstance(item, _RecordType) else Record(*item)
2540+ for item in downloads]
2541+ destinations = set(record.destination for record in records)
2542+ # Check for duplicate destinations, specifically for a local file path
2543+ # coming from two different sources. It's okay if there are duplicate
2544+ # destination records in the download request, but each of those must
2545+ # be specified by the same source url and have the same checksum.
2546+ #
2547+ # An easy quick check just asks if the set of destinations is smaller
2548+ # than the total number of requested downloads. It can't be larger.
2549+ # If it *is* smaller, then there are some duplicates, however the
2550+ # duplicates may be legitimate, so look at the details.
2551+ #
2552+ # Note though that we cannot pass duplicates destinations to udm, so we
2553+ # have to filter out legitimate duplicates. That's fine since they
2554+ # really are pointing to the same file, and will end up in the
2555+ # destination location.
2556+ if len(destinations) < len(downloads):
2557+ by_destination = dict()
2558+ unique_downloads = set()
2559+ for record in records:
2560+ by_destination.setdefault(record.destination, set()).add(
2561+ record)
2562+ unique_downloads.add(record)
2563+ duplicates = []
2564+ for dst, seen in by_destination.items():
2565+ if len(seen) > 1:
2566+ # Tuples will look better in the pretty-printed output.
2567+ duplicates.append(
2568+ (dst, sorted(tuple(dup) for dup in seen)))
2569+ if len(duplicates) > 0:
2570+ raise DuplicateDestinationError(sorted(duplicates))
2571+ # Uniquify the downloads.
2572+ records = list(unique_downloads)
2573+ return records
2574+
2575+ def _do_callback(self):
2576+ # Be defensive, so yes, use a bare except. If an exception occurs in
2577+ # the callback, log it, but continue onward.
2578+ for callback in self.callbacks:
2579+ try:
2580+ callback(self.received, self.total)
2581+ except:
2582+ log.exception('Exception in progress callback')
2583+
2584+ def cancel(self):
2585+ """Cancel any current downloads."""
2586+ self._queued_cancel = True
2587+
2588+ def pause(self):
2589+ """Pause the download, but only if one is in progress."""
2590+ pass # pragma: no cover
2591+
2592+ def resume(self):
2593+ """Resume the download, but only if one is in progress."""
2594+ pass # pragma: no cover
2595+
2596+ def _get_files(self, records, pausable):
2597+ raise NotImplementedError # pragma: no cover
2598
2599 def get_files(self, downloads, *, pausable=False):
2600 """Download a bunch of files concurrently.
2601@@ -204,52 +182,16 @@
2602 :raises: DuplicateDestinationError if more than one source url is
2603 downloaded to the same destination file.
2604 """
2605- assert self._iface is None
2606 if self._queued_cancel:
2607 # A cancel is queued, so don't actually download anything.
2608 raise Canceled
2609 if len(downloads) == 0:
2610 # Nothing to download. See LP: #1245597.
2611 return
2612- # Convert the downloads items to download records.
2613- records = [item if isinstance(item, _RecordType) else Record(*item)
2614- for item in downloads]
2615- destinations = set(record.destination for record in records)
2616- # Check for duplicate destinations, specifically for a local file path
2617- # coming from two different sources. It's okay if there are duplicate
2618- # destination records in the download request, but each of those must
2619- # be specified by the same source url and have the same checksum.
2620- #
2621- # An easy quick check just asks if the set of destinations is smaller
2622- # than the total number of requested downloads. It can't be larger.
2623- # If it *is* smaller, then there are some duplicates, however the
2624- # duplicates may be legitimate, so look at the details.
2625- #
2626- # Note though that we cannot pass duplicates destinations to udm,
2627- # so we have to filter out legitimate duplicates. That's fine since
2628- # they really are pointing to the same file, and will end up in the
2629- # destination location.
2630- if len(destinations) < len(downloads):
2631- by_destination = dict()
2632- unique_downloads = set()
2633- for record in records:
2634- by_destination.setdefault(record.destination, set()).add(
2635- record)
2636- unique_downloads.add(record)
2637- duplicates = []
2638- for dst, seen in by_destination.items():
2639- if len(seen) > 1:
2640- # Tuples will look better in the pretty-printed output.
2641- duplicates.append(
2642- (dst, sorted(tuple(dup) for dup in seen)))
2643- if len(duplicates) > 0:
2644- raise DuplicateDestinationError(sorted(duplicates))
2645- # Uniquify the downloads.
2646- records = list(unique_downloads)
2647- bus = dbus.SystemBus()
2648- service = bus.get_object(DOWNLOADER_INTERFACE, '/')
2649- iface = dbus.Interface(service, MANAGER_INTERFACE)
2650- # Better logging of the requested downloads.
2651+ records = self._get_download_records(downloads)
2652+ # Better logging of the requested downloads. However, we want the
2653+ # entire block of multiline log output to appear under a single
2654+ # timestamp.
2655 fp = StringIO()
2656 print('[0x{:x}] Requesting group download:'.format(id(self)), file=fp)
2657 for record in records:
2658@@ -258,69 +200,38 @@
2659 else:
2660 print('\t{} [{}] -> {}'.format(*record), file=fp)
2661 log.info('{}'.format(fp.getvalue()))
2662- object_path = iface.createDownloadGroup(
2663- records,
2664- 'sha256',
2665- False, # Don't allow GSM yet.
2666- # https://bugs.freedesktop.org/show_bug.cgi?id=55594
2667- dbus.Dictionary(signature='sv'),
2668- _headers())
2669- download = bus.get_object(OBJECT_NAME, object_path)
2670- self._iface = dbus.Interface(download, OBJECT_INTERFACE)
2671- # Are GSM downloads allowed? Yes, except if auto_download is set to 1
2672- # (i.e. wifi-only).
2673- allow_gsm = Settings().get('auto_download') != '1'
2674- DBusDownloadManager._set_gsm(self._iface, allow_gsm=allow_gsm)
2675- # Start the download.
2676- reactor = DownloadReactor(bus, self.callback, pausable)
2677- reactor.schedule(self._iface.start)
2678- log.info('[0x{:x}] Running group download reactor', id(self))
2679- reactor.run()
2680- # This download is complete so the object path is no longer
2681- # applicable. Setting this to None will cause subsequent cancels to
2682- # be queued.
2683- self._iface = None
2684- log.info('[0x{:x}] Group download reactor done', id(self))
2685- if reactor.error is not None:
2686- log.error('Reactor error: {}'.format(reactor.error))
2687- if reactor.canceled:
2688- log.info('Reactor canceled')
2689- # Report any other problems.
2690- if reactor.error is not None:
2691- raise FileNotFoundError(reactor.error)
2692- if reactor.canceled:
2693- raise Canceled
2694- if reactor.timed_out:
2695- raise TimeoutError
2696- # For sanity.
2697- for record in records:
2698- assert os.path.exists(record.destination), (
2699- 'Missing destination: {}'.format(record))
2700-
2701- @staticmethod
2702- def _set_gsm(iface, *, allow_gsm):
2703- # This is a separate method for easier testing via mocks.
2704- iface.allowGSMDownload(allow_gsm)
2705-
2706- def cancel(self):
2707- """Cancel any current downloads."""
2708- if self._iface is None:
2709- # Since there's no download in progress right now, there's nothing
2710- # to cancel. Setting this flag queues the cancel signal once the
2711- # reactor starts running again. Yes, this is a bit weird, but if
2712- # we don't do it this way, the caller will immediately get a
2713- # Canceled exception, which isn't helpful because it's expecting
2714- # one when the next download begins.
2715- self._queued_cancel = True
2716+ self._get_files(records, pausable)
2717+
2718+
2719+def get_download_manager(*args):
2720+ # We have to avoid circular imports since both download managers import
2721+ # various things from this module.
2722+ from systemimage.curl import CurlDownloadManager
2723+ from systemimage.udm import DOWNLOADER_INTERFACE, UDMDownloadManager
2724+ # Detect if we have ubuntu-download-manager.
2725+ #
2726+ # Use PyCURL based downloader if no udm is found, or if the environment
2727+ # variable is set. However, if we're told to use PyCURL and it's
2728+ # unavailable, throw an exception.
2729+ cls = None
2730+ use_pycurl = os.environ.get('SYSTEMIMAGE_PYCURL')
2731+ if use_pycurl is None:
2732+ # Auto-detect. For backward compatibility, use udm if it's available,
2733+ # otherwise use PyCURL.
2734+ try:
2735+ bus = dbus.SystemBus()
2736+ bus.get_object(DOWNLOADER_INTERFACE, '/')
2737+ udm_available = True
2738+ except dbus.exceptions.DBusException:
2739+ udm_available = False
2740+ if udm_available:
2741+ cls = UDMDownloadManager
2742+ elif pycurl is None:
2743+ raise ImportError('No module named {}'.format('pycurl'))
2744 else:
2745- self._iface.cancel()
2746-
2747- def pause(self):
2748- """Pause the download, but only if one is in progress."""
2749- if self._iface is not None: # pragma: no branch
2750- self._iface.pause()
2751-
2752- def resume(self):
2753- """Resume the download, but only if one is in progress."""
2754- if self._iface is not None: # pragma: no branch
2755- self._iface.resume()
2756+ cls = CurlDownloadManager
2757+ else:
2758+ cls = (CurlDownloadManager
2759+ if use_pycurl.lower() in ('1', 'yes', 'true')
2760+ else UDMDownloadManager)
2761+ return cls(*args)
2762
2763=== modified file 'systemimage/gpg.py'
2764--- systemimage/gpg.py 2014-09-17 13:41:31 +0000
2765+++ systemimage/gpg.py 2015-05-20 14:55:53 +0000
2766@@ -1,4 +1,4 @@
2767-# Copyright (C) 2013-2014 Canonical Ltd.
2768+# Copyright (C) 2013-2015 Canonical Ltd.
2769 # Author: Barry Warsaw <barry@ubuntu.com>
2770
2771 # This program is free software: you can redistribute it and/or modify
2772@@ -188,6 +188,11 @@
2773 :type data_path: str
2774 :return: bool
2775 """
2776+ # For testing on some systems that are connecting to test servers, GPG
2777+ # verification isn't possible. The s-i-cli supports a switch to
2778+ # disable all GPG checks.
2779+ if config.skip_gpg_verification:
2780+ return True
2781 with open(signature_path, 'rb') as sig_fp:
2782 verified = self._ctx.verify_file(sig_fp, data_path)
2783 # If the file is properly signed, we'll be able to get back a set of
2784
2785=== modified file 'systemimage/helpers.py'
2786--- systemimage/helpers.py 2014-09-17 13:41:31 +0000
2787+++ systemimage/helpers.py 2015-05-20 14:55:53 +0000
2788@@ -1,4 +1,4 @@
2789-# Copyright (C) 2013-2014 Canonical Ltd.
2790+# Copyright (C) 2013-2015 Canonical Ltd.
2791 # Author: Barry Warsaw <barry@ubuntu.com>
2792
2793 # This program is free software: you can redistribute it and/or modify
2794@@ -20,6 +20,8 @@
2795 'MiB',
2796 'as_loglevel',
2797 'as_object',
2798+ 'as_port',
2799+ 'as_stripped',
2800 'as_timedelta',
2801 'atomic',
2802 'calculate_signature',
2803@@ -34,7 +36,6 @@
2804
2805 import os
2806 import re
2807-import time
2808 import random
2809 import shutil
2810 import logging
2811@@ -46,11 +47,12 @@
2812 from importlib import import_module
2813
2814
2815+UNIQUE_MACHINE_ID_FILES = ['/var/lib/dbus/machine-id', '/etc/machine-id']
2816 LAST_UPDATE_FILE = '/userdata/.last_update'
2817-UNIQUE_MACHINE_ID_FILE = '/var/lib/dbus/machine-id'
2818 DEFAULT_DIRMODE = 0o02700
2819 MiB = 1 << 20
2820 EMPTYSTRING = ''
2821+NO_PORT = object()
2822
2823
2824 def calculate_signature(fp, hash_class=None):
2825@@ -79,7 +81,7 @@
2826 """Like os.remove() but don't complain if the file doesn't exist."""
2827 try:
2828 os.remove(path)
2829- except (FileNotFoundError, IsADirectoryError):
2830+ except (FileNotFoundError, IsADirectoryError, PermissionError):
2831 pass
2832
2833
2834@@ -195,13 +197,26 @@
2835 dbus = 'ERROR'
2836 main_level = getattr(logging, main, None)
2837 if main_level is None or not isinstance(main_level, int):
2838- raise ValueError
2839+ raise ValueError(value)
2840 dbus_level = getattr(logging, dbus, None)
2841 if dbus_level is None or not isinstance(dbus_level, int):
2842- raise ValueError
2843+ raise ValueError(value)
2844 return main_level, dbus_level
2845
2846
2847+def as_port(value):
2848+ if value.lower() in ('disabled', 'disable'):
2849+ return NO_PORT
2850+ result = int(value)
2851+ if result < 0:
2852+ raise ValueError(value)
2853+ return result
2854+
2855+
2856+def as_stripped(value):
2857+ return value.strip()
2858+
2859+
2860 @contextmanager
2861 def temporary_directory(*args, **kws):
2862 """A context manager that creates a temporary directory.
2863@@ -227,30 +242,24 @@
2864 def last_update_date():
2865 """Return the last update date.
2866
2867- Taken from the mtime of the following files, in order:
2868-
2869- - /userdata/.last_update
2870- - /etc/system-image/channel.ini
2871- - /etc/ubuntu-build
2872-
2873- First existing path wins.
2874+ If /userdata/.last_update exists, we use this file's mtime. If it doesn't
2875+ exist, then we use the latest mtime of any of the files in
2876+ /etc/system-image/config.d/*.ini (or whatever directory was given with the
2877+ -C/--config option).
2878 """
2879 # Avoid circular imports.
2880 from systemimage.config import config
2881- channel_ini = os.path.join(
2882- os.path.dirname(config.config_file), 'channel.ini')
2883- ubuntu_build = config.system.build_file
2884- for path in (LAST_UPDATE_FILE, channel_ini, ubuntu_build):
2885- try:
2886- # Local time, since we can't know the timezone.
2887- timestamp = datetime.fromtimestamp(os.stat(path).st_mtime)
2888- # Seconds resolution.
2889- timestamp = timestamp.replace(microsecond=0)
2890- return str(timestamp)
2891- except (FileNotFoundError, PermissionError):
2892- pass
2893- else:
2894- return 'Unknown'
2895+ try:
2896+ timestamp = datetime.fromtimestamp(os.stat(LAST_UPDATE_FILE).st_mtime)
2897+ except (FileNotFoundError, PermissionError):
2898+ # We fall back to the latest mtime of the config.d/*.ini files.
2899+ timestamps = sorted(
2900+ datetime.fromtimestamp(path.stat().st_mtime)
2901+ for path in config.ini_files)
2902+ if len(timestamps) == 0:
2903+ return 'Unknown'
2904+ timestamp = timestamps[-1]
2905+ return str(timestamp.replace(microsecond=0))
2906
2907
2908 def version_detail(details_string=None):
2909@@ -270,19 +279,20 @@
2910 return details
2911
2912
2913-_pp_cache = None
2914-
2915-def phased_percentage(*, reset=False):
2916- global _pp_cache
2917- if _pp_cache is None:
2918- with open(UNIQUE_MACHINE_ID_FILE, 'rb') as fp:
2919- data = fp.read()
2920- now = str(time.time()).encode('us-ascii')
2921- r = random.Random()
2922- r.seed(data + now)
2923- _pp_cache = r.randint(0, 100)
2924- try:
2925- return _pp_cache
2926- finally:
2927- if reset:
2928- _pp_cache = None
2929+def phased_percentage(channel, target):
2930+ # Avoid circular imports.
2931+ from systemimage.config import config
2932+ if config.phase_override is not None:
2933+ return config.phase_override
2934+ for path in UNIQUE_MACHINE_ID_FILES:
2935+ try:
2936+ with open(path, 'r', encoding='utf-8') as fp:
2937+ machine_id = fp.read().strip()
2938+ break # pragma: no branch
2939+ except FileNotFoundError:
2940+ pass
2941+ else:
2942+ raise RuntimeError('No machine-id file found')
2943+ r = random.Random()
2944+ r.seed('{}.{}.{}'.format(channel, target, machine_id))
2945+ return r.randint(0, 100)
2946
2947=== modified file 'systemimage/image.py'
2948--- systemimage/image.py 2014-09-17 13:41:31 +0000
2949+++ systemimage/image.py 2015-05-20 14:55:53 +0000
2950@@ -1,4 +1,4 @@
2951-# Copyright (C) 2013-2014 Canonical Ltd.
2952+# Copyright (C) 2013-2015 Canonical Ltd.
2953 # Author: Barry Warsaw <barry@ubuntu.com>
2954
2955 # This program is free software: you can redistribute it and/or modify
2956@@ -74,3 +74,7 @@
2957 @property
2958 def phased_percentage(self):
2959 return self.__untranslated__.get('phased-percentage', 100)
2960+
2961+ @property
2962+ def version_detail(self):
2963+ return self.__untranslated__.get('version_detail', '')
2964
2965=== modified file 'systemimage/index.py'
2966--- systemimage/index.py 2014-02-20 23:03:24 +0000
2967+++ systemimage/index.py 2015-05-20 14:55:53 +0000
2968@@ -1,4 +1,4 @@
2969-# Copyright (C) 2013-2014 Canonical Ltd.
2970+# Copyright (C) 2013-2015 Canonical Ltd.
2971 # Author: Barry Warsaw <barry@ubuntu.com>
2972
2973 # This program is free software: you can redistribute it and/or modify
2974@@ -24,7 +24,6 @@
2975
2976 from datetime import datetime, timezone
2977 from systemimage.bag import Bag
2978-from systemimage.helpers import phased_percentage
2979 from systemimage.image import Image
2980
2981
2982@@ -49,7 +48,6 @@
2983 global_ = Bag(generated_at=generated_at)
2984 # Parse the images.
2985 images = []
2986- percentage = phased_percentage()
2987 for image_data in mapping['images']:
2988 # Descriptions can be any of:
2989 #
2990@@ -70,6 +68,5 @@
2991 image = Image(files=bundles,
2992 descriptions=descriptions,
2993 **image_data)
2994- if percentage <= image.phased_percentage:
2995- images.append(image)
2996+ images.append(image)
2997 return cls(global_=global_, images=images)
2998
2999=== modified file 'systemimage/keyring.py'
3000--- systemimage/keyring.py 2014-02-20 23:03:24 +0000
3001+++ systemimage/keyring.py 2015-05-20 14:55:53 +0000
3002@@ -1,4 +1,4 @@
3003-# Copyright (C) 2013-2014 Canonical Ltd.
3004+# Copyright (C) 2013-2015 Canonical Ltd.
3005 # Author: Barry Warsaw <barry@ubuntu.com>
3006
3007 # This program is free software: you can redistribute it and/or modify
3008@@ -30,7 +30,7 @@
3009 from contextlib import ExitStack
3010 from datetime import datetime, timezone
3011 from systemimage.config import config
3012-from systemimage.download import DBusDownloadManager
3013+from systemimage.download import get_download_manager
3014 from systemimage.gpg import Context
3015 from systemimage.helpers import makedirs, safe_remove
3016 from urllib.parse import urljoin
3017@@ -86,8 +86,8 @@
3018 else:
3019 srcurl = urls
3020 ascurl = urls + '.asc'
3021- tarxz_src = urljoin(config.service.https_base, srcurl)
3022- ascxz_src = urljoin(config.service.https_base, ascurl)
3023+ tarxz_src = urljoin(config.https_base, srcurl)
3024+ ascxz_src = urljoin(config.https_base, ascurl)
3025 # Calculate the local paths to the temporary download files. The
3026 # blacklist goes to the data partition and all the other files go to the
3027 # cache partition.
3028@@ -102,7 +102,7 @@
3029 safe_remove(ascxz_dst)
3030 with ExitStack() as stack:
3031 # Let FileNotFoundError percolate up.
3032- DBusDownloadManager().get_files([
3033+ get_download_manager().get_files([
3034 (tarxz_src, tarxz_dst),
3035 (ascxz_src, ascxz_dst),
3036 ])
3037
3038=== modified file 'systemimage/logging.py'
3039--- systemimage/logging.py 2014-09-17 13:41:31 +0000
3040+++ systemimage/logging.py 2015-05-20 14:55:53 +0000
3041@@ -1,4 +1,4 @@
3042-# Copyright (C) 2013-2014 Canonical Ltd.
3043+# Copyright (C) 2013-2015 Canonical Ltd.
3044 # Author: Barry Warsaw <barry@ubuntu.com>
3045
3046 # This program is free software: you can redistribute it and/or modify
3047@@ -80,7 +80,9 @@
3048 def initialize(*, verbosity=0):
3049 """Initialize the loggers."""
3050 main, dbus = config.system.loglevel
3051- for name, loglevel in (('systemimage', main), ('systemimage.dbus', dbus)):
3052+ for name, loglevel in (('systemimage', main),
3053+ ('systemimage.dbus', dbus),
3054+ ('dbus.proxies', dbus)):
3055 level = {
3056 0: logging.ERROR,
3057 1: logging.INFO,
3058
3059=== modified file 'systemimage/main.py'
3060--- systemimage/main.py 2014-09-26 14:36:34 +0000
3061+++ systemimage/main.py 2015-05-20 14:55:53 +0000
3062@@ -1,4 +1,4 @@
3063-# Copyright (C) 2013-2014 Canonical Ltd.
3064+# Copyright (C) 2013-2015 Canonical Ltd.
3065 # Author: Barry Warsaw <barry@ubuntu.com>
3066
3067 # This program is free software: you can redistribute it and/or modify
3068@@ -21,18 +21,19 @@
3069 ]
3070
3071
3072-import os
3073 import sys
3074+import json
3075 import logging
3076 import argparse
3077
3078 from dbus.mainloop.glib import DBusGMainLoop
3079 from pkg_resources import resource_string as resource_bytes
3080+from systemimage.apply import factory_reset, production_reset
3081 from systemimage.candidates import delta_filter, full_filter
3082 from systemimage.config import config
3083-from systemimage.helpers import last_update_date, makedirs, version_detail
3084+from systemimage.helpers import (
3085+ last_update_date, makedirs, phased_percentage, version_detail)
3086 from systemimage.logging import initialize
3087-from systemimage.reboot import factory_reset
3088 from systemimage.settings import Settings
3089 from systemimage.state import State
3090 from textwrap import dedent
3091@@ -41,12 +42,47 @@
3092 __version__ = resource_bytes(
3093 'systemimage', 'version.txt').decode('utf-8').strip()
3094
3095-DEFAULT_CONFIG_FILE = '/etc/system-image/client.ini'
3096+DEFAULT_CONFIG_D = '/etc/system-image/config.d'
3097 COLON = ':'
3098+LINE_LENGTH = 78
3099+
3100+
3101+class _DotsProgress:
3102+ def __init__(self):
3103+ self._dot_count = 0
3104+
3105+ def callback(self, received, total):
3106+ received = int(received)
3107+ total = int(total)
3108+ sys.stderr.write('.')
3109+ sys.stderr.flush()
3110+ self._dot_count += 1
3111+ show_dots = self._dot_count % LINE_LENGTH == 0
3112+ if show_dots or received >= total: # pragma: no cover
3113+ sys.stderr.write('\n')
3114+ sys.stderr.flush()
3115+
3116+
3117+class _LogfileProgress:
3118+ def __init__(self, log):
3119+ self._log = log
3120+
3121+ def callback(self, received, total):
3122+ self._log.debug('received: {} of {} bytes', received, total)
3123+
3124+
3125+def _json_progress(received, total):
3126+ # For use with --progress=json output. LP: #1423622
3127+ message = json.dumps(dict(
3128+ type='progress',
3129+ now=received,
3130+ total=total))
3131+ sys.stdout.write(message)
3132+ sys.stdout.write('\n')
3133+ sys.stdout.flush()
3134
3135
3136 def main():
3137- global config
3138 parser = argparse.ArgumentParser(
3139 prog='system-image-cli',
3140 description='Ubuntu System Image Upgrader')
3141@@ -54,10 +90,10 @@
3142 action='version',
3143 version='system-image-cli {}'.format(__version__))
3144 parser.add_argument('-C', '--config',
3145- default=DEFAULT_CONFIG_FILE, action='store',
3146- metavar='FILE',
3147- help="""Use the given configuration file instead of
3148- the default""")
3149+ default=DEFAULT_CONFIG_D, action='store',
3150+ metavar='DIRECTORY',
3151+ help="""Use the given configuration directory instead
3152+ of the default""")
3153 parser.add_argument('-b', '--build',
3154 default=None, action='store',
3155 help="""Override the current build number just
3156@@ -76,12 +112,16 @@
3157 full updates or only delta updates. The
3158 argument to this option must be either `full`
3159 or `delta`""")
3160- parser.add_argument('-g', '--no-reboot',
3161+ parser.add_argument('-g', '--no-apply',
3162 default=False, action='store_true',
3163 help="""Download (i.e. "get") all the data files and
3164 prepare for updating, but don't actually
3165 reboot the device into recovery to apply the
3166 update""")
3167+ # Deprecated since si 3.0.
3168+ parser.add_argument('--no-reboot',
3169+ default=False, action='store_true',
3170+ help="""Deprecated; use -g/--no-apply""")
3171 parser.add_argument('-i', '--info',
3172 default=False, action='store_true',
3173 help="""Show some information about the current
3174@@ -94,6 +134,15 @@
3175 parser.add_argument('-v', '--verbose',
3176 default=0, action='count',
3177 help='Increase verbosity')
3178+ parser.add_argument('--progress',
3179+ default=[], action='append',
3180+ help="""Add a progress meter. Available meters are:
3181+ dots, logfile, and json. Multiple --progress
3182+ options are allowed.""")
3183+ parser.add_argument('-p', '--percentage',
3184+ default=None, action='store',
3185+ help="""Override the device's phased percentage value
3186+ during upgrade candidate calculation.""")
3187 parser.add_argument('--list-channels',
3188 default=False, action='store_true',
3189 help="""List all available channels, then exit""")
3190@@ -102,6 +151,12 @@
3191 help="""Perform a destructive factory reset and
3192 reboot. WARNING: this will wipe all user data
3193 on the device!""")
3194+ parser.add_argument('--production-reset',
3195+ default=False, action='store_true',
3196+ help="""Perform a destructive production reset
3197+ (similar to factory reset) and reboot.
3198+ WARNING: this will wipe all user data
3199+ on the device!""")
3200 parser.add_argument('--switch',
3201 default=None, action='store', metavar='CHANNEL',
3202 help="""Switch to the given channel. This is
3203@@ -128,27 +183,36 @@
3204 help="""Delete the key and its value. It is a no-op
3205 if the key does not exist. Multiple
3206 --del arguments can be given.""")
3207+ # Hidden system-image-cli only feature for testing purposes. LP: #1333414
3208+ parser.add_argument('--skip-gpg-verification',
3209+ default=False, action='store_true',
3210+ help=argparse.SUPPRESS)
3211
3212 args = parser.parse_args(sys.argv[1:])
3213 try:
3214 config.load(args.config)
3215- except FileNotFoundError as error:
3216- parser.error('\nConfiguration file not found: {}'.format(error))
3217+ except (TypeError, FileNotFoundError):
3218+ parser.error('\nConfiguration directory not found: {}'.format(
3219+ args.config))
3220 assert 'parser.error() does not return' # pragma: no cover
3221- # Load the optional channel.ini file, which must live next to the
3222- # configuration file. It's okay if this file does not exist.
3223- channel_ini = os.path.join(os.path.dirname(args.config), 'channel.ini')
3224- try:
3225- config.load(channel_ini, override=True)
3226- except FileNotFoundError:
3227- pass
3228-
3229- # Perform a factory reset.
3230+
3231+ if args.skip_gpg_verification:
3232+ print("""\
3233+WARNING: All GPG signature verifications have been disabled.
3234+Your upgrades are INSECURE.""", file=sys.stderr)
3235+ config.skip_gpg_verification = True
3236+
3237+ # Perform factory and production resets.
3238 if args.factory_reset:
3239 factory_reset()
3240 # We should never get here, except possibly during the testing
3241 # process, so just return as normal.
3242 return 0
3243+ if args.production_reset:
3244+ production_reset()
3245+ # We should never get here, except possibly during the testing
3246+ # process, so just return as normal.
3247+ return 0
3248
3249 # Handle all settings arguments. They are mutually exclusive.
3250 if sum(bool(arg) for arg in
3251@@ -213,6 +277,8 @@
3252 config.channel = args.channel
3253 if args.device is not None:
3254 config.device = args.device
3255+ if args.percentage is not None:
3256+ config.phase_override = args.percentage
3257
3258 if args.info:
3259 alias = getattr(config.service, 'channel_target', None)
3260@@ -245,11 +311,16 @@
3261 print('version {}: {}'.format(key, details[key]))
3262 return 0
3263
3264+ DBusGMainLoop(set_as_default=True)
3265+
3266 if args.list_channels:
3267 state = State()
3268 try:
3269 state.run_thru('get_channel')
3270 except Exception:
3271+ print('Exception occurred during channel search; '
3272+ 'see log file for details',
3273+ file=sys.stderr)
3274 log.exception('system-image-cli exception')
3275 return 1
3276 print('Available channels:')
3277@@ -261,33 +332,26 @@
3278 print(' {} (alias for: {})'.format(key, alias))
3279 return 0
3280
3281- # When verbosity is at 1, logging every progress signal from
3282- # ubuntu-download-manager would be way too noisy. OTOH, not printing
3283- # anything leads some folks to think the process is just hung, since it
3284- # can take a long time to download all the data files. As a compromise,
3285- # we'll output some dots to stderr at verbosity 1, but we won't log these
3286- # dots since they would just be noise. This doesn't have to be perfect.
3287- if args.verbose == 1: # pragma: no cover
3288- dot_count = 0
3289- def callback(received, total):
3290- nonlocal dot_count
3291- sys.stderr.write('.')
3292- sys.stderr.flush()
3293- dot_count += 1
3294- if dot_count % 78 == 0 or received >= total:
3295- sys.stderr.write('\n')
3296- sys.stderr.flush()
3297- else:
3298- def callback(received, total):
3299- log.debug('received: {} of {} bytes', received, total)
3300-
3301- DBusGMainLoop(set_as_default=True)
3302 state = State(candidate_filter=candidate_filter)
3303- state.downloader.callback = callback
3304+
3305+ for meter in args.progress:
3306+ if meter == 'dots':
3307+ state.downloader.callbacks.append(_DotsProgress().callback)
3308+ elif meter == 'json':
3309+ state.downloader.callbacks.append(_json_progress)
3310+ elif meter == 'logfile':
3311+ state.downloader.callbacks.append(_LogfileProgress(log).callback)
3312+ else:
3313+ parser.error('Unknown progress meter: {}'.format(meter))
3314+ assert 'parser.error() does not return' # pragma: no cover
3315+
3316 if args.dry_run:
3317 try:
3318 state.run_until('download_files')
3319 except Exception:
3320+ print('Exception occurred during dry-run; '
3321+ 'see log file for details',
3322+ file=sys.stderr)
3323 log.exception('system-image-cli exception')
3324 return 1
3325 # Say -c <no-such-channel> was given. This will fail.
3326@@ -296,16 +360,20 @@
3327 else:
3328 winning_path = [str(image.version) for image in state.winner]
3329 kws = dict(path=COLON.join(winning_path))
3330+ target_build = state.winner[-1].version
3331 if state.channel_switch is None:
3332 # We're not switching channels due to an alias change.
3333 template = 'Upgrade path is {path}'
3334+ percentage = phased_percentage(config.channel, target_build)
3335 else:
3336 # This upgrade changes the channel that our alias is mapped
3337 # to, so include that information in the output.
3338 template = 'Upgrade path is {path} ({from} -> {to})'
3339 kws['from'], kws['to'] = state.channel_switch
3340+ percentage = phased_percentage(kws['to'], target_build)
3341 print(template.format(**kws))
3342- return
3343+ print('Target phase: {}%'.format(percentage))
3344+ return 0
3345 else:
3346 # Run the state machine to conclusion. Suppress all exceptions, but
3347 # note that the state machine will log them. If an exception occurs,
3348@@ -313,13 +381,15 @@
3349 log.info('running state machine [{}/{}]',
3350 config.channel, config.device)
3351 try:
3352- if args.no_reboot:
3353- state.run_until('reboot')
3354+ if args.no_apply or args.no_reboot:
3355+ state.run_until('apply')
3356 else:
3357 list(state)
3358 except KeyboardInterrupt: # pragma: no cover
3359 return 0
3360 except Exception:
3361+ print('Exception occurred during update; see log file for details',
3362+ file=sys.stderr)
3363 log.exception('system-image-cli exception')
3364 return 1
3365 else:
3366
3367=== modified file 'systemimage/reactor.py'
3368--- systemimage/reactor.py 2014-09-17 13:41:31 +0000
3369+++ systemimage/reactor.py 2015-05-20 14:55:53 +0000
3370@@ -1,4 +1,4 @@
3371-# Copyright (C) 2013-2014 Canonical Ltd.
3372+# Copyright (C) 2013-2015 Canonical Ltd.
3373 # Author: Barry Warsaw <barry@ubuntu.com>
3374
3375 # This program is free software: you can redistribute it and/or modify
3376@@ -76,11 +76,14 @@
3377 self._quitter = GLib.timeout_add_seconds(
3378 self._active_timeout, self._quit_with_error)
3379
3380- def react_to(self, signal):
3381+ def react_to(self, signal, object_path=None):
3382 signal_match = self._bus.add_signal_receiver(
3383- self._handle_signal, signal_name=signal,
3384+ self._handle_signal,
3385+ signal_name=signal,
3386+ path=object_path,
3387 member_keyword='member',
3388- path_keyword='path')
3389+ path_keyword='path',
3390+ )
3391 self._signal_matches.append(signal_match)
3392
3393 def schedule(self, method, milliseconds=50):
3394
3395=== modified file 'systemimage/scores.py'
3396--- systemimage/scores.py 2014-09-17 13:41:31 +0000
3397+++ systemimage/scores.py 2015-05-20 14:55:53 +0000
3398@@ -1,4 +1,4 @@
3399-# Copyright (C) 2013-2014 Canonical Ltd.
3400+# Copyright (C) 2013-2015 Canonical Ltd.
3401 # Author: Barry Warsaw <barry@ubuntu.com>
3402
3403 # This program is free software: you can redistribute it and/or modify
3404@@ -26,19 +26,18 @@
3405
3406 import logging
3407
3408-from io import StringIO
3409 from itertools import count
3410-from systemimage.helpers import MiB
3411+from systemimage.helpers import MiB, phased_percentage
3412+
3413
3414 log = logging.getLogger('systemimage')
3415-
3416 COLON = ':'
3417
3418
3419 class Scorer:
3420 """Abstract base class providing an API for candidate selection."""
3421
3422- def choose(self, candidates):
3423+ def choose(self, candidates, channel):
3424 """Choose the candidate upgrade paths.
3425
3426 Lowest score wins.
3427@@ -47,10 +46,14 @@
3428 the device from the current version to the latest version, sorted
3429 in order from oldest verson to newest.
3430 :type candidates: list of lists
3431+ :param channel: The channel being upgraded to. This is used in the
3432+ phased update calculate.
3433+ :type channel: str
3434 :return: The chosen path.
3435 :rtype: list
3436 """
3437 if len(candidates) == 0:
3438+ log.debug('No candidates, so no winner')
3439 return []
3440 # We want to zip together the score for each candidate path, plus the
3441 # candidate path, so that when we sort the sequence, we'll always get
3442@@ -68,17 +71,31 @@
3443 # Be sure that after all is said and done we return the list of Images
3444 # though!
3445 scores = sorted(zip(self.score(candidates), count(), candidates))
3446- fp = StringIO()
3447- print('{} path scores (last one wins):'.format(
3448- self.__class__.__name__),
3449- file=fp)
3450- for score, i, candidate in reversed(scores):
3451- print('\t[{:4d}] -> {}'.format(
3452+ # Calculate the phase percentage for the device. Use the highest
3453+ # available build number as input into the random seed.
3454+ max_target_number = -1
3455+ for score, i, path in scores:
3456+ # The last image will be the target image.
3457+ assert len(path) > 0, 'Empty upgrade candidate path?'
3458+ max_target_number = max(max_target_number, path[-1].version)
3459+ assert max_target_number != -1, 'No max target version?'
3460+ device_percentage = phased_percentage(channel, max_target_number)
3461+ log.debug('Device phased percentage: {}%'.format(device_percentage))
3462+ log.debug('{} path scores:'.format(self.__class__.__name__))
3463+ # Log the candidate paths, their scores, and their phases.
3464+ for score, i, path in reversed(scores):
3465+ log.debug('\t[{:4d}] -> {} ({}%)'.format(
3466 score,
3467- COLON.join(str(image.version) for image in candidate)),
3468- file=fp)
3469- log.debug('{}'.format(fp.getvalue()))
3470- return scores[0][2]
3471+ COLON.join(str(image.version) for image in path),
3472+ (path[-1].phased_percentage if len(path) > 0 else '--')
3473+ ))
3474+ for score, i, path in scores:
3475+ image_percentage = path[-1].phased_percentage
3476+ # An image percentage of 0 means that it's been pulled.
3477+ if image_percentage > 0 and device_percentage <= image_percentage:
3478+ return path
3479+ # No upgrade path.
3480+ return []
3481
3482 def score(self, candidates): # pragma: no cover
3483 """Like `choose()` except returns the candidate path scores.
3484
3485=== modified file 'systemimage/service.py'
3486--- systemimage/service.py 2014-09-17 13:41:31 +0000
3487+++ systemimage/service.py 2015-05-20 14:55:53 +0000
3488@@ -1,4 +1,4 @@
3489-# Copyright (C) 2013-2014 Canonical Ltd.
3490+# Copyright (C) 2013-2015 Canonical Ltd.
3491 # Author: Barry Warsaw <barry@ubuntu.com>
3492
3493 # This program is free software: you can redistribute it and/or modify
3494@@ -20,7 +20,6 @@
3495 ]
3496
3497
3498-import os
3499 import sys
3500 import dbus
3501 import logging
3502@@ -33,7 +32,8 @@
3503 from systemimage.dbus import Loop
3504 from systemimage.helpers import makedirs
3505 from systemimage.logging import initialize
3506-from systemimage.main import DEFAULT_CONFIG_FILE
3507+from systemimage.main import DEFAULT_CONFIG_D
3508+
3509
3510 # --testing is only enabled when the systemimage.testing package is
3511 # available. This will be the case for the upstream source package, and when
3512@@ -60,32 +60,28 @@
3513 action='version',
3514 version='system-image-dbus {}'.format(__version__))
3515 parser.add_argument('-C', '--config',
3516- default=DEFAULT_CONFIG_FILE, action='store',
3517- metavar='FILE',
3518- help="""Use the given configuration file instead of
3519- the default""")
3520+ default=DEFAULT_CONFIG_D, action='store',
3521+ metavar='DIRECTORY',
3522+ help="""Use the given configuration directory instead
3523+ of the default""")
3524 parser.add_argument('-v', '--verbose',
3525 default=0, action='count',
3526 help='Increase verbosity')
3527 # Hidden argument for special setup required by test environment.
3528 if instrument is not None: # pragma: no branch
3529 parser.add_argument('--testing',
3530- default=False, action='store',
3531+ default=None, action='store',
3532+ help=argparse.SUPPRESS)
3533+ parser.add_argument('--self-signed-cert',
3534+ default=None, action='store',
3535 help=argparse.SUPPRESS)
3536
3537 args = parser.parse_args(sys.argv[1:])
3538 try:
3539 config.load(args.config)
3540- except FileNotFoundError as error:
3541- parser.error('\nConfiguration file not found: {}'.format(error))
3542+ except TypeError as error:
3543+ parser.error('\nConfiguration directory not found: {}'.format(error))
3544 assert 'parser.error() does not return' # pragma: no cover
3545- # Load the optional channel.ini file, which must live next to the
3546- # configuration file. It's okay if this file does not exist.
3547- channel_ini = os.path.join(os.path.dirname(args.config), 'channel.ini')
3548- try:
3549- config.load(channel_ini, override=True)
3550- except FileNotFoundError:
3551- pass
3552
3553 # Create the temporary directory if it doesn't exist.
3554 makedirs(config.system.tempdir)
3555@@ -112,12 +108,13 @@
3556 loop = Loop()
3557 testing_mode = getattr(args, 'testing', None)
3558 if testing_mode:
3559- instrument(config, stack)
3560+ instrument(config, stack, args.self_signed_cert)
3561 config.dbus_service = get_service(
3562 testing_mode, system_bus, '/Service', loop)
3563 else:
3564 from systemimage.dbus import Service
3565 config.dbus_service = Service(system_bus, '/Service', loop)
3566+
3567 try:
3568 loop.run()
3569 except KeyboardInterrupt: # pragma: no cover
3570
3571=== modified file 'systemimage/settings.py'
3572--- systemimage/settings.py 2014-09-17 13:41:31 +0000
3573+++ systemimage/settings.py 2015-05-20 14:55:53 +0000
3574@@ -1,4 +1,4 @@
3575-# Copyright (C) 2013-2014 Canonical Ltd.
3576+# Copyright (C) 2013-2015 Canonical Ltd.
3577 # Author: Barry Warsaw <barry@ubuntu.com>
3578
3579 # This program is free software: you can redistribute it and/or modify
3580
3581=== modified file 'systemimage/state.py'
3582--- systemimage/state.py 2014-09-17 13:41:31 +0000
3583+++ systemimage/state.py 2015-05-20 14:55:53 +0000
3584@@ -1,4 +1,4 @@
3585-# Copyright (C) 2013-2014 Canonical Ltd.
3586+# Copyright (C) 2013-2015 Canonical Ltd.
3587 # Author: Barry Warsaw <barry@ubuntu.com>
3588
3589 # This program is free software: you can redistribute it and/or modify
3590@@ -35,7 +35,7 @@
3591 from systemimage.candidates import get_candidates, iter_path
3592 from systemimage.channel import Channels
3593 from systemimage.config import config
3594-from systemimage.download import DBusDownloadManager, Record
3595+from systemimage.download import Record, get_download_manager
3596 from systemimage.gpg import Context, SignatureError
3597 from systemimage.helpers import (
3598 atomic, calculate_signature, makedirs, safe_remove, temporary_directory)
3599@@ -58,6 +58,9 @@
3600 self.got = got
3601 self.expected = checksum
3602
3603+ def __str__(self): # pragma: no cover
3604+ return 'got:{0.got} != exp:{0.expected}: {0.destination}'.format(self)
3605+
3606
3607 def _copy_if_missing(src, dstdir):
3608 dst_path = os.path.join(dstdir, os.path.basename(src))
3609@@ -110,7 +113,7 @@
3610 self.files = []
3611 self.channel_switch = None
3612 # Other public attributes.
3613- self.downloader = DBusDownloadManager()
3614+ self.downloader = get_download_manager()
3615 self._next.append(self._cleanup)
3616
3617 def __iter__(self):
3618@@ -211,7 +214,7 @@
3619 # I think it makes no sense to check the blacklist when we're
3620 # downloading a blacklist file.
3621 log.info('Looking for blacklist: {}'.format(
3622- urljoin(config.service.https_base, url)))
3623+ urljoin(config.https_base, url)))
3624 get_keyring('blacklist', url, 'image-master')
3625 except SignatureError:
3626 log.exception('No signed blacklist found')
3627@@ -247,7 +250,7 @@
3628 url = 'gpg/blacklist.tar.xz'
3629 try:
3630 log.info('Looking for blacklist again: {}',
3631- urljoin(config.service.https_base, url))
3632+ urljoin(config.https_base, url))
3633 get_keyring('blacklist', url, 'image-master')
3634 except FileNotFoundError:
3635 log.info('No blacklist found on second attempt')
3636@@ -273,9 +276,9 @@
3637 get_keyring(
3638 'image-signing', 'gpg/image-signing.tar.xz', 'image-master',
3639 self.blacklist)
3640- channels_url = urljoin(config.service.https_base, 'channels.json')
3641+ channels_url = urljoin(config.https_base, 'channels.json')
3642 channels_path = os.path.join(config.tempdir, 'channels.json')
3643- asc_url = urljoin(config.service.https_base, 'channels.json.asc')
3644+ asc_url = urljoin(config.https_base, 'channels.json.asc')
3645 asc_path = os.path.join(config.tempdir, 'channels.json.asc')
3646 log.info('Looking for: {}', channels_url)
3647 with ExitStack() as stack:
3648@@ -329,8 +332,8 @@
3649 self._next.append(partial(self._get_index, device.index))
3650
3651 def _get_device_keyring(self, keyring):
3652- keyring_url = urljoin(config.service.https_base, keyring.path)
3653- asc_url = urljoin(config.service.https_base, keyring.signature)
3654+ keyring_url = urljoin(config.https_base, keyring.path)
3655+ asc_url = urljoin(config.https_base, keyring.signature)
3656 log.info('getting device keyring: {}', keyring_url)
3657 get_keyring(
3658 'device-signing', (keyring_url, asc_url), 'image-signing',
3659@@ -378,7 +381,7 @@
3660
3661 def _get_index(self, index):
3662 """Get and verify the index.json file."""
3663- index_url = urljoin(config.service.https_base, index)
3664+ index_url = urljoin(config.https_base, index)
3665 asc_url = index_url + '.asc'
3666 index_path = os.path.join(config.tempdir, 'index.json')
3667 asc_path = index_path + '.asc'
3668@@ -410,7 +413,7 @@
3669 # winner. Otherwise, trust the configured build number.
3670 channel = self.channels[config.channel]
3671 # channel_target is the channel we're on based on the alias mapping in
3672- # our channel.ini file. channel_alias is the alias mapping in the
3673+ # our config files. channel_alias is the alias mapping in the
3674 # channel.json file, i.e. the channel an update will put us on.
3675 channel_target = getattr(config.service, 'channel_target', None)
3676 channel_alias = getattr(channel, 'alias', None)
3677@@ -418,17 +421,23 @@
3678 channel_target is None or
3679 channel_alias == channel_target):
3680 build_number = config.build_number
3681- elif config.build_number_cli is not None:
3682- # An explicit --build on the command line still takes precedence.
3683- build_number = config.build_number_cli
3684 else:
3685- # This is a channel switch caused by a new alias.
3686- build_number = 0
3687+ # This is a channel switch caused by a new alias. Unless the
3688+ # build number has been explicitly overridden on the command line
3689+ # via --build/-b, use build number 0 to force a full update.
3690+ build_number = (config.build_number
3691+ if config.build_number_override
3692+ else 0)
3693 self.channel_switch = (channel_target, channel_alias)
3694 candidates = get_candidates(self.index, build_number)
3695+ log.debug('Candidates from build# {}: {}'.format(
3696+ build_number, len(candidates)))
3697 if self._filter is not None:
3698 candidates = self._filter(candidates)
3699- self.winner = config.hooks.scorer().choose(candidates)
3700+ self.winner = config.hooks.scorer().choose(
3701+ candidates, (channel_target
3702+ if channel_alias is None
3703+ else channel_alias))
3704 # If there is no winning upgrade candidate, then there's nothing more
3705 # to do. We can skip everything between downloading the files and
3706 # doing the reboot.
3707@@ -474,11 +483,11 @@
3708 else:
3709 # Add the data file, which has a checksum.
3710 downloads.append(Record(
3711- urljoin(config.service.http_base, filerec.path),
3712+ urljoin(config.http_base, filerec.path),
3713 dst, checksum))
3714 # Add the signature file, which does not have a checksum.
3715 downloads.append(Record(
3716- urljoin(config.service.http_base, filerec.signature),
3717+ urljoin(config.http_base, filerec.signature),
3718 asc))
3719 signatures.append((dst, asc))
3720 checksums.append((dst, checksum))
3721@@ -603,9 +612,9 @@
3722 file=fp)
3723 # The filesystem must be unmounted.
3724 print('unmount system', file=fp)
3725- self._next.append(self._reboot)
3726+ self._next.append(self._apply)
3727
3728- def _reboot(self):
3729- log.info('rebooting')
3730- config.hooks.reboot().reboot()
3731+ def _apply(self):
3732+ log.info('applying')
3733+ config.hooks.apply().apply()
3734 # Nothing more to do.
3735
3736=== modified file 'systemimage/testing/controller.py'
3737--- systemimage/testing/controller.py 2014-09-17 13:41:31 +0000
3738+++ systemimage/testing/controller.py 2015-05-20 14:55:53 +0000
3739@@ -1,4 +1,4 @@
3740-# Copyright (C) 2013-2014 Canonical Ltd.
3741+# Copyright (C) 2013-2015 Canonical Ltd.
3742 # Author: Barry Warsaw <barry@ubuntu.com>
3743
3744 # This program is free software: you can redistribute it and/or modify
3745@@ -29,24 +29,31 @@
3746 import psutil
3747 import subprocess
3748
3749+try:
3750+ import pycurl
3751+except ImportError:
3752+ pycurl = None
3753+
3754 from contextlib import ExitStack
3755 from distutils.spawn import find_executable
3756 from pkg_resources import resource_string as resource_bytes
3757 from systemimage.helpers import temporary_directory
3758 from systemimage.testing.helpers import (
3759- data_path, find_dbus_process, reset_envar)
3760+ data_path, find_dbus_process, makedirs, reset_envar, wait_for_service)
3761+from unittest.mock import patch
3762
3763
3764 SPACE = ' '
3765-OVERRIDE = os.environ.get('SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS')
3766-HUP_SLEEP = (0 if OVERRIDE is None else int(OVERRIDE))
3767+DLSERVICE = os.environ.get(
3768+ 'SYSTEMIMAGE_DLSERVICE',
3769+ '/usr/bin/ubuntu-download-manager'
3770+ # For debugging the in-tree version of u-d-m.
3771+ #'/bin/sh $HOME/projects/phone/trunk/tools/runme.sh'
3772+ )
3773
3774
3775 def start_system_image(controller):
3776- bus = dbus.SystemBus()
3777- service = bus.get_object('com.canonical.SystemImage', '/Service')
3778- iface = dbus.Interface(service, 'com.canonical.SystemImage')
3779- iface.Info()
3780+ wait_for_service(reload=False)
3781 process = find_dbus_process(controller.ini_path)
3782 if process is None:
3783 raise RuntimeError('Could not start system-image-dbus')
3784@@ -78,12 +85,14 @@
3785
3786
3787 def start_downloader(controller):
3788- bus = dbus.SystemBus()
3789- service = bus.get_object('com.canonical.applications.Downloader', '/')
3790- iface = dbus.Interface(
3791- service, 'com.canonical.applications.DownloadManager')
3792+ service = dbus.SystemBus().get_object('org.freedesktop.DBus', '/')
3793+ iface = dbus.Interface(service, 'org.freedesktop.DBus')
3794+ reply = 0
3795+ while reply != 2:
3796+ reply = iface.StartServiceByName(
3797+ 'com.canonical.applications.Downloader', 0)
3798+ time.sleep(0.1)
3799 # Something innocuous.
3800- iface.defaultThrottle()
3801 process = _find_udm_process()
3802 if process is None:
3803 raise RuntimeError('Could not start ubuntu-download-manager')
3804@@ -105,24 +114,30 @@
3805 process.wait(60)
3806
3807
3808-DLSERVICE = '/usr/bin/ubuntu-download-manager'
3809-# For debugging the in-tree version of u-d-m.
3810-#DLSERVICE = '/bin/sh /home/barry/projects/phone/runme'
3811-
3812-
3813 SERVICES = [
3814 ('com.canonical.SystemImage',
3815- '{python} -m {self.MODULE} -C {self.ini_path} --testing {self.mode}',
3816+ '{python} -m {self.MODULE} -C {self.ini_path} '
3817+ '{self.curl_cert} --testing {self.mode}',
3818 start_system_image,
3819 stop_system_image,
3820 ),
3821- ('com.canonical.applications.Downloader',
3822+ ]
3823+
3824+
3825+if pycurl is None:
3826+ USING_PYCURL = False
3827+else:
3828+ USING_PYCURL = int(os.environ.get('SYSTEMIMAGE_PYCURL', '0'))
3829+
3830+if not USING_PYCURL:
3831+ SERVICES.append(
3832+ ('com.canonical.applications.Downloader',
3833 DLSERVICE +
3834- ' {self.certs} -disable-timeout -stoppable -log-dir {self.tmpdir}',
3835+ ' {self.udm_certs} -disable-timeout -stoppable -log-dir {self.tmpdir}',
3836 start_downloader,
3837 stop_downloader,
3838- ),
3839- ]
3840+ )
3841+ )
3842
3843
3844 class Controller:
3845@@ -131,17 +146,19 @@
3846 MODULE = 'systemimage.testing.service'
3847
3848 def __init__(self, logfile=None, loglevel='info'):
3849+ self.loglevel = loglevel
3850 # Non-public.
3851 self._stack = ExitStack()
3852 self._stoppers = []
3853 # Public.
3854 self.tmpdir = self._stack.enter_context(temporary_directory())
3855 self.config_path = os.path.join(self.tmpdir, 'dbus-system.conf')
3856- self.ini_path = None
3857 self.serverdir = self._stack.enter_context(temporary_directory())
3858 self.daemon_pid = None
3859 self.mode = 'live'
3860- self.certs = ''
3861+ self.udm_certs = ''
3862+ self.curl_cert = ''
3863+ self.patcher = None
3864 # Set up the dbus-daemon system configuration file.
3865 path = data_path('dbus-system.conf.in')
3866 with open(path, 'r', encoding='utf-8') as fp:
3867@@ -151,19 +168,27 @@
3868 with open(self.config_path, 'w', encoding='utf-8') as fp:
3869 fp.write(config)
3870 # We need a client.ini file for the subprocess.
3871- ini_tmpdir = self._stack.enter_context(temporary_directory())
3872- ini_vardir = self._stack.enter_context(temporary_directory())
3873- ini_logfile = (os.path.join(ini_tmpdir, 'client.log')
3874- if logfile is None
3875- else logfile)
3876- self.ini_path = os.path.join(self.tmpdir, 'client.ini')
3877+ self.ini_tmpdir = self._stack.enter_context(temporary_directory())
3878+ self.ini_vardir = self._stack.enter_context(temporary_directory())
3879+ self.ini_logfile = (os.path.join(self.ini_tmpdir, 'client.log')
3880+ if logfile is None
3881+ else logfile)
3882+ self.ini_path = os.path.join(self.tmpdir, 'config.d')
3883+ makedirs(self.ini_path)
3884+ self._reset_configs()
3885+
3886+ def _reset_configs(self):
3887+ for filename in os.listdir(self.ini_path):
3888+ if filename.endswith('.ini'):
3889+ os.remove(os.path.join(self.ini_path, filename))
3890 template = resource_bytes(
3891- 'systemimage.tests.data', 'config_03.ini').decode('utf-8')
3892- with open(self.ini_path, 'w', encoding='utf-8') as fp:
3893- print(template.format(tmpdir=ini_tmpdir,
3894- vardir=ini_vardir,
3895- logfile=ini_logfile,
3896- loglevel=loglevel),
3897+ 'systemimage.tests.data', '01.ini').decode('utf-8')
3898+ defaults = os.path.join(self.ini_path, '00_defaults.ini')
3899+ with open(defaults, 'w', encoding='utf-8') as fp:
3900+ print(template.format(tmpdir=self.ini_tmpdir,
3901+ vardir=self.ini_vardir,
3902+ logfile=self.ini_logfile,
3903+ loglevel=self.loglevel),
3904 file=fp)
3905
3906 def _configure_services(self):
3907@@ -184,16 +209,41 @@
3908 self._stoppers.append(stopper)
3909 # If the dbus-daemon is running, reload its configuration files.
3910 if self.daemon_pid is not None:
3911- service = dbus.SystemBus().get_object('org.freedesktop.DBus', '/')
3912- iface = dbus.Interface(service, 'org.freedesktop.DBus')
3913- iface.ReloadConfig()
3914- time.sleep(HUP_SLEEP)
3915+ wait_for_service()
3916+
3917+ def _set_udm_certs(self, cert_pem, certificate_path):
3918+ self.udm_certs = (
3919+ '' if cert_pem is None
3920+ else '-self-signed-certs ' + certificate_path)
3921+
3922+ def _set_curl_certs(self, cert_pem, certificate_path):
3923+ # We have to set up the PyCURL downloader's self-signed certificate for
3924+ # the test in two ways. First, because we might be spawning the D-Bus
3925+ # service, we have to pass the path to the cert to that service...
3926+ self.curl_cert = (
3927+ '' if cert_pem is None
3928+ else '--self-signed-cert ' + certificate_path)
3929+ # ...but the controller is also used to set the mode for foreground
3930+ # tests, such as test_download.py. Here we don't spawn any D-Bus
3931+ # processes, but we still have to mock make_testable() in curl.py so
3932+ # that the PyCURL object accepts the self-signed cert.
3933+ if self.patcher is not None:
3934+ self.patcher.stop()
3935+ self.patcher = None
3936+ if cert_pem is not None:
3937+ def self_sign(c):
3938+ c.setopt(pycurl.CAINFO, certificate_path)
3939+ self.patcher = patch('systemimage.curl.make_testable', self_sign)
3940+ self.patcher.start()
3941
3942 def set_mode(self, *, cert_pem=None, service_mode=''):
3943 self.mode = service_mode
3944- self.certs = (
3945- '' if cert_pem is None
3946- else '-self-signed-certs ' + data_path(cert_pem))
3947+ certificate_path = data_path(cert_pem)
3948+ if USING_PYCURL:
3949+ self._set_curl_certs(cert_pem, certificate_path)
3950+ else:
3951+ self._set_udm_certs(cert_pem, certificate_path)
3952+ self._reset_configs()
3953 self._configure_services()
3954
3955 def _start(self):
3956@@ -213,7 +263,7 @@
3957 daemon_exe,
3958 #'/usr/lib/x86_64-linux-gnu/dbus-1.0/debug-build/bin/dbus-daemon',
3959 '--fork',
3960- '--config-file=' + self.config_path,
3961+ '--config-file=' + str(self.config_path),
3962 # Return the address and pid on stdout.
3963 '--print-address=1',
3964 '--print-pid=1',
3965
3966=== modified file 'systemimage/testing/dbus.py'
3967--- systemimage/testing/dbus.py 2014-09-26 14:36:34 +0000
3968+++ systemimage/testing/dbus.py 2015-05-20 14:55:53 +0000
3969@@ -1,4 +1,4 @@
3970-# Copyright (C) 2013-2014 Canonical Ltd.
3971+# Copyright (C) 2013-2015 Canonical Ltd.
3972 # Author: Barry Warsaw <barry@ubuntu.com>
3973
3974 # This program is free software: you can redistribute it and/or modify
3975@@ -23,6 +23,11 @@
3976
3977 import os
3978
3979+try:
3980+ import pycurl
3981+except ImportError:
3982+ pycurl = None
3983+
3984 from dbus.service import method, signal
3985 from gi.repository import GLib
3986 from systemimage.api import Mediator
3987@@ -45,7 +50,7 @@
3988 fp.write(SPACE.join(args[0]).strip())
3989
3990
3991-def instrument(config, stack):
3992+def instrument(config, stack, cert_file):
3993 """Instrument the system for testing."""
3994 # Ensure the destination directories exist.
3995 makedirs(config.updater.data_partition)
3996@@ -54,9 +59,16 @@
3997 # file which the testing parent process can open and read.
3998 safe_reboot = _ActionLog('reboot.log')
3999 stack.enter_context(
4000- patch('systemimage.reboot.check_call', safe_reboot.write))
4001+ patch('systemimage.apply.check_call', safe_reboot.write))
4002 stack.enter_context(
4003 patch('systemimage.device.check_output', return_value='nexus7'))
4004+ # If available, patch the PyCURL downloader to accept self-signed
4005+ # certificates.
4006+ if pycurl is not None:
4007+ def self_sign(c):
4008+ c.setopt(pycurl.CAINFO, cert_file)
4009+ stack.enter_context(
4010+ patch('systemimage.curl.make_testable', self_sign))
4011
4012
4013 class _LiveTestableService(Service):
4014@@ -65,6 +77,7 @@
4015 @log_and_exit
4016 @method('com.canonical.SystemImage')
4017 def Reset(self):
4018+ config.reload()
4019 self._api = Mediator()
4020 try:
4021 self._checking.release()
4022@@ -72,7 +85,6 @@
4023 # Lock is already released.
4024 pass
4025 self._update = None
4026- self._downloading = False
4027 self._rebootable = False
4028 self._failure_count = 0
4029 del config.build_number
4030@@ -189,9 +201,9 @@
4031 @method('com.canonical.SystemImage')
4032 def ApplyUpdate(self):
4033 # Always succeeds.
4034- def _rebooting():
4035- self.Rebooting(True)
4036- GLib.timeout_add(50, _rebooting)
4037+ def _applied():
4038+ self.Applied(True)
4039+ GLib.timeout_add(50, _applied)
4040
4041
4042 class _UpdateManualSuccess(_UpdateAutoSuccess):
4043@@ -259,9 +271,9 @@
4044 @method('com.canonical.SystemImage')
4045 def ApplyUpdate(self):
4046 # The update cannot be applied.
4047- def _rebooting():
4048- self.Rebooting(False)
4049- GLib.timeout_add(50, _rebooting)
4050+ def _applied():
4051+ self.Applied(False)
4052+ GLib.timeout_add(50, _applied)
4053
4054
4055 class _FailResume(Service):
4056
4057=== modified file 'systemimage/testing/demo.py'
4058--- systemimage/testing/demo.py 2014-02-20 23:03:24 +0000
4059+++ systemimage/testing/demo.py 2015-05-20 14:55:53 +0000
4060@@ -1,4 +1,4 @@
4061-# Copyright (C) 2013-2014 Canonical Ltd.
4062+# Copyright (C) 2013-2015 Canonical Ltd.
4063 # Author: Barry Warsaw <barry@ubuntu.com>
4064
4065 # This program is free software: you can redistribute it and/or modify
4066@@ -22,12 +22,12 @@
4067 ]
4068
4069
4070+from systemimage.apply import BaseApply
4071 from systemimage.device import BaseDevice
4072-from systemimage.reboot import BaseReboot
4073-
4074-
4075-class DemoReboot(BaseReboot):
4076- def reboot(self):
4077+
4078+
4079+class DemoReboot(BaseApply):
4080+ def apply(self):
4081 print("If I was a phone, I'd be rebooting right about now.")
4082
4083
4084
4085=== modified file 'systemimage/testing/helpers.py'
4086--- systemimage/testing/helpers.py 2014-09-17 13:41:31 +0000
4087+++ systemimage/testing/helpers.py 2015-05-20 14:55:53 +0000
4088@@ -1,4 +1,4 @@
4089-# Copyright (C) 2013-2014 Canonical Ltd.
4090+# Copyright (C) 2013-2015 Canonical Ltd.
4091 # Author: Barry Warsaw <barry@ubuntu.com>
4092
4093 # This program is free software: you can redistribute it and/or modify
4094@@ -23,6 +23,7 @@
4095 'data_path',
4096 'debug',
4097 'debuggable',
4098+ 'descriptions',
4099 'find_dbus_process',
4100 'get_channels',
4101 'get_index',
4102@@ -32,14 +33,18 @@
4103 'setup_keyring_txz',
4104 'setup_keyrings',
4105 'sign',
4106+ 'terminate_service',
4107 'touch_build',
4108+ 'wait_for_service',
4109 'write_bytes',
4110 ]
4111
4112
4113 import os
4114 import ssl
4115+import dbus
4116 import json
4117+import time
4118 import gnupg
4119 import psutil
4120 import shutil
4121@@ -47,8 +52,8 @@
4122 import tarfile
4123 import unittest
4124
4125-from contextlib import ExitStack, contextmanager
4126-from functools import partial, wraps
4127+from contextlib import ExitStack, contextmanager, suppress
4128+from functools import partial, partialmethod, wraps
4129 from http.server import HTTPServer, SimpleHTTPRequestHandler
4130 from pathlib import Path
4131 from pkg_resources import resource_filename, resource_string as resource_bytes
4132@@ -57,7 +62,6 @@
4133 from systemimage.config import Configuration, config
4134 from systemimage.helpers import MiB, atomic, makedirs, temporary_directory
4135 from systemimage.index import Index
4136-from systemimage.state import State
4137 from threading import Thread
4138 from unittest.mock import patch
4139
4140@@ -78,7 +82,7 @@
4141
4142 def data_path(filename):
4143 return os.path.abspath(
4144- resource_filename('systemimage.tests.data', filename))
4145+ resource_filename('systemimage.tests.data', filename))
4146
4147
4148 def make_http_server(directory, port, certpem=None, keypem=None):
4149@@ -117,6 +121,17 @@
4150 except ConnectionResetError:
4151 super().handle_one_request()
4152
4153+ def do_HEAD(self):
4154+ # Just tell the client we have the magic file.
4155+ if self.path == '/user-agent.txt':
4156+ self.send_response(200)
4157+ self.end_headers()
4158+ else:
4159+ # Canceling a download can cause our internal server to
4160+ # see various ignorable errors. No worries.
4161+ with suppress(BrokenPipeError, ConnectionResetError):
4162+ super().do_HEAD()
4163+
4164 def do_GET(self):
4165 # If we requested the magic 'user-agent.txt' file, send back the
4166 # value of the User-Agent header. Otherwise, vend as normal.
4167@@ -127,12 +142,10 @@
4168 self.end_headers()
4169 self.wfile.write(user_agent.encode('utf-8'))
4170 else:
4171- try:
4172+ # Canceling a download can cause our internal server to
4173+ # see various ignorable errors. No worries.
4174+ with suppress(BrokenPipeError, ConnectionResetError):
4175 super().do_GET()
4176- except (BrokenPipeError, ConnectionResetError):
4177- # Canceling a download can cause our internal server to
4178- # see various ignorable errors. No worries.
4179- pass
4180 # Create the server in the main thread, but start it in the sub-thread.
4181 # This lets the main thread call .shutdown() to stop everything. Return
4182 # just the shutdown method to the caller.
4183@@ -198,47 +211,98 @@
4184 return resources.pop_all()
4185
4186
4187-def configuration(function):
4188- """Decorator that produces a temporary configuration for testing.
4189-
4190- The config_00.ini template is copied to a temporary file and the the
4191- various file system locations are filled in with the location for a,
4192- er, temporary temporary directory. This temporary configuration
4193- file is loaded up and the global configuration object is patched so
4194- that all other code will see it instead of the default global
4195- configuration object.
4196-
4197- Everything is properly cleaned up after the test method exits.
4198- """
4199- @wraps(function)
4200- def wrapper(*args, **kws):
4201- with ExitStack() as resources:
4202- etc_dir = resources.enter_context(temporary_directory())
4203- ini_file = os.path.join(etc_dir, 'client.ini')
4204- temp_tmpdir = resources.enter_context(temporary_directory())
4205- temp_vardir = resources.enter_context(temporary_directory())
4206+# This defines the @configuration decorator used in various test suites to
4207+# create a temporary config.d/ directory for a test. This is all fairly
4208+# complicated, but here's what's going on.
4209+#
4210+# The _wrapper() function is the inner part of the decorator, and it does the
4211+# heart of the operation, which is to create a temporary directory for
4212+# config.d, along with temporary var and tmp directories. These latter two
4213+# will be interpolated into any configuration file copied into config.d.
4214+#
4215+# The outer decorator function differs depending on whether @configuration was
4216+# given without arguments, or called with arguments at the time of the
4217+# function definition.
4218+#
4219+# In the former case, e.g.
4220+#
4221+# @configuration
4222+# def test_something(self):
4223+#
4224+# The default 00.ini file is interpolated and copied into config.d. Simple.
4225+#
4226+# In the latter case, e.g.
4227+#
4228+# @configuration('some-config.ini')
4229+# def test_something(self):
4230+#
4231+# There's actually another level of interior function, because the outer
4232+# decorator itself is getting called. Here, any named configuration file is
4233+# additionally copied to the config.d directory, renaming it sequentionally to
4234+# something like 01_override.ini, with the numeric part incrementing
4235+# monotonically.
4236+#
4237+# The implementation is tricky because we want the call sites to be simple.
4238+def _wrapper(self, function, ini_files, *args, **kws):
4239+ start = 0
4240+ with ExitStack() as resources:
4241+ # Create the config.d directory and copy all the source ini files to
4242+ # this directory in sequential order, interpolating in the temporary
4243+ # tmp and var directories.
4244+ config_d = resources.enter_context(temporary_directory())
4245+ temp_tmpdir = resources.enter_context(temporary_directory())
4246+ temp_vardir = resources.enter_context(temporary_directory())
4247+ for ini_file in ini_files:
4248+ dst = os.path.join(config_d, '{:02d}_override.ini'.format(start))
4249+ start += 1
4250 template = resource_bytes(
4251- 'systemimage.tests.data', 'config_00.ini').decode('utf-8')
4252- with atomic(ini_file) as fp:
4253+ 'systemimage.tests.data', ini_file).decode('utf-8')
4254+ with atomic(dst) as fp:
4255 print(template.format(tmpdir=temp_tmpdir,
4256 vardir=temp_vardir), file=fp)
4257- config = Configuration(ini_file)
4258- resources.enter_context(
4259- patch('systemimage.config._config', config))
4260- resources.enter_context(
4261- patch('systemimage.device.check_output',
4262- return_value='nexus7'))
4263- # Make sure the cache_partition and data_partition exist.
4264- makedirs(config.updater.cache_partition)
4265- makedirs(config.updater.data_partition)
4266- # The method under test is allowed to specify some additional
4267- # keyword arguments, in order to pass some variables in from the
4268- # wrapper.
4269- signature = inspect.signature(function)
4270- if 'ini_file' in signature.parameters:
4271- kws['ini_file'] = ini_file
4272- return function(*args, **kws)
4273- return wrapper
4274+ # Patch the global configuration object so that it can be used
4275+ # directly, which is good enough in most cases. Also patch the bit of
4276+ # code that detects the device name.
4277+ config = Configuration(config_d)
4278+ resources.enter_context(
4279+ patch('systemimage.config._config', config))
4280+ resources.enter_context(
4281+ patch('systemimage.device.check_output',
4282+ return_value='nexus7'))
4283+ # Make sure the cache_partition and data_partition exist.
4284+ makedirs(config.updater.cache_partition)
4285+ makedirs(config.updater.data_partition)
4286+ # The method under test is allowed to specify some additional
4287+ # keyword arguments, in order to pass some variables in from the
4288+ # wrapper.
4289+ signature = inspect.signature(function)
4290+ if 'config_d' in signature.parameters:
4291+ kws['config_d'] = config_d
4292+ if 'config' in signature.parameters:
4293+ kws['config'] = config
4294+ # Call the function with the given arguments and return the result.
4295+ return function(self, *args, **kws)
4296+
4297+
4298+def configuration(*args):
4299+ """Outer decorator which can be called or not at function definition time.
4300+
4301+ If called, the arguments are positional only, and name the test data .ini
4302+ files which are to be copied to config.d directory. If none are given,
4303+ then 00.ini is used.
4304+ """
4305+ if len(args) == 1 and callable(args[0]):
4306+ # We assume this was the bare @configuration decorator flavor.
4307+ function = args[0]
4308+ inner = partialmethod(_wrapper, function, ('00.ini',))
4309+ return wraps(function)(inner)
4310+ else:
4311+ # We assume this was the called @configuration(...) decorator flavor,
4312+ # so create the actual decorator that wraps the _wrapper function.
4313+ def decorator(function):
4314+ inner = partialmethod(_wrapper, function, args)
4315+ return wraps(function)(inner)
4316+ return decorator
4317
4318
4319 def sign(filename, pubkey_ring):
4320@@ -249,6 +313,8 @@
4321 with. This keyring must contain only one key, and its key id must
4322 exist in the master secret keyring.
4323 """
4324+ # filename could be a Path object. For now, just str-ify it.
4325+ filename = str(filename)
4326 with ExitStack() as resources:
4327 home = resources.enter_context(temporary_directory())
4328 secring = data_path('master-secring.gpg')
4329@@ -268,7 +334,7 @@
4330
4331 def copy(filename, todir, dst=None):
4332 src = data_path(filename)
4333- dst = os.path.join(todir, filename if dst is None else dst)
4334+ dst = os.path.join(str(todir), filename if dst is None else dst)
4335 makedirs(os.path.dirname(dst))
4336 shutil.copy(src, dst)
4337
4338@@ -395,21 +461,24 @@
4339 os.chmod(path, old_mode)
4340
4341
4342-def touch_build(version, timestamp=None):
4343+def touch_build(version, timestamp=None, use_config=None):
4344 # LP: #1220238 - assert that no old-style version numbers are being used.
4345 assert 0 <= version < (1 << 16), (
4346- 'old style version number: {}'.format(version))
4347- with open(config.system.build_file, 'w', encoding='utf-8') as fp:
4348- print(version, file=fp)
4349+ 'Old style version number: {}'.format(version))
4350+ if use_config is None:
4351+ use_config = config
4352+ override = Path(use_config.config_d) / '99_build.ini'
4353+ with override.open('wt', encoding='utf-8') as fp:
4354+ print("""\
4355+[service]
4356+build_number: {}
4357+""".format(version), file=fp)
4358+ # We have to touch the mtimes for all the files in the config directory.
4359 if timestamp is not None:
4360 timestamp = int(timestamp)
4361- os.utime(config.system.build_file, (timestamp, timestamp))
4362- channel_ini = os.path.join(
4363- os.path.dirname(config.config_file), 'channel.ini')
4364- try:
4365- os.utime(channel_ini, (timestamp, timestamp))
4366- except FileNotFoundError:
4367- pass
4368+ for path in Path(use_config.config_d).iterdir():
4369+ os.utime(str(path), (timestamp, timestamp))
4370+ use_config.reload()
4371
4372
4373 def write_bytes(path, size_in_mib):
4374@@ -431,13 +500,13 @@
4375
4376
4377 @contextmanager
4378-def debug(*, check_flag=False):
4379+def debug(*, check_flag=False, end='\n'):
4380 if not check_flag or os.path.exists('/tmp/debug.enabled'):
4381 path = Path('/tmp/debug.log')
4382 else:
4383 path = Path(os.devnull)
4384 with path.open('a', encoding='utf-8') as fp:
4385- function = partial(print, file=fp)
4386+ function = partial(print, file=fp, end=end)
4387 function.fp = fp
4388 yield function
4389 fp.flush()
4390@@ -487,6 +556,8 @@
4391 SystemImagePlugin.controller.set_mode(cert_pem='cert.pem')
4392
4393 def setUp(self):
4394+ # Avoid circular imports.
4395+ from systemimage.state import State
4396 self._resources = ExitStack()
4397 self._state = State()
4398 try:
4399@@ -544,3 +615,44 @@
4400 dict(type='device-signing'),
4401 os.path.join(self._serverdir, self.CHANNEL, self.DEVICE,
4402 'device-signing.tar.xz'))
4403+
4404+
4405+def descriptions(path):
4406+ descriptions = []
4407+ for image in path:
4408+ # There's only one description per image so order doesn't
4409+ # matter.
4410+ descriptions.extend(image.descriptions.values())
4411+ return descriptions
4412+
4413+
4414+def wait_for_service(*, restart=False, reload=True):
4415+ bus = dbus.SystemBus()
4416+ if restart:
4417+ service = bus.get_object('com.canonical.SystemImage', '/Service')
4418+ iface = dbus.Interface(service, 'com.canonical.SystemImage')
4419+ iface.Exit()
4420+ service = dbus.SystemBus().get_object('org.freedesktop.DBus', '/')
4421+ iface = dbus.Interface(service, 'org.freedesktop.DBus')
4422+ if reload:
4423+ iface.ReloadConfig()
4424+ # Wait until the system-image-dbus process is actually running.
4425+ # http://people.freedesktop.org/~david/eggdbus-20091014/eggdbus-interface-org.freedesktop.DBus.html#eggdbus-method-org.freedesktop.DBus.StartServiceByName
4426+ reply = 0
4427+ # 2015-03-09 BAW: This could potentially spin forever, but we'll assume
4428+ # D-Bus eventually is successful in starting the service.
4429+ while reply != 2:
4430+ reply = iface.StartServiceByName('com.canonical.SystemImage', 0)
4431+ time.sleep(0.1)
4432+
4433+
4434+def terminate_service():
4435+ # Avoid circular imports.
4436+ from systemimage.testing.nose import SystemImagePlugin
4437+ proc = find_dbus_process(SystemImagePlugin.controller.ini_path)
4438+ if proc is not None:
4439+ bus = dbus.SystemBus()
4440+ service = bus.get_object('com.canonical.SystemImage', '/Service')
4441+ iface = dbus.Interface(service, 'com.canonical.SystemImage')
4442+ iface.Exit()
4443+ proc.wait()
4444
4445=== modified file 'systemimage/testing/nose.py'
4446--- systemimage/testing/nose.py 2014-09-26 14:36:34 +0000
4447+++ systemimage/testing/nose.py 2015-05-20 14:55:53 +0000
4448@@ -1,4 +1,4 @@
4449-# Copyright (C) 2013-2014 Canonical Ltd.
4450+# Copyright (C) 2013-2015 Canonical Ltd.
4451 # Author: Barry Warsaw <barry@ubuntu.com>
4452
4453 # This program is free software: you can redistribute it and/or modify
4454@@ -90,7 +90,7 @@
4455 'Set the log file for the test run',
4456 nargs=1)
4457 def set_dbus_loglevel(level):
4458- self.log_level = 'info:{}'.format(level[0])
4459+ self.log_level = level[0]
4460 self.addOption(set_dbus_loglevel, 'M', 'loglevel',
4461 'Set the systemimage.dbus log level',
4462 nargs=1)
4463@@ -149,3 +149,8 @@
4464 ## from systemimage.testing.helpers import debug
4465 ## with debug() as dlog:
4466 ## dlog('^^^^^', event.test)
4467+
4468+ def describeTest(self, event):
4469+ # This is fucked up.
4470+ if 'partial' in event.description:
4471+ event.description = event.description[:-73]
4472
4473=== added file 'systemimage/testing/service.py'
4474--- systemimage/testing/service.py 1970-01-01 00:00:00 +0000
4475+++ systemimage/testing/service.py 2015-05-20 14:55:53 +0000
4476@@ -0,0 +1,65 @@
4477+# Copyright (C) 2014-2015 Canonical Ltd.
4478+# Author: Barry Warsaw <barry@ubuntu.com>
4479+
4480+# This program is free software: you can redistribute it and/or modify
4481+# it under the terms of the GNU General Public License as published by
4482+# the Free Software Foundation; version 3 of the License.
4483+#
4484+# This program is distributed in the hope that it will be useful,
4485+# but WITHOUT ANY WARRANTY; without even the implied warranty of
4486+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
4487+# GNU General Public License for more details.
4488+#
4489+# You should have received a copy of the GNU General Public License
4490+# along with this program. If not, see <http://www.gnu.org/licenses/>.
4491+
4492+"""DBus service testing pre-load module.
4493+
4494+This is arranged so that the test suite can enable code coverage data
4495+collection as early as possible in the private bus D-Bus activated processes.
4496+"""
4497+
4498+# Uncomment this if the controller won't start. There's no other good way to
4499+# get debugging information about the D-Bus activated process, since their
4500+# stderr just seems to get lost.
4501+## import sys
4502+## sys.stderr = open('/tmp/debug.log', 'a', encoding='utf-8')
4503+
4504+
4505+import os
4506+
4507+# Set this environment variable if the controller won't start. There's no
4508+# other good way to get debugging information about the D-Bus activated
4509+# process, since their stderr just seems to get lost.
4510+if os.environ.get('SYSTEMIMAGE_DEBUG_DBUS_ACTIVATION'):
4511+ import sys
4512+ sys.stderr = open('/tmp/debug.log', 'a', encoding='utf-8')
4513+
4514+
4515+# It's okay if this module isn't available.
4516+try:
4517+ from coverage.control import coverage as _Coverage
4518+except ImportError:
4519+ _Coverage = None
4520+
4521+
4522+def main():
4523+ # Enable code coverage.
4524+ ini_file = os.environ.get('COVERAGE_PROCESS_START')
4525+ if _Coverage is not None and ini_file is not None:
4526+ coverage =_Coverage(config_file=ini_file, auto_data=True)
4527+ # Stolen from coverage.process_startup()
4528+ coverage.erase()
4529+ coverage.start()
4530+ coverage._warn_no_data = False
4531+ coverage._warn_unimported_source = False
4532+ # All systemimage imports happen here so that we have the best possible
4533+ # chance of instrumenting all relevant code.
4534+ from systemimage.service import main as real_main
4535+ # Now run the actual D-Bus service.
4536+ return real_main()
4537+
4538+
4539+if __name__ == '__main__':
4540+ import sys
4541+ sys.exit(main())
4542
4543=== removed file 'systemimage/testing/service.py'
4544--- systemimage/testing/service.py 2014-09-17 02:58:58 +0000
4545+++ systemimage/testing/service.py 1970-01-01 00:00:00 +0000
4546@@ -1,50 +0,0 @@
4547-# Copyright (C) 2014 Canonical Ltd.
4548-# Author: Barry Warsaw <barry@ubuntu.com>
4549-
4550-# This program is free software: you can redistribute it and/or modify
4551-# it under the terms of the GNU General Public License as published by
4552-# the Free Software Foundation; version 3 of the License.
4553-#
4554-# This program is distributed in the hope that it will be useful,
4555-# but WITHOUT ANY WARRANTY; without even the implied warranty of
4556-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
4557-# GNU General Public License for more details.
4558-#
4559-# You should have received a copy of the GNU General Public License
4560-# along with this program. If not, see <http://www.gnu.org/licenses/>.
4561-
4562-"""DBus service testing pre-load module.
4563-
4564-This is arranged so that the test suite can enable code coverage data
4565-collection as early as possible in the private bus D-Bus activated processes.
4566-"""
4567-
4568-import os
4569-
4570-# It's okay if this module isn't available.
4571-try:
4572- from coverage.control import coverage as _Coverage
4573-except ImportError:
4574- _Coverage = None
4575-
4576-
4577-def main():
4578- # Enable code coverage.
4579- ini_file = os.environ.get('COVERAGE_PROCESS_START')
4580- if _Coverage is not None and ini_file is not None:
4581- coverage =_Coverage(config_file=ini_file, auto_data=True)
4582- # Stolen from coverage.process_startup()
4583- coverage.erase()
4584- coverage.start()
4585- coverage._warn_no_data = False
4586- coverage._warn_unimported_source = False
4587- # All systemimage imports happen here so that we have the best possible
4588- # chance of instrumenting all relevant code.
4589- from systemimage.service import main as real_main
4590- # Now run the actual D-Bus service.
4591- return real_main()
4592-
4593-
4594-if __name__ == '__main__':
4595- import sys
4596- sys.exit(main())
4597
4598=== renamed file 'systemimage/tests/data/config_00.ini' => 'systemimage/tests/data/00.ini'
4599--- systemimage/tests/data/config_00.ini 2014-01-30 15:41:03 +0000
4600+++ systemimage/tests/data/00.ini 2015-05-20 14:55:53 +0000
4601@@ -12,7 +12,6 @@
4602
4603 [system]
4604 timeout: 1s
4605-build_file: {tmpdir}/ubuntu-build
4606 tempdir: {tmpdir}/tmp
4607 logfile: {tmpdir}/client.log
4608 loglevel: info
4609@@ -31,7 +30,7 @@
4610 [hooks]
4611 device: systemimage.device.SystemProperty
4612 scorer: systemimage.scores.WeightedScorer
4613-reboot: systemimage.reboot.Reboot
4614+apply: systemimage.apply.Reboot
4615
4616 [dbus]
4617 lifetime: 2m
4618
4619=== renamed file 'systemimage/tests/data/config_03.ini' => 'systemimage/tests/data/01.ini'
4620--- systemimage/tests/data/config_03.ini 2014-09-17 13:41:31 +0000
4621+++ systemimage/tests/data/01.ini 2015-05-20 14:55:53 +0000
4622@@ -12,14 +12,13 @@
4623
4624 [system]
4625 timeout: 1s
4626-build_file: {tmpdir}/ubuntu-build
4627 tempdir: {tmpdir}/tmp
4628 logfile: {logfile}
4629 loglevel: {loglevel}
4630 settings_db: {vardir}/settings.db
4631
4632 [gpg]
4633-archive_master: {vardir}/etc/archive-master.tar.xz
4634+archive_master: {vardir}/usr/share/system-image/archive-master.tar.xz
4635 image_master: {vardir}/keyrings/image-master.tar.xz
4636 image_signing: {vardir}/keyrings/image-signing.tar.xz
4637 device_signing: {vardir}/keyrings/device-signing.tar.xz
4638@@ -31,7 +30,7 @@
4639 [hooks]
4640 device: systemimage.testing.demo.TestingDevice
4641 scorer: systemimage.scores.WeightedScorer
4642-reboot: systemimage.reboot.Reboot
4643+apply: systemimage.apply.Reboot
4644
4645 [dbus]
4646 lifetime: 5m
4647
4648=== added file 'systemimage/tests/data/api.channels_01.json'
4649--- systemimage/tests/data/api.channels_01.json 1970-01-01 00:00:00 +0000
4650+++ systemimage/tests/data/api.channels_01.json 2015-05-20 14:55:53 +0000
4651@@ -0,0 +1,13 @@
4652+{
4653+ "stable": {
4654+ "devices": {
4655+ "nexus7": {
4656+ "index": "/stable/nexus7/index.json",
4657+ "keyring": {
4658+ "path": "/stable/nexus7/device-signing.tar.xz",
4659+ "signature": "/stable/nexus7/device-signing.tar.xz.asc"
4660+ }
4661+ }
4662+ }
4663+ }
4664+}
4665
4666=== added file 'systemimage/tests/data/api.index_01.json'
4667--- systemimage/tests/data/api.index_01.json 1970-01-01 00:00:00 +0000
4668+++ systemimage/tests/data/api.index_01.json 2015-05-20 14:55:53 +0000
4669@@ -0,0 +1,36 @@
4670+{
4671+ "global": {
4672+ "generated_at": "Thu Aug 01 08:01:00 UTC 2013"
4673+ },
4674+ "images": [
4675+ {
4676+ "description": "Full",
4677+ "files": [
4678+ {
4679+ "checksum": "da70dfa4d9f95ac979f921e8e623358236313f334afcd06cddf8a5621cf6a1e9",
4680+ "order": 3,
4681+ "path": "/3/4/5.txt",
4682+ "signature": "/3/4/5.txt.asc",
4683+ "size": 104857600
4684+ },
4685+ {
4686+ "checksum": "b3a8e0e1f9ab1bfe3a36f231f676f78bb30a519d2b21e6c530c0eee8ebb4a5d0",
4687+ "order": 1,
4688+ "path": "/4/5/6.txt",
4689+ "signature": "/4/5/6.txt.asc",
4690+ "size": 104857600
4691+ },
4692+ {
4693+ "checksum": "97a6d21df7c51e8289ac1a8c026aaac143e15aa1957f54f42e30d8f8a85c3a55",
4694+ "order": 2,
4695+ "path": "/5/6/7.txt",
4696+ "signature": "/5/6/7.txt.asc",
4697+ "size": 104857600
4698+ }
4699+ ],
4700+ "type": "full",
4701+ "version": 1600,
4702+ "bootme": true
4703+ }
4704+ ]
4705+}
4706
4707=== added file 'systemimage/tests/data/api.index_02.json'
4708--- systemimage/tests/data/api.index_02.json 1970-01-01 00:00:00 +0000
4709+++ systemimage/tests/data/api.index_02.json 2015-05-20 14:55:53 +0000
4710@@ -0,0 +1,251 @@
4711+{
4712+ "global": {
4713+ "generated_at": "Mon Apr 29 18:45:27 UTC 2013"
4714+ },
4715+ "images": [
4716+ {
4717+ "bootme": true,
4718+ "description": "Full A",
4719+ "files": [
4720+ {
4721+ "checksum": "abc",
4722+ "order": 1,
4723+ "path": "/a/b/c.txt",
4724+ "signature": "/a/b/c.txt.asc",
4725+ "size": 104857600
4726+
4727+ },
4728+ {
4729+ "checksum": "bcd",
4730+ "order": 1,
4731+ "path": "/b/c/d.txt",
4732+ "signature": "/b/c/d.txt.asc",
4733+ "size": 104857600
4734+ },
4735+ {
4736+ "checksum": "cde",
4737+ "order": 1,
4738+ "path": "/c/d/e.txt",
4739+ "signature": "/c/d/e.txt.asc",
4740+ "size": 104857600
4741+ }
4742+ ],
4743+ "type": "full",
4744+ "version": 1300
4745+ },
4746+ {
4747+ "base": 1300,
4748+ "bootme": true,
4749+ "description": "Delta A.1",
4750+ "files": [
4751+ {
4752+ "checksum": "def",
4753+ "order": 1,
4754+ "path": "/d/e/f.txt",
4755+ "signature": "/d/e/f.txt.asc",
4756+ "size": 104857600
4757+ },
4758+ {
4759+ "checksum": "ef0",
4760+ "order": 1,
4761+ "path": "/e/f/0.txt",
4762+ "signature": "/e/f/0.txt.asc",
4763+ "size": 104857600
4764+ },
4765+ {
4766+ "checksum": "f01",
4767+ "order": 1,
4768+ "path": "/f/e/1.txt",
4769+ "signature": "/f/e/1.txt.asc",
4770+ "size": 104857600
4771+ }
4772+ ],
4773+ "type": "delta",
4774+ "version": 1301
4775+ },
4776+ {
4777+ "base": 1301,
4778+ "bootme": true,
4779+ "description": "Delta A.2",
4780+ "files": [
4781+ {
4782+ "checksum": "012",
4783+ "order": 1,
4784+ "path": "/0/1/2.txt",
4785+ "signature": "/0/1/2.txt.asc",
4786+ "size": 104857600
4787+ },
4788+ {
4789+ "checksum": "123",
4790+ "order": 1,
4791+ "path": "/1/2/3.txt",
4792+ "signature": "/1/2/3.txt.asc",
4793+ "size": 104857600
4794+ },
4795+ {
4796+ "checksum": "234",
4797+ "order": 1,
4798+ "path": "/2/3/4.txt",
4799+ "signature": "/2/3/4.txt.asc",
4800+ "size": 104857600
4801+ }
4802+ ],
4803+ "type": "delta",
4804+ "version": 1304
4805+ },
4806+
4807+ {
4808+ "description": "Full B",
4809+ "description-en": "The full B",
4810+ "files": [
4811+ {
4812+ "checksum": "da70dfa4d9f95ac979f921e8e623358236313f334afcd06cddf8a5621cf6a1e9",
4813+ "order": 1,
4814+ "path": "/3/4/5.txt",
4815+ "signature": "/3/4/5.txt.asc",
4816+ "size": 10000
4817+ },
4818+ {
4819+ "checksum": "b3a8e0e1f9ab1bfe3a36f231f676f78bb30a519d2b21e6c530c0eee8ebb4a5d0",
4820+ "order": 1,
4821+ "path": "/4/5/6.txt",
4822+ "signature": "/4/5/6.txt.asc",
4823+ "size": 10001
4824+ },
4825+ {
4826+ "checksum": "97a6d21df7c51e8289ac1a8c026aaac143e15aa1957f54f42e30d8f8a85c3a55",
4827+ "order": 1,
4828+ "path": "/5/6/7.txt",
4829+ "signature": "/5/6/7.txt.asc",
4830+ "size": 10002
4831+ }
4832+ ],
4833+ "type": "full",
4834+ "version": 1200
4835+ },
4836+ {
4837+ "base": 1200,
4838+ "description": "Delta B.1",
4839+ "description-en_US": "This is the delta B.1",
4840+ "description-xx": "XX This is the delta B.1",
4841+ "description-yy": "YY This is the delta B.1",
4842+ "description-yy_ZZ": "YY-ZZ This is the delta B.1",
4843+ "files": [
4844+ {
4845+ "checksum": "cebe3d9d614ba5c19f633566104315854a11353a333bf96f16b5afa0e90abdc4",
4846+ "order": 1,
4847+ "path": "/6/7/8.txt",
4848+ "signature": "/6/7/8.txt.asc",
4849+ "size": 20000
4850+ },
4851+ {
4852+ "checksum": "35a9e381b1a27567549b5f8a6f783c167ebf809f1c4d6a9e367240484d8ce281",
4853+ "order": 1,
4854+ "path": "/7/8/9.txt",
4855+ "signature": "/7/8/9.txt.asc",
4856+ "size": 20001
4857+ },
4858+ {
4859+ "checksum": "6bd6c3f7808391e8b74f5c2d58810809eda5c134aaa7f1b27ddf4b445c421ac5",
4860+ "order": 1,
4861+ "path": "/8/9/a.txt",
4862+ "signature": "/8/9/a.txt.asc",
4863+ "size": 20002
4864+ }
4865+ ],
4866+ "type": "delta",
4867+ "version": 1201
4868+ },
4869+ {
4870+ "base": 1201,
4871+ "description": "Delta B.2",
4872+ "description-xx": "Oh delta, my delta",
4873+ "description-xx_CC": "This hyar is the delta B.2",
4874+ "files": [
4875+ {
4876+ "checksum": "8c43d75d5b9f1aa9fc3fabb6b60b6c06553324352399a33febce95a1b588d1d6",
4877+ "order": 1,
4878+ "path": "/9/a/b.txt",
4879+ "signature": "/9/a/b.txt.asc",
4880+ "size": 30000
4881+ },
4882+ {
4883+ "checksum": "20e796c128096d229ba89bf412a53c3151d170a409c2c8c1dd8e414087b7ffae",
4884+ "order": 1,
4885+ "path": "/f/e/d.txt",
4886+ "signature": "/f/e/d.txt.asc",
4887+ "size": 30001
4888+ },
4889+ {
4890+ "checksum": "278238e8bafa4709c77aa723e168101acd6ee1fb9fcc1b6eca4762e5c7dad768",
4891+ "order": 1,
4892+ "path": "/e/d/c.txt",
4893+ "signature": "/e/d/c.txt.asc",
4894+ "size": 30002
4895+
4896+ }
4897+ ],
4898+ "type": "delta",
4899+ "version": 1304
4900+ },
4901+
4902+ {
4903+ "description": "Full C",
4904+ "files": [
4905+ {
4906+ "checksum": "dcb",
4907+ "order": 1,
4908+ "path": "/d/c/b.txt",
4909+ "signature": "/d/c/b.txt.asc",
4910+ "size": 104857600
4911+ },
4912+ {
4913+ "checksum": "cba",
4914+ "order": 1,
4915+ "path": "/c/b/a.txt",
4916+ "signature": "/c/b/a.txt.asc",
4917+ "size": 104857600
4918+ },
4919+ {
4920+ "checksum": "ba9",
4921+ "order": 1,
4922+ "path": "/b/a/9.txt",
4923+ "signature": "/b/a/9.txt.asc",
4924+ "size": 104857600
4925+ }
4926+ ],
4927+ "type": "full",
4928+ "version": 1100
4929+ },
4930+ {
4931+ "base": 1100,
4932+ "description": "Delta C.1",
4933+ "files": [
4934+ {
4935+ "checksum": "a98",
4936+ "order": 1,
4937+ "path": "/a/9/8.txt",
4938+ "signature": "/a/9/8.txt.asc",
4939+ "size": 104857600
4940+ },
4941+ {
4942+ "checksum": "987",
4943+ "order": 1,
4944+ "path": "/9/8/7.txt",
4945+ "signature": "/9/8/7.txt.asc",
4946+ "size": 104857600
4947+ },
4948+ {
4949+ "checksum": "876",
4950+ "order": 1,
4951+ "path": "/8/7/6.txt",
4952+ "signature": "/8/7/6.txt.asc",
4953+ "size": 838860800
4954+
4955+ }
4956+ ],
4957+ "type": "delta",
4958+ "version": 1303
4959+ }
4960+ ]
4961+}
4962
4963=== added file 'systemimage/tests/data/api.index_03.json'
4964--- systemimage/tests/data/api.index_03.json 1970-01-01 00:00:00 +0000
4965+++ systemimage/tests/data/api.index_03.json 2015-05-20 14:55:53 +0000
4966@@ -0,0 +1,37 @@
4967+{
4968+ "global": {
4969+ "generated_at": "Thu Aug 01 08:01:00 UTC 2013"
4970+ },
4971+ "images": [
4972+ {
4973+ "description": "Full",
4974+ "files": [
4975+ {
4976+ "checksum": "da70dfa4d9f95ac979f921e8e623358236313f334afcd06cddf8a5621cf6a1e9",
4977+ "order": 3,
4978+ "path": "/3/4/5.txt",
4979+ "signature": "/3/4/5.txt.asc",
4980+ "size": 104857600
4981+ },
4982+ {
4983+ "checksum": "b3a8e0e1f9ab1bfe3a36f231f676f78bb30a519d2b21e6c530c0eee8ebb4a5d0",
4984+ "order": 1,
4985+ "path": "/4/5/6.txt",
4986+ "signature": "/4/5/6.txt.asc",
4987+ "size": 104857600
4988+ },
4989+ {
4990+ "checksum": "97a6d21df7c51e8289ac1a8c026aaac143e15aa1957f54f42e30d8f8a85c3a55",
4991+ "order": 2,
4992+ "path": "/5/6/7.txt",
4993+ "signature": "/5/6/7.txt.asc",
4994+ "size": 104857600
4995+ }
4996+ ],
4997+ "type": "full",
4998+ "version": 1600,
4999+ "version_detail": "ubuntu=101,raw-device=201,version=301",
5000+ "bootme": true
The diff has been truncated for viewing.

Subscribers

People subscribed via source and target branches