Ubuntu system image

Merge lp:~cjwatson/ubuntu-system-image/cdimage-custom into lp:ubuntu-system-image/server

  1. cdimage-custom
  2. Merge into server
Proposed by Colin Watson
Status: Merged
Merged at revision: 248
Proposed branch: lp:~cjwatson/ubuntu-system-image/cdimage-custom
Merge into: lp:ubuntu-system-image/server
Diff against target: 250 lines (+222/-0)
2 files modified
lib/systemimage/generators.py (+123/-0)
tests/test_generators.py (+99/-0)
To merge this branch: bzr merge lp:~cjwatson/ubuntu-system-image/cdimage-custom
Reviewer Review Type Date Requested Status
Registry Administrators Pending
Review via email: mp+237942@code.launchpad.net

This proposal supersedes a proposal from 2014-10-10.

Commit message

Add a new cdimage-custom generator.

Description of the change

Add a new cdimage-custom generator.

This is basically just a clone-and-hack of cdimage-ubuntu, simplified somewhat. It goes with recent changes to ubuntu-cdimage, and is all with the aim of being able to fix bug 1367332 (moving some click packages to /custom) in a single step for the community Ubuntu images.

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/systemimage/generators.py'
2--- lib/systemimage/generators.py 2014-09-26 15:31:15 +0000
3+++ lib/systemimage/generators.py 2014-10-10 11:11:53 +0000
4@@ -142,6 +142,8 @@
5 path = generate_file_cdimage_device(conf, arguments, environment)
6 elif generator == "cdimage-ubuntu":
7 path = generate_file_cdimage_ubuntu(conf, arguments, environment)
8+ elif generator == "cdimage-custom":
9+ path = generate_file_cdimage_custom(conf, arguments, environment)
10 elif generator == "http":
11 path = generate_file_http(conf, arguments, environment)
12 elif generator == "keyring":
13@@ -561,6 +563,127 @@
14 return None
15
16
17+def generate_file_cdimage_custom(conf, arguments, environment):
18+ """
19+ Scan a cdimage tree for new custom files.
20+ """
21+
22+ # We need at least a path and a series
23+ if len(arguments) < 2:
24+ return None
25+
26+ # Read the arguments
27+ cdimage_path = arguments[0]
28+ series = arguments[1]
29+
30+ options = {}
31+ if len(arguments) > 2:
32+ options = unpack_arguments(arguments[2])
33+
34+ arch = "armhf"
35+ if environment['device_name'] in ("generic_x86", "generic_i386"):
36+ arch = "i386"
37+ elif environment['device_name'] in ("generic_amd64",):
38+ arch = "amd64"
39+
40+ # Check that the directory exists
41+ if not os.path.exists(cdimage_path):
42+ return None
43+
44+ versions = sorted([version for version in os.listdir(cdimage_path)
45+ if version not in ("pending", "current")],
46+ reverse=True)
47+
48+ for version in versions:
49+ # Skip directory without checksums
50+ if not os.path.exists(os.path.join(cdimage_path, version,
51+ "SHA256SUMS")):
52+ continue
53+
54+ # Check for the custom tarball
55+ custom_path = os.path.join(cdimage_path, version,
56+ "%s-preinstalled-%s-%s.custom.tar.gz" %
57+ (series, options.get("product", "touch"),
58+ arch))
59+ if not os.path.exists(custom_path):
60+ continue
61+
62+ # Check if we should only import tested images
63+ if options.get("import", "any") == "good":
64+ if not os.path.exists(os.path.join(cdimage_path, version,
65+ ".marked_good")):
66+ continue
67+
68+ # Set the version_detail string
69+ version_detail = "custom=%s" % version
70+
71+ # Extract the hash
72+ custom_hash = None
73+ with open(os.path.join(cdimage_path, version,
74+ "SHA256SUMS"), "r") as fd:
75+ for line in fd:
76+ line = line.strip()
77+ if line.endswith(custom_path.split("/")[-1]):
78+ custom_hash = line.split()[0]
79+ break
80+
81+ if not custom_hash:
82+ continue
83+
84+ # Generate the path
85+ path = os.path.join(conf.publish_path, "pool",
86+ "custom-%s.tar.xz" % custom_hash)
87+
88+ # Return pre-existing entries
89+ if os.path.exists(path):
90+ # Get the real version number (in case it got copied)
91+ if os.path.exists(path.replace(".tar.xz", ".json")):
92+ with open(path.replace(".tar.xz", ".json"), "r") as fd:
93+ metadata = json.loads(fd.read())
94+
95+ if "version_detail" in metadata:
96+ version_detail = metadata['version_detail']
97+
98+ environment['version_detail'].append(version_detail)
99+ return path
100+
101+ temp_dir = tempfile.mkdtemp()
102+
103+ # Unpack the source tarball
104+ tools.gzip_uncompress(custom_path, os.path.join(temp_dir,
105+ "source.tar"))
106+
107+ # Create the pool if it doesn't exist
108+ if not os.path.exists(os.path.join(conf.publish_path, "pool")):
109+ os.makedirs(os.path.join(conf.publish_path, "pool"))
110+
111+ # Compress the target tarball and sign it
112+ tools.xz_compress(os.path.join(temp_dir, "source.tar"), path)
113+ gpg.sign_file(conf, "image-signing", path)
114+
115+ # Generate the metadata file
116+ metadata = {}
117+ metadata['generator'] = "cdimage-custom"
118+ metadata['version'] = version
119+ metadata['version_detail'] = version_detail
120+ metadata['series'] = series
121+ metadata['custom_path'] = custom_path
122+ metadata['custom_checksum'] = custom_hash
123+
124+ with open(path.replace(".tar.xz", ".json"), "w+") as fd:
125+ fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
126+ indent=4, separators=(',', ': ')))
127+ gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
128+
129+ # Cleanup
130+ shutil.rmtree(temp_dir)
131+
132+ environment['version_detail'].append(version_detail)
133+ return path
134+
135+ return None
136+
137+
138 def generate_file_http(conf, arguments, environment):
139 """
140 Grab, cache and returns a file using http/https.
141
142=== modified file 'tests/test_generators.py'
143--- tests/test_generators.py 2014-09-10 19:31:04 +0000
144+++ tests/test_generators.py 2014-10-10 11:11:53 +0000
145@@ -424,6 +424,105 @@
146
147 @unittest.skipIf(not os.path.exists("tests/keys/generated"),
148 "No GPG testing keys present. Run tests/generate-keys")
149+ def test_generate_file_cdimage_custom(self):
150+ environment = {}
151+ environment['channel_name'] = "test"
152+ environment['device'] = self.device
153+ environment['device_name'] = "generic_x86"
154+ environment['new_files'] = []
155+ environment['version'] = 1234
156+ environment['version_detail'] = []
157+
158+ # Check the path and series requirement
159+ self.assertEquals(
160+ generators.generate_file_cdimage_custom(self.config, [],
161+ environment),
162+ None)
163+
164+ # Check behaviour on invalid cdimage path
165+ self.assertEquals(
166+ generators.generate_file_cdimage_custom(
167+ self.config, ['invalid-path', 'invalid-series'],
168+ environment),
169+ None)
170+
171+ # Check behaviour on empty tree
172+ cdimage_tree = os.path.join(self.temp_directory, "cdimage")
173+ os.mkdir(cdimage_tree)
174+ self.assertEquals(
175+ generators.generate_file_cdimage_custom(
176+ self.config, [cdimage_tree, 'series'],
177+ environment),
178+ None)
179+
180+ # Check behaviour on missing hash
181+ version_path = os.path.join(cdimage_tree, "1234")
182+ os.mkdir(version_path)
183+ self.assertEquals(
184+ generators.generate_file_cdimage_custom(
185+ self.config, [cdimage_tree, 'series'],
186+ environment),
187+ None)
188+
189+ # Check behaviour on missing files
190+ for filename in ("SHA256SUMS",
191+ "series-preinstalled-touch-i386.custom.tar.gz",
192+ ".marked_good"):
193+ open(os.path.join(version_path, filename), "w+").close()
194+ self.assertEquals(
195+ generators.generate_file_cdimage_custom(
196+ self.config, [cdimage_tree, 'series', 'import=good'],
197+ environment),
198+ None)
199+
200+ # Working run
201+ for device_arch, cdimage_arch, cdimage_product in (
202+ ("generic_x86", "i386", "touch"),
203+ ("generic_i386", "i386", "core"),
204+ ("generic_amd64", "amd64", "core")):
205+ environment['device_name'] = device_arch
206+
207+ for filename in ("SHA256SUMS",
208+ "series-preinstalled-%s-%s.custom.tar.gz" %
209+ (cdimage_product, cdimage_arch),
210+ ".marked_good"):
211+ open(os.path.join(version_path, filename), "w+").close()
212+
213+ with open(os.path.join(version_path, "SHA256SUMS"), "w+") as fd:
214+ fd.write("HASH *series-preinstalled-%s-%s.custom.tar.gz\n" %
215+ (cdimage_product, cdimage_arch))
216+
217+ tarball = os.path.join(version_path,
218+ "series-preinstalled-%s-%s.custom.tar.gz" %
219+ (cdimage_product, cdimage_arch))
220+ os.remove(tarball)
221+ tarball_obj = tarfile.open(tarball, "w:gz")
222+ tarball_obj.close()
223+
224+ self.assertEquals(
225+ generators.generate_file(
226+ self.config, "cdimage-custom",
227+ [cdimage_tree, 'series', 'product=%s' % cdimage_product],
228+ environment),
229+ os.path.join(self.config.publish_path, "pool",
230+ "custom-HASH.tar.xz"))
231+
232+ # Cached run
233+ self.assertEquals(
234+ generators.generate_file_cdimage_custom(
235+ self.config, [cdimage_tree, 'series',
236+ 'product=%s' % cdimage_product],
237+ environment),
238+ os.path.join(self.config.publish_path, "pool",
239+ "custom-HASH.tar.xz"))
240+
241+ for entry in ("custom-HASH.tar.xz", "custom-HASH.tar.xz.asc",
242+ "custom-HASH.json", "custom-HASH.json.asc"):
243+ os.remove(os.path.join(self.config.publish_path,
244+ "pool", entry))
245+
246+ @unittest.skipIf(not os.path.exists("tests/keys/generated"),
247+ "No GPG testing keys present. Run tests/generate-keys")
248 @mock.patch("systemimage.generators.urlretrieve")
249 @mock.patch("systemimage.generators.urlopen")
250 def test_generate_file_http(self, mock_urlopen, mock_urlretrieve):

Subscribers

People subscribed via source and target branches

to all changes: