Ubuntu system image

Merge lp:~cjwatson/ubuntu-system-image/cdimage-custom into lp:ubuntu-system-image/server

  1. cdimage-custom
  2. Merge into server
Proposed by Colin Watson
Status: Merged
Merged at revision: 248
Proposed branch: lp:~cjwatson/ubuntu-system-image/cdimage-custom
Merge into: lp:ubuntu-system-image/server
Diff against target: 250 lines (+222/-0)
2 files modified
lib/systemimage/generators.py (+123/-0)
tests/test_generators.py (+99/-0)
To merge this branch: bzr merge lp:~cjwatson/ubuntu-system-image/cdimage-custom
Reviewer Review Type Date Requested Status
Registry Administrators Pending
Review via email: mp+237942@code.launchpad.net

This proposal supersedes a proposal from 2014-10-10.

Commit message

Add a new cdimage-custom generator.

Description of the change

Add a new cdimage-custom generator.

This is basically just a clone-and-hack of cdimage-ubuntu, simplified somewhat. It goes with recent changes to ubuntu-cdimage, and is all with the aim of being able to fix bug 1367332 (moving some click packages to /custom) in a single step for the community Ubuntu images.

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/systemimage/generators.py'
--- lib/systemimage/generators.py 2014-09-26 15:31:15 +0000
+++ lib/systemimage/generators.py 2014-10-10 11:11:53 +0000
@@ -142,6 +142,8 @@
142 path = generate_file_cdimage_device(conf, arguments, environment)142 path = generate_file_cdimage_device(conf, arguments, environment)
143 elif generator == "cdimage-ubuntu":143 elif generator == "cdimage-ubuntu":
144 path = generate_file_cdimage_ubuntu(conf, arguments, environment)144 path = generate_file_cdimage_ubuntu(conf, arguments, environment)
145 elif generator == "cdimage-custom":
146 path = generate_file_cdimage_custom(conf, arguments, environment)
145 elif generator == "http":147 elif generator == "http":
146 path = generate_file_http(conf, arguments, environment)148 path = generate_file_http(conf, arguments, environment)
147 elif generator == "keyring":149 elif generator == "keyring":
@@ -561,6 +563,127 @@
561 return None563 return None
562564
563565
566def generate_file_cdimage_custom(conf, arguments, environment):
567 """
568 Scan a cdimage tree for new custom files.
569 """
570
571 # We need at least a path and a series
572 if len(arguments) < 2:
573 return None
574
575 # Read the arguments
576 cdimage_path = arguments[0]
577 series = arguments[1]
578
579 options = {}
580 if len(arguments) > 2:
581 options = unpack_arguments(arguments[2])
582
583 arch = "armhf"
584 if environment['device_name'] in ("generic_x86", "generic_i386"):
585 arch = "i386"
586 elif environment['device_name'] in ("generic_amd64",):
587 arch = "amd64"
588
589 # Check that the directory exists
590 if not os.path.exists(cdimage_path):
591 return None
592
593 versions = sorted([version for version in os.listdir(cdimage_path)
594 if version not in ("pending", "current")],
595 reverse=True)
596
597 for version in versions:
598 # Skip directory without checksums
599 if not os.path.exists(os.path.join(cdimage_path, version,
600 "SHA256SUMS")):
601 continue
602
603 # Check for the custom tarball
604 custom_path = os.path.join(cdimage_path, version,
605 "%s-preinstalled-%s-%s.custom.tar.gz" %
606 (series, options.get("product", "touch"),
607 arch))
608 if not os.path.exists(custom_path):
609 continue
610
611 # Check if we should only import tested images
612 if options.get("import", "any") == "good":
613 if not os.path.exists(os.path.join(cdimage_path, version,
614 ".marked_good")):
615 continue
616
617 # Set the version_detail string
618 version_detail = "custom=%s" % version
619
620 # Extract the hash
621 custom_hash = None
622 with open(os.path.join(cdimage_path, version,
623 "SHA256SUMS"), "r") as fd:
624 for line in fd:
625 line = line.strip()
626 if line.endswith(custom_path.split("/")[-1]):
627 custom_hash = line.split()[0]
628 break
629
630 if not custom_hash:
631 continue
632
633 # Generate the path
634 path = os.path.join(conf.publish_path, "pool",
635 "custom-%s.tar.xz" % custom_hash)
636
637 # Return pre-existing entries
638 if os.path.exists(path):
639 # Get the real version number (in case it got copied)
640 if os.path.exists(path.replace(".tar.xz", ".json")):
641 with open(path.replace(".tar.xz", ".json"), "r") as fd:
642 metadata = json.loads(fd.read())
643
644 if "version_detail" in metadata:
645 version_detail = metadata['version_detail']
646
647 environment['version_detail'].append(version_detail)
648 return path
649
650 temp_dir = tempfile.mkdtemp()
651
652 # Unpack the source tarball
653 tools.gzip_uncompress(custom_path, os.path.join(temp_dir,
654 "source.tar"))
655
656 # Create the pool if it doesn't exist
657 if not os.path.exists(os.path.join(conf.publish_path, "pool")):
658 os.makedirs(os.path.join(conf.publish_path, "pool"))
659
660 # Compress the target tarball and sign it
661 tools.xz_compress(os.path.join(temp_dir, "source.tar"), path)
662 gpg.sign_file(conf, "image-signing", path)
663
664 # Generate the metadata file
665 metadata = {}
666 metadata['generator'] = "cdimage-custom"
667 metadata['version'] = version
668 metadata['version_detail'] = version_detail
669 metadata['series'] = series
670 metadata['custom_path'] = custom_path
671 metadata['custom_checksum'] = custom_hash
672
673 with open(path.replace(".tar.xz", ".json"), "w+") as fd:
674 fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
675 indent=4, separators=(',', ': ')))
676 gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
677
678 # Cleanup
679 shutil.rmtree(temp_dir)
680
681 environment['version_detail'].append(version_detail)
682 return path
683
684 return None
685
686
564def generate_file_http(conf, arguments, environment):687def generate_file_http(conf, arguments, environment):
565 """688 """
566 Grab, cache and returns a file using http/https.689 Grab, cache and returns a file using http/https.
567690
=== modified file 'tests/test_generators.py'
--- tests/test_generators.py 2014-09-10 19:31:04 +0000
+++ tests/test_generators.py 2014-10-10 11:11:53 +0000
@@ -424,6 +424,105 @@
424424
425 @unittest.skipIf(not os.path.exists("tests/keys/generated"),425 @unittest.skipIf(not os.path.exists("tests/keys/generated"),
426 "No GPG testing keys present. Run tests/generate-keys")426 "No GPG testing keys present. Run tests/generate-keys")
427 def test_generate_file_cdimage_custom(self):
428 environment = {}
429 environment['channel_name'] = "test"
430 environment['device'] = self.device
431 environment['device_name'] = "generic_x86"
432 environment['new_files'] = []
433 environment['version'] = 1234
434 environment['version_detail'] = []
435
436 # Check the path and series requirement
437 self.assertEquals(
438 generators.generate_file_cdimage_custom(self.config, [],
439 environment),
440 None)
441
442 # Check behaviour on invalid cdimage path
443 self.assertEquals(
444 generators.generate_file_cdimage_custom(
445 self.config, ['invalid-path', 'invalid-series'],
446 environment),
447 None)
448
449 # Check behaviour on empty tree
450 cdimage_tree = os.path.join(self.temp_directory, "cdimage")
451 os.mkdir(cdimage_tree)
452 self.assertEquals(
453 generators.generate_file_cdimage_custom(
454 self.config, [cdimage_tree, 'series'],
455 environment),
456 None)
457
458 # Check behaviour on missing hash
459 version_path = os.path.join(cdimage_tree, "1234")
460 os.mkdir(version_path)
461 self.assertEquals(
462 generators.generate_file_cdimage_custom(
463 self.config, [cdimage_tree, 'series'],
464 environment),
465 None)
466
467 # Check behaviour on missing files
468 for filename in ("SHA256SUMS",
469 "series-preinstalled-touch-i386.custom.tar.gz",
470 ".marked_good"):
471 open(os.path.join(version_path, filename), "w+").close()
472 self.assertEquals(
473 generators.generate_file_cdimage_custom(
474 self.config, [cdimage_tree, 'series', 'import=good'],
475 environment),
476 None)
477
478 # Working run
479 for device_arch, cdimage_arch, cdimage_product in (
480 ("generic_x86", "i386", "touch"),
481 ("generic_i386", "i386", "core"),
482 ("generic_amd64", "amd64", "core")):
483 environment['device_name'] = device_arch
484
485 for filename in ("SHA256SUMS",
486 "series-preinstalled-%s-%s.custom.tar.gz" %
487 (cdimage_product, cdimage_arch),
488 ".marked_good"):
489 open(os.path.join(version_path, filename), "w+").close()
490
491 with open(os.path.join(version_path, "SHA256SUMS"), "w+") as fd:
492 fd.write("HASH *series-preinstalled-%s-%s.custom.tar.gz\n" %
493 (cdimage_product, cdimage_arch))
494
495 tarball = os.path.join(version_path,
496 "series-preinstalled-%s-%s.custom.tar.gz" %
497 (cdimage_product, cdimage_arch))
498 os.remove(tarball)
499 tarball_obj = tarfile.open(tarball, "w:gz")
500 tarball_obj.close()
501
502 self.assertEquals(
503 generators.generate_file(
504 self.config, "cdimage-custom",
505 [cdimage_tree, 'series', 'product=%s' % cdimage_product],
506 environment),
507 os.path.join(self.config.publish_path, "pool",
508 "custom-HASH.tar.xz"))
509
510 # Cached run
511 self.assertEquals(
512 generators.generate_file_cdimage_custom(
513 self.config, [cdimage_tree, 'series',
514 'product=%s' % cdimage_product],
515 environment),
516 os.path.join(self.config.publish_path, "pool",
517 "custom-HASH.tar.xz"))
518
519 for entry in ("custom-HASH.tar.xz", "custom-HASH.tar.xz.asc",
520 "custom-HASH.json", "custom-HASH.json.asc"):
521 os.remove(os.path.join(self.config.publish_path,
522 "pool", entry))
523
524 @unittest.skipIf(not os.path.exists("tests/keys/generated"),
525 "No GPG testing keys present. Run tests/generate-keys")
427 @mock.patch("systemimage.generators.urlretrieve")526 @mock.patch("systemimage.generators.urlretrieve")
428 @mock.patch("systemimage.generators.urlopen")527 @mock.patch("systemimage.generators.urlopen")
429 def test_generate_file_http(self, mock_urlopen, mock_urlretrieve):528 def test_generate_file_http(self, mock_urlopen, mock_urlretrieve):

Subscribers

People subscribed via source and target branches

to all changes: