Hi William, As Julian has already reviewed the logic in this branch, I'll just do a style/convention review. I really have very little to say, and that which I have said is pretty trivial. It's inline with the diff below. Running `make lint` generated a lot of warnings. Please can you clean up those that make sense. Thanks, Gavin. > === modified file 'database/schema/security.cfg' > --- database/schema/security.cfg 2009-11-06 01:16:21 +0000 > +++ database/schema/security.cfg 2009-11-11 14:24:18 +0000 > @@ -271,6 +271,7 @@ > public.shippingrun = SELECT, INSERT, UPDATE > public.sourcepackagepublishinghistory = SELECT > public.seriessourcepackagebranch = SELECT, INSERT, UPDATE, DELETE > +public.sourcepackageformatselection = SELECT > public.specificationbranch = SELECT, INSERT, UPDATE, DELETE > public.specificationbug = SELECT, INSERT, DELETE > public.specificationdependency = SELECT, INSERT, DELETE > @@ -986,6 +987,7 @@ > public.section = SELECT, INSERT, UPDATE > public.sectionselection = SELECT, INSERT, UPDATE > public.signedcodeofconduct = SELECT, INSERT, UPDATE > +public.sourcepackageformatselection = SELECT, INSERT This should go after the following line. Wow, how pedantic I am :) > public.sourcepackagefilepublishing = SELECT, INSERT, UPDATE > public.sourcepackagename = SELECT, INSERT, UPDATE > public.sourcepackagepublishinghistory = SELECT > @@ -1103,6 +1105,7 @@ > public.archivepermission = SELECT > public.processor = SELECT > public.processorfamily = SELECT > +public.sourcepackageformatselection = SELECT > > # Source and Binary packages and builds > public.sourcepackagename = SELECT, INSERT > > === modified file 'lib/canonical/launchpad/helpers.py' > --- lib/canonical/launchpad/helpers.py 2009-07-17 00:26:05 +0000 > +++ lib/canonical/launchpad/helpers.py 2009-11-11 14:24:18 +0000 > @@ -477,9 +477,9 @@ > if fname.endswith(".diff.gz"): > return SourcePackageFileType.DIFF > if fname.endswith(".orig.tar.gz"): > - return SourcePackageFileType.ORIG > + return SourcePackageFileType.ORIG_TARBALL > if fname.endswith(".tar.gz"): > - return SourcePackageFileType.TARBALL > + return SourcePackageFileType.NATIVE_TARBALL > > > BINARYPACKAGE_EXTENSIONS = { > > === modified file 'lib/lp/archiveuploader/dscfile.py' > --- lib/lp/archiveuploader/dscfile.py 2009-06-24 23:33:29 +0000 > +++ lib/lp/archiveuploader/dscfile.py 2009-11-11 14:24:18 +0000 > @@ -31,10 +31,13 @@ > parse_tagfile, TagFileParseError) > from lp.archiveuploader.utils import ( > prefix_multi_line_string, safe_fix_maintainer, ParseMaintError, > - re_valid_pkg_name, re_valid_version, re_issource) > + re_valid_pkg_name, re_valid_version, re_issource, > + determine_source_file_type) > from canonical.encoding import guess as guess_encoding > from lp.registry.interfaces.person import IPersonSet, PersonCreationRationale > +from lp.registry.interfaces.sourcepackage import SourcePackageFileType > from lp.soyuz.interfaces.archive import ArchivePurpose > +from lp.soyuz.interfaces.sourcepackageformat import SourcePackageFormat > from canonical.launchpad.interfaces import ( > GPGVerificationError, IGPGHandler, IGPGKeySet, > ISourcePackageNameSet, NotFoundError) > @@ -228,6 +231,9 @@ > This method is an error generator, i.e, it returns an iterator over all > exceptions that are generated while processing DSC file checks. > """ > + # Avoid circular imports. > + from lp.archiveuploader.nascentupload import EarlyReturnUploadError > + > for error in SourceUploadFile.verify(self): > yield error > > @@ -265,10 +271,17 @@ > yield UploadError( > "%s: invalid version %s" % (self.filename, self.dsc_version)) > > - if self.format != "1.0": > + try: > + format_term = SourcePackageFormat.getTermByToken(self.format) > + except LookupError: > + raise EarlyReturnUploadError( > + "Unsupported source format: %s" % self.format) > + > + if not self.policy.distroseries.isSourcePackageFormatPermitted( > + format_term.value): > yield UploadError( > - "%s: Format is not 1.0. This is incompatible with " > - "dpkg-source." % self.filename) > + "%s: format '%s' is not permitted in %s." % > + (self.filename, self.format, self.policy.distroseries.name)) > > # Validate the build dependencies > for field_name in ['build-depends', 'build-depends-indep']: > @@ -324,7 +337,8 @@ > :raise: `NotFoundError` when the wanted file could not be found. > """ > if (self.policy.archive.purpose == ArchivePurpose.PPA and > - filename.endswith('.orig.tar.gz')): > + determine_source_file_type(filename) == > + SourcePackageFileType.ORIG_TARBALL): > archives = [self.policy.archive, self.policy.distro.main_archive] > else: > archives = [self.policy.archive] > @@ -348,11 +362,25 @@ > We don't use the NascentUploadFile.verify here, only verify size > and checksum. > """ > - has_tar = False > + > + diff_count = 0 > + orig_tar_count = 0 > + native_tar_count = 0 > + > files_missing = False > for sub_dsc_file in self.files: > - if sub_dsc_file.filename.endswith("tar.gz"): > - has_tar = True > + filetype = determine_source_file_type(sub_dsc_file.filename) > + > + if filetype == SourcePackageFileType.DIFF: > + diff_count += 1 > + elif filetype == SourcePackageFileType.ORIG_TARBALL: > + orig_tar_count += 1 > + elif filetype == SourcePackageFileType.NATIVE_TARBALL: > + native_tar_count += 1 > + else: > + yield UploadError('Unknown file: ' + sub_dsc_file.filename) > + continue > + > try: > library_file, file_archive = self._getFileByName( > sub_dsc_file.filename) > @@ -397,11 +425,37 @@ > yield error > files_missing = True > > - > - if not has_tar: > - yield UploadError( > - "%s: does not mention any tar.gz or orig.tar.gz." > - % self.filename) > + # Reject if we have more than one file of any type. > + if orig_tar_count > 1: > + yield UploadError( > + "%s: has more than one orig.tar.*." > + % self.filename) > + if native_tar_count > 1: > + yield UploadError( > + "%s: has more than one tar.*." > + % self.filename) > + if diff_count > 1: > + yield UploadError( > + "%s: has more than one diff.gz." > + % self.filename) > + > + if ((orig_tar_count == 0 and native_tar_count == 0) or > + (orig_tar_count > 0 and native_tar_count > 0)): > + yield UploadError( > + "%s: must have exactly one tar.* or orig.tar.*." > + % self.filename) > + > + # Format 1.0 must be native (exactly one tar.gz), or > + # have an orig.tar.gz and a diff.gz. It cannot have > + # compression types other than 'gz'. > + if self.format == '1.0': > + if ((diff_count == 0 and native_tar_count == 0) or > + (diff_count > 0 and native_tar_count > 0)): > + yield UploadError( > + "%s: must have exactly one diff.gz or tar.gz." > + % self.filename) > + else: > + raise AssertionError("Unknown source format.") > > if files_missing: > yield UploadError( > > === modified file 'lib/lp/archiveuploader/nascentupload.py' > --- lib/lp/archiveuploader/nascentupload.py 2009-09-04 08:35:20 +0000 > +++ lib/lp/archiveuploader/nascentupload.py 2009-11-11 14:24:18 +0000 > @@ -28,8 +28,10 @@ > from lp.archiveuploader.nascentuploadfile import ( > UploadError, UploadWarning, CustomUploadFile, SourceUploadFile, > BaseBinaryUploadFile) > +from lp.archiveuploader.utils import determine_source_file_type > from lp.archiveuploader.permission import verify_upload > from lp.registry.interfaces.pocket import PackagePublishingPocket > +from lp.registry.interfaces.sourcepackage import SourcePackageFileType > from lp.soyuz.interfaces.archive import ArchivePurpose, MAIN_ARCHIVE_PURPOSES > from canonical.launchpad.interfaces import ( > IBinaryPackageNameSet, IDistributionSet, ILibraryFileAliasSet, > @@ -302,53 +304,38 @@ > """Heuristic checks on a sourceful upload. > > Raises AssertionError when called for a non-sourceful upload. > - Ensures a sourceful upload has, at least: > - > - * One DSC > - * One or none DIFF > - * One or none ORIG > - * One or none TAR > - * If no DIFF is present it must have a TAR (native) > - > - 'hasorig' and 'native' attributes are set when an ORIG and/or an > - TAR file, respectively, are present. > + Ensures a sourceful upload has exactly one DSC. > """ > assert self.sourceful, ( > "Source consistency check called for a non-source upload") > > dsc = 0 > - diff = 0 > - orig = 0 > - tar = 0 > + native_tarball = 0 > + orig_tarball = 0 > > for uploaded_file in self.changes.files: > - if uploaded_file.filename.endswith(".dsc"): > + filetype = determine_source_file_type(uploaded_file.filename) > + if filetype == SourcePackageFileType.DSC: > dsc += 1 > - elif uploaded_file.filename.endswith(".diff.gz"): > - diff += 1 > - elif uploaded_file.filename.endswith(".orig.tar.gz"): > - orig += 1 > - elif (uploaded_file.filename.endswith(".tar.gz") > + elif (filetype == SourcePackageFileType.NATIVE_TARBALL > and not isinstance(uploaded_file, CustomUploadFile)): We don't (afaik) have a policy to prefer isinstance() or interfaces, but consider here if it's better to use ISomeInterface.providedBy() instead of isinstance(). > - tar += 1 > - > - # Okay, let's check the sanity of the upload. > + native_tarball += 1 > + elif filetype == SourcePackageFileType.ORIG_TARBALL: > + orig_tarball += 1 > + > + > + # It is never sane to upload more than one source at a time. > + # XXX: What about orphaned files? How will that work? > + # I think we might need to verify that all source files are > + # claimed by a dsc. Julian commented on this. If you decide to leave the XXX in, please file a bug for it, and make sure it's formatted according to https://dev.launchpad.net/XXXPolicy. > if dsc > 1: > self.reject("Changes file lists more than one .dsc") > - if diff > 1: > - self.reject("Changes file lists more than one .diff.gz") > - if orig > 1: > - self.reject("Changes file lists more than one orig.tar.gz") > - if tar > 1: > - self.reject("Changes file lists more than one native tar.gz") > > if dsc == 0: > self.reject("Sourceful upload without a .dsc") > - if diff == 0 and tar == 0: > - self.reject("Sourceful upload without a diff or native tar") > > - self.native = bool(tar) > - self.hasorig = bool(orig) > + self.native = bool(native_tarball) > + self.hasorig = bool(orig_tarball) > > def _check_binaryful_consistency(self): > """Heuristic checks on a binaryful upload. > > === modified file 'lib/lp/archiveuploader/nascentuploadfile.py' > --- lib/lp/archiveuploader/nascentuploadfile.py 2009-07-17 00:26:05 +0000 > +++ lib/lp/archiveuploader/nascentuploadfile.py 2009-11-11 14:24:18 +0000 > @@ -33,8 +33,9 @@ > from lp.archiveuploader.utils import ( > prefix_multi_line_string, re_taint_free, re_isadeb, re_issource, > re_no_epoch, re_no_revision, re_valid_version, re_valid_pkg_name, > - re_extract_src_version) > + re_extract_src_version, determine_source_file_type) > from canonical.encoding import guess as guess_encoding > +from lp.registry.interfaces.sourcepackage import SourcePackageFileType > from lp.soyuz.interfaces.binarypackagename import ( > IBinaryPackageNameSet) > from lp.soyuz.interfaces.binarypackagerelease import ( > @@ -351,7 +352,8 @@ > "Architecture field." % (self.filename)) > > version_chopped = re_no_epoch.sub('', self.version) > - if self.filename.endswith("orig.tar.gz"): > + if determine_source_file_type(self.filename) == ( > + SourcePackageFileType.ORIG_TARBALL): > version_chopped = re_no_revision.sub('', version_chopped) > > source_match = re_issource.match(self.filename) > > === modified file 'lib/lp/archiveuploader/tests/test_uploadprocessor.py' > --- lib/lp/archiveuploader/tests/test_uploadprocessor.py 2009-09-04 08:35:20 +0000 > +++ lib/lp/archiveuploader/tests/test_uploadprocessor.py 2009-11-11 14:24:18 +0000 > @@ -49,6 +49,7 @@ > from lp.soyuz.interfaces.archivepermission import ( > ArchivePermissionType, IArchivePermissionSet) > from lp.soyuz.interfaces.component import IComponentSet > +from lp.soyuz.interfaces.sourcepackageformat import SourcePackageFormat > from lp.registry.interfaces.person import IPersonSet > from lp.registry.interfaces.sourcepackagename import ( > ISourcePackageNameSet) > @@ -157,7 +158,7 @@ > excName = str(excClass) > raise self.failureException, "%s not raised" % excName > > - def setupBreezy(self, name="breezy"): > + def setupBreezy(self, name="breezy", permitted_formats=None): > """Create a fresh distroseries in ubuntu. > > Use *initialiseFromParent* procedure to create 'breezy' > @@ -168,6 +169,8 @@ > > :param name: supply the name of the distroseries if you don't want > it to be called "breezy" > + :param permitted_formats: list of SourcePackageFormats to allow > + in the new distroseries. Only permits '1.0' by default. > """ > self.ubuntu = getUtility(IDistributionSet).getByName('ubuntu') > bat = self.ubuntu['breezy-autotest'] > @@ -185,6 +188,12 @@ > self.breezy.changeslist = '