lpci

Merge ~artivis/lpci:feature/yaml-include into lpci:main

  1. Git
  2. lp:~artivis/lpci
  3. feature/yaml-include
  4. Merge into main
Proposed by jeremie
Status: Needs review
Proposed branch: ~artivis/lpci:feature/yaml-include
Merge into: lpci:main
Prerequisite: ~artivis/lpci:feature/yaml-alias-filter
Diff against target: 388 lines (+304/-4) (has conflicts)
4 files modified
lpcraft/config.py (+1/-1)
lpcraft/tests/test_config.py (+211/-2)
lpcraft/tests/test_utils.py (+21/-1)
lpcraft/utils.py (+71/-0)
Conflict in lpcraft/tests/test_config.py
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+435460@code.launchpad.net

Description of the change

[Do Not Merge Yet]

Note: This is based on top of another branch (https://git.launchpad.net/~artivis/lpcraft/tree/?h=feature/yaml-alias-filter) which may replace MP https://code.launchpad.net/~artivis/lpcraft/+git/lpcraft/+merge/435370

Add support for yaml inclusion.

This allows for including other yaml files to a configuration file using the 'include' tag located at the root. Included file paths can be relative or absolute. Since yaml anchors are lost during the inclusion, this MP also introduce the 'extends' tag that essentially does the work of mapping.

Hereafter is an example of what that looks like,

# .included.launchpad.yaml
.test:
  series: bionic
  architectures: [amd64]

# .launchpad.yaml
pipeline:
  - test

include:
  - .included.launchpad.yaml

jobs:
  test-a:
      series: focal
      architectures: [amd64]
  test-b:
      extends: .test # maps '.test' nested entries
      packages: [file] # can be further extended with new entries

[Do Not Merge Yet]

To post a comment you must log in.
Revision history for this message
Jürgen Gmach (jugmac00) wrote :

Similar to the notes on YAML aliases, let's discuss whether an external tool / script could be a viable solution.
https://chat.canonical.com/canonical/pl/r71cgk3ejtr1zmbfnfitbi187c

Reply

Unmerged commits

1bee04c... by jeremie

test yaml inclusion

Signed-off-by: artivis <email address hidden>

Succeeded
[SUCCEEDED] test:0 (build)
[SUCCEEDED] build:0 (build)
1 → 2 of 2 results First • Previous • Next • Last
e1c0554... by jeremie

add support for yaml inclusion

Signed-off-by: artivis <email address hidden>

5e3aa4f... by jeremie

couple tests yaml anchor/alias

Signed-off-by: artivis <email address hidden>

Succeeded
[SUCCEEDED] test:0 (build)
[SUCCEEDED] build:0 (build)
1 → 2 of 2 results First • Previous • Next • Last
31e792a... by jeremie

pydantic filter out extra fields with prefix

Signed-off-by: artivis <email address hidden>

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lpcraft/config.py b/lpcraft/config.py
2index 3564d9c..1409cde 100644
3--- a/lpcraft/config.py
4+++ b/lpcraft/config.py
5@@ -373,7 +373,7 @@ class Config(ModelConfigDefaults):
6 """Filter out extension field."""
7 expanded_values = values.copy()
8 for k, v in values.items():
9- if k.startswith("."):
10+ if k.startswith(".") or k == "include":
11 expanded_values.pop(k)
12 return expanded_values
13
14diff --git a/lpcraft/tests/test_config.py b/lpcraft/tests/test_config.py
15index 8695520..6ca9de8 100644
16--- a/lpcraft/tests/test_config.py
17+++ b/lpcraft/tests/test_config.py
18@@ -16,6 +16,7 @@ from testtools.matchers import (
19 MatchesStructure,
20 )
21
22+<<<<<<< lpcraft/tests/test_config.py
23 from lpcraft.config import (
24 LAUNCHPAD_PPA_BASE_URL,
25 Config,
26@@ -25,6 +26,10 @@ from lpcraft.config import (
27 get_ppa_url_parts,
28 )
29 from lpcraft.errors import CommandError
30+=======
31+from lpcraft.config import Config, OutputDistributeEnum, PackageRepository
32+from lpcraft.errors import CommandError, ConfigurationError
33+>>>>>>> lpcraft/tests/test_config.py
34
35
36 class TestConfig(TestCase):
37@@ -35,8 +40,8 @@ class TestConfig(TestCase):
38 # So switch to the created project directory.
39 os.chdir(self.tempdir)
40
41- def create_config(self, text):
42- path = self.tempdir / ".launchpad.yaml"
43+ def create_config(self, text, filename=".launchpad.yaml"):
44+ path = self.tempdir / filename
45 path.write_text(text)
46 return path
47
48@@ -909,3 +914,207 @@ class TestConfig(TestCase):
49 ),
50 ),
51 )
52+
53+ def test_yaml_include(self):
54+ # Include a config file given a relative path
55+ self.create_config(
56+ dedent(
57+ """
58+ .test:
59+ series: bionic
60+ architectures: [amd64]
61+ """
62+ ),
63+ ".included.launchpad.yaml",
64+ )
65+
66+ path = self.create_config(
67+ dedent(
68+ """
69+ pipeline:
70+ - test
71+
72+ include:
73+ - .included.launchpad.yaml
74+
75+ jobs:
76+ test-a:
77+ series: focal
78+ architectures: [amd64]
79+
80+ test-b:
81+ extends: .test
82+ packages: [file]
83+ """
84+ )
85+ )
86+
87+ config = Config.load(path)
88+
89+ self.assertThat(
90+ config,
91+ MatchesStructure(
92+ pipeline=Equals([["test"]]),
93+ jobs=MatchesDict(
94+ {
95+ "test-a": MatchesListwise(
96+ [
97+ MatchesStructure.byEquality(
98+ series="focal",
99+ architectures=["amd64"],
100+ )
101+ ]
102+ ),
103+ "test-b": MatchesListwise(
104+ [
105+ MatchesStructure.byEquality(
106+ series="bionic",
107+ architectures=["amd64"],
108+ packages=["file"],
109+ )
110+ ]
111+ ),
112+ }
113+ ),
114+ ),
115+ )
116+
117+ def test_yaml_include_abs_path(self):
118+ # Include a config file given an absolute path
119+ tempdir = Path(self.useFixture(TempDir()).path)
120+ included_config_path = tempdir / ".included.launchpad.yaml"
121+ included_config_path.write_text(
122+ dedent(
123+ """
124+ .test:
125+ series: bionic
126+ architectures: [amd64]
127+ """
128+ )
129+ )
130+
131+ base_config_path = self.create_config(
132+ dedent(
133+ f"""
134+ pipeline:
135+ - test
136+
137+ include:
138+ - '{included_config_path}'
139+
140+ jobs:
141+ test-a:
142+ series: focal
143+ architectures: [amd64]
144+
145+ test-b:
146+ extends: .test
147+ packages: [file]
148+ """
149+ )
150+ )
151+
152+ config = Config.load(base_config_path)
153+
154+ self.assertNotEqual(
155+ base_config_path.parent, included_config_path.parent
156+ )
157+
158+ self.assertThat(
159+ config,
160+ MatchesStructure(
161+ pipeline=Equals([["test"]]),
162+ jobs=MatchesDict(
163+ {
164+ "test-a": MatchesListwise(
165+ [
166+ MatchesStructure.byEquality(
167+ series="focal",
168+ architectures=["amd64"],
169+ )
170+ ]
171+ ),
172+ "test-b": MatchesListwise(
173+ [
174+ MatchesStructure.byEquality(
175+ series="bionic",
176+ architectures=["amd64"],
177+ packages=["file"],
178+ )
179+ ]
180+ ),
181+ }
182+ ),
183+ ),
184+ )
185+
186+ def test_yaml_bad_include(self):
187+ # Include a config file that doesn't exists.
188+ path = self.create_config(
189+ dedent(
190+ """
191+ pipeline:
192+ - test
193+
194+ include:
195+ - bad_include.launchpad.yaml
196+
197+ jobs:
198+ test:
199+ series: focal
200+ architectures: [amd64]
201+ """
202+ )
203+ )
204+
205+ self.assertRaisesRegex(
206+ ConfigurationError,
207+ "Couldn't find config file 'bad_include.launchpad.yaml'.",
208+ Config.load,
209+ path,
210+ )
211+
212+ def test_yaml_include_not_mapping(self):
213+ # Include a config file that is not defining a mapping
214+ self.create_config(
215+ dedent(
216+ """
217+ - foo
218+ """
219+ ),
220+ ".launchpad.included.yaml",
221+ )
222+
223+ path = self.create_config(
224+ dedent(
225+ """
226+ include:
227+ - .launchpad.included.yaml
228+ """
229+ )
230+ )
231+
232+ self.assertRaisesRegex(
233+ ConfigurationError,
234+ "Config file '.launchpad.included.yaml' does not define a mapping",
235+ Config.load,
236+ path,
237+ )
238+
239+ def test_yaml_bad_extends(self):
240+ # Extends a node that doesn't exists.
241+ path = self.create_config(
242+ dedent(
243+ """
244+ test:
245+ extends: .test
246+ """
247+ )
248+ )
249+
250+ self.assertRaisesRegex(
251+ ConfigurationError,
252+ "Couldn't find extension '.test'.",
253+ Config.load,
254+ path,
255+ )
256diff --git a/lpcraft/tests/test_utils.py b/lpcraft/tests/test_utils.py
257index 52b42e3..86cbda1 100644
258--- a/lpcraft/tests/test_utils.py
259+++ b/lpcraft/tests/test_utils.py
260@@ -11,7 +11,7 @@ from systemfixtures import FakeProcesses
261 from testtools import TestCase
262
263 from lpcraft.errors import ConfigurationError
264-from lpcraft.utils import ask_user, get_host_architecture, load_yaml
265+from lpcraft.utils import ask_user, find, get_host_architecture, load_yaml
266
267
268 class TestLoadYAML(TestCase):
269@@ -66,6 +66,26 @@ class TestLoadYAML(TestCase):
270 path,
271 )
272
273+ def test_find(self):
274+ self.assertEqual(42, next(find({"a": 42}, "a")))
275+ self.assertEqual(42, next(find({"b": 41, "a": 42}, "a")))
276+
277+ self.assertEqual(42, next(find({"b": {"a": 42}}, "a")))
278+
279+ self.assertEqual(42, next(find([{"b": 41, "a": 42}], "a")))
280+ self.assertEqual(42, next(find([{"b": 41}, {"a": 42}], "a")))
281+ self.assertEqual(42, next(find([("b", {"a": 42})], "a")))
282+
283+ def find_raises_map():
284+ next(find({"a": 42}, "b"))
285+
286+ self.assertRaises(StopIteration, find_raises_map)
287+
288+ def find_raises_list():
289+ next(find([1, 2, 3], "b"))
290+
291+ self.assertRaises(StopIteration, find_raises_list)
292+
293
294 class TestGetHostArchitecture(TestCase):
295 def setUp(self):
296diff --git a/lpcraft/utils.py b/lpcraft/utils.py
297index a369cbd..9b5ef19 100644
298--- a/lpcraft/utils.py
299+++ b/lpcraft/utils.py
300@@ -18,6 +18,71 @@ import yaml
301 from lpcraft.errors import ConfigurationError
302
303
304+def find(d: Any, key: Any) -> Any:
305+ """Recursively search in dict/list/tuple."""
306+ if isinstance(d, dict):
307+ if key in d:
308+ yield d[key]
309+ for v in d.values():
310+ for i in find(v, key):
311+ yield i
312+ if isinstance(d, (list, tuple)):
313+ for v in d:
314+ for i in find(v, key):
315+ yield i
316+
317+
318+def expand_yaml_extends(loaded: Any, root: Dict[Any, Any]) -> None:
319+ """Expand 'extends' entries.
320+
321+ extends entries are searched for throughout the entire configuration.
322+ When found, their associated value (pseudo-alias) is then
323+ searched for throughout the entire configuration.
324+ The value associated to the pseudo-alias is copied
325+ at the level the initial 'extends' key was found at.
326+ Finally the 'extends' entry is entirely removed.
327+ """
328+ tag = "extends"
329+ if isinstance(loaded, dict):
330+ for _, v in loaded.items():
331+ if isinstance(v, (dict, list, tuple)):
332+ expand_yaml_extends(v, root)
333+ if tag in loaded.keys():
334+ v = loaded.pop(tag)
335+
336+ try:
337+ val = next(find(root, v))
338+ except StopIteration:
339+ raise ConfigurationError(
340+ f"Couldn't find extension {str(v)!r}."
341+ )
342+
343+ loaded.update(val)
344+ elif isinstance(loaded, (list, tuple)):
345+ for v in loaded:
346+ expand_yaml_extends(v, root)
347+
348+
349+def load_yaml_includes(loaded: Dict[str, Any]) -> None:
350+ """Load all configuration files listed as include."""
351+ for include in loaded.get("include", []):
352+ include_path = Path(include)
353+ if not include_path.is_file():
354+ raise ConfigurationError(
355+ f"Couldn't find config file {str(include_path)!r}."
356+ )
357+ with include_path.open("rb") as f:
358+
359+ included = yaml.safe_load(f)
360+
361+ if not isinstance(included, dict):
362+ raise ConfigurationError(
363+ f"Config file {str(include_path)!r} does not define a mapping" # noqa: E501
364+ )
365+
366+ loaded.update(included)
367+
368+
369 def load_yaml(path: Path) -> Dict[Any, Any]:
370 """Return the content of a YAML file."""
371 if not path.is_file():
372@@ -25,10 +90,16 @@ def load_yaml(path: Path) -> Dict[Any, Any]:
373 try:
374 with path.open("rb") as f:
375 loaded = yaml.safe_load(f)
376+
377 if not isinstance(loaded, dict):
378 raise ConfigurationError(
379 f"Config file {str(path)!r} does not define a mapping"
380 )
381+
382+ load_yaml_includes(loaded)
383+
384+ expand_yaml_extends(loaded, loaded)
385+
386 return loaded
387 except (yaml.error.YAMLError, OSError) as e:
388 raise ConfigurationError(

Subscribers

People subscribed via source and target branches