Juju Charms Collection
turnipcake package

Merge lp:~cjwatson/charms/trusty/turnipcake/build-label into lp:~canonical-launchpad-branches/charms/trusty/turnipcake/devel

  1. Trusty Tahr (14.04)
  2. build-label
  3. Merge into devel
Proposed by Colin Watson
Status: Merged
Merged at revision: 22
Proposed branch: lp:~cjwatson/charms/trusty/turnipcake/build-label
Merge into: lp:~canonical-launchpad-branches/charms/trusty/turnipcake/devel
Diff against target: 449 lines (+220/-82)
6 files modified
.bzrignore (+1/-2)
Makefile.common (+24/-21)
config.yaml (+28/-0)
deploy-requirements.txt (+0/-1)
hooks/actions.py (+165/-55)
hooks/services.py (+2/-3)
To merge this branch: bzr merge lp:~cjwatson/charms/trusty/turnipcake/build-label
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+275469@code.launchpad.net

Commit message

Allow updating the code payload separately from the charm using a build label.

Description of the change

Allow updating the code payload separately from the charm using a build label. This parallels https://code.launchpad.net/~cjwatson/charms/trusty/turnip/build-label/+merge/275468.

The approach used here is based heavily on the software-center-agent charm. The preferred approach is to store the payload in Swift (Canonical developers can use their Canonistack credentials for this), but you can also use "make deploy" or "make rollout" to do an initial deployment or an updated code rollout respectively and it'll push the payload around manually. The previous payload is kept in place for the sake of quick rollouts, but at the moment we prune all but the previous and current payloads: this has the benefit of not needing to worry about working out which the newest ones are.

The virtualenv moves inside the payload directory so that each payload gets its own. /srv/turnipcake/code remains as a symlink to the current payload, which is convenient.

There are no tests directly here, but it'll at least get integration testing by way of the corresponding changes to the Mojo spec.

To post a comment you must log in.
lp:~cjwatson/charms/trusty/turnipcake/build-label updated
23. By Colin Watson

Fix over-indentation.

Revision history for this message
Colin Watson (cjwatson) wrote :

Approving as a parallel of https://code.launchpad.net/~cjwatson/charms/trusty/turnip/build-label/+merge/275468.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file '.bzrignore'
2--- .bzrignore 2015-02-11 01:09:24 +0000
3+++ .bzrignore 2015-11-02 11:16:30 +0000
4@@ -1,3 +1,2 @@
5 lib/charmhelpers
6-files/*.tar.gz
7-files/*.tgz
8+files/*
9
10=== modified file 'Makefile.common'
11--- Makefile.common 2015-03-26 02:18:32 +0000
12+++ Makefile.common 2015-11-02 11:16:30 +0000
13@@ -7,13 +7,13 @@
14 TEST_PREFIX := PYTHONPATH=$(HOOKS_DIR)
15 TEST_DIR := $(PWD)/tests
16 SOURCE_DIR ?= $(shell dirname $(PWD))/.source/$(APP_NAME)
17-PIP_CACHE := $(PWD)/files/pip-cache
18+FILES_DIR := $(PWD)/files
19
20-ifeq ($(PIP_SOURCE_DIR),)
21-PIP_CACHE_ARGS :=
22-else
23-PIP_CACHE_ARGS := --no-index --find-links=file://$(PIP_SOURCE_DIR)
24-endif
25+BUILD_LABEL = $(shell bzr log -rlast: --show-ids $(SOURCE_DIR) | sed -n 's/^revision-id: //p')
26+TARBALL = $(APP_NAME).tar.gz
27+ASSET = $(FILES_DIR)/$(BUILD_LABEL)/$(TARBALL)
28+UNIT = $(APP_NAME)/0
29+CHARM_UNIT_PATH := /var/lib/juju/agents/unit-$(APP_NAME)-0/charm
30
31 all: setup lint test
32
33@@ -22,9 +22,22 @@
34 @juju upgrade-charm --repository=../.. $(APP_NAME)
35
36
37-deploy: tarball pip-cache
38+deploy: payload
39 @echo "Deploying $(APP_NAME)..."
40 @juju deploy --repository=../.. local:trusty/$(APP_NAME)
41+ @$(MAKE) rollout SKIP_BUILD=true
42+
43+
44+# deploy a new revision/branch
45+rollout: _PATH=$(CHARM_UNIT_PATH)/files/$(BUILD_LABEL)
46+rollout:
47+ifneq ($(SKIP_BUILD),true)
48+ $(MAKE) payload
49+endif
50+ # manually copy our asset to be in the right place, rather than upgrade-charm
51+ juju scp $(ASSET) $(UNIT):$(TARBALL)
52+ juju ssh $(UNIT) 'sudo mkdir -p $(_PATH) && sudo mv $(TARBALL) $(_PATH)/'
53+ juju set $(APP_NAME) build_label=$(BUILD_LABEL)
54
55
56 ifeq ($(NO_FETCH_CODE),)
57@@ -41,23 +54,14 @@
58 endif
59
60
61-pip-cache: fetch-code
62- @echo "Updating python dependency cache..."
63- @mkdir -p $(PIP_CACHE)
64- @pip install $(PIP_CACHE_ARGS) --no-use-wheel --download $(PIP_CACHE) \
65- -r $(SOURCE_DIR)/requirements.txt \
66- -r deploy-requirements.txt
67-
68-
69 check-rev:
70 ifndef REV
71 $(error Revision number required to fetch source: e.g. $ REV=10 make deploy)
72 endif
73
74-tarball: fetch-code
75- @echo "Creating tarball for deploy..."
76- @mkdir -p files/
77- @tar czf files/$(APP_NAME).tar.gz -C $(SOURCE_DIR) .
78+payload: fetch-code
79+ @echo "Building asset for $(BUILD_LABEL)..."
80+ @$(MAKE) -C $(SOURCE_DIR) build-tarball TARBALL_BUILDS_DIR=$(FILES_DIR)
81
82
83 # The following targets are for charm maintenance.
84@@ -67,7 +71,6 @@
85 @find . -depth -name '__pycache__' -exec rm -rf '{}' \;
86 @rm -f .coverage
87 @rm -rf $(SOURCE_DIR)
88- @rm -rf $(PIP_CACHE)
89 @rm -rf .venv
90
91
92@@ -100,4 +103,4 @@
93 @echo "Starting tests..."
94 @$(TEST_PREFIX) .venv/bin/coverage run -m unittest discover -s unit_tests
95
96-.PHONY: clean lint setup tarball test upgrade
97+.PHONY: clean lint setup payload test upgrade
98
99=== modified file 'config.yaml'
100--- config.yaml 2015-03-26 02:18:32 +0000
101+++ config.yaml 2015-11-02 11:16:30 +0000
102@@ -3,6 +3,10 @@
103 type: string
104 default: 'turnipcake'
105 description: Name of this application.
106+ build_label:
107+ type: string
108+ default: ""
109+ description: Build label to run.
110 nagios_context:
111 default: "juju"
112 type: string
113@@ -37,3 +41,27 @@
114 type: string
115 default: turnipcake
116 description: The service will run under this group.
117+ swift_username:
118+ type: string
119+ default: ""
120+ description: Username to use when accessing Swift.
121+ swift_password:
122+ type: string
123+ default: ""
124+ description: Password to use when accessing Swift.
125+ swift_auth_url:
126+ type: string
127+ default: ""
128+ description: URL for authenticating against Keystone.
129+ swift_region_name:
130+ type: string
131+ default: ""
132+ description: Swift region.
133+ swift_tenant_name:
134+ type: string
135+ default: ""
136+ description: Entity that owns resources.
137+ swift_container_name:
138+ type: string
139+ default: ""
140+ description: Container to put objects in.
141
142=== removed file 'deploy-requirements.txt'
143--- deploy-requirements.txt 2015-03-27 01:39:30 +0000
144+++ deploy-requirements.txt 1970-01-01 00:00:00 +0000
145@@ -1,1 +0,0 @@
146-gunicorn==19.3.0
147
148=== modified file 'hooks/actions.py'
149--- hooks/actions.py 2015-03-27 07:41:51 +0000
150+++ hooks/actions.py 2015-11-02 11:16:30 +0000
151@@ -1,6 +1,8 @@
152+import errno
153 import grp
154 import os
155 import pwd
156+import shutil
157 import subprocess
158
159 from charmhelpers import fetch
160@@ -16,14 +18,22 @@
161
162 # Globals
163 CHARM_FILES_DIR = os.path.join(hookenv.charm_dir(), 'files')
164-REQUIRED_PACKAGES = ['python-virtualenv', 'python-dev', 'make']
165+REQUIRED_PACKAGES = [
166+ 'python-virtualenv',
167+ 'python-dev',
168+ 'make',
169+ 'python-swiftclient',
170+ ]
171 BASE_DIR = config['base_dir']
172+PAYLOADS_DIR = os.path.join(BASE_DIR, 'payloads')
173 CODE_DIR = os.path.join(BASE_DIR, 'code')
174-VENV_DIR = os.path.join(BASE_DIR, 'venv')
175+VENV_DIR = os.path.join(CODE_DIR, 'env')
176+OLD_VENV_DIR = os.path.join(BASE_DIR, 'venv')
177 LOGS_DIR = os.path.join(BASE_DIR, 'logs')
178 DATA_DIR = os.path.join(BASE_DIR, 'data')
179 # XXX: Should really move this outside the code dir.
180 DB_DIR = os.path.join(BASE_DIR, 'code', 'db')
181+CODE_TARBALL = 'turnipcake.tar.gz'
182
183 CODE_USER = config['code_user']
184 CODE_GROUP = config['code_group']
185@@ -43,7 +53,7 @@
186 def make_srv_location():
187 hookenv.log('Creating directories...')
188
189- for dir in (BASE_DIR, CODE_DIR):
190+ for dir in (BASE_DIR, PAYLOADS_DIR):
191 host.mkdir(dir, owner=CODE_USER, group=CODE_GROUP, perms=0o755)
192 for dir in (LOGS_DIR, DATA_DIR):
193 host.mkdir(dir, owner=USER, group=GROUP, perms=0o755)
194@@ -63,28 +73,159 @@
195 host.add_user_to_group(USER, GROUP)
196
197
198-def unpack_source(service_name):
199- hookenv.log('Deploying source...')
200-
201+def get_swift_creds(config):
202+ return {
203+ 'user': config['swift_username'],
204+ 'project': config['swift_tenant_name'],
205+ 'password': config['swift_password'],
206+ 'authurl': config['swift_auth_url'],
207+ 'region': config['swift_region_name'],
208+ }
209+
210+
211+def swift_base_cmd(**swift_creds):
212+ return [
213+ 'swift',
214+ '--os-username=' + swift_creds['user'],
215+ '--os-tenant-name=' + swift_creds['project'],
216+ '--os-password=' + swift_creds['password'],
217+ '--os-auth-url=' + swift_creds['authurl'],
218+ '--os-region-name=' + swift_creds['region'],
219+ ]
220+
221+
222+def swift_get_etag(name, container=None, **swift_creds):
223+ cmd = swift_base_cmd(**swift_creds) + ['stat', container, name]
224+ file_stat = subprocess.check_output(cmd).splitlines()
225+ for line in file_stat:
226+ words = line.split()
227+ if words[0] == 'ETag:':
228+ return words[1]
229+
230+
231+def swift_fetch(source, target, container=None, **swift_creds):
232+ cmd = swift_base_cmd(**swift_creds) + [
233+ 'download', '--output=' + target, container, source]
234+ subprocess.check_call(cmd)
235+
236+
237+def unlink_force(path):
238+ """Unlink path, without worrying about whether it exists."""
239+ try:
240+ os.unlink(path)
241+ except OSError as e:
242+ if e.errno != errno.ENOENT:
243+ raise
244+
245+
246+def symlink_force(source, link_name):
247+ """Create symlink link_name -> source, even if link_name exists."""
248+ unlink_force(link_name)
249+ os.symlink(source, link_name)
250+
251+
252+def install_python_packages(target_dir):
253+ hookenv.log('Installing Python dependencies...')
254+ subprocess.check_call(
255+ ['sudo', '-u', CODE_USER, 'make', '-C', target_dir, 'build',
256+ 'PIP_SOURCE_DIR=%s' % os.path.join(target_dir, 'pip-cache')])
257+
258+
259+def prune_payloads(keep):
260+ for entry in os.listdir(PAYLOADS_DIR):
261+ if entry in keep:
262+ continue
263+ entry_path = os.path.join(PAYLOADS_DIR, entry)
264+ if os.path.isdir(entry_path):
265+ hookenv.log('Purging old build in %s...' % entry_path)
266+ shutil.rmtree(entry_path)
267+
268+
269+def migrate_db():
270+ hookenv.log('Migrating database...')
271+ path = '%s:%s' % (os.path.join(VENV_DIR, 'bin'), os.environ['PATH'])
272+
273+ with host.chdir(CODE_DIR):
274+ subprocess.check_call([
275+ 'sudo', '-u', USER, 'PATH=%s' % path, 'make', 'migrate'])
276+
277+
278+def deploy_code(service_name):
279 make_srv_location()
280
281+ current_build_label = None
282+ if os.path.islink(CODE_DIR):
283+ current_build_label = os.path.basename(os.path.realpath(CODE_DIR))
284+ elif os.path.isdir(os.path.join(CODE_DIR, '.bzr')):
285+ log_output = subprocess.check_output(
286+ ['bzr', 'log', '-rlast:', '--show-ids', CODE_DIR])
287+ for line in log_output.splitlines():
288+ if line.startswith('revision-id: '):
289+ current_build_label = line[len('revision-id: '):]
290+ desired_build_label = config['build_label']
291+ if not desired_build_label:
292+ if current_build_label is not None:
293+ hookenv.log(
294+ 'No desired build label, but build %s is already deployed' %
295+ current_build_label)
296+ # We're probably upgrading from a charm that used old-style code
297+ # assets, so make sure we at least have a virtualenv available
298+ # from the current preferred location.
299+ if not os.path.isdir(VENV_DIR) and os.path.isdir(OLD_VENV_DIR):
300+ os.symlink(OLD_VENV_DIR, VENV_DIR)
301+ return
302+ else:
303+ raise AssertionError('Build label unset, so cannot deploy code')
304+ if current_build_label == desired_build_label:
305+ hookenv.log('Build %s already deployed' % desired_build_label)
306+ return
307+ hookenv.log('Deploying build %s...' % desired_build_label)
308+
309 # Copy source archive
310- archive_path = os.path.join(BASE_DIR, 'turnipcake.tar.gz')
311-
312- with open(os.path.join(CHARM_FILES_DIR, 'turnipcake.tar.gz')) as file:
313- host.write_file(archive_path, file.read(), perms=0o644)
314-
315- # Unpack source
316- archive.extract_tarfile(archive_path, CODE_DIR)
317- os.chown(
318- CODE_DIR,
319- pwd.getpwnam(CODE_USER).pw_uid, grp.getgrnam(CODE_GROUP).gr_gid)
320- host.lchownr(CODE_DIR, CODE_USER, CODE_GROUP)
321-
322- # Ensure the DB is writable by the app user. It really shouldn't
323- # live in the code tree.
324- os.chown(DB_DIR, pwd.getpwnam(USER).pw_uid, grp.getgrnam(GROUP).gr_gid)
325- host.lchownr(DB_DIR, USER, GROUP)
326+ archive_path = os.path.join(PAYLOADS_DIR, desired_build_label + '.tar.gz')
327+ object_name = os.path.join(desired_build_label, CODE_TARBALL)
328+
329+ try:
330+ if config['swift_container_name']:
331+ swift_creds = get_swift_creds(config)
332+ swift_container = config['swift_container_name']
333+ swift_fetch(
334+ os.path.join('turnipcake-builds', object_name), archive_path,
335+ container=swift_container, **swift_creds)
336+ else:
337+ with open(os.path.join(CHARM_FILES_DIR, object_name)) as file:
338+ host.write_file(archive_path, file.read(), perms=0o644)
339+
340+ # Unpack source
341+ target_dir = os.path.join(PAYLOADS_DIR, desired_build_label)
342+ if os.path.isdir(target_dir):
343+ shutil.rmtree(target_dir)
344+ archive.extract_tarfile(archive_path, target_dir)
345+ os.chown(
346+ target_dir,
347+ pwd.getpwnam(CODE_USER).pw_uid, grp.getgrnam(CODE_GROUP).gr_gid)
348+ host.lchownr(target_dir, CODE_USER, CODE_GROUP)
349+
350+ # Ensure the DB is writable by the app user. It really shouldn't
351+ # live in the code tree.
352+ os.chown(DB_DIR, pwd.getpwnam(USER).pw_uid, grp.getgrnam(GROUP).gr_gid)
353+ host.lchownr(DB_DIR, USER, GROUP)
354+
355+ install_python_packages(target_dir)
356+
357+ if not os.path.islink(CODE_DIR) and os.path.isdir(CODE_DIR):
358+ old_payload_dir = os.path.join(PAYLOADS_DIR, current_build_label)
359+ if os.path.exists(old_payload_dir):
360+ shutil.rmtree(CODE_DIR)
361+ else:
362+ os.rename(CODE_DIR, old_payload_dir)
363+ symlink_force(
364+ os.path.relpath(target_dir, os.path.dirname(CODE_DIR)), CODE_DIR)
365+ prune_payloads([desired_build_label, current_build_label])
366+ finally:
367+ unlink_force(archive_path)
368+
369+ migrate_db()
370
371
372 def install_packages(service_name):
373@@ -93,42 +234,11 @@
374 fetch.apt_install(REQUIRED_PACKAGES, fatal=True)
375
376
377-def install_python_packages(service_name):
378- hookenv.log('Installing Python dependencies...')
379- pip_cache = os.path.join(CHARM_FILES_DIR, 'pip-cache')
380- code_reqs = os.path.join(CODE_DIR, 'requirements.txt')
381- deploy_reqs = os.path.join(hookenv.charm_dir(), 'deploy-requirements.txt')
382-
383- pip_bin = os.path.join(VENV_DIR, 'bin', 'pip')
384-
385- subprocess.call([
386- 'sudo', '-u', CODE_USER, 'virtualenv', '--system-site-packages',
387- VENV_DIR])
388- subprocess.check_call([
389- 'sudo', '-u', CODE_USER, pip_bin, 'install', '--no-index',
390- '--find-links={}'.format(pip_cache), '-r', code_reqs,
391- '-r', deploy_reqs])
392- subprocess.check_call([
393- 'sudo', '-u', CODE_USER, pip_bin, 'install', '--no-deps',
394- '-e', CODE_DIR])
395-
396-
397-def migrate_db(service_name):
398- hookenv.log('Migrating database...')
399- path = '%s:%s' % (os.path.join(VENV_DIR, 'bin'), os.environ['PATH'])
400-
401- with host.chdir(CODE_DIR):
402- subprocess.check_call([
403- 'sudo', '-u', USER, 'PATH=%s' % path, 'make', 'migrate'])
404-
405-
406 def publish_wsgi_relations(self):
407 # Publish the wsgi-file relation so the gunicorn subordinate can
408 # serve us. Other WSGI containers could be made to work, as the most
409 # gunicorn-specific thing is the --paste hack.
410 config = hookenv.config()
411- code_dir = os.path.join(config['base_dir'], 'code')
412- venv_bin = os.path.join(config['base_dir'], 'venv', 'bin')
413 # XXX We only support a single related turnip-api unit so far.
414 turnip_api_rid = sorted(hookenv.relation_ids('turnip-api'))[0]
415 turnip_api_unit = sorted(hookenv.related_units(turnip_api_rid))[0]
416@@ -136,7 +246,7 @@
417 rid=turnip_api_rid, unit=turnip_api_unit)
418
419 env = {
420- 'PATH': '%s:$PATH' % venv_bin,
421+ 'PATH': '%s:$PATH' % os.path.join(VENV_DIR, 'bin'),
422 'TURNIP_ENDPOINT': 'http://%s:%s' % (
423 turnip_api_data['turnip_api_host'],
424 turnip_api_data['turnip_api_port']),
425@@ -144,7 +254,7 @@
426 for relid in hookenv.relation_ids('wsgi-file'):
427 hookenv.relation_set(
428 relid,
429- working_dir=code_dir,
430+ working_dir=CODE_DIR,
431 wsgi_wsgi_file='--paste turnipcake.ini', # XXX: Gross.
432 wsgi_user=config['user'],
433 wsgi_group=config['group'],
434
435=== modified file 'hooks/services.py'
436--- hooks/services.py 2015-03-27 02:43:39 +0000
437+++ hooks/services.py 2015-11-02 11:16:30 +0000
438@@ -12,9 +12,8 @@
439 actions.execd_preinstall('turnipcake')
440 actions.install_packages('turnipcake')
441 actions.create_users('turnipcake')
442- actions.unpack_source('turnipcake')
443- actions.install_python_packages('turnipcake')
444- actions.migrate_db('turnipcake')
445+ if hookenv.hook_name() in ('install', 'upgrade-charm', 'config-changed'):
446+ actions.deploy_code('turnipcake')
447
448 config = hookenv.config()
449 manager = ServiceManager([

Subscribers

People subscribed via source and target branches

to all changes: