Juju Charms Collection
rutabaga package

Merge lp:~blr/charms/trusty/rutabaga/code-asset-and-auth into lp:~canonical-launchpad-branches/charms/trusty/rutabaga/devel

  1. Trusty Tahr (14.04)
  2. code-asset-and-auth
  3. Merge into devel
Proposed by Kit Randel
Status: Merged
Merged at revision: 16
Proposed branch: lp:~blr/charms/trusty/rutabaga/code-asset-and-auth
Merge into: lp:~canonical-launchpad-branches/charms/trusty/rutabaga/devel
Diff against target: 486 lines (+238/-72)
11 files modified
.bzrignore (+1/-2)
Makefile.common (+34/-22)
config.yaml (+45/-1)
deploy-requirements.txt (+0/-2)
hooks/actions.py (+146/-41)
hooks/hooks.py (+1/-1)
hooks/services.py (+7/-2)
templates/envs/ADMIN_API_SECRET.j2 (+1/-0)
templates/envs/ADMIN_API_USERNAME.j2 (+1/-0)
templates/envs/ADMIN_API_USERNAME_PREFIX.j2 (+1/-0)
templates/rutabaga-cron.j2 (+1/-1)
To merge this branch: bzr merge lp:~blr/charms/trusty/rutabaga/code-asset-and-auth
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+276950@code.launchpad.net

Description of the change

This branch aligns rutabaga with the recent payload/swift handling in turnip's charm.

Additionally, several config keys for basic auth support have been added (admin_api_*).

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) wrote :

This will need to be brought up to date with more recent changes to the turnip charm, or else you'll end up running into the same problems I did sorting out Swift credentials. I've tried to catch them individually below, but see http://bazaar.launchpad.net/~canonical-launchpad-branches/charms/trusty/turnip/devel/revision/83?remember=81&compare_revid=81 for the full diff.

review: Approve
lp:~blr/charms/trusty/rutabaga/code-asset-and-auth updated
21. By Kit Randel

* Build juju temp config in more portable manner.
* Add anonymous swift support/storageurl.
* Remove PYVENV envvar.

22. By Kit Randel

Add make to required packages.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file '.bzrignore'
2--- .bzrignore 2015-08-03 02:30:33 +0000
3+++ .bzrignore 2015-11-25 03:09:17 +0000
4@@ -1,6 +1,5 @@
5 *.pyc
6 .coverage
7 .venv
8-files/pip-cache
9-files/rutabaga.tar.gz
10+files/*
11 src
12
13=== modified file 'Makefile.common'
14--- Makefile.common 2015-08-20 08:45:53 +0000
15+++ Makefile.common 2015-11-25 03:09:17 +0000
16@@ -1,14 +1,17 @@
17+# -*- mode: makefile -*-
18+
19 PYTHON := /usr/bin/env python3
20 PWD := $(shell pwd)
21 HOOKS_DIR := $(PWD)/hooks
22 SOURCE_DIR ?= $(shell dirname $(PWD))/.source/$(APP_NAME)
23-PIP_CACHE := $(PWD)/files/pip-cache
24+FILES_DIR := $(PWD)/files
25
26-ifeq ($(PIP_SOURCE_DIR),)
27-PIP_CACHE_ARGS :=
28-else
29-PIP_CACHE_ARGS := --no-index --find-links=file://$(PIP_SOURCE_DIR)
30-endif
31+BUILD_LABEL = $(shell bzr log -rlast: --show-ids $(SOURCE_DIR) | sed -n 's/^revision-id: //p')
32+TARBALL = $(APP_NAME).tar.gz
33+ASSET = $(FILES_DIR)/$(BUILD_LABEL)/$(TARBALL)
34+UNIT = $(APP_NAME)/0
35+CHARM_UNIT_PATH := /var/lib/juju/agents/unit-$(APP_NAME)-0/charm
36+TMP_JUJU_CFG = tmp_juju_cfg.yaml
37
38 all: setup lint test
39
40@@ -17,9 +20,27 @@
41 @juju upgrade-charm --repository=../.. $(APP_NAME)
42
43
44-deploy: tarball pip-cache
45+deploy: payload
46 @echo "Deploying $(APP_NAME)..."
47- @juju deploy --repository=../.. local:trusty/$(APP_NAME)
48+ @printf 'rutabaga:\n build_label: %s' "$(BUILD_LABEL)" > $(TMP_JUJU_CFG)
49+ @juju deploy --repository=../.. local:trusty/$(APP_NAME) --config $(TMP_JUJU_CFG)
50+ @echo "Waiting 60s for unit to become available..."
51+ @sleep 60
52+ @rm -f $(TMP_JUJU_CFG)
53+ @$(MAKE) rollout SKIP_BUILD=true
54+
55+
56+# deploy a new revision/branch
57+rollout: _PATH=$(CHARM_UNIT_PATH)/files/$(BUILD_LABEL)
58+rollout:
59+ifneq ($(SKIP_BUILD),true)
60+ $(MAKE) payload
61+endif
62+# manually copy our asset to be in the right place, rather
63+# than upgrade-charm
64+ juju scp $(ASSET) $(UNIT):$(TARBALL)
65+ juju ssh $(UNIT) 'sudo mkdir -p $(_PATH) && sudo mv $(TARBALL) $(_PATH)/'
66+ juju set $(APP_NAME) build_label=$(BUILD_LABEL)
67
68
69 ifeq ($(NO_FETCH_CODE),)
70@@ -36,24 +57,14 @@
71 endif
72
73
74-pip-cache: fetch-code
75- @echo "Updating python dependency cache..."
76- @mkdir -p $(PIP_CACHE)
77- @pip3 install $(PIP_CACHE_ARGS) --no-use-wheel --download $(PIP_CACHE) \
78- -r $(SOURCE_DIR)/requirements.txt \
79- -r deploy-requirements.txt
80-
81-
82 check-rev:
83 ifndef REV
84 $(error Revision number required to fetch source: e.g. $ REV=10 make deploy)
85 endif
86
87-tarball: fetch-code
88- @echo "Creating tarball for deploy..."
89- @mkdir -p files/
90- @tar czf files/$(APP_NAME).tar.gz -C $(SOURCE_DIR) .
91-
92+payload: fetch-code
93+ @echo "Building asset for $(BUILD_LABEL)..."
94+ @$(MAKE) -C $(SOURCE_DIR) build-tarball TARBALL_BUILDS_DIR=$(FILES_DIR)
95
96 # The following targets are for charm maintenance.
97
98@@ -63,6 +74,7 @@
99 @rm -f .coverage
100 @rm -rf $(SOURCE_DIR)
101 @rm -rf $(PIP_CACHE)
102+ @rm -f $(TMP_JUJU_CFG)
103 @rm -rf .venv
104
105
106@@ -91,4 +103,4 @@
107 @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py > /tmp/charm_helpers_sync.py
108
109
110-.PHONY: clean lint setup tarball test upgrade
111+.PHONY: clean lint setup payload test upgrade
112
113=== modified file 'config.yaml'
114--- config.yaml 2015-08-06 02:58:34 +0000
115+++ config.yaml 2015-11-25 03:09:17 +0000
116@@ -3,10 +3,26 @@
117 type: string
118 default: 'rutabaga'
119 description: Name of this application.
120+ build_label:
121+ type: string
122+ default: ""
123+ description: Build label to run.
124 token_purge_minutes:
125 type: string
126- default: '5'
127+ default: '20'
128 description: Period of token purging cron job.
129+ admin_api_username_prefix:
130+ type: string
131+ default: 'admin-'
132+ description: Admin account username prefix.
133+ admin_api_username:
134+ type: string
135+ default: 'launchpad.net'
136+ description: Admin account username.
137+ admin_api_secret:
138+ type: string
139+ default: ''
140+ description: Admin api secret.
141 nagios_context:
142 default: "juju"
143 type: string
144@@ -59,6 +75,34 @@
145 description: |
146 Hosts that should be allowed to rsync logs. Note that this relies on
147 basenode.
148+ swift_username:
149+ type: string
150+ default: ""
151+ description: Username to use when accessing Swift.
152+ swift_password:
153+ type: string
154+ default: ""
155+ description: Password to use when accessing Swift.
156+ swift_auth_url:
157+ type: string
158+ default: ""
159+ description: URL for authenticating against Keystone.
160+ swift_storage_url:
161+ type: string
162+ default: ""
163+ description: URL for fetching from Swift (for anonymous access).
164+ swift_region_name:
165+ type: string
166+ default: ""
167+ description: Swift region.
168+ swift_tenant_name:
169+ type: string
170+ default: ""
171+ description: Entity that owns resources.
172+ swift_container_name:
173+ type: string
174+ default: ""
175+ description: Container to put objects in.
176
177 # apt configuration used by charmhelpers.
178 install_sources:
179
180=== removed file 'deploy-requirements.txt'
181--- deploy-requirements.txt 2015-07-21 21:46:39 +0000
182+++ deploy-requirements.txt 1970-01-01 00:00:00 +0000
183@@ -1,2 +0,0 @@
184-envdir==0.7
185-gunicorn==19.3.0
186
187=== modified file 'hooks/actions.py'
188--- hooks/actions.py 2015-08-06 03:09:06 +0000
189+++ hooks/actions.py 2015-11-25 03:09:17 +0000
190@@ -1,7 +1,9 @@
191+import errno
192 import grp
193 import os
194 import pwd
195 import sqlite3
196+import shutil
197 import subprocess
198 import time
199
200@@ -20,15 +22,23 @@
201
202 # Globals
203 CHARM_FILES_DIR = os.path.join(hookenv.charm_dir(), 'files')
204-REQUIRED_PACKAGES = ['python-virtualenv', 'python3-dev',
205- 'libsqlite3-dev', 'python3-jinja2']
206+REQUIRED_PACKAGES = [
207+ 'make',
208+ 'python-virtualenv',
209+ 'python3-dev',
210+ 'python3.4-venv',
211+ 'libsqlite3-dev',
212+ 'python3-jinja2',
213+ ]
214
215 BASE_DIR = config['base_dir']
216+PAYLOADS_DIR = os.path.join(BASE_DIR, 'payloads')
217 CODE_DIR = os.path.join(BASE_DIR, 'code')
218 DB_DIR = os.path.join(BASE_DIR, 'db')
219 ENV_DIR = os.path.join(BASE_DIR, 'envs') # environment variables
220-VENV_DIR = os.path.join(BASE_DIR, 'venv')
221+VENV_DIR = os.path.join(CODE_DIR, 'env')
222 LOGS_DIR = os.path.join(BASE_DIR, 'logs')
223+CODE_TARBALL = 'rutabaga.tar.gz'
224
225 CODE_USER = config['code_user']
226 CODE_GROUP = config['code_group']
227@@ -44,7 +54,7 @@
228 def make_srv_location():
229 hookenv.log('Creating directories...')
230
231- for dir in (BASE_DIR, CODE_DIR):
232+ for dir in (BASE_DIR, PAYLOADS_DIR):
233 host.mkdir(dir, owner=CODE_USER, group=CODE_GROUP, perms=0o755)
234 for dir in (LOGS_DIR, DB_DIR):
235 host.mkdir(dir, owner=USER, group=GROUP, perms=0o755)
236@@ -77,51 +87,146 @@
237 host.chownr(DB_DIR, USER, GROUP)
238
239
240-def unpack_source(service_name):
241- tarball = 'rutabaga.tar.gz'
242- hookenv.log('Deploying source...')
243-
244+def get_swift_creds(config):
245+ return {
246+ 'user': config['swift_username'],
247+ 'project': config['swift_tenant_name'],
248+ 'password': config['swift_password'],
249+ 'authurl': config['swift_auth_url'],
250+ 'region': config['swift_region_name'],
251+ 'storageurl': config['swift_storage_url'],
252+ }
253+
254+
255+def swift_base_cmd(**swift_creds):
256+ return [
257+ 'swift',
258+ '--os-username=' + swift_creds['user'],
259+ '--os-tenant-name=' + swift_creds['project'],
260+ '--os-password=' + swift_creds['password'],
261+ '--os-auth-url=' + swift_creds['authurl'],
262+ '--os-region-name=' + swift_creds['region'],
263+ ]
264+
265+
266+def swift_fetch(source, target, container=None, **swift_creds):
267+ if swift_creds['user']:
268+ cmd = swift_base_cmd(**swift_creds) + [
269+ 'download', '--output=' + target, container, source]
270+ else:
271+ storage_url = swift_creds['storageurl']
272+ assert storage_url
273+ cmd = [
274+ 'wget', '-O', target,
275+ '%s/%s/%s' % (storage_url, container, source)]
276+ subprocess.check_call(cmd)
277+
278+
279+def unlink_force(path):
280+ """Unlink path, without worrying about whether it exists."""
281+ try:
282+ os.unlink(path)
283+ except OSError as e:
284+ if e.errno != errno.ENOENT:
285+ raise
286+
287+
288+def symlink_force(source, link_name):
289+ """Create symlink link_name -> source, even if link_name exists."""
290+ unlink_force(link_name)
291+ os.symlink(source, link_name)
292+
293+
294+def install_python_packages(target_dir):
295+ hookenv.log('Installing Python dependencies...')
296+ subprocess.check_call(
297+ ['sudo', '-u', CODE_USER, 'make', '-C', target_dir, 'build',
298+ 'PIP_SOURCE_DIR=%s' % os.path.join(target_dir, 'pip-cache')])
299+
300+
301+def prune_payloads(keep):
302+ for entry in os.listdir(PAYLOADS_DIR):
303+ if entry in keep:
304+ continue
305+ entry_path = os.path.join(PAYLOADS_DIR, entry)
306+ if os.path.isdir(entry_path):
307+ hookenv.log('Purging old build in %s...' % entry_path)
308+ shutil.rmtree(entry_path)
309+
310+
311+def deploy_code(service_name):
312 make_srv_location()
313
314+ current_build_label = None
315+ if os.path.islink(CODE_DIR):
316+ current_build_label = os.path.basename(os.path.realpath(CODE_DIR))
317+ elif os.path.isdir(os.path.join(CODE_DIR, '.bzr')):
318+ log_output = subprocess.check_output(
319+ ['bzr', 'log', '-rlast:', '--show-ids', CODE_DIR])
320+ for line in log_output.splitlines():
321+ if line.startswith('revision-id: '):
322+ current_build_label = line[len('revision-id: '):]
323+ desired_build_label = config['build_label']
324+ if not desired_build_label:
325+ if current_build_label is not None:
326+ hookenv.log(
327+ 'No desired build label, but build %s is already deployed' %
328+ current_build_label)
329+ return
330+ else:
331+ raise AssertionError('Build label unset, so cannot deploy code')
332+ if current_build_label == desired_build_label:
333+ hookenv.log('Build %s already deployed' % desired_build_label)
334+ return
335+ hookenv.log('Deploying build %s...' % desired_build_label)
336+
337 # Copy source archive
338- archive_path = os.path.join(BASE_DIR, tarball)
339-
340- with open(os.path.join(CHARM_FILES_DIR, tarball), 'rb') as file:
341- host.write_file(archive_path, file.read(), perms=0o644)
342-
343- # Unpack source
344- archive.extract_tarfile(archive_path, CODE_DIR)
345- os.chown(
346- CODE_DIR,
347- pwd.getpwnam(CODE_USER).pw_uid, grp.getgrnam(CODE_GROUP).gr_gid)
348- host.lchownr(CODE_DIR, CODE_USER, CODE_GROUP)
349+ archive_path = os.path.join(PAYLOADS_DIR, desired_build_label + '.tar.gz')
350+ object_name = os.path.join(desired_build_label, CODE_TARBALL)
351+
352+ try:
353+ if config['swift_container_name']:
354+ swift_creds = get_swift_creds(config)
355+ swift_container = config['swift_container_name']
356+ swift_fetch(
357+ os.path.join('rutabaga-builds', object_name), archive_path,
358+ container=swift_container, **swift_creds)
359+ else:
360+ with open(os.path.join(CHARM_FILES_DIR, object_name)) as file:
361+ host.write_file(archive_path, file.read(), perms=0o644)
362+
363+ # Unpack source
364+ target_dir = os.path.join(PAYLOADS_DIR, desired_build_label)
365+ if os.path.isdir(target_dir):
366+ shutil.rmtree(target_dir)
367+ archive.extract_tarfile(archive_path, target_dir)
368+ os.chown(
369+ target_dir,
370+ pwd.getpwnam(CODE_USER).pw_uid, grp.getgrnam(CODE_GROUP).gr_gid)
371+ host.lchownr(target_dir, CODE_USER, CODE_GROUP)
372+
373+ install_python_packages(target_dir)
374+
375+ if not os.path.islink(CODE_DIR) and os.path.isdir(CODE_DIR):
376+ old_payload_dir = os.path.join(PAYLOADS_DIR, current_build_label)
377+ if os.path.exists(old_payload_dir):
378+ shutil.rmtree(CODE_DIR)
379+ else:
380+ os.rename(CODE_DIR, old_payload_dir)
381+ symlink_force(
382+ os.path.relpath(target_dir, os.path.dirname(CODE_DIR)), CODE_DIR)
383+ prune_payloads([desired_build_label, current_build_label])
384+ finally:
385+ unlink_force(archive_path)
386
387
388 def install_packages(service_name):
389 hookenv.log('Installing system packages...')
390 fetch.configure_sources(update=True)
391- fetch.apt_install(REQUIRED_PACKAGES, fatal=True)
392-
393-
394-def install_python_packages(service_name):
395- hookenv.log('Installing Python dependencies...')
396- pip_cache = os.path.join(CHARM_FILES_DIR, 'pip-cache')
397- code_reqs = os.path.join(CODE_DIR, 'requirements.txt')
398- deploy_reqs = os.path.join(hookenv.charm_dir(), 'deploy-requirements.txt')
399-
400- pip_bin = os.path.join(VENV_DIR, 'bin', 'pip3')
401- os.environ["LANG"] = 'C.UTF-8' # workaround for envdir 0.7
402-
403- subprocess.call([
404- 'sudo', '-u', CODE_USER, 'virtualenv', '--system-site-packages',
405- VENV_DIR, '--distribute', '-p', '/usr/bin/python3'])
406- subprocess.check_call([
407- 'sudo', '-u', CODE_USER, pip_bin, 'install', '--no-index',
408- '--find-links={}'.format(pip_cache), '-r', code_reqs,
409- '-r', deploy_reqs])
410- subprocess.check_call([
411- 'sudo', '-u', CODE_USER, pip_bin, 'install', '--no-deps',
412- '-e', CODE_DIR])
413+ packages = list(REQUIRED_PACKAGES)
414+ if config['swift_username']:
415+ packages.append('python-swiftclient')
416+ fetch.apt_install(packages, fatal=True)
417
418
419 def restart_rsync(service_name):
420
421=== modified file 'hooks/hooks.py'
422--- hooks/hooks.py 2015-08-03 02:30:33 +0000
423+++ hooks/hooks.py 2015-11-25 03:09:17 +0000
424@@ -1,3 +1,3 @@
425-#!/usr/bin/env python3
426+#!/usr/bin/env python
427 import services
428 services.manage()
429
430=== modified file 'hooks/services.py'
431--- hooks/services.py 2015-08-06 02:58:34 +0000
432+++ hooks/services.py 2015-11-25 03:09:17 +0000
433@@ -37,12 +37,14 @@
434
435
436 def manage():
437+
438 if hookenv.hook_name() in ('install', 'upgrade-charm'):
439 actions.execd_preinstall('rutabaga')
440 actions.install_packages('rutabaga')
441 actions.create_users('rutabaga')
442- actions.unpack_source('rutabaga')
443- actions.install_python_packages('rutabaga')
444+
445+ if hookenv.hook_name() in ('install', 'upgrade-charm', 'config-changed'):
446+ actions.deploy_code('rutabaga')
447
448 if hookenv.hook_name() in ('install'):
449 actions.create_schema('rutabaga')
450@@ -64,6 +66,9 @@
451 render_env_template(config, 'RUTABAGA_LOG_DIR'),
452 render_env_template(config, 'SITE_NAME'),
453 render_env_template(config, 'MAIN_SITE_ROOT'),
454+ render_env_template(config, 'ADMIN_API_USERNAME_PREFIX'),
455+ render_env_template(config, 'ADMIN_API_USERNAME'),
456+ render_env_template(config, 'ADMIN_API_SECRET'),
457 helpers.render_template(
458 source='rutabaga-logrotate.j2',
459 target='/etc/logrotate.d/rutabaga',
460
461=== added file 'templates/envs/ADMIN_API_SECRET.j2'
462--- templates/envs/ADMIN_API_SECRET.j2 1970-01-01 00:00:00 +0000
463+++ templates/envs/ADMIN_API_SECRET.j2 2015-11-25 03:09:17 +0000
464@@ -0,0 +1,1 @@
465+{{ admin_api_secret }}
466
467=== added file 'templates/envs/ADMIN_API_USERNAME.j2'
468--- templates/envs/ADMIN_API_USERNAME.j2 1970-01-01 00:00:00 +0000
469+++ templates/envs/ADMIN_API_USERNAME.j2 2015-11-25 03:09:17 +0000
470@@ -0,0 +1,1 @@
471+{{ admin_api_username }}
472
473=== added file 'templates/envs/ADMIN_API_USERNAME_PREFIX.j2'
474--- templates/envs/ADMIN_API_USERNAME_PREFIX.j2 1970-01-01 00:00:00 +0000
475+++ templates/envs/ADMIN_API_USERNAME_PREFIX.j2 2015-11-25 03:09:17 +0000
476@@ -0,0 +1,1 @@
477+{{ admin_api_username_prefix }}
478
479=== modified file 'templates/rutabaga-cron.j2'
480--- templates/rutabaga-cron.j2 2015-09-30 05:59:11 +0000
481+++ templates/rutabaga-cron.j2 2015-11-25 03:09:17 +0000
482@@ -1,4 +1,4 @@
483 # /etc/cron.d/rutabaga: crontab entries for rutabaga
484
485-*/{{ token_purge_minutes }} * * * * root {{base_dir}}/venv/bin/envdir {{base_dir}}/envs/ {{ base_dir }}/venv/bin/python3 {{ base_dir }}/code/rutabaga/scripts/purge.py >> {{base_dir}}/logs/rutabaga-purge.log 2>&1
486+*/{{ token_purge_minutes }} * * * * root {{base_dir}}/code/env/bin/envdir {{base_dir}}/envs/ {{ base_dir }}/code/env/bin/python3 {{ base_dir }}/code/rutabaga/scripts/purge.py >> {{base_dir}}/logs/rutabaga-purge.log 2>&1
487

Subscribers

People subscribed via source and target branches

to all changes: