Juju Charms Collection
turnipcake package

Merge lp:~cjwatson/charms/trusty/turnipcake/build-label into lp:~canonical-launchpad-branches/charms/trusty/turnipcake/devel

  1. Trusty Tahr (14.04)
  2. build-label
  3. Merge into devel
Proposed by Colin Watson
Status: Merged
Merged at revision: 22
Proposed branch: lp:~cjwatson/charms/trusty/turnipcake/build-label
Merge into: lp:~canonical-launchpad-branches/charms/trusty/turnipcake/devel
Diff against target: 449 lines (+220/-82)
6 files modified
.bzrignore (+1/-2)
Makefile.common (+24/-21)
config.yaml (+28/-0)
deploy-requirements.txt (+0/-1)
hooks/actions.py (+165/-55)
hooks/services.py (+2/-3)
To merge this branch: bzr merge lp:~cjwatson/charms/trusty/turnipcake/build-label
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+275469@code.launchpad.net

Commit message

Allow updating the code payload separately from the charm using a build label.

Description of the change

Allow updating the code payload separately from the charm using a build label. This parallels https://code.launchpad.net/~cjwatson/charms/trusty/turnip/build-label/+merge/275468.

The approach used here is based heavily on the software-center-agent charm. The preferred approach is to store the payload in Swift (Canonical developers can use their Canonistack credentials for this), but you can also use "make deploy" or "make rollout" to do an initial deployment or an updated code rollout respectively and it'll push the payload around manually. The previous payload is kept in place for the sake of quick rollouts, but at the moment we prune all but the previous and current payloads: this has the benefit of not needing to worry about working out which the newest ones are.

The virtualenv moves inside the payload directory so that each payload gets its own. /srv/turnipcake/code remains as a symlink to the current payload, which is convenient.

There are no tests directly here, but it'll at least get integration testing by way of the corresponding changes to the Mojo spec.

To post a comment you must log in.
lp:~cjwatson/charms/trusty/turnipcake/build-label updated
23. By Colin Watson

Fix over-indentation.

Revision history for this message
Colin Watson (cjwatson) wrote :

Approving as a parallel of https://code.launchpad.net/~cjwatson/charms/trusty/turnip/build-label/+merge/275468.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file '.bzrignore'
--- .bzrignore 2015-02-11 01:09:24 +0000
+++ .bzrignore 2015-11-02 11:16:30 +0000
@@ -1,3 +1,2 @@
1lib/charmhelpers1lib/charmhelpers
2files/*.tar.gz2files/*
3files/*.tgz
43
=== modified file 'Makefile.common'
--- Makefile.common 2015-03-26 02:18:32 +0000
+++ Makefile.common 2015-11-02 11:16:30 +0000
@@ -7,13 +7,13 @@
7TEST_PREFIX := PYTHONPATH=$(HOOKS_DIR)7TEST_PREFIX := PYTHONPATH=$(HOOKS_DIR)
8TEST_DIR := $(PWD)/tests8TEST_DIR := $(PWD)/tests
9SOURCE_DIR ?= $(shell dirname $(PWD))/.source/$(APP_NAME)9SOURCE_DIR ?= $(shell dirname $(PWD))/.source/$(APP_NAME)
10PIP_CACHE := $(PWD)/files/pip-cache10FILES_DIR := $(PWD)/files
1111
12ifeq ($(PIP_SOURCE_DIR),)12BUILD_LABEL = $(shell bzr log -rlast: --show-ids $(SOURCE_DIR) | sed -n 's/^revision-id: //p')
13PIP_CACHE_ARGS :=13TARBALL = $(APP_NAME).tar.gz
14else14ASSET = $(FILES_DIR)/$(BUILD_LABEL)/$(TARBALL)
15PIP_CACHE_ARGS := --no-index --find-links=file://$(PIP_SOURCE_DIR)15UNIT = $(APP_NAME)/0
16endif16CHARM_UNIT_PATH := /var/lib/juju/agents/unit-$(APP_NAME)-0/charm
1717
18all: setup lint test18all: setup lint test
1919
@@ -22,9 +22,22 @@
22 @juju upgrade-charm --repository=../.. $(APP_NAME)22 @juju upgrade-charm --repository=../.. $(APP_NAME)
2323
2424
25deploy: tarball pip-cache25deploy: payload
26 @echo "Deploying $(APP_NAME)..."26 @echo "Deploying $(APP_NAME)..."
27 @juju deploy --repository=../.. local:trusty/$(APP_NAME)27 @juju deploy --repository=../.. local:trusty/$(APP_NAME)
28 @$(MAKE) rollout SKIP_BUILD=true
29
30
31# deploy a new revision/branch
32rollout: _PATH=$(CHARM_UNIT_PATH)/files/$(BUILD_LABEL)
33rollout:
34ifneq ($(SKIP_BUILD),true)
35 $(MAKE) payload
36endif
37 # manually copy our asset to be in the right place, rather than upgrade-charm
38 juju scp $(ASSET) $(UNIT):$(TARBALL)
39 juju ssh $(UNIT) 'sudo mkdir -p $(_PATH) && sudo mv $(TARBALL) $(_PATH)/'
40 juju set $(APP_NAME) build_label=$(BUILD_LABEL)
2841
2942
30ifeq ($(NO_FETCH_CODE),)43ifeq ($(NO_FETCH_CODE),)
@@ -41,23 +54,14 @@
41endif54endif
4255
4356
44pip-cache: fetch-code
45 @echo "Updating python dependency cache..."
46 @mkdir -p $(PIP_CACHE)
47 @pip install $(PIP_CACHE_ARGS) --no-use-wheel --download $(PIP_CACHE) \
48 -r $(SOURCE_DIR)/requirements.txt \
49 -r deploy-requirements.txt
50
51
52check-rev:57check-rev:
53ifndef REV58ifndef REV
54 $(error Revision number required to fetch source: e.g. $ REV=10 make deploy)59 $(error Revision number required to fetch source: e.g. $ REV=10 make deploy)
55endif 60endif
5661
57tarball: fetch-code62payload: fetch-code
58 @echo "Creating tarball for deploy..."63 @echo "Building asset for $(BUILD_LABEL)..."
59 @mkdir -p files/64 @$(MAKE) -C $(SOURCE_DIR) build-tarball TARBALL_BUILDS_DIR=$(FILES_DIR)
60 @tar czf files/$(APP_NAME).tar.gz -C $(SOURCE_DIR) .
6165
6266
63# The following targets are for charm maintenance.67# The following targets are for charm maintenance.
@@ -67,7 +71,6 @@
67 @find . -depth -name '__pycache__' -exec rm -rf '{}' \; 71 @find . -depth -name '__pycache__' -exec rm -rf '{}' \;
68 @rm -f .coverage72 @rm -f .coverage
69 @rm -rf $(SOURCE_DIR)73 @rm -rf $(SOURCE_DIR)
70 @rm -rf $(PIP_CACHE)
71 @rm -rf .venv74 @rm -rf .venv
7275
7376
@@ -100,4 +103,4 @@
100 @echo "Starting tests..."103 @echo "Starting tests..."
101 @$(TEST_PREFIX) .venv/bin/coverage run -m unittest discover -s unit_tests 104 @$(TEST_PREFIX) .venv/bin/coverage run -m unittest discover -s unit_tests
102105
103.PHONY: clean lint setup tarball test upgrade106.PHONY: clean lint setup payload test upgrade
104107
=== modified file 'config.yaml'
--- config.yaml 2015-03-26 02:18:32 +0000
+++ config.yaml 2015-11-02 11:16:30 +0000
@@ -3,6 +3,10 @@
3 type: string3 type: string
4 default: 'turnipcake'4 default: 'turnipcake'
5 description: Name of this application.5 description: Name of this application.
6 build_label:
7 type: string
8 default: ""
9 description: Build label to run.
6 nagios_context:10 nagios_context:
7 default: "juju"11 default: "juju"
8 type: string12 type: string
@@ -37,3 +41,27 @@
37 type: string41 type: string
38 default: turnipcake42 default: turnipcake
39 description: The service will run under this group.43 description: The service will run under this group.
44 swift_username:
45 type: string
46 default: ""
47 description: Username to use when accessing Swift.
48 swift_password:
49 type: string
50 default: ""
51 description: Password to use when accessing Swift.
52 swift_auth_url:
53 type: string
54 default: ""
55 description: URL for authenticating against Keystone.
56 swift_region_name:
57 type: string
58 default: ""
59 description: Swift region.
60 swift_tenant_name:
61 type: string
62 default: ""
63 description: Entity that owns resources.
64 swift_container_name:
65 type: string
66 default: ""
67 description: Container to put objects in.
4068
=== removed file 'deploy-requirements.txt'
--- deploy-requirements.txt 2015-03-27 01:39:30 +0000
+++ deploy-requirements.txt 1970-01-01 00:00:00 +0000
@@ -1,1 +0,0 @@
1gunicorn==19.3.0
20
=== modified file 'hooks/actions.py'
--- hooks/actions.py 2015-03-27 07:41:51 +0000
+++ hooks/actions.py 2015-11-02 11:16:30 +0000
@@ -1,6 +1,8 @@
1import errno
1import grp2import grp
2import os3import os
3import pwd4import pwd
5import shutil
4import subprocess6import subprocess
57
6from charmhelpers import fetch8from charmhelpers import fetch
@@ -16,14 +18,22 @@
1618
17# Globals19# Globals
18CHARM_FILES_DIR = os.path.join(hookenv.charm_dir(), 'files')20CHARM_FILES_DIR = os.path.join(hookenv.charm_dir(), 'files')
19REQUIRED_PACKAGES = ['python-virtualenv', 'python-dev', 'make']21REQUIRED_PACKAGES = [
22 'python-virtualenv',
23 'python-dev',
24 'make',
25 'python-swiftclient',
26 ]
20BASE_DIR = config['base_dir']27BASE_DIR = config['base_dir']
28PAYLOADS_DIR = os.path.join(BASE_DIR, 'payloads')
21CODE_DIR = os.path.join(BASE_DIR, 'code')29CODE_DIR = os.path.join(BASE_DIR, 'code')
22VENV_DIR = os.path.join(BASE_DIR, 'venv')30VENV_DIR = os.path.join(CODE_DIR, 'env')
31OLD_VENV_DIR = os.path.join(BASE_DIR, 'venv')
23LOGS_DIR = os.path.join(BASE_DIR, 'logs')32LOGS_DIR = os.path.join(BASE_DIR, 'logs')
24DATA_DIR = os.path.join(BASE_DIR, 'data')33DATA_DIR = os.path.join(BASE_DIR, 'data')
25# XXX: Should really move this outside the code dir.34# XXX: Should really move this outside the code dir.
26DB_DIR = os.path.join(BASE_DIR, 'code', 'db')35DB_DIR = os.path.join(BASE_DIR, 'code', 'db')
36CODE_TARBALL = 'turnipcake.tar.gz'
2737
28CODE_USER = config['code_user']38CODE_USER = config['code_user']
29CODE_GROUP = config['code_group']39CODE_GROUP = config['code_group']
@@ -43,7 +53,7 @@
43def make_srv_location():53def make_srv_location():
44 hookenv.log('Creating directories...')54 hookenv.log('Creating directories...')
4555
46 for dir in (BASE_DIR, CODE_DIR):56 for dir in (BASE_DIR, PAYLOADS_DIR):
47 host.mkdir(dir, owner=CODE_USER, group=CODE_GROUP, perms=0o755)57 host.mkdir(dir, owner=CODE_USER, group=CODE_GROUP, perms=0o755)
48 for dir in (LOGS_DIR, DATA_DIR):58 for dir in (LOGS_DIR, DATA_DIR):
49 host.mkdir(dir, owner=USER, group=GROUP, perms=0o755)59 host.mkdir(dir, owner=USER, group=GROUP, perms=0o755)
@@ -63,28 +73,159 @@
63 host.add_user_to_group(USER, GROUP)73 host.add_user_to_group(USER, GROUP)
6474
6575
66def unpack_source(service_name):76def get_swift_creds(config):
67 hookenv.log('Deploying source...')77 return {
6878 'user': config['swift_username'],
79 'project': config['swift_tenant_name'],
80 'password': config['swift_password'],
81 'authurl': config['swift_auth_url'],
82 'region': config['swift_region_name'],
83 }
84
85
86def swift_base_cmd(**swift_creds):
87 return [
88 'swift',
89 '--os-username=' + swift_creds['user'],
90 '--os-tenant-name=' + swift_creds['project'],
91 '--os-password=' + swift_creds['password'],
92 '--os-auth-url=' + swift_creds['authurl'],
93 '--os-region-name=' + swift_creds['region'],
94 ]
95
96
97def swift_get_etag(name, container=None, **swift_creds):
98 cmd = swift_base_cmd(**swift_creds) + ['stat', container, name]
99 file_stat = subprocess.check_output(cmd).splitlines()
100 for line in file_stat:
101 words = line.split()
102 if words[0] == 'ETag:':
103 return words[1]
104
105
106def swift_fetch(source, target, container=None, **swift_creds):
107 cmd = swift_base_cmd(**swift_creds) + [
108 'download', '--output=' + target, container, source]
109 subprocess.check_call(cmd)
110
111
112def unlink_force(path):
113 """Unlink path, without worrying about whether it exists."""
114 try:
115 os.unlink(path)
116 except OSError as e:
117 if e.errno != errno.ENOENT:
118 raise
119
120
121def symlink_force(source, link_name):
122 """Create symlink link_name -> source, even if link_name exists."""
123 unlink_force(link_name)
124 os.symlink(source, link_name)
125
126
127def install_python_packages(target_dir):
128 hookenv.log('Installing Python dependencies...')
129 subprocess.check_call(
130 ['sudo', '-u', CODE_USER, 'make', '-C', target_dir, 'build',
131 'PIP_SOURCE_DIR=%s' % os.path.join(target_dir, 'pip-cache')])
132
133
134def prune_payloads(keep):
135 for entry in os.listdir(PAYLOADS_DIR):
136 if entry in keep:
137 continue
138 entry_path = os.path.join(PAYLOADS_DIR, entry)
139 if os.path.isdir(entry_path):
140 hookenv.log('Purging old build in %s...' % entry_path)
141 shutil.rmtree(entry_path)
142
143
144def migrate_db():
145 hookenv.log('Migrating database...')
146 path = '%s:%s' % (os.path.join(VENV_DIR, 'bin'), os.environ['PATH'])
147
148 with host.chdir(CODE_DIR):
149 subprocess.check_call([
150 'sudo', '-u', USER, 'PATH=%s' % path, 'make', 'migrate'])
151
152
153def deploy_code(service_name):
69 make_srv_location()154 make_srv_location()
70155
156 current_build_label = None
157 if os.path.islink(CODE_DIR):
158 current_build_label = os.path.basename(os.path.realpath(CODE_DIR))
159 elif os.path.isdir(os.path.join(CODE_DIR, '.bzr')):
160 log_output = subprocess.check_output(
161 ['bzr', 'log', '-rlast:', '--show-ids', CODE_DIR])
162 for line in log_output.splitlines():
163 if line.startswith('revision-id: '):
164 current_build_label = line[len('revision-id: '):]
165 desired_build_label = config['build_label']
166 if not desired_build_label:
167 if current_build_label is not None:
168 hookenv.log(
169 'No desired build label, but build %s is already deployed' %
170 current_build_label)
171 # We're probably upgrading from a charm that used old-style code
172 # assets, so make sure we at least have a virtualenv available
173 # from the current preferred location.
174 if not os.path.isdir(VENV_DIR) and os.path.isdir(OLD_VENV_DIR):
175 os.symlink(OLD_VENV_DIR, VENV_DIR)
176 return
177 else:
178 raise AssertionError('Build label unset, so cannot deploy code')
179 if current_build_label == desired_build_label:
180 hookenv.log('Build %s already deployed' % desired_build_label)
181 return
182 hookenv.log('Deploying build %s...' % desired_build_label)
183
71 # Copy source archive184 # Copy source archive
72 archive_path = os.path.join(BASE_DIR, 'turnipcake.tar.gz')185 archive_path = os.path.join(PAYLOADS_DIR, desired_build_label + '.tar.gz')
73186 object_name = os.path.join(desired_build_label, CODE_TARBALL)
74 with open(os.path.join(CHARM_FILES_DIR, 'turnipcake.tar.gz')) as file:187
75 host.write_file(archive_path, file.read(), perms=0o644)188 try:
76189 if config['swift_container_name']:
77 # Unpack source190 swift_creds = get_swift_creds(config)
78 archive.extract_tarfile(archive_path, CODE_DIR)191 swift_container = config['swift_container_name']
79 os.chown(192 swift_fetch(
80 CODE_DIR,193 os.path.join('turnipcake-builds', object_name), archive_path,
81 pwd.getpwnam(CODE_USER).pw_uid, grp.getgrnam(CODE_GROUP).gr_gid)194 container=swift_container, **swift_creds)
82 host.lchownr(CODE_DIR, CODE_USER, CODE_GROUP)195 else:
83196 with open(os.path.join(CHARM_FILES_DIR, object_name)) as file:
84 # Ensure the DB is writable by the app user. It really shouldn't197 host.write_file(archive_path, file.read(), perms=0o644)
85 # live in the code tree.198
86 os.chown(DB_DIR, pwd.getpwnam(USER).pw_uid, grp.getgrnam(GROUP).gr_gid)199 # Unpack source
87 host.lchownr(DB_DIR, USER, GROUP)200 target_dir = os.path.join(PAYLOADS_DIR, desired_build_label)
201 if os.path.isdir(target_dir):
202 shutil.rmtree(target_dir)
203 archive.extract_tarfile(archive_path, target_dir)
204 os.chown(
205 target_dir,
206 pwd.getpwnam(CODE_USER).pw_uid, grp.getgrnam(CODE_GROUP).gr_gid)
207 host.lchownr(target_dir, CODE_USER, CODE_GROUP)
208
209 # Ensure the DB is writable by the app user. It really shouldn't
210 # live in the code tree.
211 os.chown(DB_DIR, pwd.getpwnam(USER).pw_uid, grp.getgrnam(GROUP).gr_gid)
212 host.lchownr(DB_DIR, USER, GROUP)
213
214 install_python_packages(target_dir)
215
216 if not os.path.islink(CODE_DIR) and os.path.isdir(CODE_DIR):
217 old_payload_dir = os.path.join(PAYLOADS_DIR, current_build_label)
218 if os.path.exists(old_payload_dir):
219 shutil.rmtree(CODE_DIR)
220 else:
221 os.rename(CODE_DIR, old_payload_dir)
222 symlink_force(
223 os.path.relpath(target_dir, os.path.dirname(CODE_DIR)), CODE_DIR)
224 prune_payloads([desired_build_label, current_build_label])
225 finally:
226 unlink_force(archive_path)
227
228 migrate_db()
88229
89230
90def install_packages(service_name):231def install_packages(service_name):
@@ -93,42 +234,11 @@
93 fetch.apt_install(REQUIRED_PACKAGES, fatal=True)234 fetch.apt_install(REQUIRED_PACKAGES, fatal=True)
94235
95236
96def install_python_packages(service_name):
97 hookenv.log('Installing Python dependencies...')
98 pip_cache = os.path.join(CHARM_FILES_DIR, 'pip-cache')
99 code_reqs = os.path.join(CODE_DIR, 'requirements.txt')
100 deploy_reqs = os.path.join(hookenv.charm_dir(), 'deploy-requirements.txt')
101
102 pip_bin = os.path.join(VENV_DIR, 'bin', 'pip')
103
104 subprocess.call([
105 'sudo', '-u', CODE_USER, 'virtualenv', '--system-site-packages',
106 VENV_DIR])
107 subprocess.check_call([
108 'sudo', '-u', CODE_USER, pip_bin, 'install', '--no-index',
109 '--find-links={}'.format(pip_cache), '-r', code_reqs,
110 '-r', deploy_reqs])
111 subprocess.check_call([
112 'sudo', '-u', CODE_USER, pip_bin, 'install', '--no-deps',
113 '-e', CODE_DIR])
114
115
116def migrate_db(service_name):
117 hookenv.log('Migrating database...')
118 path = '%s:%s' % (os.path.join(VENV_DIR, 'bin'), os.environ['PATH'])
119
120 with host.chdir(CODE_DIR):
121 subprocess.check_call([
122 'sudo', '-u', USER, 'PATH=%s' % path, 'make', 'migrate'])
123
124
125def publish_wsgi_relations(self):237def publish_wsgi_relations(self):
126 # Publish the wsgi-file relation so the gunicorn subordinate can238 # Publish the wsgi-file relation so the gunicorn subordinate can
127 # serve us. Other WSGI containers could be made to work, as the most239 # serve us. Other WSGI containers could be made to work, as the most
128 # gunicorn-specific thing is the --paste hack.240 # gunicorn-specific thing is the --paste hack.
129 config = hookenv.config()241 config = hookenv.config()
130 code_dir = os.path.join(config['base_dir'], 'code')
131 venv_bin = os.path.join(config['base_dir'], 'venv', 'bin')
132 # XXX We only support a single related turnip-api unit so far.242 # XXX We only support a single related turnip-api unit so far.
133 turnip_api_rid = sorted(hookenv.relation_ids('turnip-api'))[0]243 turnip_api_rid = sorted(hookenv.relation_ids('turnip-api'))[0]
134 turnip_api_unit = sorted(hookenv.related_units(turnip_api_rid))[0]244 turnip_api_unit = sorted(hookenv.related_units(turnip_api_rid))[0]
@@ -136,7 +246,7 @@
136 rid=turnip_api_rid, unit=turnip_api_unit)246 rid=turnip_api_rid, unit=turnip_api_unit)
137247
138 env = {248 env = {
139 'PATH': '%s:$PATH' % venv_bin,249 'PATH': '%s:$PATH' % os.path.join(VENV_DIR, 'bin'),
140 'TURNIP_ENDPOINT': 'http://%s:%s' % (250 'TURNIP_ENDPOINT': 'http://%s:%s' % (
141 turnip_api_data['turnip_api_host'],251 turnip_api_data['turnip_api_host'],
142 turnip_api_data['turnip_api_port']),252 turnip_api_data['turnip_api_port']),
@@ -144,7 +254,7 @@
144 for relid in hookenv.relation_ids('wsgi-file'):254 for relid in hookenv.relation_ids('wsgi-file'):
145 hookenv.relation_set(255 hookenv.relation_set(
146 relid,256 relid,
147 working_dir=code_dir,257 working_dir=CODE_DIR,
148 wsgi_wsgi_file='--paste turnipcake.ini', # XXX: Gross.258 wsgi_wsgi_file='--paste turnipcake.ini', # XXX: Gross.
149 wsgi_user=config['user'],259 wsgi_user=config['user'],
150 wsgi_group=config['group'],260 wsgi_group=config['group'],
151261
=== modified file 'hooks/services.py'
--- hooks/services.py 2015-03-27 02:43:39 +0000
+++ hooks/services.py 2015-11-02 11:16:30 +0000
@@ -12,9 +12,8 @@
12 actions.execd_preinstall('turnipcake')12 actions.execd_preinstall('turnipcake')
13 actions.install_packages('turnipcake')13 actions.install_packages('turnipcake')
14 actions.create_users('turnipcake')14 actions.create_users('turnipcake')
15 actions.unpack_source('turnipcake')15 if hookenv.hook_name() in ('install', 'upgrade-charm', 'config-changed'):
16 actions.install_python_packages('turnipcake')16 actions.deploy_code('turnipcake')
17 actions.migrate_db('turnipcake')
1817
19 config = hookenv.config()18 config = hookenv.config()
20 manager = ServiceManager([19 manager = ServiceManager([

Subscribers

People subscribed via source and target branches

to all changes: