Juju Charms Collection
logstash-forwarder package

Merge lp:~woutervb/charms/trusty/logstash-forwarder/focal into lp:~canonical-is-sa/charms/trusty/logstash-forwarder/trunk

  1. Trusty Tahr (14.04)
  2. focal
  3. Merge into trunk
Proposed by Wouter van Bommel
Status: Merged
Approved by: Benjamin Allot
Approved revision: 28
Merged at revision: 26
Proposed branch: lp:~woutervb/charms/trusty/logstash-forwarder/focal
Merge into: lp:~canonical-is-sa/charms/trusty/logstash-forwarder/trunk
Diff against target: 37291 lines (+34685/-683)
195 files modified
hooks/Config.py (+1/-1)
hooks/hooks.py (+14/-13)
hooks/install (+5/-5)
hooks/test_hooks.py (+10/-10)
hooks/upgrade-charm (+20/-0)
lib/charmhelpers/__init__.py (+99/-0)
lib/charmhelpers/cli/README.rst (+57/-0)
lib/charmhelpers/cli/__init__.py (+196/-0)
lib/charmhelpers/cli/benchmark.py (+34/-0)
lib/charmhelpers/cli/commands.py (+30/-0)
lib/charmhelpers/cli/hookenv.py (+21/-0)
lib/charmhelpers/cli/host.py (+29/-0)
lib/charmhelpers/cli/unitdata.py (+46/-0)
lib/charmhelpers/context.py (+205/-0)
lib/charmhelpers/contrib/__init__.py (+13/-0)
lib/charmhelpers/contrib/amulet/__init__.py (+13/-0)
lib/charmhelpers/contrib/amulet/deployment.py (+99/-0)
lib/charmhelpers/contrib/amulet/utils.py (+820/-0)
lib/charmhelpers/contrib/ansible/__init__.py (+306/-0)
lib/charmhelpers/contrib/benchmark/__init__.py (+124/-0)
lib/charmhelpers/contrib/charmhelpers/IMPORT (+4/-0)
lib/charmhelpers/contrib/charmhelpers/__init__.py (+203/-0)
lib/charmhelpers/contrib/charmsupport/IMPORT (+14/-0)
lib/charmhelpers/contrib/charmsupport/__init__.py (+13/-0)
lib/charmhelpers/contrib/charmsupport/nrpe.py (+330/-26)
lib/charmhelpers/contrib/charmsupport/volumes.py (+14/-0)
lib/charmhelpers/contrib/database/__init__.py (+11/-0)
lib/charmhelpers/contrib/database/mysql.py (+840/-0)
lib/charmhelpers/contrib/hahelpers/__init__.py (+13/-0)
lib/charmhelpers/contrib/hahelpers/apache.py (+90/-0)
lib/charmhelpers/contrib/hahelpers/cluster.py (+451/-0)
lib/charmhelpers/contrib/hardening/README.hardening.md (+38/-0)
lib/charmhelpers/contrib/hardening/__init__.py (+13/-0)
lib/charmhelpers/contrib/hardening/apache/__init__.py (+17/-0)
lib/charmhelpers/contrib/hardening/apache/checks/__init__.py (+29/-0)
lib/charmhelpers/contrib/hardening/apache/checks/config.py (+104/-0)
lib/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf (+32/-0)
lib/charmhelpers/contrib/hardening/apache/templates/alias.conf (+31/-0)
lib/charmhelpers/contrib/hardening/audits/__init__.py (+54/-0)
lib/charmhelpers/contrib/hardening/audits/apache.py (+105/-0)
lib/charmhelpers/contrib/hardening/audits/apt.py (+104/-0)
lib/charmhelpers/contrib/hardening/audits/file.py (+550/-0)
lib/charmhelpers/contrib/hardening/defaults/apache.yaml (+16/-0)
lib/charmhelpers/contrib/hardening/defaults/apache.yaml.schema (+12/-0)
lib/charmhelpers/contrib/hardening/defaults/mysql.yaml (+38/-0)
lib/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema (+15/-0)
lib/charmhelpers/contrib/hardening/defaults/os.yaml (+68/-0)
lib/charmhelpers/contrib/hardening/defaults/os.yaml.schema (+43/-0)
lib/charmhelpers/contrib/hardening/defaults/ssh.yaml (+49/-0)
lib/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema (+42/-0)
lib/charmhelpers/contrib/hardening/harden.py (+96/-0)
lib/charmhelpers/contrib/hardening/host/__init__.py (+17/-0)
lib/charmhelpers/contrib/hardening/host/checks/__init__.py (+48/-0)
lib/charmhelpers/contrib/hardening/host/checks/apt.py (+37/-0)
lib/charmhelpers/contrib/hardening/host/checks/limits.py (+53/-0)
lib/charmhelpers/contrib/hardening/host/checks/login.py (+65/-0)
lib/charmhelpers/contrib/hardening/host/checks/minimize_access.py (+50/-0)
lib/charmhelpers/contrib/hardening/host/checks/pam.py (+132/-0)
lib/charmhelpers/contrib/hardening/host/checks/profile.py (+49/-0)
lib/charmhelpers/contrib/hardening/host/checks/securetty.py (+37/-0)
lib/charmhelpers/contrib/hardening/host/checks/suid_sgid.py (+129/-0)
lib/charmhelpers/contrib/hardening/host/checks/sysctl.py (+209/-0)
lib/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf (+8/-0)
lib/charmhelpers/contrib/hardening/host/templates/99-hardening.sh (+5/-0)
lib/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf (+7/-0)
lib/charmhelpers/contrib/hardening/host/templates/login.defs (+349/-0)
lib/charmhelpers/contrib/hardening/host/templates/modules (+117/-0)
lib/charmhelpers/contrib/hardening/host/templates/passwdqc.conf (+11/-0)
lib/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh (+8/-0)
lib/charmhelpers/contrib/hardening/host/templates/securetty (+11/-0)
lib/charmhelpers/contrib/hardening/host/templates/tally2 (+14/-0)
lib/charmhelpers/contrib/hardening/mysql/__init__.py (+17/-0)
lib/charmhelpers/contrib/hardening/mysql/checks/__init__.py (+29/-0)
lib/charmhelpers/contrib/hardening/mysql/checks/config.py (+87/-0)
lib/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf (+12/-0)
lib/charmhelpers/contrib/hardening/ssh/__init__.py (+17/-0)
lib/charmhelpers/contrib/hardening/ssh/checks/__init__.py (+29/-0)
lib/charmhelpers/contrib/hardening/ssh/checks/config.py (+435/-0)
lib/charmhelpers/contrib/hardening/ssh/templates/ssh_config (+70/-0)
lib/charmhelpers/contrib/hardening/ssh/templates/sshd_config (+159/-0)
lib/charmhelpers/contrib/hardening/templating.py (+73/-0)
lib/charmhelpers/contrib/hardening/utils.py (+155/-0)
lib/charmhelpers/contrib/mellanox/__init__.py (+13/-0)
lib/charmhelpers/contrib/mellanox/infiniband.py (+153/-0)
lib/charmhelpers/contrib/network/__init__.py (+13/-0)
lib/charmhelpers/contrib/network/ip.py (+603/-0)
lib/charmhelpers/contrib/network/ovs/__init__.py (+693/-0)
lib/charmhelpers/contrib/network/ovs/ovn.py (+233/-0)
lib/charmhelpers/contrib/network/ovs/ovsdb.py (+246/-0)
lib/charmhelpers/contrib/network/ovs/utils.py (+26/-0)
lib/charmhelpers/contrib/network/ufw.py (+386/-0)
lib/charmhelpers/contrib/openstack/__init__.py (+13/-0)
lib/charmhelpers/contrib/openstack/alternatives.py (+44/-0)
lib/charmhelpers/contrib/openstack/amulet/__init__.py (+13/-0)
lib/charmhelpers/contrib/openstack/amulet/deployment.py (+387/-0)
lib/charmhelpers/contrib/openstack/amulet/utils.py (+1595/-0)
lib/charmhelpers/contrib/openstack/audits/__init__.py (+212/-0)
lib/charmhelpers/contrib/openstack/audits/openstack_security_guide.py (+270/-0)
lib/charmhelpers/contrib/openstack/cert_utils.py (+443/-0)
lib/charmhelpers/contrib/openstack/context.py (+3313/-0)
lib/charmhelpers/contrib/openstack/deferred_events.py (+416/-0)
lib/charmhelpers/contrib/openstack/exceptions.py (+26/-0)
lib/charmhelpers/contrib/openstack/files/__init__.py (+16/-0)
lib/charmhelpers/contrib/openstack/files/check_haproxy.sh (+34/-0)
lib/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh (+30/-0)
lib/charmhelpers/contrib/openstack/files/policy_rc_d_script.py (+196/-0)
lib/charmhelpers/contrib/openstack/ha/__init__.py (+13/-0)
lib/charmhelpers/contrib/openstack/ha/utils.py (+348/-0)
lib/charmhelpers/contrib/openstack/ip.py (+235/-0)
lib/charmhelpers/contrib/openstack/keystone.py (+178/-0)
lib/charmhelpers/contrib/openstack/neutron.py (+359/-0)
lib/charmhelpers/contrib/openstack/policy_rcd.py (+173/-0)
lib/charmhelpers/contrib/openstack/policyd.py (+801/-0)
lib/charmhelpers/contrib/openstack/ssh_migrations.py (+412/-0)
lib/charmhelpers/contrib/openstack/templates/__init__.py (+16/-0)
lib/charmhelpers/contrib/openstack/templates/ceph.conf (+28/-0)
lib/charmhelpers/contrib/openstack/templates/git.upstart (+17/-0)
lib/charmhelpers/contrib/openstack/templates/haproxy.cfg (+77/-0)
lib/charmhelpers/contrib/openstack/templates/logrotate (+9/-0)
lib/charmhelpers/contrib/openstack/templates/memcached.conf (+53/-0)
lib/charmhelpers/contrib/openstack/templates/openstack_https_frontend (+35/-0)
lib/charmhelpers/contrib/openstack/templates/section-ceph-bluestore-compression (+28/-0)
lib/charmhelpers/contrib/openstack/templates/section-keystone-authtoken (+12/-0)
lib/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy (+10/-0)
lib/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka (+22/-0)
lib/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-v3only (+9/-0)
lib/charmhelpers/contrib/openstack/templates/section-oslo-cache (+6/-0)
lib/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit (+10/-0)
lib/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit-ocata (+10/-0)
lib/charmhelpers/contrib/openstack/templates/section-oslo-middleware (+5/-0)
lib/charmhelpers/contrib/openstack/templates/section-oslo-notifications (+15/-0)
lib/charmhelpers/contrib/openstack/templates/section-placement (+20/-0)
lib/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo (+22/-0)
lib/charmhelpers/contrib/openstack/templates/section-zeromq (+14/-0)
lib/charmhelpers/contrib/openstack/templates/vendor_data.json (+1/-0)
lib/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf (+91/-0)
lib/charmhelpers/contrib/openstack/templating.py (+379/-0)
lib/charmhelpers/contrib/openstack/utils.py (+2684/-0)
lib/charmhelpers/contrib/openstack/vaultlocker.py (+179/-0)
lib/charmhelpers/contrib/peerstorage/__init__.py (+267/-0)
lib/charmhelpers/contrib/python.py (+21/-0)
lib/charmhelpers/contrib/saltstack/__init__.py (+116/-0)
lib/charmhelpers/contrib/ssl/__init__.py (+92/-0)
lib/charmhelpers/contrib/ssl/service.py (+277/-0)
lib/charmhelpers/contrib/storage/__init__.py (+13/-0)
lib/charmhelpers/contrib/storage/linux/__init__.py (+13/-0)
lib/charmhelpers/contrib/storage/linux/bcache.py (+74/-0)
lib/charmhelpers/contrib/storage/linux/ceph.py (+2381/-0)
lib/charmhelpers/contrib/storage/linux/loopback.py (+92/-0)
lib/charmhelpers/contrib/storage/linux/lvm.py (+182/-0)
lib/charmhelpers/contrib/storage/linux/utils.py (+128/-0)
lib/charmhelpers/contrib/templating/__init__.py (+13/-0)
lib/charmhelpers/contrib/templating/contexts.py (+137/-0)
lib/charmhelpers/contrib/templating/jinja.py (+51/-0)
lib/charmhelpers/contrib/templating/pyformat.py (+27/-0)
lib/charmhelpers/contrib/unison/__init__.py (+316/-0)
lib/charmhelpers/coordinator.py (+606/-0)
lib/charmhelpers/core/__init__.py (+13/-0)
lib/charmhelpers/core/decorators.py (+93/-0)
lib/charmhelpers/core/files.py (+43/-0)
lib/charmhelpers/core/fstab.py (+28/-12)
lib/charmhelpers/core/hookenv.py (+1191/-75)
lib/charmhelpers/core/host.py (+1082/-163)
lib/charmhelpers/core/host_factory/centos.py (+72/-0)
lib/charmhelpers/core/host_factory/ubuntu.py (+120/-0)
lib/charmhelpers/core/hugepage.py (+69/-0)
lib/charmhelpers/core/kernel.py (+72/-0)
lib/charmhelpers/core/kernel_factory/centos.py (+17/-0)
lib/charmhelpers/core/kernel_factory/ubuntu.py (+13/-0)
lib/charmhelpers/core/services/__init__.py (+16/-2)
lib/charmhelpers/core/services/base.py (+80/-26)
lib/charmhelpers/core/services/helpers.py (+176/-11)
lib/charmhelpers/core/strutils.py (+129/-0)
lib/charmhelpers/core/sysctl.py (+75/-0)
lib/charmhelpers/core/templating.py (+60/-18)
lib/charmhelpers/core/unitdata.py (+525/-0)
lib/charmhelpers/fetch/__init__.py (+90/-276)
lib/charmhelpers/fetch/archiveurl.py (+121/-19)
lib/charmhelpers/fetch/bzrurl.py (+52/-26)
lib/charmhelpers/fetch/centos.py (+171/-0)
lib/charmhelpers/fetch/giturl.py (+69/-0)
lib/charmhelpers/fetch/python/__init__.py (+13/-0)
lib/charmhelpers/fetch/python/debug.py (+54/-0)
lib/charmhelpers/fetch/python/packages.py (+154/-0)
lib/charmhelpers/fetch/python/rpdb.py (+56/-0)
lib/charmhelpers/fetch/python/version.py (+32/-0)
lib/charmhelpers/fetch/snap.py (+150/-0)
lib/charmhelpers/fetch/ubuntu.py (+863/-0)
lib/charmhelpers/fetch/ubuntu_apt_pkg.py (+312/-0)
lib/charmhelpers/osplatform.py (+46/-0)
lib/charmhelpers/payload/__init__.py (+15/-0)
lib/charmhelpers/payload/archive.py (+71/-0)
lib/charmhelpers/payload/execd.py (+65/-0)
metadata.yaml (+1/-0)
templates/etcdefault.j2 (+5/-0)
To merge this branch: bzr merge lp:~woutervb/charms/trusty/logstash-forwarder/focal
Reviewer Review Type Date Requested Status
Benjamin Allot Approve
William Grant (community) Approve
Review via email: mp+404554@code.launchpad.net
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve
lp:~woutervb/charms/trusty/logstash-forwarder/focal updated
26. By Wouter van Bommel

Bump charmhelpers to the current release

Make sure that charmhelpers is new enough that it works on all our LTS series since trusty

27. By Wouter van Bommel

Added support for focal

Changed the python template library to jinja2 (from cheeta), so that we can use python3 on all platforms.
Als made sure that make is installed during an upgrade of the charm.

Revision history for this message
Benjamin Allot (ballot) wrote :

Some questions/typo regarding mako/jinja and casting to a list.

Also, isn't filebeat what we want to use instead of logstash-forwarder now ?

review: Needs Fixing
Revision history for this message
Wouter van Bommel (woutervb) wrote :

> Some questions/typo regarding mako/jinja and casting to a list.
>
> Also, isn't filebeat what we want to use instead of logstash-forwarder now ?

Thanks for having a look, updated the comment and removed the list (the list was a 2to3 leftover).

Indeed we would like to go to filebeat, but the charm lacks some features used in the sca spec, so we now 'fixed' logstash-forwarder so we can at least rollout on focal for the services that support it.

lp:~woutervb/charms/trusty/logstash-forwarder/focal updated
28. By Wouter van Bommel

Update based on feedback

Als realized that the updated template was not committed

Revision history for this message
Benjamin Allot (ballot) wrote :

LGTM

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== removed file 'charm-helpers.yaml'
2=== modified file 'hooks/Config.py'
3--- hooks/Config.py 2018-04-23 22:00:12 +0000
4+++ hooks/Config.py 2021-06-24 14:42:02 +0000
5@@ -1,4 +1,4 @@
6-#!/usr/bin/python
7+#!/usr/bin/python3
8 #
9 # Copyright 2014 Canonical Ltd. All rights reserved
10 # Author: Chris Stratford <chris.stratford@canonical.com>
11
12=== modified file 'hooks/hooks.py'
13--- hooks/hooks.py 2021-05-13 02:17:53 +0000
14+++ hooks/hooks.py 2021-06-24 14:42:02 +0000
15@@ -1,4 +1,4 @@
16-#!/usr/bin/python
17+#!/usr/bin/python3
18 #
19 # Copyright 2014 Canonical Ltd. All rights reserved
20 # Author: Chris Stratford <chris.stratford@canonical.com>
21@@ -20,7 +20,7 @@
22 from charmhelpers.core.host import mkdir, service_start, service_stop, service_restart
23 from charmhelpers.core.services import RelationContext
24 from charmhelpers.fetch import apt_install, apt_update, add_source
25-from Cheetah.Template import Template
26+import jinja2
27 from Config import Config
28 #from nrpe import update_nrpe_checks
29 from charmhelpers.contrib.charmsupport import nrpe
30@@ -69,7 +69,7 @@
31 src = os.path.join(charm_dir(), "files", "init.sh")
32 dest = "/etc/init.d/logstash-forwarder"
33 shutil.copyfile(src, dest)
34- os.chmod(dest, 0755)
35+ os.chmod(dest, 0o755)
36 if get_distrib_codename() == 'bionic':
37 subprocess.call(['systemctl', 'daemon-reload'])
38
39@@ -78,33 +78,34 @@
40 tmplData = {}
41 tmplData["config_file"] = conf.configFile()
42 tmplData["spool_size"] = conf.spoolSize()
43- templateFile = os.path.join(charm_dir(), "templates", "etcdefault.tmpl")
44- t = Template(file=templateFile, searchList=tmplData)
45+ templateLoader = jinja2.FileSystemLoader(searchpath="./templates")
46+ templateEnv = jinja2.Environment(loader=templateLoader)
47+ t = templateEnv.get_template("etcdefault.j2")
48 with open("/etc/default/logstash-forwarder", "w") as f:
49- os.chmod("/etc/default/logstash-forwarder", 0644)
50+ os.chmod("/etc/default/logstash-forwarder", 0o644)
51 f.write(jujuHeader())
52- f.write(str(t))
53+ f.write(t.render(**tmplData))
54
55 def writeLogrotate():
56 src = os.path.join(charm_dir(), "files", "logstash-forwarder.logrotate")
57 dest = "/etc/logrotate.d/logstash-forwarder"
58 shutil.copyfile(src, dest)
59- os.chmod(dest, 0644)
60+ os.chmod(dest, 0o644)
61
62 def writeSSL():
63 if not os.path.exists(conf.configDir()):
64 mkdir(conf.configDir())
65 if conf.sslCert():
66 with open(conf.sslCertFile(), "w") as f:
67- os.chmod(conf.sslCertFile(), 0644)
68+ os.chmod(conf.sslCertFile(), 0o644)
69 f.write(base64.b64decode(conf.sslCert()))
70 if conf.sslKey():
71 with open(conf.sslKeyFile(), "w") as f:
72- os.chmod(conf.sslKeyFile(), 0600)
73+ os.chmod(conf.sslKeyFile(), 0o600)
74 f.write(base64.b64decode(conf.sslKey()))
75 if conf.sslCACert():
76 with open(conf.sslCACertFile(), "w") as f:
77- os.chmod(conf.sslCACertFile(), 0644)
78+ os.chmod(conf.sslCACertFile(), 0o644)
79 f.write(base64.b64decode(conf.sslCACert()))
80
81
82@@ -245,7 +246,7 @@
83 src = os.path.join(charm_dir(), "files", script)
84 dest = os.path.join(script_dir, script)
85 shutil.copyfile(src, dest)
86- os.chmod(dest, 0755)
87+ os.chmod(dest, 0o755)
88
89
90 @hooks.hook("install.real")
91@@ -256,7 +257,7 @@
92 writeLogrotate()
93
94
95-@hooks.hook("upgrade-charm")
96+@hooks.hook("upgrade-charm.real")
97 def upgrade_charm():
98 log("CHARM: Upgrading {}".format(conf.appName()))
99 writeLogrotate()
100
101=== modified file 'hooks/install'
102--- hooks/install 2017-03-13 13:57:24 +0000
103+++ hooks/install 2021-06-24 14:42:02 +0000
104@@ -1,8 +1,8 @@
105 #!/bin/bash
106-# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
107-# by default.
108-
109-declare -a DEPS=('yaml' 'cheetah')
110+
111+# Ensure that the correct packages get installed
112+
113+declare -a DEPS=('yaml' 'jinja2')
114
115 check_and_install() {
116 pkg="${1}-${2}"
117@@ -11,7 +11,7 @@
118 fi
119 }
120
121-PYTHON="python"
122+PYTHON="python3"
123
124 for dep in ${DEPS[@]}; do
125 check_and_install ${PYTHON} ${dep}
126
127=== modified file 'hooks/test_hooks.py'
128--- hooks/test_hooks.py 2017-03-21 10:10:40 +0000
129+++ hooks/test_hooks.py 2021-06-24 14:42:02 +0000
130@@ -105,11 +105,11 @@
131 conf = json.loads(host.write_file.call_args[0][1])
132 self.assertEqual("file1", conf["files"][0]["paths"][0])
133 expected_fields = {
134- u'juju_service': u'remote',
135- u'juju_unit': u'remote-0',
136- u'juju_model': u'test-model',
137- u'hostname': u'HOSTNAME',
138- u'type': u'type1',
139+ 'juju_service': 'remote',
140+ 'juju_unit': 'remote-0',
141+ 'juju_model': 'test-model',
142+ 'hostname': 'HOSTNAME',
143+ 'type': 'type1',
144 }
145 self.assertEqual(expected_fields, conf["files"][0]["fields"])
146 host.service_restart.assert_called_once_with('logstash-forwarder')
147@@ -123,11 +123,11 @@
148 conf = json.loads(host.write_file.call_args[0][1])
149 self.assertEqual("file1", conf["files"][0]["paths"][0])
150 expected_fields = {
151- u'juju_service': u'logs-remote',
152- u'juju_unit': u'logs-remote-0',
153- u'juju_model': u'test-model',
154- u'hostname': u'HOSTNAME',
155- u'type': u'type1',
156+ 'juju_service': 'logs-remote',
157+ 'juju_unit': 'logs-remote-0',
158+ 'juju_model': 'test-model',
159+ 'hostname': 'HOSTNAME',
160+ 'type': 'type1',
161 }
162 self.assertEqual(expected_fields, conf["files"][0]["fields"])
163 host.service_restart.assert_called_once_with('logstash-forwarder')
164
165=== modified symlink 'hooks/upgrade-charm' (properties changed: -x to +x)
166=== target was 'hooks.py'
167--- hooks/upgrade-charm 1970-01-01 00:00:00 +0000
168+++ hooks/upgrade-charm 2021-06-24 14:42:02 +0000
169@@ -0,0 +1,20 @@
170+#!/bin/bash
171+
172+# Wrapper to ensure that we can upgrade from cheeta to jinja2
173+
174+declare -a DEPS=('yaml' 'jinja2')
175+
176+check_and_install() {
177+ pkg="${1}-${2}"
178+ if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
179+ apt-get -y install ${pkg}
180+ fi
181+}
182+
183+PYTHON="python3"
184+
185+for dep in ${DEPS[@]}; do
186+ check_and_install ${PYTHON} ${dep}
187+done
188+
189+exec ./hooks/upgrade-charm.real
190
191=== added symlink 'hooks/upgrade-charm.real'
192=== target is 'hooks.py'
193=== modified file 'lib/charmhelpers/__init__.py'
194--- lib/charmhelpers/__init__.py 2014-09-22 12:14:13 +0000
195+++ lib/charmhelpers/__init__.py 2021-06-24 14:42:02 +0000
196@@ -0,0 +1,99 @@
197+# Copyright 2014-2015 Canonical Limited.
198+#
199+# Licensed under the Apache License, Version 2.0 (the "License");
200+# you may not use this file except in compliance with the License.
201+# You may obtain a copy of the License at
202+#
203+# http://www.apache.org/licenses/LICENSE-2.0
204+#
205+# Unless required by applicable law or agreed to in writing, software
206+# distributed under the License is distributed on an "AS IS" BASIS,
207+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
208+# See the License for the specific language governing permissions and
209+# limitations under the License.
210+
211+# Bootstrap charm-helpers, installing its dependencies if necessary using
212+# only standard libraries.
213+from __future__ import print_function
214+from __future__ import absolute_import
215+
216+import functools
217+import inspect
218+import subprocess
219+import sys
220+
221+try:
222+ import six # NOQA:F401
223+except ImportError:
224+ if sys.version_info.major == 2:
225+ subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
226+ else:
227+ subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
228+ import six # NOQA:F401
229+
230+try:
231+ import yaml # NOQA:F401
232+except ImportError:
233+ if sys.version_info.major == 2:
234+ subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
235+ else:
236+ subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
237+ import yaml # NOQA:F401
238+
239+
240+# Holds a list of mapping of mangled function names that have been deprecated
241+# using the @deprecate decorator below. This is so that the warning is only
242+# printed once for each usage of the function.
243+__deprecated_functions = {}
244+
245+
246+def deprecate(warning, date=None, log=None):
247+ """Add a deprecation warning the first time the function is used.
248+
249+ The date which is a string in semi-ISO8660 format indicates the year-month
250+ that the function is officially going to be removed.
251+
252+ usage:
253+
254+ @deprecate('use core/fetch/add_source() instead', '2017-04')
255+ def contributed_add_source_thing(...):
256+ ...
257+
258+ And it then prints to the log ONCE that the function is deprecated.
259+ The reason for passing the logging function (log) is so that hookenv.log
260+ can be used for a charm if needed.
261+
262+ :param warning: String to indicate what is to be used instead.
263+ :param date: Optional string in YYYY-MM format to indicate when the
264+ function will definitely (probably) be removed.
265+ :param log: The log function to call in order to log. If None, logs to
266+ stdout
267+ """
268+ def wrap(f):
269+
270+ @functools.wraps(f)
271+ def wrapped_f(*args, **kwargs):
272+ try:
273+ module = inspect.getmodule(f)
274+ file = inspect.getsourcefile(f)
275+ lines = inspect.getsourcelines(f)
276+ f_name = "{}-{}-{}..{}-{}".format(
277+ module.__name__, file, lines[0], lines[-1], f.__name__)
278+ except (IOError, TypeError):
279+ # assume it was local, so just use the name of the function
280+ f_name = f.__name__
281+ if f_name not in __deprecated_functions:
282+ __deprecated_functions[f_name] = True
283+ s = "DEPRECATION WARNING: Function {} is being removed".format(
284+ f.__name__)
285+ if date:
286+ s = "{} on/around {}".format(s, date)
287+ if warning:
288+ s = "{} : {}".format(s, warning)
289+ if log:
290+ log(s)
291+ else:
292+ print(s)
293+ return f(*args, **kwargs)
294+ return wrapped_f
295+ return wrap
296
297=== added directory 'lib/charmhelpers/cli'
298=== added file 'lib/charmhelpers/cli/README.rst'
299--- lib/charmhelpers/cli/README.rst 1970-01-01 00:00:00 +0000
300+++ lib/charmhelpers/cli/README.rst 2021-06-24 14:42:02 +0000
301@@ -0,0 +1,57 @@
302+==========
303+Commandant
304+==========
305+
306+-----------------------------------------------------
307+Automatic command-line interfaces to Python functions
308+-----------------------------------------------------
309+
310+One of the benefits of ``libvirt`` is the uniformity of the interface: the C API (as well as the bindings in other languages) is a set of functions that accept parameters that are nearly identical to the command-line arguments. If you run ``virsh``, you get an interactive command prompt that supports all of the same commands that your shell scripts use as ``virsh`` subcommands.
311+
312+Command execution and stdio manipulation is the greatest common factor across all development systems in the POSIX environment. By exposing your functions as commands that manipulate streams of text, you can make life easier for all the Ruby and Erlang and Go programmers in your life.
313+
314+Goals
315+=====
316+
317+* Single decorator to expose a function as a command.
318+ * now two decorators - one "automatic" and one that allows authors to manipulate the arguments for fine-grained control.(MW)
319+* Automatic analysis of function signature through ``inspect.getargspec()`` on python 2 or ``inspect.getfullargspec()`` on python 3
320+* Command argument parser built automatically with ``argparse``
321+* Interactive interpreter loop object made with ``Cmd``
322+* Options to output structured return value data via ``pprint``, ``yaml`` or ``json`` dumps.
323+
324+Other Important Features that need writing
325+------------------------------------------
326+
327+* Help and Usage documentation can be automatically generated, but it will be important to let users override this behaviour
328+* The decorator should allow specifying further parameters to the parser's add_argument() calls, to specify types or to make arguments behave as boolean flags, etc.
329+ - Filename arguments are important, as good practice is for functions to accept file objects as parameters.
330+ - choices arguments help to limit bad input before the function is called
331+* Some automatic behaviour could make for better defaults, once the user can override them.
332+ - We could automatically detect arguments that default to False or True, and automatically support --no-foo for foo=True.
333+ - We could automatically support hyphens as alternates for underscores
334+ - Arguments defaulting to sequence types could support the ``append`` action.
335+
336+
337+-----------------------------------------------------
338+Implementing subcommands
339+-----------------------------------------------------
340+
341+(WIP)
342+
343+So as to avoid dependencies on the cli module, subcommands should be defined separately from their implementations. The recommmendation would be to place definitions into separate modules near the implementations which they expose.
344+
345+Some examples::
346+
347+ from charmhelpers.cli import CommandLine
348+ from charmhelpers.payload import execd
349+ from charmhelpers.foo import bar
350+
351+ cli = CommandLine()
352+
353+ cli.subcommand(execd.execd_run)
354+
355+ @cli.subcommand_builder("bar", help="Bar baz qux")
356+ def barcmd_builder(subparser):
357+ subparser.add_argument('argument1', help="yackety")
358+ return bar
359
360=== added file 'lib/charmhelpers/cli/__init__.py'
361--- lib/charmhelpers/cli/__init__.py 1970-01-01 00:00:00 +0000
362+++ lib/charmhelpers/cli/__init__.py 2021-06-24 14:42:02 +0000
363@@ -0,0 +1,196 @@
364+# Copyright 2014-2015 Canonical Limited.
365+#
366+# Licensed under the Apache License, Version 2.0 (the "License");
367+# you may not use this file except in compliance with the License.
368+# You may obtain a copy of the License at
369+#
370+# http://www.apache.org/licenses/LICENSE-2.0
371+#
372+# Unless required by applicable law or agreed to in writing, software
373+# distributed under the License is distributed on an "AS IS" BASIS,
374+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
375+# See the License for the specific language governing permissions and
376+# limitations under the License.
377+
378+import inspect
379+import argparse
380+import sys
381+
382+import six
383+from six.moves import zip
384+
385+import charmhelpers.core.unitdata
386+
387+
388+class OutputFormatter(object):
389+ def __init__(self, outfile=sys.stdout):
390+ self.formats = (
391+ "raw",
392+ "json",
393+ "py",
394+ "yaml",
395+ "csv",
396+ "tab",
397+ )
398+ self.outfile = outfile
399+
400+ def add_arguments(self, argument_parser):
401+ formatgroup = argument_parser.add_mutually_exclusive_group()
402+ choices = self.supported_formats
403+ formatgroup.add_argument("--format", metavar='FMT',
404+ help="Select output format for returned data, "
405+ "where FMT is one of: {}".format(choices),
406+ choices=choices, default='raw')
407+ for fmt in self.formats:
408+ fmtfunc = getattr(self, fmt)
409+ formatgroup.add_argument("-{}".format(fmt[0]),
410+ "--{}".format(fmt), action='store_const',
411+ const=fmt, dest='format',
412+ help=fmtfunc.__doc__)
413+
414+ @property
415+ def supported_formats(self):
416+ return self.formats
417+
418+ def raw(self, output):
419+ """Output data as raw string (default)"""
420+ if isinstance(output, (list, tuple)):
421+ output = '\n'.join(map(str, output))
422+ self.outfile.write(str(output))
423+
424+ def py(self, output):
425+ """Output data as a nicely-formatted python data structure"""
426+ import pprint
427+ pprint.pprint(output, stream=self.outfile)
428+
429+ def json(self, output):
430+ """Output data in JSON format"""
431+ import json
432+ json.dump(output, self.outfile)
433+
434+ def yaml(self, output):
435+ """Output data in YAML format"""
436+ import yaml
437+ yaml.safe_dump(output, self.outfile)
438+
439+ def csv(self, output):
440+ """Output data as excel-compatible CSV"""
441+ import csv
442+ csvwriter = csv.writer(self.outfile)
443+ csvwriter.writerows(output)
444+
445+ def tab(self, output):
446+ """Output data in excel-compatible tab-delimited format"""
447+ import csv
448+ csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab)
449+ csvwriter.writerows(output)
450+
451+ def format_output(self, output, fmt='raw'):
452+ fmtfunc = getattr(self, fmt)
453+ fmtfunc(output)
454+
455+
456+class CommandLine(object):
457+ argument_parser = None
458+ subparsers = None
459+ formatter = None
460+ exit_code = 0
461+
462+ def __init__(self):
463+ if not self.argument_parser:
464+ self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks')
465+ if not self.formatter:
466+ self.formatter = OutputFormatter()
467+ self.formatter.add_arguments(self.argument_parser)
468+ if not self.subparsers:
469+ self.subparsers = self.argument_parser.add_subparsers(help='Commands')
470+
471+ def subcommand(self, command_name=None):
472+ """
473+ Decorate a function as a subcommand. Use its arguments as the
474+ command-line arguments"""
475+ def wrapper(decorated):
476+ cmd_name = command_name or decorated.__name__
477+ subparser = self.subparsers.add_parser(cmd_name,
478+ description=decorated.__doc__)
479+ for args, kwargs in describe_arguments(decorated):
480+ subparser.add_argument(*args, **kwargs)
481+ subparser.set_defaults(func=decorated)
482+ return decorated
483+ return wrapper
484+
485+ def test_command(self, decorated):
486+ """
487+ Subcommand is a boolean test function, so bool return values should be
488+ converted to a 0/1 exit code.
489+ """
490+ decorated._cli_test_command = True
491+ return decorated
492+
493+ def no_output(self, decorated):
494+ """
495+ Subcommand is not expected to return a value, so don't print a spurious None.
496+ """
497+ decorated._cli_no_output = True
498+ return decorated
499+
500+ def subcommand_builder(self, command_name, description=None):
501+ """
502+ Decorate a function that builds a subcommand. Builders should accept a
503+ single argument (the subparser instance) and return the function to be
504+ run as the command."""
505+ def wrapper(decorated):
506+ subparser = self.subparsers.add_parser(command_name)
507+ func = decorated(subparser)
508+ subparser.set_defaults(func=func)
509+ subparser.description = description or func.__doc__
510+ return wrapper
511+
512+ def run(self):
513+ "Run cli, processing arguments and executing subcommands."
514+ arguments = self.argument_parser.parse_args()
515+ if six.PY2:
516+ argspec = inspect.getargspec(arguments.func)
517+ else:
518+ argspec = inspect.getfullargspec(arguments.func)
519+ vargs = []
520+ for arg in argspec.args:
521+ vargs.append(getattr(arguments, arg))
522+ if argspec.varargs:
523+ vargs.extend(getattr(arguments, argspec.varargs))
524+ output = arguments.func(*vargs)
525+ if getattr(arguments.func, '_cli_test_command', False):
526+ self.exit_code = 0 if output else 1
527+ output = ''
528+ if getattr(arguments.func, '_cli_no_output', False):
529+ output = ''
530+ self.formatter.format_output(output, arguments.format)
531+ if charmhelpers.core.unitdata._KV:
532+ charmhelpers.core.unitdata._KV.flush()
533+
534+
535+cmdline = CommandLine()
536+
537+
538+def describe_arguments(func):
539+ """
540+ Analyze a function's signature and return a data structure suitable for
541+ passing in as arguments to an argparse parser's add_argument() method."""
542+
543+ if six.PY2:
544+ argspec = inspect.getargspec(func)
545+ else:
546+ argspec = inspect.getfullargspec(func)
547+ # we should probably raise an exception somewhere if func includes **kwargs
548+ if argspec.defaults:
549+ positional_args = argspec.args[:-len(argspec.defaults)]
550+ keyword_names = argspec.args[-len(argspec.defaults):]
551+ for arg, default in zip(keyword_names, argspec.defaults):
552+ yield ('--{}'.format(arg),), {'default': default}
553+ else:
554+ positional_args = argspec.args
555+
556+ for arg in positional_args:
557+ yield (arg,), {}
558+ if argspec.varargs:
559+ yield (argspec.varargs,), {'nargs': '*'}
560
561=== added file 'lib/charmhelpers/cli/benchmark.py'
562--- lib/charmhelpers/cli/benchmark.py 1970-01-01 00:00:00 +0000
563+++ lib/charmhelpers/cli/benchmark.py 2021-06-24 14:42:02 +0000
564@@ -0,0 +1,34 @@
565+# Copyright 2014-2015 Canonical Limited.
566+#
567+# Licensed under the Apache License, Version 2.0 (the "License");
568+# you may not use this file except in compliance with the License.
569+# You may obtain a copy of the License at
570+#
571+# http://www.apache.org/licenses/LICENSE-2.0
572+#
573+# Unless required by applicable law or agreed to in writing, software
574+# distributed under the License is distributed on an "AS IS" BASIS,
575+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
576+# See the License for the specific language governing permissions and
577+# limitations under the License.
578+
579+from . import cmdline
580+from charmhelpers.contrib.benchmark import Benchmark
581+
582+
583+@cmdline.subcommand(command_name='benchmark-start')
584+def start():
585+ Benchmark.start()
586+
587+
588+@cmdline.subcommand(command_name='benchmark-finish')
589+def finish():
590+ Benchmark.finish()
591+
592+
593+@cmdline.subcommand_builder('benchmark-composite', description="Set the benchmark composite score")
594+def service(subparser):
595+ subparser.add_argument("value", help="The composite score.")
596+ subparser.add_argument("units", help="The units the composite score represents, i.e., 'reads/sec'.")
597+ subparser.add_argument("direction", help="'asc' if a lower score is better, 'desc' if a higher score is better.")
598+ return Benchmark.set_composite_score
599
600=== added file 'lib/charmhelpers/cli/commands.py'
601--- lib/charmhelpers/cli/commands.py 1970-01-01 00:00:00 +0000
602+++ lib/charmhelpers/cli/commands.py 2021-06-24 14:42:02 +0000
603@@ -0,0 +1,30 @@
604+# Copyright 2014-2015 Canonical Limited.
605+#
606+# Licensed under the Apache License, Version 2.0 (the "License");
607+# you may not use this file except in compliance with the License.
608+# You may obtain a copy of the License at
609+#
610+# http://www.apache.org/licenses/LICENSE-2.0
611+#
612+# Unless required by applicable law or agreed to in writing, software
613+# distributed under the License is distributed on an "AS IS" BASIS,
614+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
615+# See the License for the specific language governing permissions and
616+# limitations under the License.
617+
618+"""
619+This module loads sub-modules into the python runtime so they can be
620+discovered via the inspect module. In order to prevent flake8 from (rightfully)
621+telling us these are unused modules, throw a ' # noqa' at the end of each import
622+so that the warning is suppressed.
623+"""
624+
625+from . import CommandLine # noqa
626+
627+"""
628+Import the sub-modules which have decorated subcommands to register with chlp.
629+"""
630+from . import host # noqa
631+from . import benchmark # noqa
632+from . import unitdata # noqa
633+from . import hookenv # noqa
634
635=== added file 'lib/charmhelpers/cli/hookenv.py'
636--- lib/charmhelpers/cli/hookenv.py 1970-01-01 00:00:00 +0000
637+++ lib/charmhelpers/cli/hookenv.py 2021-06-24 14:42:02 +0000
638@@ -0,0 +1,21 @@
639+# Copyright 2014-2015 Canonical Limited.
640+#
641+# Licensed under the Apache License, Version 2.0 (the "License");
642+# you may not use this file except in compliance with the License.
643+# You may obtain a copy of the License at
644+#
645+# http://www.apache.org/licenses/LICENSE-2.0
646+#
647+# Unless required by applicable law or agreed to in writing, software
648+# distributed under the License is distributed on an "AS IS" BASIS,
649+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
650+# See the License for the specific language governing permissions and
651+# limitations under the License.
652+
653+from . import cmdline
654+from charmhelpers.core import hookenv
655+
656+
657+cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped)
658+cmdline.subcommand('service-name')(hookenv.service_name)
659+cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped)
660
661=== added file 'lib/charmhelpers/cli/host.py'
662--- lib/charmhelpers/cli/host.py 1970-01-01 00:00:00 +0000
663+++ lib/charmhelpers/cli/host.py 2021-06-24 14:42:02 +0000
664@@ -0,0 +1,29 @@
665+# Copyright 2014-2015 Canonical Limited.
666+#
667+# Licensed under the Apache License, Version 2.0 (the "License");
668+# you may not use this file except in compliance with the License.
669+# You may obtain a copy of the License at
670+#
671+# http://www.apache.org/licenses/LICENSE-2.0
672+#
673+# Unless required by applicable law or agreed to in writing, software
674+# distributed under the License is distributed on an "AS IS" BASIS,
675+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
676+# See the License for the specific language governing permissions and
677+# limitations under the License.
678+
679+from . import cmdline
680+from charmhelpers.core import host
681+
682+
683+@cmdline.subcommand()
684+def mounts():
685+ "List mounts"
686+ return host.mounts()
687+
688+
689+@cmdline.subcommand_builder('service', description="Control system services")
690+def service(subparser):
691+ subparser.add_argument("action", help="The action to perform (start, stop, etc...)")
692+ subparser.add_argument("service_name", help="Name of the service to control")
693+ return host.service
694
695=== added file 'lib/charmhelpers/cli/unitdata.py'
696--- lib/charmhelpers/cli/unitdata.py 1970-01-01 00:00:00 +0000
697+++ lib/charmhelpers/cli/unitdata.py 2021-06-24 14:42:02 +0000
698@@ -0,0 +1,46 @@
699+# Copyright 2014-2015 Canonical Limited.
700+#
701+# Licensed under the Apache License, Version 2.0 (the "License");
702+# you may not use this file except in compliance with the License.
703+# You may obtain a copy of the License at
704+#
705+# http://www.apache.org/licenses/LICENSE-2.0
706+#
707+# Unless required by applicable law or agreed to in writing, software
708+# distributed under the License is distributed on an "AS IS" BASIS,
709+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
710+# See the License for the specific language governing permissions and
711+# limitations under the License.
712+
713+from . import cmdline
714+from charmhelpers.core import unitdata
715+
716+
717+@cmdline.subcommand_builder('unitdata', description="Store and retrieve data")
718+def unitdata_cmd(subparser):
719+ nested = subparser.add_subparsers()
720+
721+ get_cmd = nested.add_parser('get', help='Retrieve data')
722+ get_cmd.add_argument('key', help='Key to retrieve the value of')
723+ get_cmd.set_defaults(action='get', value=None)
724+
725+ getrange_cmd = nested.add_parser('getrange', help='Retrieve multiple data')
726+ getrange_cmd.add_argument('key', metavar='prefix',
727+ help='Prefix of the keys to retrieve')
728+ getrange_cmd.set_defaults(action='getrange', value=None)
729+
730+ set_cmd = nested.add_parser('set', help='Store data')
731+ set_cmd.add_argument('key', help='Key to set')
732+ set_cmd.add_argument('value', help='Value to store')
733+ set_cmd.set_defaults(action='set')
734+
735+ def _unitdata_cmd(action, key, value):
736+ if action == 'get':
737+ return unitdata.kv().get(key)
738+ elif action == 'getrange':
739+ return unitdata.kv().getrange(key)
740+ elif action == 'set':
741+ unitdata.kv().set(key, value)
742+ unitdata.kv().flush()
743+ return ''
744+ return _unitdata_cmd
745
746=== added file 'lib/charmhelpers/context.py'
747--- lib/charmhelpers/context.py 1970-01-01 00:00:00 +0000
748+++ lib/charmhelpers/context.py 2021-06-24 14:42:02 +0000
749@@ -0,0 +1,205 @@
750+# Copyright 2015 Canonical Limited.
751+#
752+# Licensed under the Apache License, Version 2.0 (the "License");
753+# you may not use this file except in compliance with the License.
754+# You may obtain a copy of the License at
755+#
756+# http://www.apache.org/licenses/LICENSE-2.0
757+#
758+# Unless required by applicable law or agreed to in writing, software
759+# distributed under the License is distributed on an "AS IS" BASIS,
760+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
761+# See the License for the specific language governing permissions and
762+# limitations under the License.
763+
764+'''
765+A Pythonic API to interact with the charm hook environment.
766+
767+:author: Stuart Bishop <stuart.bishop@canonical.com>
768+'''
769+
770+import six
771+
772+from charmhelpers.core import hookenv
773+
774+from collections import OrderedDict
775+if six.PY3:
776+ from collections import UserDict # pragma: nocover
777+else:
778+ from UserDict import IterableUserDict as UserDict # pragma: nocover
779+
780+
781+class Relations(OrderedDict):
782+ '''Mapping relation name -> relation id -> Relation.
783+
784+ >>> rels = Relations()
785+ >>> rels['sprog']['sprog:12']['client/6']['widget']
786+ 'remote widget'
787+ >>> rels['sprog']['sprog:12'].local['widget'] = 'local widget'
788+ >>> rels['sprog']['sprog:12'].local['widget']
789+ 'local widget'
790+ >>> rels.peer.local['widget']
791+ 'local widget on the peer relation'
792+ '''
793+ def __init__(self):
794+ super(Relations, self).__init__()
795+ for relname in sorted(hookenv.relation_types()):
796+ self[relname] = OrderedDict()
797+ relids = hookenv.relation_ids(relname)
798+ relids.sort(key=lambda x: int(x.split(':', 1)[-1]))
799+ for relid in relids:
800+ self[relname][relid] = Relation(relid)
801+
802+ @property
803+ def peer(self):
804+ peer_relid = hookenv.peer_relation_id()
805+ for rels in self.values():
806+ if peer_relid in rels:
807+ return rels[peer_relid]
808+
809+
810+class Relation(OrderedDict):
811+ '''Mapping of unit -> remote RelationInfo for a relation.
812+
813+ This is an OrderedDict mapping, ordered numerically by
814+ by unit number.
815+
816+ Also provides access to the local RelationInfo, and peer RelationInfo
817+ instances by the 'local' and 'peers' attributes.
818+
819+ >>> r = Relation('sprog:12')
820+ >>> r.keys()
821+ ['client/9', 'client/10'] # Ordered numerically
822+ >>> r['client/10']['widget'] # A remote RelationInfo setting
823+ 'remote widget'
824+ >>> r.local['widget'] # The local RelationInfo setting
825+ 'local widget'
826+ '''
827+ relid = None # The relation id.
828+ relname = None # The relation name (also known as relation type).
829+ service = None # The remote service name, if known.
830+ local = None # The local end's RelationInfo.
831+ peers = None # Map of peer -> RelationInfo. None if no peer relation.
832+
833+ def __init__(self, relid):
834+ remote_units = hookenv.related_units(relid)
835+ remote_units.sort(key=lambda u: int(u.split('/', 1)[-1]))
836+ super(Relation, self).__init__((unit, RelationInfo(relid, unit))
837+ for unit in remote_units)
838+
839+ self.relname = relid.split(':', 1)[0]
840+ self.relid = relid
841+ self.local = RelationInfo(relid, hookenv.local_unit())
842+
843+ for relinfo in self.values():
844+ self.service = relinfo.service
845+ break
846+
847+ # If we have peers, and they have joined both the provided peer
848+ # relation and this relation, we can peek at their data too.
849+ # This is useful for creating consensus without leadership.
850+ peer_relid = hookenv.peer_relation_id()
851+ if peer_relid and peer_relid != relid:
852+ peers = hookenv.related_units(peer_relid)
853+ if peers:
854+ peers.sort(key=lambda u: int(u.split('/', 1)[-1]))
855+ self.peers = OrderedDict((peer, RelationInfo(relid, peer))
856+ for peer in peers)
857+ else:
858+ self.peers = OrderedDict()
859+ else:
860+ self.peers = None
861+
862+ def __str__(self):
863+ return '{} ({})'.format(self.relid, self.service)
864+
865+
866+class RelationInfo(UserDict):
867+ '''The bag of data at an end of a relation.
868+
869+ Every unit participating in a relation has a single bag of
870+ data associated with that relation. This is that bag.
871+
872+ The bag of data for the local unit may be updated. Remote data
873+ is immutable and will remain static for the duration of the hook.
874+
875+ Changes made to the local units relation data only become visible
876+ to other units after the hook completes successfully. If the hook
877+ does not complete successfully, the changes are rolled back.
878+
879+ Unlike standard Python mappings, setting an item to None is the
880+ same as deleting it.
881+
882+ >>> relinfo = RelationInfo('db:12') # Default is the local unit.
883+ >>> relinfo['user'] = 'fred'
884+ >>> relinfo['user']
885+ 'fred'
886+ >>> relinfo['user'] = None
887+ >>> 'fred' in relinfo
888+ False
889+
890+ This class wraps hookenv.relation_get and hookenv.relation_set.
891+ All caching is left up to these two methods to avoid synchronization
892+ issues. Data is only loaded on demand.
893+ '''
894+ relid = None # The relation id.
895+ relname = None # The relation name (also know as the relation type).
896+ unit = None # The unit id.
897+ number = None # The unit number (integer).
898+ service = None # The service name.
899+
900+ def __init__(self, relid, unit):
901+ self.relname = relid.split(':', 1)[0]
902+ self.relid = relid
903+ self.unit = unit
904+ self.service, num = self.unit.split('/', 1)
905+ self.number = int(num)
906+
907+ def __str__(self):
908+ return '{} ({})'.format(self.relid, self.unit)
909+
910+ @property
911+ def data(self):
912+ return hookenv.relation_get(rid=self.relid, unit=self.unit)
913+
914+ def __setitem__(self, key, value):
915+ if self.unit != hookenv.local_unit():
916+ raise TypeError('Attempting to set {} on remote unit {}'
917+ ''.format(key, self.unit))
918+ if value is not None and not isinstance(value, six.string_types):
919+ # We don't do implicit casting. This would cause simple
920+ # types like integers to be read back as strings in subsequent
921+ # hooks, and mutable types would require a lot of wrapping
922+ # to ensure relation-set gets called when they are mutated.
923+ raise ValueError('Only string values allowed')
924+ hookenv.relation_set(self.relid, {key: value})
925+
926+ def __delitem__(self, key):
927+ # Deleting a key and setting it to null is the same thing in
928+ # Juju relations.
929+ self[key] = None
930+
931+
932+class Leader(UserDict):
933+ def __init__(self):
934+ pass # Don't call superclass initializer, as it will nuke self.data
935+
936+ @property
937+ def data(self):
938+ return hookenv.leader_get()
939+
940+ def __setitem__(self, key, value):
941+ if not hookenv.is_leader():
942+ raise TypeError('Not the leader. Cannot change leader settings.')
943+ if value is not None and not isinstance(value, six.string_types):
944+ # We don't do implicit casting. This would cause simple
945+ # types like integers to be read back as strings in subsequent
946+ # hooks, and mutable types would require a lot of wrapping
947+ # to ensure leader-set gets called when they are mutated.
948+ raise ValueError('Only string values allowed')
949+ hookenv.leader_set({key: value})
950+
951+ def __delitem__(self, key):
952+ # Deleting a key and setting it to null is the same thing in
953+ # Juju leadership settings.
954+ self[key] = None
955
956=== modified file 'lib/charmhelpers/contrib/__init__.py'
957--- lib/charmhelpers/contrib/__init__.py 2014-09-22 12:14:13 +0000
958+++ lib/charmhelpers/contrib/__init__.py 2021-06-24 14:42:02 +0000
959@@ -0,0 +1,13 @@
960+# Copyright 2014-2015 Canonical Limited.
961+#
962+# Licensed under the Apache License, Version 2.0 (the "License");
963+# you may not use this file except in compliance with the License.
964+# You may obtain a copy of the License at
965+#
966+# http://www.apache.org/licenses/LICENSE-2.0
967+#
968+# Unless required by applicable law or agreed to in writing, software
969+# distributed under the License is distributed on an "AS IS" BASIS,
970+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
971+# See the License for the specific language governing permissions and
972+# limitations under the License.
973
974=== added directory 'lib/charmhelpers/contrib/amulet'
975=== added file 'lib/charmhelpers/contrib/amulet/__init__.py'
976--- lib/charmhelpers/contrib/amulet/__init__.py 1970-01-01 00:00:00 +0000
977+++ lib/charmhelpers/contrib/amulet/__init__.py 2021-06-24 14:42:02 +0000
978@@ -0,0 +1,13 @@
979+# Copyright 2014-2015 Canonical Limited.
980+#
981+# Licensed under the Apache License, Version 2.0 (the "License");
982+# you may not use this file except in compliance with the License.
983+# You may obtain a copy of the License at
984+#
985+# http://www.apache.org/licenses/LICENSE-2.0
986+#
987+# Unless required by applicable law or agreed to in writing, software
988+# distributed under the License is distributed on an "AS IS" BASIS,
989+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
990+# See the License for the specific language governing permissions and
991+# limitations under the License.
992
993=== added file 'lib/charmhelpers/contrib/amulet/deployment.py'
994--- lib/charmhelpers/contrib/amulet/deployment.py 1970-01-01 00:00:00 +0000
995+++ lib/charmhelpers/contrib/amulet/deployment.py 2021-06-24 14:42:02 +0000
996@@ -0,0 +1,99 @@
997+# Copyright 2014-2015 Canonical Limited.
998+#
999+# Licensed under the Apache License, Version 2.0 (the "License");
1000+# you may not use this file except in compliance with the License.
1001+# You may obtain a copy of the License at
1002+#
1003+# http://www.apache.org/licenses/LICENSE-2.0
1004+#
1005+# Unless required by applicable law or agreed to in writing, software
1006+# distributed under the License is distributed on an "AS IS" BASIS,
1007+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1008+# See the License for the specific language governing permissions and
1009+# limitations under the License.
1010+
1011+import amulet
1012+import os
1013+import six
1014+
1015+
1016+class AmuletDeployment(object):
1017+ """Amulet deployment.
1018+
1019+ This class provides generic Amulet deployment and test runner
1020+ methods.
1021+ """
1022+
1023+ def __init__(self, series=None):
1024+ """Initialize the deployment environment."""
1025+ self.series = None
1026+
1027+ if series:
1028+ self.series = series
1029+ self.d = amulet.Deployment(series=self.series)
1030+ else:
1031+ self.d = amulet.Deployment()
1032+
1033+ def _add_services(self, this_service, other_services):
1034+ """Add services.
1035+
1036+ Add services to the deployment where this_service is the local charm
1037+ that we're testing and other_services are the other services that
1038+ are being used in the local amulet tests.
1039+ """
1040+ if this_service['name'] != os.path.basename(os.getcwd()):
1041+ s = this_service['name']
1042+ msg = "The charm's root directory name needs to be {}".format(s)
1043+ amulet.raise_status(amulet.FAIL, msg=msg)
1044+
1045+ if 'units' not in this_service:
1046+ this_service['units'] = 1
1047+
1048+ self.d.add(this_service['name'], units=this_service['units'],
1049+ constraints=this_service.get('constraints'),
1050+ storage=this_service.get('storage'))
1051+
1052+ for svc in other_services:
1053+ if 'location' in svc:
1054+ branch_location = svc['location']
1055+ elif self.series:
1056+ branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
1057+ else:
1058+ branch_location = None
1059+
1060+ if 'units' not in svc:
1061+ svc['units'] = 1
1062+
1063+ self.d.add(svc['name'], charm=branch_location, units=svc['units'],
1064+ constraints=svc.get('constraints'),
1065+ storage=svc.get('storage'))
1066+
1067+ def _add_relations(self, relations):
1068+ """Add all of the relations for the services."""
1069+ for k, v in six.iteritems(relations):
1070+ self.d.relate(k, v)
1071+
1072+ def _configure_services(self, configs):
1073+ """Configure all of the services."""
1074+ for service, config in six.iteritems(configs):
1075+ self.d.configure(service, config)
1076+
1077+ def _deploy(self):
1078+ """Deploy environment and wait for all hooks to finish executing."""
1079+ timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 900))
1080+ try:
1081+ self.d.setup(timeout=timeout)
1082+ self.d.sentry.wait(timeout=timeout)
1083+ except amulet.helpers.TimeoutError:
1084+ amulet.raise_status(
1085+ amulet.FAIL,
1086+ msg="Deployment timed out ({}s)".format(timeout)
1087+ )
1088+ except Exception:
1089+ raise
1090+
1091+ def run_tests(self):
1092+ """Run all of the methods that are prefixed with 'test_'."""
1093+ for test in dir(self):
1094+ if test.startswith('test_'):
1095+ getattr(self, test)()
1096
1097=== added file 'lib/charmhelpers/contrib/amulet/utils.py'
1098--- lib/charmhelpers/contrib/amulet/utils.py 1970-01-01 00:00:00 +0000
1099+++ lib/charmhelpers/contrib/amulet/utils.py 2021-06-24 14:42:02 +0000
1100@@ -0,0 +1,820 @@
1101+# Copyright 2014-2015 Canonical Limited.
1102+#
1103+# Licensed under the Apache License, Version 2.0 (the "License");
1104+# you may not use this file except in compliance with the License.
1105+# You may obtain a copy of the License at
1106+#
1107+# http://www.apache.org/licenses/LICENSE-2.0
1108+#
1109+# Unless required by applicable law or agreed to in writing, software
1110+# distributed under the License is distributed on an "AS IS" BASIS,
1111+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1112+# See the License for the specific language governing permissions and
1113+# limitations under the License.
1114+
1115+import io
1116+import json
1117+import logging
1118+import os
1119+import re
1120+import socket
1121+import subprocess
1122+import sys
1123+import time
1124+import uuid
1125+
1126+import amulet
1127+import distro_info
1128+import six
1129+from six.moves import configparser
1130+if six.PY3:
1131+ from urllib import parse as urlparse
1132+else:
1133+ import urlparse
1134+
1135+
1136+class AmuletUtils(object):
1137+ """Amulet utilities.
1138+
1139+ This class provides common utility functions that are used by Amulet
1140+ tests.
1141+ """
1142+
1143+ def __init__(self, log_level=logging.ERROR):
1144+ self.log = self.get_logger(level=log_level)
1145+ self.ubuntu_releases = self.get_ubuntu_releases()
1146+
1147+ def get_logger(self, name="amulet-logger", level=logging.DEBUG):
1148+ """Get a logger object that will log to stdout."""
1149+ log = logging
1150+ logger = log.getLogger(name)
1151+ fmt = log.Formatter("%(asctime)s %(funcName)s "
1152+ "%(levelname)s: %(message)s")
1153+
1154+ handler = log.StreamHandler(stream=sys.stdout)
1155+ handler.setLevel(level)
1156+ handler.setFormatter(fmt)
1157+
1158+ logger.addHandler(handler)
1159+ logger.setLevel(level)
1160+
1161+ return logger
1162+
1163+ def valid_ip(self, ip):
1164+ if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
1165+ return True
1166+ else:
1167+ return False
1168+
1169+ def valid_url(self, url):
1170+ p = re.compile(
1171+ r'^(?:http|ftp)s?://'
1172+ r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa
1173+ r'localhost|'
1174+ r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
1175+ r'(?::\d+)?'
1176+ r'(?:/?|[/?]\S+)$',
1177+ re.IGNORECASE)
1178+ if p.match(url):
1179+ return True
1180+ else:
1181+ return False
1182+
1183+ def get_ubuntu_release_from_sentry(self, sentry_unit):
1184+ """Get Ubuntu release codename from sentry unit.
1185+
1186+ :param sentry_unit: amulet sentry/service unit pointer
1187+ :returns: list of strings - release codename, failure message
1188+ """
1189+ msg = None
1190+ cmd = 'lsb_release -cs'
1191+ release, code = sentry_unit.ssh(cmd)
1192+ if code == 0:
1193+ self.log.debug('{} lsb_release: {}'.format(
1194+ sentry_unit.info['unit_name'], release))
1195+ else:
1196+ msg = ('{} `{}` returned {} '
1197+ '{}'.format(sentry_unit.info['unit_name'],
1198+ cmd, release, code))
1199+ if release not in self.ubuntu_releases:
1200+ msg = ("Release ({}) not found in Ubuntu releases "
1201+ "({})".format(release, self.ubuntu_releases))
1202+ return release, msg
1203+
1204+ def validate_services(self, commands):
1205+ """Validate that lists of commands succeed on service units. Can be
1206+ used to verify system services are running on the corresponding
1207+ service units.
1208+
1209+ :param commands: dict with sentry keys and arbitrary command list vals
1210+ :returns: None if successful, Failure string message otherwise
1211+ """
1212+ self.log.debug('Checking status of system services...')
1213+
1214+ # /!\ DEPRECATION WARNING (beisner):
1215+ # New and existing tests should be rewritten to use
1216+ # validate_services_by_name() as it is aware of init systems.
1217+ self.log.warn('DEPRECATION WARNING: use '
1218+ 'validate_services_by_name instead of validate_services '
1219+ 'due to init system differences.')
1220+
1221+ for k, v in six.iteritems(commands):
1222+ for cmd in v:
1223+ output, code = k.run(cmd)
1224+ self.log.debug('{} `{}` returned '
1225+ '{}'.format(k.info['unit_name'],
1226+ cmd, code))
1227+ if code != 0:
1228+ return "command `{}` returned {}".format(cmd, str(code))
1229+ return None
1230+
1231+ def validate_services_by_name(self, sentry_services):
1232+ """Validate system service status by service name, automatically
1233+ detecting init system based on Ubuntu release codename.
1234+
1235+ :param sentry_services: dict with sentry keys and svc list values
1236+ :returns: None if successful, Failure string message otherwise
1237+ """
1238+ self.log.debug('Checking status of system services...')
1239+
1240+ # Point at which systemd became a thing
1241+ systemd_switch = self.ubuntu_releases.index('vivid')
1242+
1243+ for sentry_unit, services_list in six.iteritems(sentry_services):
1244+ # Get lsb_release codename from unit
1245+ release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
1246+ if ret:
1247+ return ret
1248+
1249+ for service_name in services_list:
1250+ if (self.ubuntu_releases.index(release) >= systemd_switch or
1251+ service_name in ['rabbitmq-server', 'apache2',
1252+ 'memcached']):
1253+ # init is systemd (or regular sysv)
1254+ cmd = 'sudo service {} status'.format(service_name)
1255+ output, code = sentry_unit.run(cmd)
1256+ service_running = code == 0
1257+ elif self.ubuntu_releases.index(release) < systemd_switch:
1258+ # init is upstart
1259+ cmd = 'sudo status {}'.format(service_name)
1260+ output, code = sentry_unit.run(cmd)
1261+ service_running = code == 0 and "start/running" in output
1262+
1263+ self.log.debug('{} `{}` returned '
1264+ '{}'.format(sentry_unit.info['unit_name'],
1265+ cmd, code))
1266+ if not service_running:
1267+ return u"command `{}` returned {} {}".format(
1268+ cmd, output, str(code))
1269+ return None
1270+
1271+ def _get_config(self, unit, filename):
1272+ """Get a ConfigParser object for parsing a unit's config file."""
1273+ file_contents = unit.file_contents(filename)
1274+
1275+ # NOTE(beisner): by default, ConfigParser does not handle options
1276+ # with no value, such as the flags used in the mysql my.cnf file.
1277+ # https://bugs.python.org/issue7005
1278+ config = configparser.ConfigParser(allow_no_value=True)
1279+ config.readfp(io.StringIO(file_contents))
1280+ return config
1281+
1282+ def validate_config_data(self, sentry_unit, config_file, section,
1283+ expected):
1284+ """Validate config file data.
1285+
1286+ Verify that the specified section of the config file contains
1287+ the expected option key:value pairs.
1288+
1289+ Compare expected dictionary data vs actual dictionary data.
1290+ The values in the 'expected' dictionary can be strings, bools, ints,
1291+ longs, or can be a function that evaluates a variable and returns a
1292+ bool.
1293+ """
1294+ self.log.debug('Validating config file data ({} in {} on {})'
1295+ '...'.format(section, config_file,
1296+ sentry_unit.info['unit_name']))
1297+ config = self._get_config(sentry_unit, config_file)
1298+
1299+ if section != 'DEFAULT' and not config.has_section(section):
1300+ return "section [{}] does not exist".format(section)
1301+
1302+ for k in expected.keys():
1303+ if not config.has_option(section, k):
1304+ return "section [{}] is missing option {}".format(section, k)
1305+
1306+ actual = config.get(section, k)
1307+ v = expected[k]
1308+ if (isinstance(v, six.string_types) or
1309+ isinstance(v, bool) or
1310+ isinstance(v, six.integer_types)):
1311+ # handle explicit values
1312+ if actual != v:
1313+ return "section [{}] {}:{} != expected {}:{}".format(
1314+ section, k, actual, k, expected[k])
1315+ # handle function pointers, such as not_null or valid_ip
1316+ elif not v(actual):
1317+ return "section [{}] {}:{} != expected {}:{}".format(
1318+ section, k, actual, k, expected[k])
1319+ return None
1320+
1321+ def _validate_dict_data(self, expected, actual):
1322+ """Validate dictionary data.
1323+
1324+ Compare expected dictionary data vs actual dictionary data.
1325+ The values in the 'expected' dictionary can be strings, bools, ints,
1326+ longs, or can be a function that evaluates a variable and returns a
1327+ bool.
1328+ """
1329+ self.log.debug('actual: {}'.format(repr(actual)))
1330+ self.log.debug('expected: {}'.format(repr(expected)))
1331+
1332+ for k, v in six.iteritems(expected):
1333+ if k in actual:
1334+ if (isinstance(v, six.string_types) or
1335+ isinstance(v, bool) or
1336+ isinstance(v, six.integer_types)):
1337+ # handle explicit values
1338+ if v != actual[k]:
1339+ return "{}:{}".format(k, actual[k])
1340+ # handle function pointers, such as not_null or valid_ip
1341+ elif not v(actual[k]):
1342+ return "{}:{}".format(k, actual[k])
1343+ else:
1344+ return "key '{}' does not exist".format(k)
1345+ return None
1346+
1347+ def validate_relation_data(self, sentry_unit, relation, expected):
1348+ """Validate actual relation data based on expected relation data."""
1349+ actual = sentry_unit.relation(relation[0], relation[1])
1350+ return self._validate_dict_data(expected, actual)
1351+
1352+ def _validate_list_data(self, expected, actual):
1353+ """Compare expected list vs actual list data."""
1354+ for e in expected:
1355+ if e not in actual:
1356+ return "expected item {} not found in actual list".format(e)
1357+ return None
1358+
1359+ def not_null(self, string):
1360+ if string is not None:
1361+ return True
1362+ else:
1363+ return False
1364+
1365+ def _get_file_mtime(self, sentry_unit, filename):
1366+ """Get last modification time of file."""
1367+ return sentry_unit.file_stat(filename)['mtime']
1368+
1369+ def _get_dir_mtime(self, sentry_unit, directory):
1370+ """Get last modification time of directory."""
1371+ return sentry_unit.directory_stat(directory)['mtime']
1372+
1373+ def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
1374+ """Get start time of a process based on the last modification time
1375+ of the /proc/pid directory.
1376+
1377+ :sentry_unit: The sentry unit to check for the service on
1378+ :service: service name to look for in process table
1379+ :pgrep_full: [Deprecated] Use full command line search mode with pgrep
1380+ :returns: epoch time of service process start
1381+ :param commands: list of bash commands
1382+ :param sentry_units: list of sentry unit pointers
1383+ :returns: None if successful; Failure message otherwise
1384+ """
1385+ pid_list = self.get_process_id_list(
1386+ sentry_unit, service, pgrep_full=pgrep_full)
1387+ pid = pid_list[0]
1388+ proc_dir = '/proc/{}'.format(pid)
1389+ self.log.debug('Pid for {} on {}: {}'.format(
1390+ service, sentry_unit.info['unit_name'], pid))
1391+
1392+ return self._get_dir_mtime(sentry_unit, proc_dir)
1393+
1394+ def service_restarted(self, sentry_unit, service, filename,
1395+ pgrep_full=None, sleep_time=20):
1396+ """Check if service was restarted.
1397+
1398+ Compare a service's start time vs a file's last modification time
1399+ (such as a config file for that service) to determine if the service
1400+ has been restarted.
1401+ """
1402+ # /!\ DEPRECATION WARNING (beisner):
1403+ # This method is prone to races in that no before-time is known.
1404+ # Use validate_service_config_changed instead.
1405+
1406+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
1407+ # used instead of pgrep. pgrep_full is still passed through to ensure
1408+ # deprecation WARNS. lp1474030
1409+ self.log.warn('DEPRECATION WARNING: use '
1410+ 'validate_service_config_changed instead of '
1411+ 'service_restarted due to known races.')
1412+
1413+ time.sleep(sleep_time)
1414+ if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
1415+ self._get_file_mtime(sentry_unit, filename)):
1416+ return True
1417+ else:
1418+ return False
1419+
1420+ def service_restarted_since(self, sentry_unit, mtime, service,
1421+ pgrep_full=None, sleep_time=20,
1422+ retry_count=30, retry_sleep_time=10):
1423+ """Check if service was been started after a given time.
1424+
1425+ Args:
1426+ sentry_unit (sentry): The sentry unit to check for the service on
1427+ mtime (float): The epoch time to check against
1428+ service (string): service name to look for in process table
1429+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
1430+ sleep_time (int): Initial sleep time (s) before looking for file
1431+ retry_sleep_time (int): Time (s) to sleep between retries
1432+ retry_count (int): If file is not found, how many times to retry
1433+
1434+ Returns:
1435+ bool: True if service found and its start time it newer than mtime,
1436+ False if service is older than mtime or if service was
1437+ not found.
1438+ """
1439+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
1440+ # used instead of pgrep. pgrep_full is still passed through to ensure
1441+ # deprecation WARNS. lp1474030
1442+
1443+ unit_name = sentry_unit.info['unit_name']
1444+ self.log.debug('Checking that %s service restarted since %s on '
1445+ '%s' % (service, mtime, unit_name))
1446+ time.sleep(sleep_time)
1447+ proc_start_time = None
1448+ tries = 0
1449+ while tries <= retry_count and not proc_start_time:
1450+ try:
1451+ proc_start_time = self._get_proc_start_time(sentry_unit,
1452+ service,
1453+ pgrep_full)
1454+ self.log.debug('Attempt {} to get {} proc start time on {} '
1455+ 'OK'.format(tries, service, unit_name))
1456+ except IOError as e:
1457+ # NOTE(beisner) - race avoidance, proc may not exist yet.
1458+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
1459+ self.log.debug('Attempt {} to get {} proc start time on {} '
1460+ 'failed\n{}'.format(tries, service,
1461+ unit_name, e))
1462+ time.sleep(retry_sleep_time)
1463+ tries += 1
1464+
1465+ if not proc_start_time:
1466+ self.log.warn('No proc start time found, assuming service did '
1467+ 'not start')
1468+ return False
1469+ if proc_start_time >= mtime:
1470+ self.log.debug('Proc start time is newer than provided mtime'
1471+ '(%s >= %s) on %s (OK)' % (proc_start_time,
1472+ mtime, unit_name))
1473+ return True
1474+ else:
1475+ self.log.warn('Proc start time (%s) is older than provided mtime '
1476+ '(%s) on %s, service did not '
1477+ 'restart' % (proc_start_time, mtime, unit_name))
1478+ return False
1479+
1480+ def config_updated_since(self, sentry_unit, filename, mtime,
1481+ sleep_time=20, retry_count=30,
1482+ retry_sleep_time=10):
1483+ """Check if file was modified after a given time.
1484+
1485+ Args:
1486+ sentry_unit (sentry): The sentry unit to check the file mtime on
1487+ filename (string): The file to check mtime of
1488+ mtime (float): The epoch time to check against
1489+ sleep_time (int): Initial sleep time (s) before looking for file
1490+ retry_sleep_time (int): Time (s) to sleep between retries
1491+ retry_count (int): If file is not found, how many times to retry
1492+
1493+ Returns:
1494+ bool: True if file was modified more recently than mtime, False if
1495+ file was modified before mtime, or if file not found.
1496+ """
1497+ unit_name = sentry_unit.info['unit_name']
1498+ self.log.debug('Checking that %s updated since %s on '
1499+ '%s' % (filename, mtime, unit_name))
1500+ time.sleep(sleep_time)
1501+ file_mtime = None
1502+ tries = 0
1503+ while tries <= retry_count and not file_mtime:
1504+ try:
1505+ file_mtime = self._get_file_mtime(sentry_unit, filename)
1506+ self.log.debug('Attempt {} to get {} file mtime on {} '
1507+ 'OK'.format(tries, filename, unit_name))
1508+ except IOError as e:
1509+ # NOTE(beisner) - race avoidance, file may not exist yet.
1510+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
1511+ self.log.debug('Attempt {} to get {} file mtime on {} '
1512+ 'failed\n{}'.format(tries, filename,
1513+ unit_name, e))
1514+ time.sleep(retry_sleep_time)
1515+ tries += 1
1516+
1517+ if not file_mtime:
1518+ self.log.warn('Could not determine file mtime, assuming '
1519+ 'file does not exist')
1520+ return False
1521+
1522+ if file_mtime >= mtime:
1523+ self.log.debug('File mtime is newer than provided mtime '
1524+ '(%s >= %s) on %s (OK)' % (file_mtime,
1525+ mtime, unit_name))
1526+ return True
1527+ else:
1528+ self.log.warn('File mtime is older than provided mtime'
1529+ '(%s < on %s) on %s' % (file_mtime,
1530+ mtime, unit_name))
1531+ return False
1532+
1533+ def validate_service_config_changed(self, sentry_unit, mtime, service,
1534+ filename, pgrep_full=None,
1535+ sleep_time=20, retry_count=30,
1536+ retry_sleep_time=10):
1537+ """Check service and file were updated after mtime
1538+
1539+ Args:
1540+ sentry_unit (sentry): The sentry unit to check for the service on
1541+ mtime (float): The epoch time to check against
1542+ service (string): service name to look for in process table
1543+ filename (string): The file to check mtime of
1544+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
1545+ sleep_time (int): Initial sleep in seconds to pass to test helpers
1546+ retry_count (int): If service is not found, how many times to retry
1547+ retry_sleep_time (int): Time in seconds to wait between retries
1548+
1549+ Typical Usage:
1550+ u = OpenStackAmuletUtils(ERROR)
1551+ ...
1552+ mtime = u.get_sentry_time(self.cinder_sentry)
1553+ self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
1554+ if not u.validate_service_config_changed(self.cinder_sentry,
1555+ mtime,
1556+ 'cinder-api',
1557+ '/etc/cinder/cinder.conf')
1558+ amulet.raise_status(amulet.FAIL, msg='update failed')
1559+ Returns:
1560+ bool: True if both service and file where updated/restarted after
1561+ mtime, False if service is older than mtime or if service was
1562+ not found or if filename was modified before mtime.
1563+ """
1564+
1565+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
1566+ # used instead of pgrep. pgrep_full is still passed through to ensure
1567+ # deprecation WARNS. lp1474030
1568+
1569+ service_restart = self.service_restarted_since(
1570+ sentry_unit, mtime,
1571+ service,
1572+ pgrep_full=pgrep_full,
1573+ sleep_time=sleep_time,
1574+ retry_count=retry_count,
1575+ retry_sleep_time=retry_sleep_time)
1576+
1577+ config_update = self.config_updated_since(
1578+ sentry_unit,
1579+ filename,
1580+ mtime,
1581+ sleep_time=sleep_time,
1582+ retry_count=retry_count,
1583+ retry_sleep_time=retry_sleep_time)
1584+
1585+ return service_restart and config_update
1586+
1587+ def get_sentry_time(self, sentry_unit):
1588+ """Return current epoch time on a sentry"""
1589+ cmd = "date +'%s'"
1590+ return float(sentry_unit.run(cmd)[0])
1591+
1592+ def relation_error(self, name, data):
1593+ return 'unexpected relation data in {} - {}'.format(name, data)
1594+
1595+ def endpoint_error(self, name, data):
1596+ return 'unexpected endpoint data in {} - {}'.format(name, data)
1597+
1598+ def get_ubuntu_releases(self):
1599+ """Return a list of all Ubuntu releases in order of release."""
1600+ _d = distro_info.UbuntuDistroInfo()
1601+ _release_list = _d.all
1602+ return _release_list
1603+
1604+ def file_to_url(self, file_rel_path):
1605+ """Convert a relative file path to a file URL."""
1606+ _abs_path = os.path.abspath(file_rel_path)
1607+ return urlparse.urlparse(_abs_path, scheme='file').geturl()
1608+
1609+ def check_commands_on_units(self, commands, sentry_units):
1610+ """Check that all commands in a list exit zero on all
1611+ sentry units in a list.
1612+
1613+ :param commands: list of bash commands
1614+ :param sentry_units: list of sentry unit pointers
1615+ :returns: None if successful; Failure message otherwise
1616+ """
1617+ self.log.debug('Checking exit codes for {} commands on {} '
1618+ 'sentry units...'.format(len(commands),
1619+ len(sentry_units)))
1620+ for sentry_unit in sentry_units:
1621+ for cmd in commands:
1622+ output, code = sentry_unit.run(cmd)
1623+ if code == 0:
1624+ self.log.debug('{} `{}` returned {} '
1625+ '(OK)'.format(sentry_unit.info['unit_name'],
1626+ cmd, code))
1627+ else:
1628+ return ('{} `{}` returned {} '
1629+ '{}'.format(sentry_unit.info['unit_name'],
1630+ cmd, code, output))
1631+ return None
1632+
1633+ def get_process_id_list(self, sentry_unit, process_name,
1634+ expect_success=True, pgrep_full=False):
1635+ """Get a list of process ID(s) from a single sentry juju unit
1636+ for a single process name.
1637+
1638+ :param sentry_unit: Amulet sentry instance (juju unit)
1639+ :param process_name: Process name
1640+ :param expect_success: If False, expect the PID to be missing,
1641+ raise if it is present.
1642+ :returns: List of process IDs
1643+ """
1644+ if pgrep_full:
1645+ cmd = 'pgrep -f "{}"'.format(process_name)
1646+ else:
1647+ cmd = 'pidof -x "{}"'.format(process_name)
1648+ if not expect_success:
1649+ cmd += " || exit 0 && exit 1"
1650+ output, code = sentry_unit.run(cmd)
1651+ if code != 0:
1652+ msg = ('{} `{}` returned {} '
1653+ '{}'.format(sentry_unit.info['unit_name'],
1654+ cmd, code, output))
1655+ amulet.raise_status(amulet.FAIL, msg=msg)
1656+ return str(output).split()
1657+
1658+ def get_unit_process_ids(
1659+ self, unit_processes, expect_success=True, pgrep_full=False):
1660+ """Construct a dict containing unit sentries, process names, and
1661+ process IDs.
1662+
1663+ :param unit_processes: A dictionary of Amulet sentry instance
1664+ to list of process names.
1665+ :param expect_success: if False expect the processes to not be
1666+ running, raise if they are.
1667+ :returns: Dictionary of Amulet sentry instance to dictionary
1668+ of process names to PIDs.
1669+ """
1670+ pid_dict = {}
1671+ for sentry_unit, process_list in six.iteritems(unit_processes):
1672+ pid_dict[sentry_unit] = {}
1673+ for process in process_list:
1674+ pids = self.get_process_id_list(
1675+ sentry_unit, process, expect_success=expect_success,
1676+ pgrep_full=pgrep_full)
1677+ pid_dict[sentry_unit].update({process: pids})
1678+ return pid_dict
1679+
1680+ def validate_unit_process_ids(self, expected, actual):
1681+ """Validate process id quantities for services on units."""
1682+ self.log.debug('Checking units for running processes...')
1683+ self.log.debug('Expected PIDs: {}'.format(expected))
1684+ self.log.debug('Actual PIDs: {}'.format(actual))
1685+
1686+ if len(actual) != len(expected):
1687+ return ('Unit count mismatch. expected, actual: {}, '
1688+ '{} '.format(len(expected), len(actual)))
1689+
1690+ for (e_sentry, e_proc_names) in six.iteritems(expected):
1691+ e_sentry_name = e_sentry.info['unit_name']
1692+ if e_sentry in actual.keys():
1693+ a_proc_names = actual[e_sentry]
1694+ else:
1695+ return ('Expected sentry ({}) not found in actual dict data.'
1696+ '{}'.format(e_sentry_name, e_sentry))
1697+
1698+ if len(e_proc_names.keys()) != len(a_proc_names.keys()):
1699+ return ('Process name count mismatch. expected, actual: {}, '
1700+ '{}'.format(len(expected), len(actual)))
1701+
1702+ for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
1703+ zip(e_proc_names.items(), a_proc_names.items()):
1704+ if e_proc_name != a_proc_name:
1705+ return ('Process name mismatch. expected, actual: {}, '
1706+ '{}'.format(e_proc_name, a_proc_name))
1707+
1708+ a_pids_length = len(a_pids)
1709+ fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
1710+ '{}, {} ({})'.format(e_sentry_name, e_proc_name,
1711+ e_pids, a_pids_length,
1712+ a_pids))
1713+
1714+ # If expected is a list, ensure at least one PID quantity match
1715+ if isinstance(e_pids, list) and \
1716+ a_pids_length not in e_pids:
1717+ return fail_msg
1718+ # If expected is not bool and not list,
1719+ # ensure PID quantities match
1720+ elif not isinstance(e_pids, bool) and \
1721+ not isinstance(e_pids, list) and \
1722+ a_pids_length != e_pids:
1723+ return fail_msg
1724+ # If expected is bool True, ensure 1 or more PIDs exist
1725+ elif isinstance(e_pids, bool) and \
1726+ e_pids is True and a_pids_length < 1:
1727+ return fail_msg
1728+ # If expected is bool False, ensure 0 PIDs exist
1729+ elif isinstance(e_pids, bool) and \
1730+ e_pids is False and a_pids_length != 0:
1731+ return fail_msg
1732+ else:
1733+ self.log.debug('PID check OK: {} {} {}: '
1734+ '{}'.format(e_sentry_name, e_proc_name,
1735+ e_pids, a_pids))
1736+ return None
1737+
1738+ def validate_list_of_identical_dicts(self, list_of_dicts):
1739+ """Check that all dicts within a list are identical."""
1740+ hashes = []
1741+ for _dict in list_of_dicts:
1742+ hashes.append(hash(frozenset(_dict.items())))
1743+
1744+ self.log.debug('Hashes: {}'.format(hashes))
1745+ if len(set(hashes)) == 1:
1746+ self.log.debug('Dicts within list are identical')
1747+ else:
1748+ return 'Dicts within list are not identical'
1749+
1750+ return None
1751+
1752+ def validate_sectionless_conf(self, file_contents, expected):
1753+ """A crude conf parser. Useful to inspect configuration files which
1754+ do not have section headers (as would be necessary in order to use
1755+ the configparser). Such as openstack-dashboard or rabbitmq confs."""
1756+ for line in file_contents.split('\n'):
1757+ if '=' in line:
1758+ args = line.split('=')
1759+ if len(args) <= 1:
1760+ continue
1761+ key = args[0].strip()
1762+ value = args[1].strip()
1763+ if key in expected.keys():
1764+ if expected[key] != value:
1765+ msg = ('Config mismatch. Expected, actual: {}, '
1766+ '{}'.format(expected[key], value))
1767+ amulet.raise_status(amulet.FAIL, msg=msg)
1768+
1769+ def get_unit_hostnames(self, units):
1770+ """Return a dict of juju unit names to hostnames."""
1771+ host_names = {}
1772+ for unit in units:
1773+ host_names[unit.info['unit_name']] = \
1774+ str(unit.file_contents('/etc/hostname').strip())
1775+ self.log.debug('Unit host names: {}'.format(host_names))
1776+ return host_names
1777+
1778+ def run_cmd_unit(self, sentry_unit, cmd):
1779+ """Run a command on a unit, return the output and exit code."""
1780+ output, code = sentry_unit.run(cmd)
1781+ if code == 0:
1782+ self.log.debug('{} `{}` command returned {} '
1783+ '(OK)'.format(sentry_unit.info['unit_name'],
1784+ cmd, code))
1785+ else:
1786+ msg = ('{} `{}` command returned {} '
1787+ '{}'.format(sentry_unit.info['unit_name'],
1788+ cmd, code, output))
1789+ amulet.raise_status(amulet.FAIL, msg=msg)
1790+ return str(output), code
1791+
1792+ def file_exists_on_unit(self, sentry_unit, file_name):
1793+ """Check if a file exists on a unit."""
1794+ try:
1795+ sentry_unit.file_stat(file_name)
1796+ return True
1797+ except IOError:
1798+ return False
1799+ except Exception as e:
1800+ msg = 'Error checking file {}: {}'.format(file_name, e)
1801+ amulet.raise_status(amulet.FAIL, msg=msg)
1802+
1803+ def file_contents_safe(self, sentry_unit, file_name,
1804+ max_wait=60, fatal=False):
1805+ """Get file contents from a sentry unit. Wrap amulet file_contents
1806+ with retry logic to address races where a file checks as existing,
1807+ but no longer exists by the time file_contents is called.
1808+ Return None if file not found. Optionally raise if fatal is True."""
1809+ unit_name = sentry_unit.info['unit_name']
1810+ file_contents = False
1811+ tries = 0
1812+ while not file_contents and tries < (max_wait / 4):
1813+ try:
1814+ file_contents = sentry_unit.file_contents(file_name)
1815+ except IOError:
1816+ self.log.debug('Attempt {} to open file {} from {} '
1817+ 'failed'.format(tries, file_name,
1818+ unit_name))
1819+ time.sleep(4)
1820+ tries += 1
1821+
1822+ if file_contents:
1823+ return file_contents
1824+ elif not fatal:
1825+ return None
1826+ elif fatal:
1827+ msg = 'Failed to get file contents from unit.'
1828+ amulet.raise_status(amulet.FAIL, msg)
1829+
1830+ def port_knock_tcp(self, host="localhost", port=22, timeout=15):
1831+ """Open a TCP socket to check for a listening sevice on a host.
1832+
1833+ :param host: host name or IP address, default to localhost
1834+ :param port: TCP port number, default to 22
1835+ :param timeout: Connect timeout, default to 15 seconds
1836+ :returns: True if successful, False if connect failed
1837+ """
1838+
1839+ # Resolve host name if possible
1840+ try:
1841+ connect_host = socket.gethostbyname(host)
1842+ host_human = "{} ({})".format(connect_host, host)
1843+ except socket.error as e:
1844+ self.log.warn('Unable to resolve address: '
1845+ '{} ({}) Trying anyway!'.format(host, e))
1846+ connect_host = host
1847+ host_human = connect_host
1848+
1849+ # Attempt socket connection
1850+ try:
1851+ knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1852+ knock.settimeout(timeout)
1853+ knock.connect((connect_host, port))
1854+ knock.close()
1855+ self.log.debug('Socket connect OK for host '
1856+ '{} on port {}.'.format(host_human, port))
1857+ return True
1858+ except socket.error as e:
1859+ self.log.debug('Socket connect FAIL for'
1860+ ' {} port {} ({})'.format(host_human, port, e))
1861+ return False
1862+
1863+ def port_knock_units(self, sentry_units, port=22,
1864+ timeout=15, expect_success=True):
1865+ """Open a TCP socket to check for a listening sevice on each
1866+ listed juju unit.
1867+
1868+ :param sentry_units: list of sentry unit pointers
1869+ :param port: TCP port number, default to 22
1870+ :param timeout: Connect timeout, default to 15 seconds
1871+ :expect_success: True by default, set False to invert logic
1872+ :returns: None if successful, Failure message otherwise
1873+ """
1874+ for unit in sentry_units:
1875+ host = unit.info['public-address']
1876+ connected = self.port_knock_tcp(host, port, timeout)
1877+ if not connected and expect_success:
1878+ return 'Socket connect failed.'
1879+ elif connected and not expect_success:
1880+ return 'Socket connected unexpectedly.'
1881+
1882+ def get_uuid_epoch_stamp(self):
1883+ """Returns a stamp string based on uuid4 and epoch time. Useful in
1884+ generating test messages which need to be unique-ish."""
1885+ return '[{}-{}]'.format(uuid.uuid4(), time.time())
1886+
1887+ # amulet juju action helpers:
1888+ def run_action(self, unit_sentry, action,
1889+ _check_output=subprocess.check_output,
1890+ params=None):
1891+ """Translate to amulet's built in run_action(). Deprecated.
1892+
1893+ Run the named action on a given unit sentry.
1894+
1895+ params a dict of parameters to use
1896+ _check_output parameter is no longer used
1897+
1898+ @return action_id.
1899+ """
1900+ self.log.warn('charmhelpers.contrib.amulet.utils.run_action has been '
1901+ 'deprecated for amulet.run_action')
1902+ return unit_sentry.run_action(action, action_args=params)
1903+
1904+ def wait_on_action(self, action_id, _check_output=subprocess.check_output):
1905+ """Wait for a given action, returning if it completed or not.
1906+
1907+ action_id a string action uuid
1908+ _check_output parameter is no longer used
1909+ """
1910+ data = amulet.actions.get_action_output(action_id, full_output=True)
1911+ return data.get(u"status") == "completed"
1912+
1913+ def status_get(self, unit):
1914+ """Return the current service status of this unit."""
1915+ raw_status, return_code = unit.run(
1916+ "status-get --format=json --include-data")
1917+ if return_code != 0:
1918+ return ("unknown", "")
1919+ status = json.loads(raw_status)
1920+ return (status["status"], status["message"])
1921
1922=== added directory 'lib/charmhelpers/contrib/ansible'
1923=== added file 'lib/charmhelpers/contrib/ansible/__init__.py'
1924--- lib/charmhelpers/contrib/ansible/__init__.py 1970-01-01 00:00:00 +0000
1925+++ lib/charmhelpers/contrib/ansible/__init__.py 2021-06-24 14:42:02 +0000
1926@@ -0,0 +1,306 @@
1927+# Copyright 2014-2015 Canonical Limited.
1928+#
1929+# Licensed under the Apache License, Version 2.0 (the "License");
1930+# you may not use this file except in compliance with the License.
1931+# You may obtain a copy of the License at
1932+#
1933+# http://www.apache.org/licenses/LICENSE-2.0
1934+#
1935+# Unless required by applicable law or agreed to in writing, software
1936+# distributed under the License is distributed on an "AS IS" BASIS,
1937+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1938+# See the License for the specific language governing permissions and
1939+# limitations under the License.
1940+
1941+# Copyright 2013 Canonical Ltd.
1942+#
1943+# Authors:
1944+# Charm Helpers Developers <juju@lists.ubuntu.com>
1945+"""
1946+The ansible package enables you to easily use the configuration management
1947+tool `Ansible`_ to setup and configure your charm. All of your charm
1948+configuration options and relation-data are available as regular Ansible
1949+variables which can be used in your playbooks and templates.
1950+
1951+.. _Ansible: https://www.ansible.com/
1952+
1953+Usage
1954+=====
1955+
1956+Here is an example directory structure for a charm to get you started::
1957+
1958+ charm-ansible-example/
1959+ |-- ansible
1960+ | |-- playbook.yaml
1961+ | `-- templates
1962+ | `-- example.j2
1963+ |-- config.yaml
1964+ |-- copyright
1965+ |-- icon.svg
1966+ |-- layer.yaml
1967+ |-- metadata.yaml
1968+ |-- reactive
1969+ | `-- example.py
1970+ |-- README.md
1971+
1972+Running a playbook called ``playbook.yaml`` when the ``install`` hook is run
1973+can be as simple as::
1974+
1975+ from charmhelpers.contrib import ansible
1976+ from charms.reactive import hook
1977+
1978+ @hook('install')
1979+ def install():
1980+ ansible.install_ansible_support()
1981+ ansible.apply_playbook('ansible/playbook.yaml')
1982+
1983+Here is an example playbook that uses the ``template`` module to template the
1984+file ``example.j2`` to the charm host and then uses the ``debug`` module to
1985+print out all the host and Juju variables that you can use in your playbooks.
1986+Note that you must target ``localhost`` as the playbook is run locally on the
1987+charm host::
1988+
1989+ ---
1990+ - hosts: localhost
1991+ tasks:
1992+ - name: Template a file
1993+ template:
1994+ src: templates/example.j2
1995+ dest: /tmp/example.j2
1996+
1997+ - name: Print all variables available to Ansible
1998+ debug:
1999+ var: vars
2000+
2001+Read more online about `playbooks`_ and standard Ansible `modules`_.
2002+
2003+.. _playbooks: https://docs.ansible.com/ansible/latest/user_guide/playbooks.html
2004+.. _modules: https://docs.ansible.com/ansible/latest/user_guide/modules.html
2005+
2006+A further feature of the Ansible hooks is to provide a light weight "action"
2007+scripting tool. This is a decorator that you apply to a function, and that
2008+function can now receive cli args, and can pass extra args to the playbook::
2009+
2010+ @hooks.action()
2011+ def some_action(amount, force="False"):
2012+ "Usage: some-action AMOUNT [force=True]" # <-- shown on error
2013+ # process the arguments
2014+ # do some calls
2015+ # return extra-vars to be passed to ansible-playbook
2016+ return {
2017+ 'amount': int(amount),
2018+ 'type': force,
2019+ }
2020+
2021+You can now create a symlink to hooks.py that can be invoked like a hook, but
2022+with cli params::
2023+
2024+ # link actions/some-action to hooks/hooks.py
2025+
2026+ actions/some-action amount=10 force=true
2027+
2028+Install Ansible via pip
2029+=======================
2030+
2031+If you want to install a specific version of Ansible via pip instead of
2032+``install_ansible_support`` which uses APT, consider using the layer options
2033+of `layer-basic`_ to install Ansible in a virtualenv::
2034+
2035+ options:
2036+ basic:
2037+ python_packages: ['ansible==2.9.0']
2038+ include_system_packages: true
2039+ use_venv: true
2040+
2041+.. _layer-basic: https://charmsreactive.readthedocs.io/en/latest/layer-basic.html#layer-configuration
2042+
2043+"""
2044+import os
2045+import json
2046+import stat
2047+import subprocess
2048+import functools
2049+
2050+import charmhelpers.contrib.templating.contexts
2051+import charmhelpers.core.host
2052+import charmhelpers.core.hookenv
2053+import charmhelpers.fetch
2054+
2055+
2056+charm_dir = os.environ.get('CHARM_DIR', '')
2057+ansible_hosts_path = '/etc/ansible/hosts'
2058+# Ansible will automatically include any vars in the following
2059+# file in its inventory when run locally.
2060+ansible_vars_path = '/etc/ansible/host_vars/localhost'
2061+
2062+
2063+def install_ansible_support(from_ppa=True, ppa_location='ppa:ansible/ansible'):
2064+ """Installs Ansible via APT.
2065+
2066+ By default this installs Ansible from the `PPA`_ linked from
2067+ the Ansible `website`_ or from a PPA set in ``ppa_location``.
2068+
2069+ .. _PPA: https://launchpad.net/~ansible/+archive/ubuntu/ansible
2070+ .. _website: http://docs.ansible.com/intro_installation.html#latest-releases-via-apt-ubuntu
2071+
2072+ If ``from_ppa`` is ``False``, then Ansible will be installed from
2073+ Ubuntu's Universe repositories.
2074+ """
2075+ if from_ppa:
2076+ charmhelpers.fetch.add_source(ppa_location)
2077+ charmhelpers.fetch.apt_update(fatal=True)
2078+ charmhelpers.fetch.apt_install('ansible')
2079+ with open(ansible_hosts_path, 'w+') as hosts_file:
2080+ hosts_file.write('localhost ansible_connection=local ansible_remote_tmp=/root/.ansible/tmp')
2081+
2082+
2083+def apply_playbook(playbook, tags=None, extra_vars=None):
2084+ """Run a playbook.
2085+
2086+ This helper runs a playbook with juju state variables as context,
2087+ therefore variables set in application config can be used directly.
2088+ List of tags (--tags) and dictionary with extra_vars (--extra-vars)
2089+ can be passed as additional parameters.
2090+
2091+ Read more about playbook `_variables`_ online.
2092+
2093+ .. _variables: https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html
2094+
2095+ Example::
2096+
2097+ # Run ansible/playbook.yaml with tag install and pass extra
2098+ # variables var_a and var_b
2099+ apply_playbook(
2100+ playbook='ansible/playbook.yaml',
2101+ tags=['install'],
2102+ extra_vars={'var_a': 'val_a', 'var_b': 'val_b'}
2103+ )
2104+
2105+ # Run ansible/playbook.yaml with tag config and extra variable nested,
2106+ # which is passed as json and can be used as dictionary in playbook
2107+ apply_playbook(
2108+ playbook='ansible/playbook.yaml',
2109+ tags=['config'],
2110+ extra_vars={'nested': {'a': 'value1', 'b': 'value2'}}
2111+ )
2112+
2113+ # Custom config file can be passed within extra_vars
2114+ apply_playbook(
2115+ playbook='ansible/playbook.yaml',
2116+ extra_vars="@some_file.json"
2117+ )
2118+
2119+ """
2120+ tags = tags or []
2121+ tags = ",".join(tags)
2122+ charmhelpers.contrib.templating.contexts.juju_state_to_yaml(
2123+ ansible_vars_path, namespace_separator='__',
2124+ allow_hyphens_in_keys=False, mode=(stat.S_IRUSR | stat.S_IWUSR))
2125+
2126+ # we want ansible's log output to be unbuffered
2127+ env = os.environ.copy()
2128+ proxy_settings = charmhelpers.core.hookenv.env_proxy_settings()
2129+ if proxy_settings:
2130+ env.update(proxy_settings)
2131+ env['PYTHONUNBUFFERED'] = "1"
2132+ call = [
2133+ 'ansible-playbook',
2134+ '-c',
2135+ 'local',
2136+ playbook,
2137+ ]
2138+ if tags:
2139+ call.extend(['--tags', '{}'.format(tags)])
2140+ if extra_vars:
2141+ call.extend(['--extra-vars', json.dumps(extra_vars)])
2142+ subprocess.check_call(call, env=env)
2143+
2144+
2145+class AnsibleHooks(charmhelpers.core.hookenv.Hooks):
2146+ """Run a playbook with the hook-name as the tag.
2147+
2148+ This helper builds on the standard hookenv.Hooks helper,
2149+ but additionally runs the playbook with the hook-name specified
2150+ using --tags (ie. running all the tasks tagged with the hook-name).
2151+
2152+ Example::
2153+
2154+ hooks = AnsibleHooks(playbook_path='ansible/my_machine_state.yaml')
2155+
2156+ # All the tasks within my_machine_state.yaml tagged with 'install'
2157+ # will be run automatically after do_custom_work()
2158+ @hooks.hook()
2159+ def install():
2160+ do_custom_work()
2161+
2162+ # For most of your hooks, you won't need to do anything other
2163+ # than run the tagged tasks for the hook:
2164+ @hooks.hook('config-changed', 'start', 'stop')
2165+ def just_use_playbook():
2166+ pass
2167+
2168+ # As a convenience, you can avoid the above noop function by specifying
2169+ # the hooks which are handled by ansible-only and they'll be registered
2170+ # for you:
2171+ # hooks = AnsibleHooks(
2172+ # 'ansible/my_machine_state.yaml',
2173+ # default_hooks=['config-changed', 'start', 'stop'])
2174+
2175+ if __name__ == "__main__":
2176+ # execute a hook based on the name the program is called by
2177+ hooks.execute(sys.argv)
2178+ """
2179+
2180+ def __init__(self, playbook_path, default_hooks=None):
2181+ """Register any hooks handled by ansible."""
2182+ super(AnsibleHooks, self).__init__()
2183+
2184+ self._actions = {}
2185+ self.playbook_path = playbook_path
2186+
2187+ default_hooks = default_hooks or []
2188+
2189+ def noop(*args, **kwargs):
2190+ pass
2191+
2192+ for hook in default_hooks:
2193+ self.register(hook, noop)
2194+
2195+ def register_action(self, name, function):
2196+ """Register a hook"""
2197+ self._actions[name] = function
2198+
2199+ def execute(self, args):
2200+ """Execute the hook followed by the playbook using the hook as tag."""
2201+ hook_name = os.path.basename(args[0])
2202+ extra_vars = None
2203+ if hook_name in self._actions:
2204+ extra_vars = self._actions[hook_name](args[1:])
2205+ else:
2206+ super(AnsibleHooks, self).execute(args)
2207+
2208+ charmhelpers.contrib.ansible.apply_playbook(
2209+ self.playbook_path, tags=[hook_name], extra_vars=extra_vars)
2210+
2211+ def action(self, *action_names):
2212+ """Decorator, registering them as actions"""
2213+ def action_wrapper(decorated):
2214+
2215+ @functools.wraps(decorated)
2216+ def wrapper(argv):
2217+ kwargs = dict(arg.split('=') for arg in argv)
2218+ try:
2219+ return decorated(**kwargs)
2220+ except TypeError as e:
2221+ if decorated.__doc__:
2222+ e.args += (decorated.__doc__,)
2223+ raise
2224+
2225+ self.register_action(decorated.__name__, wrapper)
2226+ if '_' in decorated.__name__:
2227+ self.register_action(
2228+ decorated.__name__.replace('_', '-'), wrapper)
2229+
2230+ return wrapper
2231+
2232+ return action_wrapper
2233
2234=== added directory 'lib/charmhelpers/contrib/benchmark'
2235=== added file 'lib/charmhelpers/contrib/benchmark/__init__.py'
2236--- lib/charmhelpers/contrib/benchmark/__init__.py 1970-01-01 00:00:00 +0000
2237+++ lib/charmhelpers/contrib/benchmark/__init__.py 2021-06-24 14:42:02 +0000
2238@@ -0,0 +1,124 @@
2239+# Copyright 2014-2015 Canonical Limited.
2240+#
2241+# Licensed under the Apache License, Version 2.0 (the "License");
2242+# you may not use this file except in compliance with the License.
2243+# You may obtain a copy of the License at
2244+#
2245+# http://www.apache.org/licenses/LICENSE-2.0
2246+#
2247+# Unless required by applicable law or agreed to in writing, software
2248+# distributed under the License is distributed on an "AS IS" BASIS,
2249+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2250+# See the License for the specific language governing permissions and
2251+# limitations under the License.
2252+
2253+import subprocess
2254+import time
2255+import os
2256+from distutils.spawn import find_executable
2257+
2258+from charmhelpers.core.hookenv import (
2259+ in_relation_hook,
2260+ relation_ids,
2261+ relation_set,
2262+ relation_get,
2263+)
2264+
2265+
2266+def action_set(key, val):
2267+ if find_executable('action-set'):
2268+ action_cmd = ['action-set']
2269+
2270+ if isinstance(val, dict):
2271+ for k, v in iter(val.items()):
2272+ action_set('%s.%s' % (key, k), v)
2273+ return True
2274+
2275+ action_cmd.append('%s=%s' % (key, val))
2276+ subprocess.check_call(action_cmd)
2277+ return True
2278+ return False
2279+
2280+
2281+class Benchmark():
2282+ """
2283+ Helper class for the `benchmark` interface.
2284+
2285+ :param list actions: Define the actions that are also benchmarks
2286+
2287+ From inside the benchmark-relation-changed hook, you would
2288+ Benchmark(['memory', 'cpu', 'disk', 'smoke', 'custom'])
2289+
2290+ Examples:
2291+
2292+ siege = Benchmark(['siege'])
2293+ siege.start()
2294+ [... run siege ...]
2295+ # The higher the score, the better the benchmark
2296+ siege.set_composite_score(16.70, 'trans/sec', 'desc')
2297+ siege.finish()
2298+
2299+
2300+ """
2301+
2302+ BENCHMARK_CONF = '/etc/benchmark.conf' # Replaced in testing
2303+
2304+ required_keys = [
2305+ 'hostname',
2306+ 'port',
2307+ 'graphite_port',
2308+ 'graphite_endpoint',
2309+ 'api_port'
2310+ ]
2311+
2312+ def __init__(self, benchmarks=None):
2313+ if in_relation_hook():
2314+ if benchmarks is not None:
2315+ for rid in sorted(relation_ids('benchmark')):
2316+ relation_set(relation_id=rid, relation_settings={
2317+ 'benchmarks': ",".join(benchmarks)
2318+ })
2319+
2320+ # Check the relation data
2321+ config = {}
2322+ for key in self.required_keys:
2323+ val = relation_get(key)
2324+ if val is not None:
2325+ config[key] = val
2326+ else:
2327+ # We don't have all of the required keys
2328+ config = {}
2329+ break
2330+
2331+ if len(config):
2332+ with open(self.BENCHMARK_CONF, 'w') as f:
2333+ for key, val in iter(config.items()):
2334+ f.write("%s=%s\n" % (key, val))
2335+
2336+ @staticmethod
2337+ def start():
2338+ action_set('meta.start', time.strftime('%Y-%m-%dT%H:%M:%SZ'))
2339+
2340+ """
2341+ If the collectd charm is also installed, tell it to send a snapshot
2342+ of the current profile data.
2343+ """
2344+ COLLECT_PROFILE_DATA = '/usr/local/bin/collect-profile-data'
2345+ if os.path.exists(COLLECT_PROFILE_DATA):
2346+ subprocess.check_output([COLLECT_PROFILE_DATA])
2347+
2348+ @staticmethod
2349+ def finish():
2350+ action_set('meta.stop', time.strftime('%Y-%m-%dT%H:%M:%SZ'))
2351+
2352+ @staticmethod
2353+ def set_composite_score(value, units, direction='asc'):
2354+ """
2355+ Set the composite score for a benchmark run. This is a single number
2356+ representative of the benchmark results. This could be the most
2357+ important metric, or an amalgamation of metric scores.
2358+ """
2359+ return action_set(
2360+ "meta.composite",
2361+ {'value': value, 'units': units, 'direction': direction}
2362+ )
2363
2364=== added directory 'lib/charmhelpers/contrib/charmhelpers'
2365=== added file 'lib/charmhelpers/contrib/charmhelpers/IMPORT'
2366--- lib/charmhelpers/contrib/charmhelpers/IMPORT 1970-01-01 00:00:00 +0000
2367+++ lib/charmhelpers/contrib/charmhelpers/IMPORT 2021-06-24 14:42:02 +0000
2368@@ -0,0 +1,4 @@
2369+Source lp:charm-tools/trunk
2370+
2371+charm-tools/helpers/python/charmhelpers/__init__.py -> charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py
2372+charm-tools/helpers/python/charmhelpers/tests/test_charmhelpers.py -> charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py
2373
2374=== added file 'lib/charmhelpers/contrib/charmhelpers/__init__.py'
2375--- lib/charmhelpers/contrib/charmhelpers/__init__.py 1970-01-01 00:00:00 +0000
2376+++ lib/charmhelpers/contrib/charmhelpers/__init__.py 2021-06-24 14:42:02 +0000
2377@@ -0,0 +1,203 @@
2378+# Copyright 2014-2015 Canonical Limited.
2379+#
2380+# Licensed under the Apache License, Version 2.0 (the "License");
2381+# you may not use this file except in compliance with the License.
2382+# You may obtain a copy of the License at
2383+#
2384+# http://www.apache.org/licenses/LICENSE-2.0
2385+#
2386+# Unless required by applicable law or agreed to in writing, software
2387+# distributed under the License is distributed on an "AS IS" BASIS,
2388+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2389+# See the License for the specific language governing permissions and
2390+# limitations under the License.
2391+
2392+import warnings
2393+warnings.warn("contrib.charmhelpers is deprecated", DeprecationWarning) # noqa
2394+
2395+import operator
2396+import tempfile
2397+import time
2398+import yaml
2399+import subprocess
2400+
2401+import six
2402+if six.PY3:
2403+ from urllib.request import urlopen
2404+ from urllib.error import (HTTPError, URLError)
2405+else:
2406+ from urllib2 import (urlopen, HTTPError, URLError)
2407+
2408+"""Helper functions for writing Juju charms in Python."""
2409+
2410+__metaclass__ = type
2411+__all__ = [
2412+ # 'get_config', # core.hookenv.config()
2413+ # 'log', # core.hookenv.log()
2414+ # 'log_entry', # core.hookenv.log()
2415+ # 'log_exit', # core.hookenv.log()
2416+ # 'relation_get', # core.hookenv.relation_get()
2417+ # 'relation_set', # core.hookenv.relation_set()
2418+ # 'relation_ids', # core.hookenv.relation_ids()
2419+ # 'relation_list', # core.hookenv.relation_units()
2420+ # 'config_get', # core.hookenv.config()
2421+ # 'unit_get', # core.hookenv.unit_get()
2422+ # 'open_port', # core.hookenv.open_port()
2423+ # 'close_port', # core.hookenv.close_port()
2424+ # 'service_control', # core.host.service()
2425+ 'unit_info', # client-side, NOT IMPLEMENTED
2426+ 'wait_for_machine', # client-side, NOT IMPLEMENTED
2427+ 'wait_for_page_contents', # client-side, NOT IMPLEMENTED
2428+ 'wait_for_relation', # client-side, NOT IMPLEMENTED
2429+ 'wait_for_unit', # client-side, NOT IMPLEMENTED
2430+]
2431+
2432+
2433+SLEEP_AMOUNT = 0.1
2434+
2435+
2436+# We create a juju_status Command here because it makes testing much,
2437+# much easier.
2438+def juju_status():
2439+ subprocess.check_call(['juju', 'status'])
2440+
2441+# re-implemented as charmhelpers.fetch.configure_sources()
2442+# def configure_source(update=False):
2443+# source = config_get('source')
2444+# if ((source.startswith('ppa:') or
2445+# source.startswith('cloud:') or
2446+# source.startswith('http:'))):
2447+# run('add-apt-repository', source)
2448+# if source.startswith("http:"):
2449+# run('apt-key', 'import', config_get('key'))
2450+# if update:
2451+# run('apt-get', 'update')
2452+
2453+
2454+# DEPRECATED: client-side only
2455+def make_charm_config_file(charm_config):
2456+ charm_config_file = tempfile.NamedTemporaryFile(mode='w+')
2457+ charm_config_file.write(yaml.dump(charm_config))
2458+ charm_config_file.flush()
2459+ # The NamedTemporaryFile instance is returned instead of just the name
2460+ # because we want to take advantage of garbage collection-triggered
2461+ # deletion of the temp file when it goes out of scope in the caller.
2462+ return charm_config_file
2463+
2464+
2465+# DEPRECATED: client-side only
2466+def unit_info(service_name, item_name, data=None, unit=None):
2467+ if data is None:
2468+ data = yaml.safe_load(juju_status())
2469+ service = data['services'].get(service_name)
2470+ if service is None:
2471+ # XXX 2012-02-08 gmb:
2472+ # This allows us to cope with the race condition that we
2473+ # have between deploying a service and having it come up in
2474+ # `juju status`. We could probably do with cleaning it up so
2475+ # that it fails a bit more noisily after a while.
2476+ return ''
2477+ units = service['units']
2478+ if unit is not None:
2479+ item = units[unit][item_name]
2480+ else:
2481+ # It might seem odd to sort the units here, but we do it to
2482+ # ensure that when no unit is specified, the first unit for the
2483+ # service (or at least the one with the lowest number) is the
2484+ # one whose data gets returned.
2485+ sorted_unit_names = sorted(units.keys())
2486+ item = units[sorted_unit_names[0]][item_name]
2487+ return item
2488+
2489+
2490+# DEPRECATED: client-side only
2491+def get_machine_data():
2492+ return yaml.safe_load(juju_status())['machines']
2493+
2494+
2495+# DEPRECATED: client-side only
2496+def wait_for_machine(num_machines=1, timeout=300):
2497+ """Wait `timeout` seconds for `num_machines` machines to come up.
2498+
2499+ This wait_for... function can be called by other wait_for functions
2500+ whose timeouts might be too short in situations where only a bare
2501+ Juju setup has been bootstrapped.
2502+
2503+ :return: A tuple of (num_machines, time_taken). This is used for
2504+ testing.
2505+ """
2506+ # You may think this is a hack, and you'd be right. The easiest way
2507+ # to tell what environment we're working in (LXC vs EC2) is to check
2508+ # the dns-name of the first machine. If it's localhost we're in LXC
2509+ # and we can just return here.
2510+ if get_machine_data()[0]['dns-name'] == 'localhost':
2511+ return 1, 0
2512+ start_time = time.time()
2513+ while True:
2514+ # Drop the first machine, since it's the Zookeeper and that's
2515+ # not a machine that we need to wait for. This will only work
2516+ # for EC2 environments, which is why we return early above if
2517+ # we're in LXC.
2518+ machine_data = get_machine_data()
2519+ non_zookeeper_machines = [
2520+ machine_data[key] for key in list(machine_data.keys())[1:]]
2521+ if len(non_zookeeper_machines) >= num_machines:
2522+ all_machines_running = True
2523+ for machine in non_zookeeper_machines:
2524+ if machine.get('instance-state') != 'running':
2525+ all_machines_running = False
2526+ break
2527+ if all_machines_running:
2528+ break
2529+ if time.time() - start_time >= timeout:
2530+ raise RuntimeError('timeout waiting for service to start')
2531+ time.sleep(SLEEP_AMOUNT)
2532+ return num_machines, time.time() - start_time
2533+
2534+
2535+# DEPRECATED: client-side only
2536+def wait_for_unit(service_name, timeout=480):
2537+ """Wait `timeout` seconds for a given service name to come up."""
2538+ wait_for_machine(num_machines=1)
2539+ start_time = time.time()
2540+ while True:
2541+ state = unit_info(service_name, 'agent-state')
2542+ if 'error' in state or state == 'started':
2543+ break
2544+ if time.time() - start_time >= timeout:
2545+ raise RuntimeError('timeout waiting for service to start')
2546+ time.sleep(SLEEP_AMOUNT)
2547+ if state != 'started':
2548+ raise RuntimeError('unit did not start, agent-state: ' + state)
2549+
2550+
2551+# DEPRECATED: client-side only
2552+def wait_for_relation(service_name, relation_name, timeout=120):
2553+ """Wait `timeout` seconds for a given relation to come up."""
2554+ start_time = time.time()
2555+ while True:
2556+ relation = unit_info(service_name, 'relations').get(relation_name)
2557+ if relation is not None and relation['state'] == 'up':
2558+ break
2559+ if time.time() - start_time >= timeout:
2560+ raise RuntimeError('timeout waiting for relation to be up')
2561+ time.sleep(SLEEP_AMOUNT)
2562+
2563+
2564+# DEPRECATED: client-side only
2565+def wait_for_page_contents(url, contents, timeout=120, validate=None):
2566+ if validate is None:
2567+ validate = operator.contains
2568+ start_time = time.time()
2569+ while True:
2570+ try:
2571+ stream = urlopen(url)
2572+ except (HTTPError, URLError):
2573+ pass
2574+ else:
2575+ page = stream.read()
2576+ if validate(page, contents):
2577+ return page
2578+ if time.time() - start_time >= timeout:
2579+ raise RuntimeError('timeout waiting for contents of ' + url)
2580+ time.sleep(SLEEP_AMOUNT)
2581
2582=== added file 'lib/charmhelpers/contrib/charmsupport/IMPORT'
2583--- lib/charmhelpers/contrib/charmsupport/IMPORT 1970-01-01 00:00:00 +0000
2584+++ lib/charmhelpers/contrib/charmsupport/IMPORT 2021-06-24 14:42:02 +0000
2585@@ -0,0 +1,14 @@
2586+Source: lp:charmsupport/trunk
2587+
2588+charmsupport/charmsupport/execd.py -> charm-helpers/charmhelpers/contrib/charmsupport/execd.py
2589+charmsupport/charmsupport/hookenv.py -> charm-helpers/charmhelpers/contrib/charmsupport/hookenv.py
2590+charmsupport/charmsupport/host.py -> charm-helpers/charmhelpers/contrib/charmsupport/host.py
2591+charmsupport/charmsupport/nrpe.py -> charm-helpers/charmhelpers/contrib/charmsupport/nrpe.py
2592+charmsupport/charmsupport/volumes.py -> charm-helpers/charmhelpers/contrib/charmsupport/volumes.py
2593+
2594+charmsupport/tests/test_execd.py -> charm-helpers/tests/contrib/charmsupport/test_execd.py
2595+charmsupport/tests/test_hookenv.py -> charm-helpers/tests/contrib/charmsupport/test_hookenv.py
2596+charmsupport/tests/test_host.py -> charm-helpers/tests/contrib/charmsupport/test_host.py
2597+charmsupport/tests/test_nrpe.py -> charm-helpers/tests/contrib/charmsupport/test_nrpe.py
2598+
2599+charmsupport/bin/charmsupport -> charm-helpers/bin/contrib/charmsupport/charmsupport
2600
2601=== modified file 'lib/charmhelpers/contrib/charmsupport/__init__.py'
2602--- lib/charmhelpers/contrib/charmsupport/__init__.py 2014-09-22 12:14:13 +0000
2603+++ lib/charmhelpers/contrib/charmsupport/__init__.py 2021-06-24 14:42:02 +0000
2604@@ -0,0 +1,13 @@
2605+# Copyright 2014-2015 Canonical Limited.
2606+#
2607+# Licensed under the Apache License, Version 2.0 (the "License");
2608+# you may not use this file except in compliance with the License.
2609+# You may obtain a copy of the License at
2610+#
2611+# http://www.apache.org/licenses/LICENSE-2.0
2612+#
2613+# Unless required by applicable law or agreed to in writing, software
2614+# distributed under the License is distributed on an "AS IS" BASIS,
2615+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2616+# See the License for the specific language governing permissions and
2617+# limitations under the License.
2618
2619=== modified file 'lib/charmhelpers/contrib/charmsupport/nrpe.py'
2620--- lib/charmhelpers/contrib/charmsupport/nrpe.py 2014-09-22 12:14:13 +0000
2621+++ lib/charmhelpers/contrib/charmsupport/nrpe.py 2021-06-24 14:42:02 +0000
2622@@ -1,26 +1,46 @@
2623+# Copyright 2014-2015 Canonical Limited.
2624+#
2625+# Licensed under the Apache License, Version 2.0 (the "License");
2626+# you may not use this file except in compliance with the License.
2627+# You may obtain a copy of the License at
2628+#
2629+# http://www.apache.org/licenses/LICENSE-2.0
2630+#
2631+# Unless required by applicable law or agreed to in writing, software
2632+# distributed under the License is distributed on an "AS IS" BASIS,
2633+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2634+# See the License for the specific language governing permissions and
2635+# limitations under the License.
2636+
2637 """Compatibility with the nrpe-external-master charm"""
2638 # Copyright 2012 Canonical Ltd.
2639 #
2640 # Authors:
2641 # Matthew Wedgwood <matthew.wedgwood@canonical.com>
2642
2643-import subprocess
2644-import pwd
2645+import glob
2646 import grp
2647 import os
2648+import pwd
2649 import re
2650 import shlex
2651+import shutil
2652+import subprocess
2653 import yaml
2654
2655 from charmhelpers.core.hookenv import (
2656 config,
2657+ hook_name,
2658 local_unit,
2659 log,
2660+ relation_get,
2661 relation_ids,
2662 relation_set,
2663+ relations_of_type,
2664 )
2665
2666 from charmhelpers.core.host import service
2667+from charmhelpers.core import host
2668
2669 # This module adds compatibility with the nrpe-external-master and plain nrpe
2670 # subordinate charms. To use it in your charm:
2671@@ -54,6 +74,12 @@
2672 # juju-myservice-0
2673 # If you're running multiple environments with the same services in them
2674 # this allows you to differentiate between them.
2675+# nagios_servicegroups:
2676+# default: ""
2677+# type: string
2678+# description: |
2679+# A comma-separated list of nagios servicegroups.
2680+# If left empty, the nagios_context will be used as the servicegroup
2681 #
2682 # 3. Add custom checks (Nagios plugins) to files/nrpe-external-master
2683 #
2684@@ -85,6 +111,13 @@
2685 # def local_monitors_relation_changed():
2686 # update_nrpe_config()
2687 #
2688+# 4.a If your charm is a subordinate charm set primary=False
2689+#
2690+# from charmsupport.nrpe import NRPE
2691+# (...)
2692+# def update_nrpe_config():
2693+# nrpe_compat = NRPE(primary=False)
2694+#
2695 # 5. ln -s hooks.py nrpe-external-master-relation-changed
2696 # ln -s hooks.py local-monitors-relation-changed
2697
2698@@ -94,7 +127,7 @@
2699
2700
2701 class Check(object):
2702- shortname_re = '[A-Za-z0-9-_]+$'
2703+ shortname_re = '[A-Za-z0-9-_.@]+$'
2704 service_template = ("""
2705 #---------------------------------------------------
2706 # This file is Juju managed
2707@@ -106,10 +139,11 @@
2708 """{description}
2709 check_command check_nrpe!{command}
2710 servicegroups {nagios_servicegroup}
2711+{service_config_overrides}
2712 }}
2713 """)
2714
2715- def __init__(self, shortname, description, check_cmd):
2716+ def __init__(self, shortname, description, check_cmd, max_check_attempts=None):
2717 super(Check, self).__init__()
2718 # XXX: could be better to calculate this from the service name
2719 if not re.match(self.shortname_re, shortname):
2720@@ -122,6 +156,14 @@
2721 # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()=
2722 self.description = description
2723 self.check_cmd = self._locate_cmd(check_cmd)
2724+ self.max_check_attempts = max_check_attempts
2725+
2726+ def _get_check_filename(self):
2727+ return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
2728+
2729+ def _get_service_filename(self, hostname):
2730+ return os.path.join(NRPE.nagios_exportdir,
2731+ 'service__{}_{}.cfg'.format(hostname, self.command))
2732
2733 def _locate_cmd(self, check_cmd):
2734 search_path = (
2735@@ -138,11 +180,30 @@
2736 log('Check command not found: {}'.format(parts[0]))
2737 return ''
2738
2739- def write(self, nagios_context, hostname):
2740- nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
2741- self.command)
2742+ def _remove_service_files(self):
2743+ if not os.path.exists(NRPE.nagios_exportdir):
2744+ return
2745+ for f in os.listdir(NRPE.nagios_exportdir):
2746+ if f.endswith('_{}.cfg'.format(self.command)):
2747+ os.remove(os.path.join(NRPE.nagios_exportdir, f))
2748+
2749+ def remove(self, hostname):
2750+ nrpe_check_file = self._get_check_filename()
2751+ if os.path.exists(nrpe_check_file):
2752+ os.remove(nrpe_check_file)
2753+ self._remove_service_files()
2754+
2755+ def write(self, nagios_context, hostname, nagios_servicegroups):
2756+ nrpe_check_file = self._get_check_filename()
2757 with open(nrpe_check_file, 'w') as nrpe_check_config:
2758 nrpe_check_config.write("# check {}\n".format(self.shortname))
2759+ if nagios_servicegroups:
2760+ nrpe_check_config.write(
2761+ "# The following header was added automatically by juju\n")
2762+ nrpe_check_config.write(
2763+ "# Modifying it will affect nagios monitoring and alerting\n")
2764+ nrpe_check_config.write(
2765+ "# servicegroups: {}\n".format(nagios_servicegroups))
2766 nrpe_check_config.write("command[{}]={}\n".format(
2767 self.command, self.check_cmd))
2768
2769@@ -150,23 +211,29 @@
2770 log('Not writing service config as {} is not accessible'.format(
2771 NRPE.nagios_exportdir))
2772 else:
2773- self.write_service_config(nagios_context, hostname)
2774-
2775- def write_service_config(self, nagios_context, hostname):
2776- for f in os.listdir(NRPE.nagios_exportdir):
2777- if re.search('.*{}.cfg'.format(self.command), f):
2778- os.remove(os.path.join(NRPE.nagios_exportdir, f))
2779-
2780+ self.write_service_config(nagios_context, hostname,
2781+ nagios_servicegroups)
2782+
2783+ def write_service_config(self, nagios_context, hostname,
2784+ nagios_servicegroups):
2785+ self._remove_service_files()
2786+
2787+ if self.max_check_attempts:
2788+ service_config_overrides = ' max_check_attempts {}'.format(
2789+ self.max_check_attempts
2790+ ) # Note indentation is here rather than in the template to avoid trailing spaces
2791+ else:
2792+ service_config_overrides = '' # empty string to avoid printing 'None'
2793 templ_vars = {
2794 'nagios_hostname': hostname,
2795- 'nagios_servicegroup': nagios_context,
2796+ 'nagios_servicegroup': nagios_servicegroups,
2797 'description': self.description,
2798 'shortname': self.shortname,
2799 'command': self.command,
2800+ 'service_config_overrides': service_config_overrides,
2801 }
2802 nrpe_service_text = Check.service_template.format(**templ_vars)
2803- nrpe_service_file = '{}/service__{}_{}.cfg'.format(
2804- NRPE.nagios_exportdir, hostname, self.command)
2805+ nrpe_service_file = self._get_service_filename(hostname)
2806 with open(nrpe_service_file, 'w') as nrpe_service_config:
2807 nrpe_service_config.write(str(nrpe_service_text))
2808
2809@@ -178,26 +245,76 @@
2810 nagios_logdir = '/var/log/nagios'
2811 nagios_exportdir = '/var/lib/nagios/export'
2812 nrpe_confdir = '/etc/nagios/nrpe.d'
2813+ homedir = '/var/lib/nagios' # home dir provided by nagios-nrpe-server
2814
2815- def __init__(self, hostname=None):
2816+ def __init__(self, hostname=None, primary=True):
2817 super(NRPE, self).__init__()
2818 self.config = config()
2819+ self.primary = primary
2820 self.nagios_context = self.config['nagios_context']
2821+ if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']:
2822+ self.nagios_servicegroups = self.config['nagios_servicegroups']
2823+ else:
2824+ self.nagios_servicegroups = self.nagios_context
2825 self.unit_name = local_unit().replace('/', '-')
2826 if hostname:
2827 self.hostname = hostname
2828 else:
2829- self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
2830+ nagios_hostname = get_nagios_hostname()
2831+ if nagios_hostname:
2832+ self.hostname = nagios_hostname
2833+ else:
2834+ self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
2835 self.checks = []
2836+ # Iff in an nrpe-external-master relation hook, set primary status
2837+ relation = relation_ids('nrpe-external-master')
2838+ if relation:
2839+ log("Setting charm primary status {}".format(primary))
2840+ for rid in relation:
2841+ relation_set(relation_id=rid, relation_settings={'primary': self.primary})
2842+ self.remove_check_queue = set()
2843+
2844+ @classmethod
2845+ def does_nrpe_conf_dir_exist(cls):
2846+ """Return True if th nrpe_confdif directory exists."""
2847+ return os.path.isdir(cls.nrpe_confdir)
2848
2849 def add_check(self, *args, **kwargs):
2850+ shortname = None
2851+ if kwargs.get('shortname') is None:
2852+ if len(args) > 0:
2853+ shortname = args[0]
2854+ else:
2855+ shortname = kwargs['shortname']
2856+
2857 self.checks.append(Check(*args, **kwargs))
2858+ try:
2859+ self.remove_check_queue.remove(shortname)
2860+ except KeyError:
2861+ pass
2862+
2863+ def remove_check(self, *args, **kwargs):
2864+ if kwargs.get('shortname') is None:
2865+ raise ValueError('shortname of check must be specified')
2866+
2867+ # Use sensible defaults if they're not specified - these are not
2868+ # actually used during removal, but they're required for constructing
2869+ # the Check object; check_disk is chosen because it's part of the
2870+ # nagios-plugins-basic package.
2871+ if kwargs.get('check_cmd') is None:
2872+ kwargs['check_cmd'] = 'check_disk'
2873+ if kwargs.get('description') is None:
2874+ kwargs['description'] = ''
2875+
2876+ check = Check(*args, **kwargs)
2877+ check.remove(self.hostname)
2878+ self.remove_check_queue.add(kwargs['shortname'])
2879
2880 def write(self):
2881 try:
2882 nagios_uid = pwd.getpwnam('nagios').pw_uid
2883 nagios_gid = grp.getgrnam('nagios').gr_gid
2884- except:
2885+ except Exception:
2886 log("Nagios user not set up, nrpe checks not updated")
2887 return
2888
2889@@ -207,13 +324,200 @@
2890
2891 nrpe_monitors = {}
2892 monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}}
2893+
2894+ # check that the charm can write to the conf dir. If not, then nagios
2895+ # probably isn't installed, and we can defer.
2896+ if not self.does_nrpe_conf_dir_exist():
2897+ return
2898+
2899 for nrpecheck in self.checks:
2900- nrpecheck.write(self.nagios_context, self.hostname)
2901+ nrpecheck.write(self.nagios_context, self.hostname,
2902+ self.nagios_servicegroups)
2903 nrpe_monitors[nrpecheck.shortname] = {
2904 "command": nrpecheck.command,
2905 }
2906-
2907- service('restart', 'nagios-nrpe-server')
2908-
2909- for rid in relation_ids("local-monitors"):
2910- relation_set(relation_id=rid, monitors=yaml.dump(monitors))
2911+ # If we were passed max_check_attempts, add that to the relation data
2912+ if nrpecheck.max_check_attempts is not None:
2913+ nrpe_monitors[nrpecheck.shortname]['max_check_attempts'] = nrpecheck.max_check_attempts
2914+
2915+ # update-status hooks are configured to firing every 5 minutes by
2916+ # default. When nagios-nrpe-server is restarted, the nagios server
2917+ # reports checks failing causing unnecessary alerts. Let's not restart
2918+ # on update-status hooks.
2919+ if not hook_name() == 'update-status':
2920+ service('restart', 'nagios-nrpe-server')
2921+
2922+ monitor_ids = relation_ids("local-monitors") + \
2923+ relation_ids("nrpe-external-master")
2924+ for rid in monitor_ids:
2925+ reldata = relation_get(unit=local_unit(), rid=rid)
2926+ if 'monitors' in reldata:
2927+ # update the existing set of monitors with the new data
2928+ old_monitors = yaml.safe_load(reldata['monitors'])
2929+ old_nrpe_monitors = old_monitors['monitors']['remote']['nrpe']
2930+ # remove keys that are in the remove_check_queue
2931+ old_nrpe_monitors = {k: v for k, v in old_nrpe_monitors.items()
2932+ if k not in self.remove_check_queue}
2933+ # update/add nrpe_monitors
2934+ old_nrpe_monitors.update(nrpe_monitors)
2935+ old_monitors['monitors']['remote']['nrpe'] = old_nrpe_monitors
2936+ # write back to the relation
2937+ relation_set(relation_id=rid, monitors=yaml.dump(old_monitors))
2938+ else:
2939+ # write a brand new set of monitors, as no existing ones.
2940+ relation_set(relation_id=rid, monitors=yaml.dump(monitors))
2941+
2942+ self.remove_check_queue.clear()
2943+
2944+
2945+def get_nagios_hostcontext(relation_name='nrpe-external-master'):
2946+ """
2947+ Query relation with nrpe subordinate, return the nagios_host_context
2948+
2949+ :param str relation_name: Name of relation nrpe sub joined to
2950+ """
2951+ for rel in relations_of_type(relation_name):
2952+ if 'nagios_host_context' in rel:
2953+ return rel['nagios_host_context']
2954+
2955+
2956+def get_nagios_hostname(relation_name='nrpe-external-master'):
2957+ """
2958+ Query relation with nrpe subordinate, return the nagios_hostname
2959+
2960+ :param str relation_name: Name of relation nrpe sub joined to
2961+ """
2962+ for rel in relations_of_type(relation_name):
2963+ if 'nagios_hostname' in rel:
2964+ return rel['nagios_hostname']
2965+
2966+
2967+def get_nagios_unit_name(relation_name='nrpe-external-master'):
2968+ """
2969+ Return the nagios unit name prepended with host_context if needed
2970+
2971+ :param str relation_name: Name of relation nrpe sub joined to
2972+ """
2973+ host_context = get_nagios_hostcontext(relation_name)
2974+ if host_context:
2975+ unit = "%s:%s" % (host_context, local_unit())
2976+ else:
2977+ unit = local_unit()
2978+ return unit
2979+
2980+
2981+def add_init_service_checks(nrpe, services, unit_name, immediate_check=True):
2982+ """
2983+ Add checks for each service in list
2984+
2985+ :param NRPE nrpe: NRPE object to add check to
2986+ :param list services: List of services to check
2987+ :param str unit_name: Unit name to use in check description
2988+ :param bool immediate_check: For sysv init, run the service check immediately
2989+ """
2990+ for svc in services:
2991+ # Don't add a check for these services from neutron-gateway
2992+ if svc in ['ext-port', 'os-charm-phy-nic-mtu']:
2993+ next
2994+
2995+ upstart_init = '/etc/init/%s.conf' % svc
2996+ sysv_init = '/etc/init.d/%s' % svc
2997+
2998+ if host.init_is_systemd(service_name=svc):
2999+ nrpe.add_check(
3000+ shortname=svc,
3001+ description='process check {%s}' % unit_name,
3002+ check_cmd='check_systemd.py %s' % svc
3003+ )
3004+ elif os.path.exists(upstart_init):
3005+ nrpe.add_check(
3006+ shortname=svc,
3007+ description='process check {%s}' % unit_name,
3008+ check_cmd='check_upstart_job %s' % svc
3009+ )
3010+ elif os.path.exists(sysv_init):
3011+ cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
3012+ checkpath = '%s/service-check-%s.txt' % (nrpe.homedir, svc)
3013+ croncmd = (
3014+ '/usr/local/lib/nagios/plugins/check_exit_status.pl '
3015+ '-e -s /etc/init.d/%s status' % svc
3016+ )
3017+ cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath)
3018+ f = open(cronpath, 'w')
3019+ f.write(cron_file)
3020+ f.close()
3021+ nrpe.add_check(
3022+ shortname=svc,
3023+ description='service check {%s}' % unit_name,
3024+ check_cmd='check_status_file.py -f %s' % checkpath,
3025+ )
3026+ # if /var/lib/nagios doesn't exist open(checkpath, 'w') will fail
3027+ # (LP: #1670223).
3028+ if immediate_check and os.path.isdir(nrpe.homedir):
3029+ f = open(checkpath, 'w')
3030+ subprocess.call(
3031+ croncmd.split(),
3032+ stdout=f,
3033+ stderr=subprocess.STDOUT
3034+ )
3035+ f.close()
3036+ os.chmod(checkpath, 0o644)
3037+
3038+
3039+def copy_nrpe_checks(nrpe_files_dir=None):
3040+ """
3041+ Copy the nrpe checks into place
3042+
3043+ """
3044+ NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
3045+ if nrpe_files_dir is None:
3046+ # determine if "charmhelpers" is in CHARMDIR or CHARMDIR/hooks
3047+ for segment in ['.', 'hooks']:
3048+ nrpe_files_dir = os.path.abspath(os.path.join(
3049+ os.getenv('CHARM_DIR'),
3050+ segment,
3051+ 'charmhelpers',
3052+ 'contrib',
3053+ 'openstack',
3054+ 'files'))
3055+ if os.path.isdir(nrpe_files_dir):
3056+ break
3057+ else:
3058+ raise RuntimeError("Couldn't find charmhelpers directory")
3059+ if not os.path.exists(NAGIOS_PLUGINS):
3060+ os.makedirs(NAGIOS_PLUGINS)
3061+ for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")):
3062+ if os.path.isfile(fname):
3063+ shutil.copy2(fname,
3064+ os.path.join(NAGIOS_PLUGINS, os.path.basename(fname)))
3065+
3066+
3067+def add_haproxy_checks(nrpe, unit_name):
3068+ """
3069+ Add checks for each service in list
3070+
3071+ :param NRPE nrpe: NRPE object to add check to
3072+ :param str unit_name: Unit name to use in check description
3073+ """
3074+ nrpe.add_check(
3075+ shortname='haproxy_servers',
3076+ description='Check HAProxy {%s}' % unit_name,
3077+ check_cmd='check_haproxy.sh')
3078+ nrpe.add_check(
3079+ shortname='haproxy_queue',
3080+ description='Check HAProxy queue depth {%s}' % unit_name,
3081+ check_cmd='check_haproxy_queue_depth.sh')
3082+
3083+
3084+def remove_deprecated_check(nrpe, deprecated_services):
3085+ """
3086+ Remove checks fro deprecated services in list
3087+
3088+ :param nrpe: NRPE object to remove check from
3089+ :type nrpe: NRPE
3090+ :param deprecated_services: List of deprecated services that are removed
3091+ :type deprecated_services: list
3092+ """
3093+ for dep_svc in deprecated_services:
3094+ log('Deprecated service: {}'.format(dep_svc))
3095+ nrpe.remove_check(shortname=dep_svc)
3096
3097=== modified file 'lib/charmhelpers/contrib/charmsupport/volumes.py'
3098--- lib/charmhelpers/contrib/charmsupport/volumes.py 2014-09-22 12:14:13 +0000
3099+++ lib/charmhelpers/contrib/charmsupport/volumes.py 2021-06-24 14:42:02 +0000
3100@@ -1,3 +1,17 @@
3101+# Copyright 2014-2015 Canonical Limited.
3102+#
3103+# Licensed under the Apache License, Version 2.0 (the "License");
3104+# you may not use this file except in compliance with the License.
3105+# You may obtain a copy of the License at
3106+#
3107+# http://www.apache.org/licenses/LICENSE-2.0
3108+#
3109+# Unless required by applicable law or agreed to in writing, software
3110+# distributed under the License is distributed on an "AS IS" BASIS,
3111+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3112+# See the License for the specific language governing permissions and
3113+# limitations under the License.
3114+
3115 '''
3116 Functions for managing volumes in juju units. One volume is supported per unit.
3117 Subordinates may have their own storage, provided it is on its own partition.
3118
3119=== added directory 'lib/charmhelpers/contrib/database'
3120=== added file 'lib/charmhelpers/contrib/database/__init__.py'
3121--- lib/charmhelpers/contrib/database/__init__.py 1970-01-01 00:00:00 +0000
3122+++ lib/charmhelpers/contrib/database/__init__.py 2021-06-24 14:42:02 +0000
3123@@ -0,0 +1,11 @@
3124+# Licensed under the Apache License, Version 2.0 (the "License");
3125+# you may not use this file except in compliance with the License.
3126+# You may obtain a copy of the License at
3127+#
3128+# http://www.apache.org/licenses/LICENSE-2.0
3129+#
3130+# Unless required by applicable law or agreed to in writing, software
3131+# distributed under the License is distributed on an "AS IS" BASIS,
3132+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3133+# See the License for the specific language governing permissions and
3134+# limitations under the License.
3135
3136=== added file 'lib/charmhelpers/contrib/database/mysql.py'
3137--- lib/charmhelpers/contrib/database/mysql.py 1970-01-01 00:00:00 +0000
3138+++ lib/charmhelpers/contrib/database/mysql.py 2021-06-24 14:42:02 +0000
3139@@ -0,0 +1,840 @@
3140+# Licensed under the Apache License, Version 2.0 (the "License");
3141+# you may not use this file except in compliance with the License.
3142+# You may obtain a copy of the License at
3143+#
3144+# http://www.apache.org/licenses/LICENSE-2.0
3145+#
3146+# Unless required by applicable law or agreed to in writing, software
3147+# distributed under the License is distributed on an "AS IS" BASIS,
3148+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3149+# See the License for the specific language governing permissions and
3150+# limitations under the License.
3151+
3152+"""Helper for working with a MySQL database"""
3153+import collections
3154+import copy
3155+import json
3156+import re
3157+import sys
3158+import platform
3159+import os
3160+import glob
3161+import six
3162+
3163+# from string import upper
3164+
3165+from charmhelpers.core.host import (
3166+ CompareHostReleases,
3167+ lsb_release,
3168+ mkdir,
3169+ pwgen,
3170+ write_file
3171+)
3172+from charmhelpers.core.hookenv import (
3173+ config as config_get,
3174+ relation_get,
3175+ related_units,
3176+ unit_get,
3177+ log,
3178+ DEBUG,
3179+ ERROR,
3180+ INFO,
3181+ WARNING,
3182+ leader_get,
3183+ leader_set,
3184+ is_leader,
3185+)
3186+from charmhelpers.fetch import (
3187+ apt_install,
3188+ apt_update,
3189+ filter_installed_packages,
3190+)
3191+from charmhelpers.contrib.network.ip import get_host_ip
3192+
3193+try:
3194+ import MySQLdb
3195+except ImportError:
3196+ apt_update(fatal=True)
3197+ if six.PY2:
3198+ apt_install(filter_installed_packages(['python-mysqldb']), fatal=True)
3199+ else:
3200+ apt_install(filter_installed_packages(['python3-mysqldb']), fatal=True)
3201+ import MySQLdb
3202+
3203+
3204+class MySQLSetPasswordError(Exception):
3205+ pass
3206+
3207+
3208+class MySQLHelper(object):
3209+
3210+ def __init__(self, rpasswdf_template, upasswdf_template, host='localhost',
3211+ migrate_passwd_to_leader_storage=True,
3212+ delete_ondisk_passwd_file=True, user="root", password=None,
3213+ port=None, connect_timeout=None):
3214+ self.user = user
3215+ self.host = host
3216+ self.password = password
3217+ self.port = port
3218+ # default timeout of 30 seconds.
3219+ self.connect_timeout = connect_timeout or 30
3220+
3221+ # Password file path templates
3222+ self.root_passwd_file_template = rpasswdf_template
3223+ self.user_passwd_file_template = upasswdf_template
3224+
3225+ self.migrate_passwd_to_leader_storage = migrate_passwd_to_leader_storage
3226+ # If we migrate we have the option to delete local copy of root passwd
3227+ self.delete_ondisk_passwd_file = delete_ondisk_passwd_file
3228+ self.connection = None
3229+
3230+ def connect(self, user='root', password=None, host=None, port=None,
3231+ connect_timeout=None):
3232+ _connection_info = {
3233+ "user": user or self.user,
3234+ "passwd": password or self.password,
3235+ "host": host or self.host
3236+ }
3237+ # set the connection timeout; for mysql8 it can hang forever, so some
3238+ # timeout is required.
3239+ timeout = connect_timeout or self.connect_timeout
3240+ if timeout:
3241+ _connection_info["connect_timeout"] = timeout
3242+ # port cannot be None but we also do not want to specify it unless it
3243+ # has been explicit set.
3244+ port = port or self.port
3245+ if port is not None:
3246+ _connection_info["port"] = port
3247+
3248+ log("Opening db connection for %s@%s" % (user, host), level=DEBUG)
3249+ try:
3250+ self.connection = MySQLdb.connect(**_connection_info)
3251+ except Exception as e:
3252+ log("Failed to connect to database due to '{}'".format(str(e)),
3253+ level=ERROR)
3254+ raise
3255+
3256+ def database_exists(self, db_name):
3257+ cursor = self.connection.cursor()
3258+ try:
3259+ cursor.execute("SHOW DATABASES")
3260+ databases = [i[0] for i in cursor.fetchall()]
3261+ finally:
3262+ cursor.close()
3263+
3264+ return db_name in databases
3265+
3266+ def create_database(self, db_name):
3267+ cursor = self.connection.cursor()
3268+ try:
3269+ cursor.execute("CREATE DATABASE `{}` CHARACTER SET UTF8"
3270+ .format(db_name))
3271+ finally:
3272+ cursor.close()
3273+
3274+ def grant_exists(self, db_name, db_user, remote_ip):
3275+ cursor = self.connection.cursor()
3276+ priv_string = "GRANT ALL PRIVILEGES ON `{}`.* " \
3277+ "TO '{}'@'{}'".format(db_name, db_user, remote_ip)
3278+ try:
3279+ cursor.execute("SHOW GRANTS for '{}'@'{}'".format(db_user,
3280+ remote_ip))
3281+ grants = [i[0] for i in cursor.fetchall()]
3282+ except MySQLdb.OperationalError:
3283+ return False
3284+ finally:
3285+ cursor.close()
3286+
3287+ # TODO: review for different grants
3288+ return priv_string in grants
3289+
3290+ def create_grant(self, db_name, db_user, remote_ip, password):
3291+ cursor = self.connection.cursor()
3292+ try:
3293+ # TODO: review for different grants
3294+ cursor.execute("GRANT ALL PRIVILEGES ON `{}`.* TO '{}'@'{}' "
3295+ "IDENTIFIED BY '{}'".format(db_name,
3296+ db_user,
3297+ remote_ip,
3298+ password))
3299+ finally:
3300+ cursor.close()
3301+
3302+ def create_admin_grant(self, db_user, remote_ip, password):
3303+ cursor = self.connection.cursor()
3304+ try:
3305+ cursor.execute("GRANT ALL PRIVILEGES ON *.* TO '{}'@'{}' "
3306+ "IDENTIFIED BY '{}'".format(db_user,
3307+ remote_ip,
3308+ password))
3309+ finally:
3310+ cursor.close()
3311+
3312+ def cleanup_grant(self, db_user, remote_ip):
3313+ cursor = self.connection.cursor()
3314+ try:
3315+ cursor.execute("DROP FROM mysql.user WHERE user='{}' "
3316+ "AND HOST='{}'".format(db_user,
3317+ remote_ip))
3318+ finally:
3319+ cursor.close()
3320+
3321+ def flush_priviledges(self):
3322+ cursor = self.connection.cursor()
3323+ try:
3324+ cursor.execute("FLUSH PRIVILEGES")
3325+ finally:
3326+ cursor.close()
3327+
3328+ def execute(self, sql):
3329+ """Execute arbitary SQL against the database."""
3330+ cursor = self.connection.cursor()
3331+ try:
3332+ cursor.execute(sql)
3333+ finally:
3334+ cursor.close()
3335+
3336+ def select(self, sql):
3337+ """
3338+ Execute arbitrary SQL select query against the database
3339+ and return the results.
3340+
3341+ :param sql: SQL select query to execute
3342+ :type sql: string
3343+ :returns: SQL select query result
3344+ :rtype: list of lists
3345+ :raises: MySQLdb.Error
3346+ """
3347+ cursor = self.connection.cursor()
3348+ try:
3349+ cursor.execute(sql)
3350+ results = [list(i) for i in cursor.fetchall()]
3351+ finally:
3352+ cursor.close()
3353+ return results
3354+
3355+ def migrate_passwords_to_leader_storage(self, excludes=None):
3356+ """Migrate any passwords storage on disk to leader storage."""
3357+ if not is_leader():
3358+ log("Skipping password migration as not the lead unit",
3359+ level=DEBUG)
3360+ return
3361+ dirname = os.path.dirname(self.root_passwd_file_template)
3362+ path = os.path.join(dirname, '*.passwd')
3363+ for f in glob.glob(path):
3364+ if excludes and f in excludes:
3365+ log("Excluding %s from leader storage migration" % (f),
3366+ level=DEBUG)
3367+ continue
3368+
3369+ key = os.path.basename(f)
3370+ with open(f, 'r') as passwd:
3371+ _value = passwd.read().strip()
3372+
3373+ try:
3374+ leader_set(settings={key: _value})
3375+
3376+ if self.delete_ondisk_passwd_file:
3377+ os.unlink(f)
3378+ except ValueError:
3379+ # NOTE cluster relation not yet ready - skip for now
3380+ pass
3381+
3382+ def get_mysql_password_on_disk(self, username=None, password=None):
3383+ """Retrieve, generate or store a mysql password for the provided
3384+ username on disk."""
3385+ if username:
3386+ template = self.user_passwd_file_template
3387+ passwd_file = template.format(username)
3388+ else:
3389+ passwd_file = self.root_passwd_file_template
3390+
3391+ _password = None
3392+ if os.path.exists(passwd_file):
3393+ log("Using existing password file '%s'" % passwd_file, level=DEBUG)
3394+ with open(passwd_file, 'r') as passwd:
3395+ _password = passwd.read().strip()
3396+ else:
3397+ log("Generating new password file '%s'" % passwd_file, level=DEBUG)
3398+ if not os.path.isdir(os.path.dirname(passwd_file)):
3399+ # NOTE: need to ensure this is not mysql root dir (which needs
3400+ # to be mysql readable)
3401+ mkdir(os.path.dirname(passwd_file), owner='root', group='root',
3402+ perms=0o770)
3403+ # Force permissions - for some reason the chmod in makedirs
3404+ # fails
3405+ os.chmod(os.path.dirname(passwd_file), 0o770)
3406+
3407+ _password = password or pwgen(length=32)
3408+ write_file(passwd_file, _password, owner='root', group='root',
3409+ perms=0o660)
3410+
3411+ return _password
3412+
3413+ def passwd_keys(self, username):
3414+ """Generator to return keys used to store passwords in peer store.
3415+
3416+ NOTE: we support both legacy and new format to support mysql
3417+ charm prior to refactor. This is necessary to avoid LP 1451890.
3418+ """
3419+ keys = []
3420+ if username == 'mysql':
3421+ log("Bad username '%s'" % (username), level=WARNING)
3422+
3423+ if username:
3424+ # IMPORTANT: *newer* format must be returned first
3425+ keys.append('mysql-%s.passwd' % (username))
3426+ keys.append('%s.passwd' % (username))
3427+ else:
3428+ keys.append('mysql.passwd')
3429+
3430+ for key in keys:
3431+ yield key
3432+
3433+ def get_mysql_password(self, username=None, password=None):
3434+ """Retrieve, generate or store a mysql password for the provided
3435+ username using peer relation cluster."""
3436+ excludes = []
3437+
3438+ # First check peer relation.
3439+ try:
3440+ for key in self.passwd_keys(username):
3441+ _password = leader_get(key)
3442+ if _password:
3443+ break
3444+
3445+ # If root password available don't update peer relation from local
3446+ if _password and not username:
3447+ excludes.append(self.root_passwd_file_template)
3448+
3449+ except ValueError:
3450+ # cluster relation is not yet started; use on-disk
3451+ _password = None
3452+
3453+ # If none available, generate new one
3454+ if not _password:
3455+ _password = self.get_mysql_password_on_disk(username, password)
3456+
3457+ # Put on wire if required
3458+ if self.migrate_passwd_to_leader_storage:
3459+ self.migrate_passwords_to_leader_storage(excludes=excludes)
3460+
3461+ return _password
3462+
3463+ def get_mysql_root_password(self, password=None):
3464+ """Retrieve or generate mysql root password for service units."""
3465+ return self.get_mysql_password(username=None, password=password)
3466+
3467+ def set_mysql_password(self, username, password, current_password=None):
3468+ """Update a mysql password for the provided username changing the
3469+ leader settings
3470+
3471+ To update root's password pass `None` in the username
3472+
3473+ :param username: Username to change password of
3474+ :type username: str
3475+ :param password: New password for user.
3476+ :type password: str
3477+ :param current_password: Existing password for user.
3478+ :type current_password: str
3479+ """
3480+
3481+ if username is None:
3482+ username = 'root'
3483+
3484+ # get root password via leader-get, it may be that in the past (when
3485+ # changes to root-password were not supported) the user changed the
3486+ # password, so leader-get is more reliable source than
3487+ # config.previous('root-password').
3488+ rel_username = None if username == 'root' else username
3489+ if not current_password:
3490+ current_password = self.get_mysql_password(rel_username)
3491+
3492+ # password that needs to be set
3493+ new_passwd = password
3494+
3495+ # update password for all users (e.g. root@localhost, root@::1, etc)
3496+ try:
3497+ self.connect(user=username, password=current_password)
3498+ cursor = self.connection.cursor()
3499+ except MySQLdb.OperationalError as ex:
3500+ raise MySQLSetPasswordError(('Cannot connect using password in '
3501+ 'leader settings (%s)') % ex, ex)
3502+
3503+ try:
3504+ # NOTE(freyes): Due to skip-name-resolve root@$HOSTNAME account
3505+ # fails when using SET PASSWORD so using UPDATE against the
3506+ # mysql.user table is needed, but changes to this table are not
3507+ # replicated across the cluster, so this update needs to run in
3508+ # all the nodes. More info at
3509+ # http://galeracluster.com/documentation-webpages/userchanges.html
3510+ release = CompareHostReleases(lsb_release()['DISTRIB_CODENAME'])
3511+ if release < 'bionic':
3512+ SQL_UPDATE_PASSWD = ("UPDATE mysql.user SET password = "
3513+ "PASSWORD( %s ) WHERE user = %s;")
3514+ else:
3515+ # PXC 5.7 (introduced in Bionic) uses authentication_string
3516+ SQL_UPDATE_PASSWD = ("UPDATE mysql.user SET "
3517+ "authentication_string = "
3518+ "PASSWORD( %s ) WHERE user = %s;")
3519+ cursor.execute(SQL_UPDATE_PASSWD, (new_passwd, username))
3520+ cursor.execute('FLUSH PRIVILEGES;')
3521+ self.connection.commit()
3522+ except MySQLdb.OperationalError as ex:
3523+ raise MySQLSetPasswordError('Cannot update password: %s' % str(ex),
3524+ ex)
3525+ finally:
3526+ cursor.close()
3527+
3528+ # check the password was changed
3529+ try:
3530+ self.connect(user=username, password=new_passwd)
3531+ self.execute('select 1;')
3532+ except MySQLdb.OperationalError as ex:
3533+ raise MySQLSetPasswordError(('Cannot connect using new password: '
3534+ '%s') % str(ex), ex)
3535+
3536+ if not is_leader():
3537+ log('Only the leader can set a new password in the relation',
3538+ level=DEBUG)
3539+ return
3540+
3541+ for key in self.passwd_keys(rel_username):
3542+ _password = leader_get(key)
3543+ if _password:
3544+ log('Updating password for %s (%s)' % (key, rel_username),
3545+ level=DEBUG)
3546+ leader_set(settings={key: new_passwd})
3547+
3548+ def set_mysql_root_password(self, password, current_password=None):
3549+ """Update mysql root password changing the leader settings
3550+
3551+ :param password: New password for user.
3552+ :type password: str
3553+ :param current_password: Existing password for user.
3554+ :type current_password: str
3555+ """
3556+ self.set_mysql_password(
3557+ 'root',
3558+ password,
3559+ current_password=current_password)
3560+
3561+ def normalize_address(self, hostname):
3562+ """Ensure that address returned is an IP address (i.e. not fqdn)"""
3563+ if config_get('prefer-ipv6'):
3564+ # TODO: add support for ipv6 dns
3565+ return hostname
3566+
3567+ if hostname != unit_get('private-address'):
3568+ return get_host_ip(hostname, fallback=hostname)
3569+
3570+ # Otherwise assume localhost
3571+ return '127.0.0.1'
3572+
3573+ def get_allowed_units(self, database, username, relation_id=None, prefix=None):
3574+ """Get list of units with access grants for database with username.
3575+
3576+ This is typically used to provide shared-db relations with a list of
3577+ which units have been granted access to the given database.
3578+ """
3579+ if not self.connection:
3580+ self.connect(password=self.get_mysql_root_password())
3581+ allowed_units = set()
3582+ if not prefix:
3583+ prefix = database
3584+ for unit in related_units(relation_id):
3585+ settings = relation_get(rid=relation_id, unit=unit)
3586+ # First check for setting with prefix, then without
3587+ for attr in ["%s_hostname" % (prefix), 'hostname']:
3588+ hosts = settings.get(attr, None)
3589+ if hosts:
3590+ break
3591+
3592+ if hosts:
3593+ # hostname can be json-encoded list of hostnames
3594+ try:
3595+ hosts = json.loads(hosts)
3596+ except ValueError:
3597+ hosts = [hosts]
3598+ else:
3599+ hosts = [settings['private-address']]
3600+
3601+ if hosts:
3602+ for host in hosts:
3603+ host = self.normalize_address(host)
3604+ if self.grant_exists(database, username, host):
3605+ log("Grant exists for host '%s' on db '%s'" %
3606+ (host, database), level=DEBUG)
3607+ if unit not in allowed_units:
3608+ allowed_units.add(unit)
3609+ else:
3610+ log("Grant does NOT exist for host '%s' on db '%s'" %
3611+ (host, database), level=DEBUG)
3612+ else:
3613+ log("No hosts found for grant check", level=INFO)
3614+
3615+ return allowed_units
3616+
3617+ def configure_db(self, hostname, database, username, admin=False):
3618+ """Configure access to database for username from hostname."""
3619+ if not self.connection:
3620+ self.connect(password=self.get_mysql_root_password())
3621+ if not self.database_exists(database):
3622+ self.create_database(database)
3623+
3624+ remote_ip = self.normalize_address(hostname)
3625+ password = self.get_mysql_password(username)
3626+ if not self.grant_exists(database, username, remote_ip):
3627+ if not admin:
3628+ self.create_grant(database, username, remote_ip, password)
3629+ else:
3630+ self.create_admin_grant(username, remote_ip, password)
3631+ self.flush_priviledges()
3632+
3633+ return password
3634+
3635+
3636+# `_singleton_config_helper` stores the instance of the helper class that is
3637+# being used during a hook invocation.
3638+_singleton_config_helper = None
3639+
3640+
3641+def get_mysql_config_helper():
3642+ global _singleton_config_helper
3643+ if _singleton_config_helper is None:
3644+ _singleton_config_helper = MySQLConfigHelper()
3645+ return _singleton_config_helper
3646+
3647+
3648+class MySQLConfigHelper(object):
3649+ """Base configuration helper for MySQL."""
3650+
3651+ # Going for the biggest page size to avoid wasted bytes.
3652+ # InnoDB page size is 16MB
3653+
3654+ DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
3655+ DEFAULT_INNODB_BUFFER_FACTOR = 0.50
3656+ DEFAULT_INNODB_BUFFER_SIZE_MAX = 512 * 1024 * 1024
3657+
3658+ # Validation and lookups for InnoDB configuration
3659+ INNODB_VALID_BUFFERING_VALUES = [
3660+ 'none',
3661+ 'inserts',
3662+ 'deletes',
3663+ 'changes',
3664+ 'purges',
3665+ 'all'
3666+ ]
3667+ INNODB_FLUSH_CONFIG_VALUES = {
3668+ 'fast': 2,
3669+ 'safest': 1,
3670+ 'unsafe': 0,
3671+ }
3672+
3673+ def human_to_bytes(self, human):
3674+ """Convert human readable configuration options to bytes."""
3675+ num_re = re.compile('^[0-9]+$')
3676+ if num_re.match(human):
3677+ return human
3678+
3679+ factors = {
3680+ 'K': 1024,
3681+ 'M': 1048576,
3682+ 'G': 1073741824,
3683+ 'T': 1099511627776
3684+ }
3685+ modifier = human[-1]
3686+ if modifier in factors:
3687+ return int(human[:-1]) * factors[modifier]
3688+
3689+ if modifier == '%':
3690+ total_ram = self.human_to_bytes(self.get_mem_total())
3691+ if self.is_32bit_system() and total_ram > self.sys_mem_limit():
3692+ total_ram = self.sys_mem_limit()
3693+ factor = int(human[:-1]) * 0.01
3694+ pctram = total_ram * factor
3695+ return int(pctram - (pctram % self.DEFAULT_PAGE_SIZE))
3696+
3697+ raise ValueError("Can only convert K,M,G, or T")
3698+
3699+ def is_32bit_system(self):
3700+ """Determine whether system is 32 or 64 bit."""
3701+ try:
3702+ return sys.maxsize < 2 ** 32
3703+ except OverflowError:
3704+ return False
3705+
3706+ def sys_mem_limit(self):
3707+ """Determine the default memory limit for the current service unit."""
3708+ if platform.machine() in ['armv7l']:
3709+ _mem_limit = self.human_to_bytes('2700M') # experimentally determined
3710+ else:
3711+ # Limit for x86 based 32bit systems
3712+ _mem_limit = self.human_to_bytes('4G')
3713+
3714+ return _mem_limit
3715+
3716+ def get_mem_total(self):
3717+ """Calculate the total memory in the current service unit."""
3718+ with open('/proc/meminfo') as meminfo_file:
3719+ for line in meminfo_file:
3720+ key, mem = line.split(':', 2)
3721+ if key == 'MemTotal':
3722+ mtot, modifier = mem.strip().split(' ')
3723+ return '%s%s' % (mtot, modifier[0].upper())
3724+
3725+ def get_innodb_flush_log_at_trx_commit(self):
3726+ """Get value for innodb_flush_log_at_trx_commit.
3727+
3728+ Use the innodb-flush-log-at-trx-commit or the tunning-level setting
3729+ translated by INNODB_FLUSH_CONFIG_VALUES to get the
3730+ innodb_flush_log_at_trx_commit value.
3731+
3732+ :returns: Numeric value for innodb_flush_log_at_trx_commit
3733+ :rtype: Union[None, int]
3734+ """
3735+ _iflatc = config_get('innodb-flush-log-at-trx-commit')
3736+ _tuning_level = config_get('tuning-level')
3737+ if _iflatc:
3738+ return _iflatc
3739+ elif _tuning_level:
3740+ return self.INNODB_FLUSH_CONFIG_VALUES.get(_tuning_level, 1)
3741+
3742+ def get_innodb_change_buffering(self):
3743+ """Get value for innodb_change_buffering.
3744+
3745+ Use the innodb-change-buffering validated against
3746+ INNODB_VALID_BUFFERING_VALUES to get the innodb_change_buffering value.
3747+
3748+ :returns: String value for innodb_change_buffering.
3749+ :rtype: Union[None, str]
3750+ """
3751+ _icb = config_get('innodb-change-buffering')
3752+ if _icb and _icb in self.INNODB_VALID_BUFFERING_VALUES:
3753+ return _icb
3754+
3755+ def get_innodb_buffer_pool_size(self):
3756+ """Get value for innodb_buffer_pool_size.
3757+
3758+ Return the number value of innodb-buffer-pool-size or dataset-size. If
3759+ neither is set, calculate a sane default based on total memory.
3760+
3761+ :returns: Numeric value for innodb_buffer_pool_size.
3762+ :rtype: int
3763+ """
3764+ total_memory = self.human_to_bytes(self.get_mem_total())
3765+
3766+ dataset_bytes = config_get('dataset-size')
3767+ innodb_buffer_pool_size = config_get('innodb-buffer-pool-size')
3768+
3769+ if innodb_buffer_pool_size:
3770+ innodb_buffer_pool_size = self.human_to_bytes(
3771+ innodb_buffer_pool_size)
3772+ elif dataset_bytes:
3773+ log("Option 'dataset-size' has been deprecated, please use"
3774+ "innodb_buffer_pool_size option instead", level="WARN")
3775+ innodb_buffer_pool_size = self.human_to_bytes(
3776+ dataset_bytes)
3777+ else:
3778+ # NOTE(jamespage): pick the smallest of 50% of RAM or 512MB
3779+ # to ensure that deployments in containers
3780+ # without constraints don't try to consume
3781+ # silly amounts of memory.
3782+ innodb_buffer_pool_size = min(
3783+ int(total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR),
3784+ self.DEFAULT_INNODB_BUFFER_SIZE_MAX
3785+ )
3786+
3787+ if innodb_buffer_pool_size > total_memory:
3788+ log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format(
3789+ innodb_buffer_pool_size,
3790+ total_memory), level='WARN')
3791+
3792+ return innodb_buffer_pool_size
3793+
3794+
3795+class PerconaClusterHelper(MySQLConfigHelper):
3796+ """Percona-cluster specific configuration helper."""
3797+
3798+ def parse_config(self):
3799+ """Parse charm configuration and calculate values for config files."""
3800+ config = config_get()
3801+ mysql_config = {}
3802+ if 'max-connections' in config:
3803+ mysql_config['max_connections'] = config['max-connections']
3804+
3805+ if 'wait-timeout' in config:
3806+ mysql_config['wait_timeout'] = config['wait-timeout']
3807+
3808+ if self.get_innodb_flush_log_at_trx_commit() is not None:
3809+ mysql_config['innodb_flush_log_at_trx_commit'] = \
3810+ self.get_innodb_flush_log_at_trx_commit()
3811+
3812+ if self.get_innodb_change_buffering() is not None:
3813+ mysql_config['innodb_change_buffering'] = config['innodb-change-buffering']
3814+
3815+ if 'innodb-io-capacity' in config:
3816+ mysql_config['innodb_io_capacity'] = config['innodb-io-capacity']
3817+
3818+ # Set a sane default key_buffer size
3819+ mysql_config['key_buffer'] = self.human_to_bytes('32M')
3820+ mysql_config['innodb_buffer_pool_size'] = self.get_innodb_buffer_pool_size()
3821+ return mysql_config
3822+
3823+
3824+class MySQL8Helper(MySQLHelper):
3825+
3826+ def grant_exists(self, db_name, db_user, remote_ip):
3827+ cursor = self.connection.cursor()
3828+ priv_string = ("GRANT ALL PRIVILEGES ON {}.* "
3829+ "TO {}@{}".format(db_name, db_user, remote_ip))
3830+ try:
3831+ cursor.execute("SHOW GRANTS FOR '{}'@'{}'".format(db_user,
3832+ remote_ip))
3833+ grants = [i[0] for i in cursor.fetchall()]
3834+ except MySQLdb.OperationalError:
3835+ return False
3836+ finally:
3837+ cursor.close()
3838+
3839+ # Different versions of MySQL use ' or `. Ignore these in the check.
3840+ return priv_string in [
3841+ i.replace("'", "").replace("`", "") for i in grants]
3842+
3843+ def create_grant(self, db_name, db_user, remote_ip, password):
3844+ if self.grant_exists(db_name, db_user, remote_ip):
3845+ return
3846+
3847+ # Make sure the user exists
3848+ # MySQL8 must create the user before the grant
3849+ self.create_user(db_user, remote_ip, password)
3850+
3851+ cursor = self.connection.cursor()
3852+ try:
3853+ cursor.execute("GRANT ALL PRIVILEGES ON `{}`.* TO '{}'@'{}'"
3854+ .format(db_name, db_user, remote_ip))
3855+ finally:
3856+ cursor.close()
3857+
3858+ def create_user(self, db_user, remote_ip, password):
3859+
3860+ SQL_USER_CREATE = (
3861+ "CREATE USER '{db_user}'@'{remote_ip}' "
3862+ "IDENTIFIED BY '{password}'")
3863+
3864+ cursor = self.connection.cursor()
3865+ try:
3866+ cursor.execute(SQL_USER_CREATE.format(
3867+ db_user=db_user,
3868+ remote_ip=remote_ip,
3869+ password=password)
3870+ )
3871+ except MySQLdb._exceptions.OperationalError:
3872+ log("DB user {} already exists.".format(db_user),
3873+ "WARNING")
3874+ finally:
3875+ cursor.close()
3876+
3877+ def create_router_grant(self, db_user, remote_ip, password):
3878+
3879+ # Make sure the user exists
3880+ # MySQL8 must create the user before the grant
3881+ self.create_user(db_user, remote_ip, password)
3882+
3883+ # Mysql-Router specific grants
3884+ cursor = self.connection.cursor()
3885+ try:
3886+ cursor.execute("GRANT CREATE USER ON *.* TO '{}'@'{}' WITH GRANT "
3887+ "OPTION".format(db_user, remote_ip))
3888+ cursor.execute("GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE ON "
3889+ "mysql_innodb_cluster_metadata.* TO '{}'@'{}'"
3890+ .format(db_user, remote_ip))
3891+ cursor.execute("GRANT SELECT ON mysql.user TO '{}'@'{}'"
3892+ .format(db_user, remote_ip))
3893+ cursor.execute("GRANT SELECT ON "
3894+ "performance_schema.replication_group_members "
3895+ "TO '{}'@'{}'".format(db_user, remote_ip))
3896+ cursor.execute("GRANT SELECT ON "
3897+ "performance_schema.replication_group_member_stats "
3898+ "TO '{}'@'{}'".format(db_user, remote_ip))
3899+ cursor.execute("GRANT SELECT ON "
3900+ "performance_schema.global_variables "
3901+ "TO '{}'@'{}'".format(db_user, remote_ip))
3902+ finally:
3903+ cursor.close()
3904+
3905+ def configure_router(self, hostname, username):
3906+
3907+ if self.connection is None:
3908+ self.connect(password=self.get_mysql_root_password())
3909+
3910+ remote_ip = self.normalize_address(hostname)
3911+ password = self.get_mysql_password(username)
3912+ self.create_user(username, remote_ip, password)
3913+ self.create_router_grant(username, remote_ip, password)
3914+
3915+ return password
3916+
3917+
3918+def get_prefix(requested, keys=None):
3919+ """Return existing prefix or None.
3920+
3921+ :param requested: Request string. i.e. novacell0_username
3922+ :type requested: str
3923+ :param keys: Keys to determine prefix. Defaults set in function.
3924+ :type keys: List of str keys
3925+ :returns: String prefix i.e. novacell0
3926+ :rtype: Union[None, str]
3927+ """
3928+ if keys is None:
3929+ # Shared-DB default keys
3930+ keys = ["_database", "_username", "_hostname"]
3931+ for key in keys:
3932+ if requested.endswith(key):
3933+ return requested[:-len(key)]
3934+
3935+
3936+def get_db_data(relation_data, unprefixed):
3937+ """Organize database requests into a collections.OrderedDict
3938+
3939+ :param relation_data: shared-db relation data
3940+ :type relation_data: dict
3941+ :param unprefixed: Prefix to use for requests without a prefix. This should
3942+ be unique for each side of the relation to avoid
3943+ conflicts.
3944+ :type unprefixed: str
3945+ :returns: Order dict of databases and users
3946+ :rtype: collections.OrderedDict
3947+ """
3948+ # Deep copy to avoid unintentionally changing relation data
3949+ settings = copy.deepcopy(relation_data)
3950+ databases = collections.OrderedDict()
3951+
3952+ # Clear non-db related elements
3953+ if "egress-subnets" in settings.keys():
3954+ settings.pop("egress-subnets")
3955+ if "ingress-address" in settings.keys():
3956+ settings.pop("ingress-address")
3957+ if "private-address" in settings.keys():
3958+ settings.pop("private-address")
3959+
3960+ singleset = {"database", "username", "hostname"}
3961+ if singleset.issubset(settings):
3962+ settings["{}_{}".format(unprefixed, "hostname")] = (
3963+ settings["hostname"])
3964+ settings.pop("hostname")
3965+ settings["{}_{}".format(unprefixed, "database")] = (
3966+ settings["database"])
3967+ settings.pop("database")
3968+ settings["{}_{}".format(unprefixed, "username")] = (
3969+ settings["username"])
3970+ settings.pop("username")
3971+
3972+ for k, v in settings.items():
3973+ db = k.split("_")[0]
3974+ x = "_".join(k.split("_")[1:])
3975+ if db not in databases:
3976+ databases[db] = collections.OrderedDict()
3977+ databases[db][x] = v
3978+
3979+ return databases
3980
3981=== added directory 'lib/charmhelpers/contrib/hahelpers'
3982=== added file 'lib/charmhelpers/contrib/hahelpers/__init__.py'
3983--- lib/charmhelpers/contrib/hahelpers/__init__.py 1970-01-01 00:00:00 +0000
3984+++ lib/charmhelpers/contrib/hahelpers/__init__.py 2021-06-24 14:42:02 +0000
3985@@ -0,0 +1,13 @@
3986+# Copyright 2014-2015 Canonical Limited.
3987+#
3988+# Licensed under the Apache License, Version 2.0 (the "License");
3989+# you may not use this file except in compliance with the License.
3990+# You may obtain a copy of the License at
3991+#
3992+# http://www.apache.org/licenses/LICENSE-2.0
3993+#
3994+# Unless required by applicable law or agreed to in writing, software
3995+# distributed under the License is distributed on an "AS IS" BASIS,
3996+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3997+# See the License for the specific language governing permissions and
3998+# limitations under the License.
3999
4000=== added file 'lib/charmhelpers/contrib/hahelpers/apache.py'
4001--- lib/charmhelpers/contrib/hahelpers/apache.py 1970-01-01 00:00:00 +0000
4002+++ lib/charmhelpers/contrib/hahelpers/apache.py 2021-06-24 14:42:02 +0000
4003@@ -0,0 +1,90 @@
4004+# Copyright 2014-2015 Canonical Limited.
4005+#
4006+# Licensed under the Apache License, Version 2.0 (the "License");
4007+# you may not use this file except in compliance with the License.
4008+# You may obtain a copy of the License at
4009+#
4010+# http://www.apache.org/licenses/LICENSE-2.0
4011+#
4012+# Unless required by applicable law or agreed to in writing, software
4013+# distributed under the License is distributed on an "AS IS" BASIS,
4014+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4015+# See the License for the specific language governing permissions and
4016+# limitations under the License.
4017+
4018+#
4019+# Copyright 2012 Canonical Ltd.
4020+#
4021+# This file is sourced from lp:openstack-charm-helpers
4022+#
4023+# Authors:
4024+# James Page <james.page@ubuntu.com>
4025+# Adam Gandelman <adamg@ubuntu.com>
4026+#
4027+
4028+import os
4029+
4030+from charmhelpers.core import host
4031+from charmhelpers.core.hookenv import (
4032+ config as config_get,
4033+ relation_get,
4034+ relation_ids,
4035+ related_units as relation_list,
4036+ log,
4037+ INFO,
4038+)
4039+
4040+# This file contains the CA cert from the charms ssl_ca configuration
4041+# option, in future the file name should be updated reflect that.
4042+CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert'
4043+
4044+
4045+def get_cert(cn=None):
4046+ # TODO: deal with multiple https endpoints via charm config
4047+ cert = config_get('ssl_cert')
4048+ key = config_get('ssl_key')
4049+ if not (cert and key):
4050+ log("Inspecting identity-service relations for SSL certificate.",
4051+ level=INFO)
4052+ cert = key = None
4053+ if cn:
4054+ ssl_cert_attr = 'ssl_cert_{}'.format(cn)
4055+ ssl_key_attr = 'ssl_key_{}'.format(cn)
4056+ else:
4057+ ssl_cert_attr = 'ssl_cert'
4058+ ssl_key_attr = 'ssl_key'
4059+ for r_id in relation_ids('identity-service'):
4060+ for unit in relation_list(r_id):
4061+ if not cert:
4062+ cert = relation_get(ssl_cert_attr,
4063+ rid=r_id, unit=unit)
4064+ if not key:
4065+ key = relation_get(ssl_key_attr,
4066+ rid=r_id, unit=unit)
4067+ return (cert, key)
4068+
4069+
4070+def get_ca_cert():
4071+ ca_cert = config_get('ssl_ca')
4072+ if ca_cert is None:
4073+ log("Inspecting identity-service relations for CA SSL certificate.",
4074+ level=INFO)
4075+ for r_id in (relation_ids('identity-service') +
4076+ relation_ids('identity-credentials')):
4077+ for unit in relation_list(r_id):
4078+ if ca_cert is None:
4079+ ca_cert = relation_get('ca_cert',
4080+ rid=r_id, unit=unit)
4081+ return ca_cert
4082+
4083+
4084+def retrieve_ca_cert(cert_file):
4085+ cert = None
4086+ if os.path.isfile(cert_file):
4087+ with open(cert_file, 'rb') as crt:
4088+ cert = crt.read()
4089+ return cert
4090+
4091+
4092+def install_ca_cert(ca_cert):
4093+ host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE)
4094
4095=== added file 'lib/charmhelpers/contrib/hahelpers/cluster.py'
4096--- lib/charmhelpers/contrib/hahelpers/cluster.py 1970-01-01 00:00:00 +0000
4097+++ lib/charmhelpers/contrib/hahelpers/cluster.py 2021-06-24 14:42:02 +0000
4098@@ -0,0 +1,451 @@
4099+# Copyright 2014-2015 Canonical Limited.
4100+#
4101+# Licensed under the Apache License, Version 2.0 (the "License");
4102+# you may not use this file except in compliance with the License.
4103+# You may obtain a copy of the License at
4104+#
4105+# http://www.apache.org/licenses/LICENSE-2.0
4106+#
4107+# Unless required by applicable law or agreed to in writing, software
4108+# distributed under the License is distributed on an "AS IS" BASIS,
4109+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4110+# See the License for the specific language governing permissions and
4111+# limitations under the License.
4112+
4113+#
4114+# Copyright 2012 Canonical Ltd.
4115+#
4116+# Authors:
4117+# James Page <james.page@ubuntu.com>
4118+# Adam Gandelman <adamg@ubuntu.com>
4119+#
4120+
4121+"""
4122+Helpers for clustering and determining "cluster leadership" and other
4123+clustering-related helpers.
4124+"""
4125+
4126+import functools
4127+import subprocess
4128+import os
4129+import time
4130+
4131+from socket import gethostname as get_unit_hostname
4132+
4133+import six
4134+
4135+from charmhelpers.core.hookenv import (
4136+ log,
4137+ relation_ids,
4138+ related_units as relation_list,
4139+ relation_get,
4140+ config as config_get,
4141+ INFO,
4142+ DEBUG,
4143+ WARNING,
4144+ unit_get,
4145+ is_leader as juju_is_leader,
4146+ status_set,
4147+)
4148+from charmhelpers.core.host import (
4149+ modulo_distribution,
4150+)
4151+from charmhelpers.core.decorators import (
4152+ retry_on_exception,
4153+)
4154+from charmhelpers.core.strutils import (
4155+ bool_from_string,
4156+)
4157+
4158+DC_RESOURCE_NAME = 'DC'
4159+
4160+
4161+class HAIncompleteConfig(Exception):
4162+ pass
4163+
4164+
4165+class HAIncorrectConfig(Exception):
4166+ pass
4167+
4168+
4169+class CRMResourceNotFound(Exception):
4170+ pass
4171+
4172+
4173+class CRMDCNotFound(Exception):
4174+ pass
4175+
4176+
4177+def is_elected_leader(resource):
4178+ """
4179+ Returns True if the charm executing this is the elected cluster leader.
4180+
4181+ It relies on two mechanisms to determine leadership:
4182+ 1. If juju is sufficiently new and leadership election is supported,
4183+ the is_leader command will be used.
4184+ 2. If the charm is part of a corosync cluster, call corosync to
4185+ determine leadership.
4186+ 3. If the charm is not part of a corosync cluster, the leader is
4187+ determined as being "the alive unit with the lowest unit numer". In
4188+ other words, the oldest surviving unit.
4189+ """
4190+ try:
4191+ return juju_is_leader()
4192+ except NotImplementedError:
4193+ log('Juju leadership election feature not enabled'
4194+ ', using fallback support',
4195+ level=WARNING)
4196+
4197+ if is_clustered():
4198+ if not is_crm_leader(resource):
4199+ log('Deferring action to CRM leader.', level=INFO)
4200+ return False
4201+ else:
4202+ peers = peer_units()
4203+ if peers and not oldest_peer(peers):
4204+ log('Deferring action to oldest service unit.', level=INFO)
4205+ return False
4206+ return True
4207+
4208+
4209+def is_clustered():
4210+ for r_id in (relation_ids('ha') or []):
4211+ for unit in (relation_list(r_id) or []):
4212+ clustered = relation_get('clustered',
4213+ rid=r_id,
4214+ unit=unit)
4215+ if clustered:
4216+ return True
4217+ return False
4218+
4219+
4220+def is_crm_dc():
4221+ """
4222+ Determine leadership by querying the pacemaker Designated Controller
4223+ """
4224+ cmd = ['crm', 'status']
4225+ try:
4226+ status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
4227+ if not isinstance(status, six.text_type):
4228+ status = six.text_type(status, "utf-8")
4229+ except subprocess.CalledProcessError as ex:
4230+ raise CRMDCNotFound(str(ex))
4231+
4232+ current_dc = ''
4233+ for line in status.split('\n'):
4234+ if line.startswith('Current DC'):
4235+ # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
4236+ current_dc = line.split(':')[1].split()[0]
4237+ if current_dc == get_unit_hostname():
4238+ return True
4239+ elif current_dc == 'NONE':
4240+ raise CRMDCNotFound('Current DC: NONE')
4241+
4242+ return False
4243+
4244+
4245+@retry_on_exception(5, base_delay=2,
4246+ exc_type=(CRMResourceNotFound, CRMDCNotFound))
4247+def is_crm_leader(resource, retry=False):
4248+ """
4249+ Returns True if the charm calling this is the elected corosync leader,
4250+ as returned by calling the external "crm" command.
4251+
4252+ We allow this operation to be retried to avoid the possibility of getting a
4253+ false negative. See LP #1396246 for more info.
4254+ """
4255+ if resource == DC_RESOURCE_NAME:
4256+ return is_crm_dc()
4257+ cmd = ['crm', 'resource', 'show', resource]
4258+ try:
4259+ status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
4260+ if not isinstance(status, six.text_type):
4261+ status = six.text_type(status, "utf-8")
4262+ except subprocess.CalledProcessError:
4263+ status = None
4264+
4265+ if status and get_unit_hostname() in status:
4266+ return True
4267+
4268+ if status and "resource %s is NOT running" % (resource) in status:
4269+ raise CRMResourceNotFound("CRM resource %s not found" % (resource))
4270+
4271+ return False
4272+
4273+
4274+def is_leader(resource):
4275+ log("is_leader is deprecated. Please consider using is_crm_leader "
4276+ "instead.", level=WARNING)
4277+ return is_crm_leader(resource)
4278+
4279+
4280+def peer_units(peer_relation="cluster"):
4281+ peers = []
4282+ for r_id in (relation_ids(peer_relation) or []):
4283+ for unit in (relation_list(r_id) or []):
4284+ peers.append(unit)
4285+ return peers
4286+
4287+
4288+def peer_ips(peer_relation='cluster', addr_key='private-address'):
4289+ '''Return a dict of peers and their private-address'''
4290+ peers = {}
4291+ for r_id in relation_ids(peer_relation):
4292+ for unit in relation_list(r_id):
4293+ peers[unit] = relation_get(addr_key, rid=r_id, unit=unit)
4294+ return peers
4295+
4296+
4297+def oldest_peer(peers):
4298+ """Determines who the oldest peer is by comparing unit numbers."""
4299+ local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1])
4300+ for peer in peers:
4301+ remote_unit_no = int(peer.split('/')[1])
4302+ if remote_unit_no < local_unit_no:
4303+ return False
4304+ return True
4305+
4306+
4307+def eligible_leader(resource):
4308+ log("eligible_leader is deprecated. Please consider using "
4309+ "is_elected_leader instead.", level=WARNING)
4310+ return is_elected_leader(resource)
4311+
4312+
4313+def https():
4314+ '''
4315+ Determines whether enough data has been provided in configuration
4316+ or relation data to configure HTTPS
4317+ .
4318+ returns: boolean
4319+ '''
4320+ use_https = config_get('use-https')
4321+ if use_https and bool_from_string(use_https):
4322+ return True
4323+ if config_get('ssl_cert') and config_get('ssl_key'):
4324+ return True
4325+ for r_id in relation_ids('certificates'):
4326+ for unit in relation_list(r_id):
4327+ ca = relation_get('ca', rid=r_id, unit=unit)
4328+ if ca:
4329+ return True
4330+ for r_id in relation_ids('identity-service'):
4331+ for unit in relation_list(r_id):
4332+ # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN
4333+ rel_state = [
4334+ relation_get('https_keystone', rid=r_id, unit=unit),
4335+ relation_get('ca_cert', rid=r_id, unit=unit),
4336+ ]
4337+ # NOTE: works around (LP: #1203241)
4338+ if (None not in rel_state) and ('' not in rel_state):
4339+ return True
4340+ return False
4341+
4342+
4343+def determine_api_port(public_port, singlenode_mode=False):
4344+ '''
4345+ Determine correct API server listening port based on
4346+ existence of HTTPS reverse proxy and/or haproxy.
4347+
4348+ public_port: int: standard public port for given service
4349+
4350+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
4351+
4352+ returns: int: the correct listening port for the API service
4353+ '''
4354+ i = 0
4355+ if singlenode_mode:
4356+ i += 1
4357+ elif len(peer_units()) > 0 or is_clustered():
4358+ i += 1
4359+ if https():
4360+ i += 1
4361+ return public_port - (i * 10)
4362+
4363+
4364+def determine_apache_port(public_port, singlenode_mode=False):
4365+ '''
4366+ Description: Determine correct apache listening port based on public IP +
4367+ state of the cluster.
4368+
4369+ public_port: int: standard public port for given service
4370+
4371+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
4372+
4373+ returns: int: the correct listening port for the HAProxy service
4374+ '''
4375+ i = 0
4376+ if singlenode_mode:
4377+ i += 1
4378+ elif len(peer_units()) > 0 or is_clustered():
4379+ i += 1
4380+ return public_port - (i * 10)
4381+
4382+
4383+determine_apache_port_single = functools.partial(
4384+ determine_apache_port, singlenode_mode=True)
4385+
4386+
4387+def get_hacluster_config(exclude_keys=None):
4388+ '''
4389+ Obtains all relevant configuration from charm configuration required
4390+ for initiating a relation to hacluster:
4391+
4392+ ha-bindiface, ha-mcastport, vip, os-internal-hostname,
4393+ os-admin-hostname, os-public-hostname, os-access-hostname
4394+
4395+ param: exclude_keys: list of setting key(s) to be excluded.
4396+ returns: dict: A dict containing settings keyed by setting name.
4397+ raises: HAIncompleteConfig if settings are missing or incorrect.
4398+ '''
4399+ settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname',
4400+ 'os-admin-hostname', 'os-public-hostname', 'os-access-hostname']
4401+ conf = {}
4402+ for setting in settings:
4403+ if exclude_keys and setting in exclude_keys:
4404+ continue
4405+
4406+ conf[setting] = config_get(setting)
4407+
4408+ if not valid_hacluster_config():
4409+ raise HAIncorrectConfig('Insufficient or incorrect config data to '
4410+ 'configure hacluster.')
4411+ return conf
4412+
4413+
4414+def valid_hacluster_config():
4415+ '''
4416+ Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname
4417+ must be set.
4418+
4419+ Note: ha-bindiface and ha-macastport both have defaults and will always
4420+ be set. We only care that either vip or dns-ha is set.
4421+
4422+ :returns: boolean: valid config returns true.
4423+ raises: HAIncompatibileConfig if settings conflict.
4424+ raises: HAIncompleteConfig if settings are missing.
4425+ '''
4426+ vip = config_get('vip')
4427+ dns = config_get('dns-ha')
4428+ if not(bool(vip) ^ bool(dns)):
4429+ msg = ('HA: Either vip or dns-ha must be set but not both in order to '
4430+ 'use high availability')
4431+ status_set('blocked', msg)
4432+ raise HAIncorrectConfig(msg)
4433+
4434+ # If dns-ha then one of os-*-hostname must be set
4435+ if dns:
4436+ dns_settings = ['os-internal-hostname', 'os-admin-hostname',
4437+ 'os-public-hostname', 'os-access-hostname']
4438+ # At this point it is unknown if one or all of the possible
4439+ # network spaces are in HA. Validate at least one is set which is
4440+ # the minimum required.
4441+ for setting in dns_settings:
4442+ if config_get(setting):
4443+ log('DNS HA: At least one hostname is set {}: {}'
4444+ ''.format(setting, config_get(setting)),
4445+ level=DEBUG)
4446+ return True
4447+
4448+ msg = ('DNS HA: At least one os-*-hostname(s) must be set to use '
4449+ 'DNS HA')
4450+ status_set('blocked', msg)
4451+ raise HAIncompleteConfig(msg)
4452+
4453+ log('VIP HA: VIP is set {}'.format(vip), level=DEBUG)
4454+ return True
4455+
4456+
4457+def canonical_url(configs, vip_setting='vip'):
4458+ '''
4459+ Returns the correct HTTP URL to this host given the state of HTTPS
4460+ configuration and hacluster.
4461+
4462+ :configs : OSTemplateRenderer: A config tempating object to inspect for
4463+ a complete https context.
4464+
4465+ :vip_setting: str: Setting in charm config that specifies
4466+ VIP address.
4467+ '''
4468+ scheme = 'http'
4469+ if 'https' in configs.complete_contexts():
4470+ scheme = 'https'
4471+ if is_clustered():
4472+ addr = config_get(vip_setting)
4473+ else:
4474+ addr = unit_get('private-address')
4475+ return '%s://%s' % (scheme, addr)
4476+
4477+
4478+def distributed_wait(modulo=None, wait=None, operation_name='operation'):
4479+ ''' Distribute operations by waiting based on modulo_distribution
4480+
4481+ If modulo and or wait are not set, check config_get for those values.
4482+ If config values are not set, default to modulo=3 and wait=30.
4483+
4484+ :param modulo: int The modulo number creates the group distribution
4485+ :param wait: int The constant time wait value
4486+ :param operation_name: string Operation name for status message
4487+ i.e. 'restart'
4488+ :side effect: Calls config_get()
4489+ :side effect: Calls log()
4490+ :side effect: Calls status_set()
4491+ :side effect: Calls time.sleep()
4492+ '''
4493+ if modulo is None:
4494+ modulo = config_get('modulo-nodes') or 3
4495+ if wait is None:
4496+ wait = config_get('known-wait') or 30
4497+ if juju_is_leader():
4498+ # The leader should never wait
4499+ calculated_wait = 0
4500+ else:
4501+ # non_zero_wait=True guarantees the non-leader who gets modulo 0
4502+ # will still wait
4503+ calculated_wait = modulo_distribution(modulo=modulo, wait=wait,
4504+ non_zero_wait=True)
4505+ msg = "Waiting {} seconds for {} ...".format(calculated_wait,
4506+ operation_name)
4507+ log(msg, DEBUG)
4508+ status_set('maintenance', msg)
4509+ time.sleep(calculated_wait)
4510+
4511+
4512+def get_managed_services_and_ports(services, external_ports,
4513+ external_services=None,
4514+ port_conv_f=determine_apache_port_single):
4515+ """Get the services and ports managed by this charm.
4516+
4517+ Return only the services and corresponding ports that are managed by this
4518+ charm. This excludes haproxy when there is a relation with hacluster. This
4519+ is because this charm passes responsability for stopping and starting
4520+ haproxy to hacluster.
4521+
4522+ Similarly, if a relation with hacluster exists then the ports returned by
4523+ this method correspond to those managed by the apache server rather than
4524+ haproxy.
4525+
4526+ :param services: List of services.
4527+ :type services: List[str]
4528+ :param external_ports: List of ports managed by external services.
4529+ :type external_ports: List[int]
4530+ :param external_services: List of services to be removed if ha relation is
4531+ present.
4532+ :type external_services: List[str]
4533+ :param port_conv_f: Function to apply to ports to calculate the ports
4534+ managed by services controlled by this charm.
4535+ :type port_convert_func: f()
4536+ :returns: A tuple containing a list of services first followed by a list of
4537+ ports.
4538+ :rtype: Tuple[List[str], List[int]]
4539+ """
4540+ if external_services is None:
4541+ external_services = ['haproxy']
4542+ if relation_ids('ha'):
4543+ for svc in external_services:
4544+ try:
4545+ services.remove(svc)
4546+ except ValueError:
4547+ pass
4548+ external_ports = [port_conv_f(p) for p in external_ports]
4549+ return services, external_ports
4550
4551=== added directory 'lib/charmhelpers/contrib/hardening'
4552=== added file 'lib/charmhelpers/contrib/hardening/README.hardening.md'
4553--- lib/charmhelpers/contrib/hardening/README.hardening.md 1970-01-01 00:00:00 +0000
4554+++ lib/charmhelpers/contrib/hardening/README.hardening.md 2021-06-24 14:42:02 +0000
4555@@ -0,0 +1,38 @@
4556+# Juju charm-helpers hardening library
4557+
4558+## Description
4559+
4560+This library provides multiple implementations of system and application
4561+hardening that conform to the standards of http://hardening.io/.
4562+
4563+Current implementations include:
4564+
4565+ * OS
4566+ * SSH
4567+ * MySQL
4568+ * Apache
4569+
4570+## Requirements
4571+
4572+* Juju Charms
4573+
4574+## Usage
4575+
4576+1. Synchronise this library into your charm and add the harden() decorator
4577+ (from contrib.hardening.harden) to any functions or methods you want to use
4578+ to trigger hardening of your application/system.
4579+
4580+2. Add a config option called 'harden' to your charm config.yaml and set it to
4581+ a space-delimited list of hardening modules you want to run e.g. "os ssh"
4582+
4583+3. Override any config defaults (contrib.hardening.defaults) by adding a file
4584+ called hardening.yaml to your charm root containing the name(s) of the
4585+ modules whose settings you want override at root level and then any settings
4586+ with overrides e.g.
4587+
4588+ os:
4589+ general:
4590+ desktop_enable: True
4591+
4592+4. Now just run your charm as usual and hardening will be applied each time the
4593+ hook runs.
4594
4595=== added file 'lib/charmhelpers/contrib/hardening/__init__.py'
4596--- lib/charmhelpers/contrib/hardening/__init__.py 1970-01-01 00:00:00 +0000
4597+++ lib/charmhelpers/contrib/hardening/__init__.py 2021-06-24 14:42:02 +0000
4598@@ -0,0 +1,13 @@
4599+# Copyright 2016 Canonical Limited.
4600+#
4601+# Licensed under the Apache License, Version 2.0 (the "License");
4602+# you may not use this file except in compliance with the License.
4603+# You may obtain a copy of the License at
4604+#
4605+# http://www.apache.org/licenses/LICENSE-2.0
4606+#
4607+# Unless required by applicable law or agreed to in writing, software
4608+# distributed under the License is distributed on an "AS IS" BASIS,
4609+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4610+# See the License for the specific language governing permissions and
4611+# limitations under the License.
4612
4613=== added directory 'lib/charmhelpers/contrib/hardening/apache'
4614=== added file 'lib/charmhelpers/contrib/hardening/apache/__init__.py'
4615--- lib/charmhelpers/contrib/hardening/apache/__init__.py 1970-01-01 00:00:00 +0000
4616+++ lib/charmhelpers/contrib/hardening/apache/__init__.py 2021-06-24 14:42:02 +0000
4617@@ -0,0 +1,17 @@
4618+# Copyright 2016 Canonical Limited.
4619+#
4620+# Licensed under the Apache License, Version 2.0 (the "License");
4621+# you may not use this file except in compliance with the License.
4622+# You may obtain a copy of the License at
4623+#
4624+# http://www.apache.org/licenses/LICENSE-2.0
4625+#
4626+# Unless required by applicable law or agreed to in writing, software
4627+# distributed under the License is distributed on an "AS IS" BASIS,
4628+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4629+# See the License for the specific language governing permissions and
4630+# limitations under the License.
4631+
4632+from os import path
4633+
4634+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')
4635
4636=== added directory 'lib/charmhelpers/contrib/hardening/apache/checks'
4637=== added file 'lib/charmhelpers/contrib/hardening/apache/checks/__init__.py'
4638--- lib/charmhelpers/contrib/hardening/apache/checks/__init__.py 1970-01-01 00:00:00 +0000
4639+++ lib/charmhelpers/contrib/hardening/apache/checks/__init__.py 2021-06-24 14:42:02 +0000
4640@@ -0,0 +1,29 @@
4641+# Copyright 2016 Canonical Limited.
4642+#
4643+# Licensed under the Apache License, Version 2.0 (the "License");
4644+# you may not use this file except in compliance with the License.
4645+# You may obtain a copy of the License at
4646+#
4647+# http://www.apache.org/licenses/LICENSE-2.0
4648+#
4649+# Unless required by applicable law or agreed to in writing, software
4650+# distributed under the License is distributed on an "AS IS" BASIS,
4651+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4652+# See the License for the specific language governing permissions and
4653+# limitations under the License.
4654+
4655+from charmhelpers.core.hookenv import (
4656+ log,
4657+ DEBUG,
4658+)
4659+from charmhelpers.contrib.hardening.apache.checks import config
4660+
4661+
4662+def run_apache_checks():
4663+ log("Starting Apache hardening checks.", level=DEBUG)
4664+ checks = config.get_audits()
4665+ for check in checks:
4666+ log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
4667+ check.ensure_compliance()
4668+
4669+ log("Apache hardening checks complete.", level=DEBUG)
4670
4671=== added file 'lib/charmhelpers/contrib/hardening/apache/checks/config.py'
4672--- lib/charmhelpers/contrib/hardening/apache/checks/config.py 1970-01-01 00:00:00 +0000
4673+++ lib/charmhelpers/contrib/hardening/apache/checks/config.py 2021-06-24 14:42:02 +0000
4674@@ -0,0 +1,104 @@
4675+# Copyright 2016 Canonical Limited.
4676+#
4677+# Licensed under the Apache License, Version 2.0 (the "License");
4678+# you may not use this file except in compliance with the License.
4679+# You may obtain a copy of the License at
4680+#
4681+# http://www.apache.org/licenses/LICENSE-2.0
4682+#
4683+# Unless required by applicable law or agreed to in writing, software
4684+# distributed under the License is distributed on an "AS IS" BASIS,
4685+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4686+# See the License for the specific language governing permissions and
4687+# limitations under the License.
4688+
4689+import os
4690+import re
4691+import six
4692+import subprocess
4693+
4694+
4695+from charmhelpers.core.hookenv import (
4696+ log,
4697+ INFO,
4698+)
4699+from charmhelpers.contrib.hardening.audits.file import (
4700+ FilePermissionAudit,
4701+ DirectoryPermissionAudit,
4702+ NoReadWriteForOther,
4703+ TemplatedFile,
4704+ DeletedFile
4705+)
4706+from charmhelpers.contrib.hardening.audits.apache import DisabledModuleAudit
4707+from charmhelpers.contrib.hardening.apache import TEMPLATES_DIR
4708+from charmhelpers.contrib.hardening import utils
4709+
4710+
4711+def get_audits():
4712+ """Get Apache hardening config audits.
4713+
4714+ :returns: dictionary of audits
4715+ """
4716+ if subprocess.call(['which', 'apache2'], stdout=subprocess.PIPE) != 0:
4717+ log("Apache server does not appear to be installed on this node - "
4718+ "skipping apache hardening", level=INFO)
4719+ return []
4720+
4721+ context = ApacheConfContext()
4722+ settings = utils.get_settings('apache')
4723+ audits = [
4724+ FilePermissionAudit(paths=os.path.join(
4725+ settings['common']['apache_dir'], 'apache2.conf'),
4726+ user='root', group='root', mode=0o0640),
4727+
4728+ TemplatedFile(os.path.join(settings['common']['apache_dir'],
4729+ 'mods-available/alias.conf'),
4730+ context,
4731+ TEMPLATES_DIR,
4732+ mode=0o0640,
4733+ user='root',
4734+ service_actions=[{'service': 'apache2',
4735+ 'actions': ['restart']}]),
4736+
4737+ TemplatedFile(os.path.join(settings['common']['apache_dir'],
4738+ 'conf-enabled/99-hardening.conf'),
4739+ context,
4740+ TEMPLATES_DIR,
4741+ mode=0o0640,
4742+ user='root',
4743+ service_actions=[{'service': 'apache2',
4744+ 'actions': ['restart']}]),
4745+
4746+ DirectoryPermissionAudit(settings['common']['apache_dir'],
4747+ user='root',
4748+ group='root',
4749+ mode=0o0750),
4750+
4751+ DisabledModuleAudit(settings['hardening']['modules_to_disable']),
4752+
4753+ NoReadWriteForOther(settings['common']['apache_dir']),
4754+
4755+ DeletedFile(['/var/www/html/index.html'])
4756+ ]
4757+
4758+ return audits
4759+
4760+
4761+class ApacheConfContext(object):
4762+ """Defines the set of key/value pairs to set in a apache config file.
4763+
4764+ This context, when called, will return a dictionary containing the
4765+ key/value pairs of setting to specify in the
4766+ /etc/apache/conf-enabled/hardening.conf file.
4767+ """
4768+ def __call__(self):
4769+ settings = utils.get_settings('apache')
4770+ ctxt = settings['hardening']
4771+
4772+ out = subprocess.check_output(['apache2', '-v'])
4773+ if six.PY3:
4774+ out = out.decode('utf-8')
4775+ ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
4776+ out).group(1)
4777+ ctxt['apache_icondir'] = '/usr/share/apache2/icons/'
4778+ return ctxt
4779
4780=== added directory 'lib/charmhelpers/contrib/hardening/apache/templates'
4781=== added file 'lib/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf'
4782--- lib/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf 1970-01-01 00:00:00 +0000
4783+++ lib/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf 2021-06-24 14:42:02 +0000
4784@@ -0,0 +1,32 @@
4785+###############################################################################
4786+# WARNING: This configuration file is maintained by Juju. Local changes may
4787+# be overwritten.
4788+###############################################################################
4789+
4790+<Location / >
4791+ <LimitExcept {{ allowed_http_methods }} >
4792+ # http://httpd.apache.org/docs/2.4/upgrading.html
4793+ {% if apache_version > '2.2' -%}
4794+ Require all granted
4795+ {% else -%}
4796+ Order Allow,Deny
4797+ Deny from all
4798+ {% endif %}
4799+ </LimitExcept>
4800+</Location>
4801+
4802+<Directory />
4803+ Options -Indexes -FollowSymLinks
4804+ AllowOverride None
4805+</Directory>
4806+
4807+<Directory /var/www/>
4808+ Options -Indexes -FollowSymLinks
4809+ AllowOverride None
4810+</Directory>
4811+
4812+TraceEnable {{ traceenable }}
4813+ServerTokens {{ servertokens }}
4814+
4815+SSLHonorCipherOrder {{ honor_cipher_order }}
4816+SSLCipherSuite {{ cipher_suite }}
4817
4818=== added file 'lib/charmhelpers/contrib/hardening/apache/templates/alias.conf'
4819--- lib/charmhelpers/contrib/hardening/apache/templates/alias.conf 1970-01-01 00:00:00 +0000
4820+++ lib/charmhelpers/contrib/hardening/apache/templates/alias.conf 2021-06-24 14:42:02 +0000
4821@@ -0,0 +1,31 @@
4822+###############################################################################
4823+# WARNING: This configuration file is maintained by Juju. Local changes may
4824+# be overwritten.
4825+###############################################################################
4826+<IfModule alias_module>
4827+ #
4828+ # Aliases: Add here as many aliases as you need (with no limit). The format is
4829+ # Alias fakename realname
4830+ #
4831+ # Note that if you include a trailing / on fakename then the server will
4832+ # require it to be present in the URL. So "/icons" isn't aliased in this
4833+ # example, only "/icons/". If the fakename is slash-terminated, then the
4834+ # realname must also be slash terminated, and if the fakename omits the
4835+ # trailing slash, the realname must also omit it.
4836+ #
4837+ # We include the /icons/ alias for FancyIndexed directory listings. If
4838+ # you do not use FancyIndexing, you may comment this out.
4839+ #
4840+ Alias /icons/ "{{ apache_icondir }}/"
4841+
4842+ <Directory "{{ apache_icondir }}">
4843+ Options -Indexes -MultiViews -FollowSymLinks
4844+ AllowOverride None
4845+{% if apache_version == '2.4' -%}
4846+ Require all granted
4847+{% else -%}
4848+ Order allow,deny
4849+ Allow from all
4850+{% endif %}
4851+ </Directory>
4852+</IfModule>
4853
4854=== added directory 'lib/charmhelpers/contrib/hardening/audits'
4855=== added file 'lib/charmhelpers/contrib/hardening/audits/__init__.py'
4856--- lib/charmhelpers/contrib/hardening/audits/__init__.py 1970-01-01 00:00:00 +0000
4857+++ lib/charmhelpers/contrib/hardening/audits/__init__.py 2021-06-24 14:42:02 +0000
4858@@ -0,0 +1,54 @@
4859+# Copyright 2016 Canonical Limited.
4860+#
4861+# Licensed under the Apache License, Version 2.0 (the "License");
4862+# you may not use this file except in compliance with the License.
4863+# You may obtain a copy of the License at
4864+#
4865+# http://www.apache.org/licenses/LICENSE-2.0
4866+#
4867+# Unless required by applicable law or agreed to in writing, software
4868+# distributed under the License is distributed on an "AS IS" BASIS,
4869+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4870+# See the License for the specific language governing permissions and
4871+# limitations under the License.
4872+
4873+
4874+class BaseAudit(object): # NO-QA
4875+ """Base class for hardening checks.
4876+
4877+ The lifecycle of a hardening check is to first check to see if the system
4878+ is in compliance for the specified check. If it is not in compliance, the
4879+ check method will return a value which will be supplied to the.
4880+ """
4881+ def __init__(self, *args, **kwargs):
4882+ self.unless = kwargs.get('unless', None)
4883+ super(BaseAudit, self).__init__()
4884+
4885+ def ensure_compliance(self):
4886+ """Checks to see if the current hardening check is in compliance or
4887+ not.
4888+
4889+ If the check that is performed is not in compliance, then an exception
4890+ should be raised.
4891+ """
4892+ pass
4893+
4894+ def _take_action(self):
4895+ """Determines whether to perform the action or not.
4896+
4897+ Checks whether or not an action should be taken. This is determined by
4898+ the truthy value for the unless parameter. If unless is a callback
4899+ method, it will be invoked with no parameters in order to determine
4900+ whether or not the action should be taken. Otherwise, the truthy value
4901+ of the unless attribute will determine if the action should be
4902+ performed.
4903+ """
4904+ # Do the action if there isn't an unless override.
4905+ if self.unless is None:
4906+ return True
4907+
4908+ # Invoke the callback if there is one.
4909+ if hasattr(self.unless, '__call__'):
4910+ return not self.unless()
4911+
4912+ return not self.unless
4913
4914=== added file 'lib/charmhelpers/contrib/hardening/audits/apache.py'
4915--- lib/charmhelpers/contrib/hardening/audits/apache.py 1970-01-01 00:00:00 +0000
4916+++ lib/charmhelpers/contrib/hardening/audits/apache.py 2021-06-24 14:42:02 +0000
4917@@ -0,0 +1,105 @@
4918+# Copyright 2016 Canonical Limited.
4919+#
4920+# Licensed under the Apache License, Version 2.0 (the "License");
4921+# you may not use this file except in compliance with the License.
4922+# You may obtain a copy of the License at
4923+#
4924+# http://www.apache.org/licenses/LICENSE-2.0
4925+#
4926+# Unless required by applicable law or agreed to in writing, software
4927+# distributed under the License is distributed on an "AS IS" BASIS,
4928+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4929+# See the License for the specific language governing permissions and
4930+# limitations under the License.
4931+
4932+import re
4933+import subprocess
4934+
4935+import six
4936+
4937+from charmhelpers.core.hookenv import (
4938+ log,
4939+ INFO,
4940+ ERROR,
4941+)
4942+
4943+from charmhelpers.contrib.hardening.audits import BaseAudit
4944+
4945+
4946+class DisabledModuleAudit(BaseAudit):
4947+ """Audits Apache2 modules.
4948+
4949+ Determines if the apache2 modules are enabled. If the modules are enabled
4950+ then they are removed in the ensure_compliance.
4951+ """
4952+ def __init__(self, modules):
4953+ if modules is None:
4954+ self.modules = []
4955+ elif isinstance(modules, six.string_types):
4956+ self.modules = [modules]
4957+ else:
4958+ self.modules = modules
4959+
4960+ def ensure_compliance(self):
4961+ """Ensures that the modules are not loaded."""
4962+ if not self.modules:
4963+ return
4964+
4965+ try:
4966+ loaded_modules = self._get_loaded_modules()
4967+ non_compliant_modules = []
4968+ for module in self.modules:
4969+ if module in loaded_modules:
4970+ log("Module '%s' is enabled but should not be." %
4971+ (module), level=INFO)
4972+ non_compliant_modules.append(module)
4973+
4974+ if len(non_compliant_modules) == 0:
4975+ return
4976+
4977+ for module in non_compliant_modules:
4978+ self._disable_module(module)
4979+ self._restart_apache()
4980+ except subprocess.CalledProcessError as e:
4981+ log('Error occurred auditing apache module compliance. '
4982+ 'This may have been already reported. '
4983+ 'Output is: %s' % e.output, level=ERROR)
4984+
4985+ @staticmethod
4986+ def _get_loaded_modules():
4987+ """Returns the modules which are enabled in Apache."""
4988+ output = subprocess.check_output(['apache2ctl', '-M'])
4989+ if six.PY3:
4990+ output = output.decode('utf-8')
4991+ modules = []
4992+ for line in output.splitlines():
4993+ # Each line of the enabled module output looks like:
4994+ # module_name (static|shared)
4995+ # Plus a header line at the top of the output which is stripped
4996+ # out by the regex.
4997+ matcher = re.search(r'^ (\S*)_module (\S*)', line)
4998+ if matcher:
4999+ modules.append(matcher.group(1))
5000+ return modules
The diff has been truncated for viewing.

Subscribers

People subscribed via source and target branches