Charm: ~yellow:oneiric/buildbot-slave
Revision: 9
Hook: local.py
# Copyright 2012 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Shared functions for the buildbot master and slave"""
__metaclass__ = type
__all__ = [
'buildbot_reconfig',
'buildslave_start',
'buildslave_stop',
'config_json',
'create_slave',
'fetch_history',
'generate_string',
'get_bucket',
'get_key',
'get_wrapper_cfg_path',
'HTTP_PORT_PROTOCOL',
'put_history',
'slave_json',
'WRAPPER_FN',
]
import os
import re
import subprocess
import uuid
from shelltoolbox import (
cd,
run,
Serializer,
su,
)
from helpers import (
get_config,
log,
)
HTTP_PORT_PROTOCOL = '8010/TCP'
WRAPPER_FN = 'wrapper.cfg'
def _get_buildbot_dir():
config = get_config()
return config.get('installdir')
def _get_tac_filename(buildbot_dir):
return os.path.join(buildbot_dir, 'buildbot.tac')
def get_wrapper_cfg_path():
return os.path.join(
_get_buildbot_dir(), WRAPPER_FN)
def generate_string(prefix=""):
"""Generate a unique string and return it."""
return prefix + uuid.uuid4().hex
def buildbot_create(buildbot_dir):
"""Create a buildbot instance in `buildbot_dir`."""
if not os.path.exists(_get_tac_filename(buildbot_dir)):
with su('buildbot'):
return run(
'buildbot', 'create-master', '--config', WRAPPER_FN,
buildbot_dir)
def buildbot_running(buildbot_dir):
pidfile = os.path.join(buildbot_dir, 'twistd.pid')
if os.path.exists(pidfile):
buildbot_pid = open(pidfile).read().strip()
try:
# Is buildbot running?
run('kill', '-0', buildbot_pid)
except subprocess.CalledProcessError:
return False
return True
return False
def buildbot_stop():
buildbot_dir = _get_buildbot_dir()
if buildbot_running(buildbot_dir):
# Buildbot is running, stop it.
log('--> Stopping buildbot')
with su('buildbot'):
run('buildbot', 'stop', buildbot_dir)
log('<-- Stopping buildbot')
def buildbot_reconfig():
buildbot_dir = _get_buildbot_dir()
pidfile = os.path.join(buildbot_dir, 'twistd.pid')
running = False
if os.path.exists(pidfile):
buildbot_pid = open(pidfile).read().strip()
try:
# Is buildbot running?
run('kill', '-0', buildbot_pid)
except subprocess.CalledProcessError:
# Buildbot isn't running, so no need to reconfigure it.
pass
else:
# Buildbot is running, reconfigure it.
log('--> Reconfiguring buildbot')
with su('buildbot'):
# Reconfig is broken in 0.8.3 (Oneiric), so we can't do this:
# run('buildbot', 'reconfig', buildbot_dir)
run('buildbot', 'stop', buildbot_dir)
run('buildbot', 'start', buildbot_dir)
log('<-- Reconfiguring buildbot')
running = True
if not running:
# Buildbot isn't running so start afresh.
if os.path.exists(os.path.join(buildbot_dir, 'master.cfg')):
log('--> Starting buildbot')
with su('buildbot'):
run('buildbot', 'start', buildbot_dir)
log('<-- Starting buildbot')
def buildslave_stop(buildbot_dir=None):
if buildbot_dir is None:
buildbot_dir = _get_buildbot_dir()
with su('buildbot'):
# In recent versions of buildbot, it provides a "buildslave" binary.
# Earlier versions have the "buildbot" binary only, providing the
# same behavior.
try:
exit_code = subprocess.call(['buildslave', 'stop', buildbot_dir])
except OSError:
exit_code = subprocess.call(['buildbot', 'stop', buildbot_dir])
tac_file = _get_tac_filename(buildbot_dir)
if os.path.exists(tac_file):
os.remove(tac_file)
return exit_code
def buildslave_start(buildbot_dir=None):
if buildbot_dir is None:
buildbot_dir = _get_buildbot_dir()
with su('buildbot'):
# In recent versions of buildbot, it provides a "buildslave" binary.
# Earlier versions have the "buildbot" binary only, providing the
# same behavior.
try:
return subprocess.call(['buildslave', 'start', buildbot_dir])
except OSError:
return subprocess.call(['buildbot', 'start', buildbot_dir])
def create_slave(name, passwd, host='localhost', buildbot_dir=None):
if buildbot_dir is None:
buildbot_dir = _get_buildbot_dir()
with su('buildbot'):
if not os.path.exists(buildbot_dir):
os.makedirs(buildbot_dir)
# In recent versions of buildbot, it provides a "buildslave" binary.
# Earlier versions have the "buildbot" binary only, providing the
# same behavior.
try:
return subprocess.call(
['buildslave',
'create-slave', buildbot_dir, host, name, passwd])
except OSError:
return subprocess.call(
['buildbot',
'create-slave', buildbot_dir, host + ":9989", name, passwd])
slave_json = Serializer('/tmp/slave_info.json')
config_json = Serializer('/tmp/config.json')
def get_bucket(config, bucket_name=None):
"""Return an S3 bucket or None."""
# Late import to ensure python-boto package has been installed.
import boto
access_key = config.get('access-key')
secret_key = config.get('secret-key')
bucket = None
if access_key and secret_key:
if bucket_name is None:
bucket_name = str(access_key + '-buildbot-history').lower()
conn = boto.connect_s3(access_key, secret_key)
bucket = conn.create_bucket(bucket_name)
log("Using bucket: " + bucket.name)
return bucket
def get_key(bucket):
"""Return an S3 key for the bucket."""
# Late import to ensure python-boto package has been installed.
from boto.s3.key import Key
key = Key(bucket)
value = os.environ['JUJU_UNIT_NAME']
key.key = value
log("Using key: " + key.key)
return key
VERSION_TO_STORE = {
'0.7': "*/builder",
'0.8': "state.sqlite",
}
def get_buildbot_version():
"""Get the major version (x.y) of buildbot and return as a string.
Return None if the output from buildbot cannot be parsed.
"""
version = None
output = run('buildbot', '--version')
match = re.search('Buildbot version: (\d+\.\d+)(\.\d+)*\n', output)
if match and len(match.groups()) > 0:
version = match.group(1)
return version
def put_history(config):
"""Put the buildbot history to an external store, if set up."""
log("put_history called")
bucket_name = config.get('bucket-name')
bucket = get_bucket(config, bucket_name)
success = False
if bucket:
key = get_key(bucket)
target = '/tmp/history-put.tgz'
version = get_buildbot_version()
store_pattern = VERSION_TO_STORE.get(version)
assert store_pattern is not None, (
"Buildbot version not supported: {}".format(version))
with cd(config['installdir']):
with su('buildbot'):
try:
run('tar', 'czf', target, store_pattern)
key.set_contents_from_filename(target)
# If would be natural to just log the success here, but we
# are su-ed to the buildbot user and that causes permission
# problems, so instead set a flag.
success = True
except subprocess.CalledProcessError as e:
print e
print e.output
raise
os.unlink(target)
else:
log("Bucket not found: " + bucket_name)
if success:
log("History stored to S3.")
def fetch_history(config, diff):
"""Fetch the buildbot history from an external store."""
# Currently only S3 is supported.
restart_required = False
log("fetching history")
if 'secret-key' not in diff.added_or_changed:
log("skipping fetch of history")
return restart_required
bucket_name = config.get('bucket-name')
bucket = get_bucket(config, bucket_name)
if bucket:
key = get_key(bucket)
if key.exists():
target = '/tmp/history-fetched.tgz'
key.get_contents_to_filename(target)
with cd(config['installdir']):
with su('buildbot'):
try:
run('tar', 'xzf', target)
except subprocess.CalledProcessError as e:
print e
print e.output
raise
os.unlink(target)
log("History fetched from S3.")
restart_required = True
else:
log("Key does not exist: " + key.key)
else:
log("Bucket not found: " + bucket_name)
return restart_required