[Xfce4-commits] <xfce-buildbot-scripts:master> Implement XMLRPC service to distribute build requests sent by the GIT hook script to the appropriate buildbot instances
Enrico Tröger
noreply at xfce.org
Sun Oct 11 23:46:01 CEST 2009
Updating branch refs/heads/master
to 6c3d5f6a08b358cb54ab50eda63c3ce639b0c698 (commit)
from 091dedc7629ed5f1a34cd2e535fd56e137eeb53d (commit)
commit 6c3d5f6a08b358cb54ab50eda63c3ce639b0c698
Author: Enrico Tröger <enrico.troeger at uvena.de>
Date: Sun Oct 11 23:42:21 2009 +0200
Implement XMLRPC service to distribute build requests sent by the GIT hook script to the appropriate buildbot instances
The new buildbot infrastructure will use separate buildbot instances
for each module because each module has its own GIT repository but
Buildbot seems to not support such an infrastructure.
To be able to keep one buildbot hook script for all GIT repositories,
this hook script sends the change request to this service which then
forward the request to the approrpiate buildbot instance.
xfbuildservice/buildbot_client.py | 67 +++++++++++++++++++++++++
xfbuildservice/contants.py | 12 +++++
xfbuildservice/doc.txt | 87 +++++++++++++++++++++++++++++++++
xfbuildservice/xfbuildservice.conf | 6 ++
xfbuildservice/xfbuildservice.py | 94 ++++++++++++++++++++++++++++++++++++
5 files changed, 266 insertions(+), 0 deletions(-)
diff --git a/xfbuildservice/buildbot_client.py b/xfbuildservice/buildbot_client.py
new file mode 100644
index 0000000..ed852be
--- /dev/null
+++ b/xfbuildservice/buildbot_client.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Part of the Xfce Buildbot Service
+# Send data to a buildbot instance to trigger a new build. The data is received
+# via a XMLRPC service which is called from GIT hook scripts.
+#
+# Copyright 2009 Enrico Tröger <enrico(at)xfce(dot)org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# Largely based on Buildbot's contrib/git_buildbot.py script, mainly stripped down unneeded parts
+
+from twisted.spread import pb
+from twisted.cred import credentials
+from twisted.internet import reactor
+from optparse import OptionParser
+
+changes = []
+
+#----------------------------------------------------------------------
+def send_changes(host, port, user, passwd, _changes):
+ global changes
+
+ changes = _changes
+
+ # perform login
+ f = pb.PBClientFactory()
+ d = f.login(credentials.UsernamePassword(user, passwd))
+ reactor.connectTCP(host, port, f)
+
+ # connect callbacks and start
+ d.addCallback(connected)
+ d.addBoth(cleanup)
+ reactor.run()
+
+#----------------------------------------------------------------------
+def addChange(dummy, remote, changei):
+ try:
+ c = changei.next()
+ except StopIteration:
+ remote.broker.transport.loseConnection()
+ return None
+
+ d = remote.callRemote('addChange', c)
+ d.addCallback(addChange, remote, changei)
+ return d
+
+#----------------------------------------------------------------------
+def connected(remote):
+ return addChange(None, remote, changes.__iter__())
+
+#----------------------------------------------------------------------
+def cleanup(res):
+ reactor.stop()
+
diff --git a/xfbuildservice/contants.py b/xfbuildservice/contants.py
new file mode 100644
index 0000000..619bc4b
--- /dev/null
+++ b/xfbuildservice/contants.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+
+
+XF_RESULT_OK = 0
+XF_RESULT_INVALID_INPUT = 1
+
+result_codes = {
+ XF_RESULT_OK: 'Success',
+ XF_RESULT_INVALID_INPUT: 'Invalid input'
+}
diff --git a/xfbuildservice/doc.txt b/xfbuildservice/doc.txt
new file mode 100644
index 0000000..8a3a1f6
--- /dev/null
+++ b/xfbuildservice/doc.txt
@@ -0,0 +1,87 @@
+Xfce Buildbot Task Distribution Service
+=======================================
+
+
+Rationale
+---------
+
+For some reasons (e.g Xfce uses multiple GIT repositories within one URL)
+we need to use separate buildbot instances for each module in the GIT
+repositories to get them working reliable and correctly.
+
+This change will require that the GIT hook scripts would connect to their
+corresponding buildbot in order to trigger a new build after a commit has
+been done. To avoid too heavy changes on the GIT hook scripts side, we
+decided to deploy a new service which collects the hook scripts triggers
+from the various modules and deliver them to the appropriate buildbot
+instances.
+
+This way the GIT hook scripts don't need to know anything about the
+buildbot infrastructure, so less action and logic is necessary. Furthermore,
+this ensures the only communication between the Xfce GIT server and
+the Buildbot server is between the GIT hook scripts and the service described
+here which will run on the buildbot server.
+
+The service receives calls the necessary information (module name,
+old revision, new revision, branch, ...) from the GIT hook scripts via XMLRPC.
+Then the received information are passed to the buildbot instance responsible
+for the corresponding module.
+
+
+
+Implementation
+--------------
+
+The service will be implemented as a XMLRPC service using Python 2.5.
+It will run on the Xfce Buildbot server (buildbot.xfce.org) and listen
+for incoming requests sent by the GIT hook scripts.
+To ensure only valid requests are accepted, only connections from the Xfce GIT
+server are accepted. Additionally, the communication will be encrypted via SSL.
+
+The received build requests are forwarded to the corresponding buildbot
+instances running on localhost (i.e. buildbot.xfce.org) to actually
+trigger the build. The corresponding buildbot instance is determined by the
+passed module name inside the request.
+
+
+Request data
+^^^^^^^^^^^^
+
+The following information are transmitted in the XMLRPC calls from the
+GIT hook scripts to the service:
+
+'module name' - (str) mandatory, e.g. "xfce4-panel"
+'revision' - (str) mandatory (buildbot specific)
+'comments' - (str) mandatory (buildbot specific)
+'branch' - (str) mandatory (buildbot specific)
+'category' - (str) optional (buildbot specific)
+'files' - (str) optional (buildbot specific)
+'who' - (str) optional (buildbot specific)
+
+The above fields are packed into a list of structures (like a Python dictionary or
+an associative array). For example::
+
+ [
+ {
+ 'module_name' => 'thunar',
+ 'revision' => 'ab12cd34',
+ 'branch' => 'master',
+ 'comments' => 'First commit message'
+ },
+ {
+ 'module_name' => 'thunar',
+ 'revision' => 'fe5678ab',
+ 'branch' => 'master',
+ 'comments' => 'Second commit message'
+ }
+ {
+ 'module_name' => 'thunar',
+ 'revision' => 'ba4712f2',
+ 'branch' => 'master',
+ 'comments' => 'Rewind branch'
+ }
+ ]
+
+The XMLRPC is called "request_build" and returns a structure containing
+two fields "result_code" (int) and "result_message" (string) describing
+whether the operation was successful or not.
diff --git a/xfbuildservice/xfbuildservice.conf b/xfbuildservice/xfbuildservice.conf
new file mode 100644
index 0000000..7cdf914
--- /dev/null
+++ b/xfbuildservice/xfbuildservice.conf
@@ -0,0 +1,6 @@
+# $Id: xnmapi.conf 1084 2009-10-05 12:34:39Z etroeger $
+
+[buildbot_modulename]
+host: localhost:9990
+user: username
+passwd: passwd
diff --git a/xfbuildservice/xfbuildservice.py b/xfbuildservice/xfbuildservice.py
new file mode 100644
index 0000000..563a700
--- /dev/null
+++ b/xfbuildservice/xfbuildservice.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+
+from SimpleXMLRPCServer import CGIXMLRPCRequestHandler
+from ConfigParser import SafeConfigParser
+from optparse import OptionParser
+import os
+
+from contants import *
+from buildbot_client import send_changes
+
+
+class XfBuildService(object):
+ #----------------------------------------------------------------------
+ def __init__(self, config):
+ self.config = config
+
+ #----------------------------------------------------------------------
+ def request_build(self, data):
+ """
+ XMLRPC call to receive data from the caller and pass it to the
+ appropriate buildbot instance
+
+ @param data (list)
+ """
+ if not data or not isinstance(data, list):
+ return get_result(XF_RESULT_INVALID_INPUT)
+
+ modules = {}
+ for change in data:
+ module = change['module_name']
+ del change['module_name']
+ try:
+ modules[module].append(change)
+ except KeyError:
+ # create item
+ modules[module] = [ change ]
+
+ for module, changes in modules.items():
+ section = 'buildbot_%s' % module
+ send_changes(
+ self.config.get(section, 'host'),
+ self.config.getint(section, 'port'),
+ self.config.get(section, 'user'),
+ self.config.get(section, 'passwd'),
+ changes
+ )
+
+ return { 'result_code': 0, 'result_message': 'success' }
+
+#----------------------------------------------------------------------
+def get_result(code):
+ """
+ Get result dictionary for the XMLRPC reply
+
+ @param result (dict)
+ """
+ return { 'result_code': code, 'result_message': result_codes[code] }
+
+#----------------------------------------------------------------------
+def setup_options(parser):
+ """
+ Set up options and defaults
+
+ @param parser (optparse.OptionParser())
+ """
+ parser.add_option(
+ "-c", dest="config",
+ default=os.path.join(os.path.dirname(__file__), 'xfbuildservice.conf'),
+ help=u"configuration file")
+
+#----------------------------------------------------------------------
+def main():
+ # arguments
+ option_parser = OptionParser()
+ setup_options(option_parser)
+ arg_options = option_parser.parse_args()[0]
+
+ # configuration
+ config = SafeConfigParser()
+ if not os.path.exists(arg_options.config):
+ raise RuntimeError(u'Configuration file does not exist')
+ config.read(arg_options.config)
+
+ # XMLRPC handler
+ xfbs = XfBuildService(config)
+ handler = CGIXMLRPCRequestHandler()
+ handler.register_instance(xfbs)
+ handler.register_introspection_functions()
+ handler.handle_request()
+
+
+main()
More information about the Xfce4-commits
mailing list