path: root/phase2/master.cfg

# -*- python -*-
# ex: set syntax=python:

import os
import re
import sys
import base64
import subprocess
import configparser

from dateutil.tz import tzutc
from datetime import datetime, timedelta

from twisted.internet import defer
from twisted.python import log

from buildbot import locks
from buildbot.data import resultspec
from buildbot.changes import filter
from buildbot.changes.gitpoller import GitPoller
from buildbot.config import BuilderConfig
from buildbot.plugins import schedulers
from buildbot.plugins import steps
from buildbot.plugins import util
from buildbot.process import results
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Property
from buildbot.process.properties import Interpolate
from buildbot.process import properties
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.shell import SetPropertyFromCommand
from buildbot.steps.shell import ShellCommand
from buildbot.steps.transfer import FileDownload
from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import StringDownload
from buildbot.worker import Worker


if not os.path.exists("twistd.pid"):
    with open("twistd.pid", "w") as pidfile:
        pidfile.write("{}".format(os.getpid()))

ini = configparser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))

buildbot_url = ini.get("phase2", "buildbot_url")

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### BUILDWORKERS

# The 'workers' list defines the set of recognized buildworkers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.

worker_port = 9990
persistent = False

if ini.has_option("phase2", "port"):
	worker_port = ini.get("phase2", "port")

if ini.has_option("phase2", "persistent"):
	persistent = ini.getboolean("phase2", "persistent")

feeds_host_override = ini.get("phase2", "feeds_host_override", fallback="").strip()

c['workers'] = []

for section in ini.sections():
	if section.startswith("worker "):
		if ini.has_option(section, "name") and ini.has_option(section, "password") and \
			ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
			name = ini.get(section, "name")
			password = ini.get(section, "password")
			sl_props = { 'shared_wd': True }

			if ini.has_option(section, "shared_wd"):
				sl_props['shared_wd'] = ini.getboolean(section, "shared_wd")

			c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))

# 'workerPortnum' defines the TCP port to listen on for connections from workers.
# This must match the value configured into the buildworkers (with their
# --master option)
c['protocols'] = {'pb': {'port': worker_port}}

# coalesce builds
c['collapseRequests'] = True

# Reduce amount of backlog data
c['configurators'] = [util.JanitorConfigurator(
    logHorizon=timedelta(days=3),
    hour=6,
)]

####### CHANGESOURCES

work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
scripts_dir = os.path.abspath("../scripts")

rsync_bin_url = ini.get("rsync", "binary_url")
rsync_bin_key = ini.get("rsync", "binary_password")

rsync_src_url = None
rsync_src_key = None

if ini.has_option("rsync", "source_url"):
	rsync_src_url = ini.get("rsync", "source_url")
	rsync_src_key = ini.get("rsync", "source_password")

rsync_sdk_url = None
rsync_sdk_key = None
rsync_sdk_pat = "openwrt-sdk-*.tar.*"

if ini.has_option("rsync", "sdk_url"):
	rsync_sdk_url = ini.get("rsync", "sdk_url")

if ini.has_option("rsync", "sdk_password"):
	rsync_sdk_key = ini.get("rsync", "sdk_password")

if ini.has_option("rsync", "sdk_pattern"):
	rsync_sdk_pat = ini.get("rsync", "sdk_pattern")

rsync_defopts = ["-4", "-v", "--timeout=120"]

repo_url = ini.get("repo", "url")
repo_branch = "main"

if ini.has_option("repo", "branch"):
	repo_branch = ini.get("repo", "branch")

if feeds_host_override:
	repo_url = re.sub(r"git\.openwrt\.org/openwrt", feeds_host_override, repo_url)

usign_key = None
usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"

if ini.has_option("usign", "key"):
	usign_key = ini.get("usign", "key")

if ini.has_option("usign", "comment"):
	usign_comment = ini.get("usign", "comment")


# find arches
arches = [ ]
archnames = [ ]

if not os.path.isdir(work_dir+'/source.git'):
	subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
else:
	subprocess.call(["git", "remote", "set-url", "origin", repo_url], cwd = work_dir+'/source.git')
	subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')

os.makedirs(work_dir+'/source.git/tmp', exist_ok=True)
findarches = subprocess.Popen(['./scripts/dump-target-info.pl', 'architectures'],
	stdout = subprocess.PIPE, cwd = work_dir+'/source.git')

while True:
	line = findarches.stdout.readline()
	if not line:
		break
	at = line.decode().strip().split()
	arches.append(at)
	archnames.append(at[0])


# find feeds
feeds = []
feedbranches = dict()

c['change_source'] = []

def parse_feed_entry(line):
	parts = line.strip().split()
	if parts[0].startswith("src-git"):
		feeds.append(parts)
		url = parts[2].strip().split(';')
		branches = [url[1]] if len(url) > 1 else ['main', 'master']
		feedbranches[url[0]] = branches
		c['change_source'].append(GitPoller(url[0], branches=branches, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollInterval=300))

make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source.git/target/sdk/', 'val.BASE_FEED'],
	env = dict(os.environ, TOPDIR=work_dir+'/source.git'), stdout = subprocess.PIPE)

line = make.stdout.readline()
if line:
	parse_feed_entry(str(line, 'utf-8'))

with open(work_dir+'/source.git/feeds.conf.default', 'r', encoding='utf-8') as f:
	for line in f:
		parse_feed_entry(line)

if len(c['change_source']) == 0:
	log.err("FATAL ERROR: no change_sources defined, aborting!")
	sys.exit(-1)

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'basebuild' build

c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
	name            = "all",
	change_filter   = filter.ChangeFilter(
		filter_fn = lambda change: change.branch in feedbranches[change.repository]
	),
	treeStableTimer = 60,
	builderNames    = archnames))

c['schedulers'].append(ForceScheduler(
	name         = "force",
	buttonName   = "Force builds",
	label        = "Force build details",
	builderNames = [ "00_force_build" ],

	codebases = [
		util.CodebaseParameter(
			"",
			label      = "Repository",
			branch     = util.FixedParameter(name = "branch",     default = ""),
			revision   = util.FixedParameter(name = "revision",   default = ""),
			repository = util.FixedParameter(name = "repository", default = ""),
			project    = util.FixedParameter(name = "project",    default = "")
		)
	],

	reason = util.StringParameter(
		name     = "reason",
		label    = "Reason",
		default  = "Trigger build",
		required = True,
		size     = 80
	),

	properties = [
		util.NestedParameter(
			name="options",
			label="Build Options",
			layout="vertical",
			fields=[
				util.ChoiceStringParameter(
					name    = "architecture",
					label   = "Build architecture",
					default = "all",
					choices = [ "all" ] + archnames
				)
			]
		)
	]
))

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them.  Note that any particular build will
# only take place on one worker.

@properties.renderer
def GetDirectorySuffix(props):
	verpat = re.compile(r'^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
	if props.hasProperty("release_version"):
		m = verpat.match(props["release_version"])
		if m is not None:
			return "-%02d.%02d" %(int(m.group(1)), int(m.group(2)))
	return ""

@properties.renderer
def GetCwd(props):
	if props.hasProperty("builddir"):
		return props["builddir"]
	elif props.hasProperty("workdir"):
		return props["workdir"]
	else:
		return "/"

def IsArchitectureSelected(target):
	def CheckArchitectureProperty(step):
		try:
			options = step.getProperty("options")
			if isinstance(options, dict):
				selected_arch = options.get("architecture", "all")
				if selected_arch != "all" and selected_arch != target:
					return False
		except KeyError:
			pass

		return True

	return CheckArchitectureProperty

def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
	try:
		seckey = base64.b64decode(seckey)
	except Exception:
		return None

	return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
		base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))

def IsSharedWorkdir(step):
	return bool(step.getProperty("shared_wd"))

def IsUsignEnabled(step):
	return ini.has_option("usign", "key")

def IsApkSigningEnabled(step):
	return ini.has_option("apk", "key")

# gpg_key - contains the key in PGP format
# gpg_keyid - contains the keyid of the key on the nk3
def IsGpgSigningEnabled(step):
	return ini.has_option("gpg", "key") or ini.has_option("gpg", "keyid")

def IsSignEnabled(step):
	return (
		IsUsignEnabled(step) or IsApkSigningEnabled(step) or IsGpgSigningEnabled(step)
	)

def IsFeedsHostOverrideEnabled(step):
	return bool(feeds_host_override)

@util.renderer
def GetFeedsHostOverride(props):
	return feeds_host_override

@defer.inlineCallbacks
def getNewestCompleteTime(bldr):
	"""Returns the complete_at of the latest completed and not SKIPPED
	build request for this builder, or None if there are no such build
	requests. We need to filter out SKIPPED requests because we're
	using collapseRequests=True which is unfortunately marking all
	previous requests as complete when new buildset is created.

	@returns: datetime instance or None, via Deferred
	"""

	bldrid = yield bldr.getBuilderId()
	completed = yield bldr.master.data.get(
			('builders', bldrid, 'buildrequests'),
			[
				resultspec.Filter('complete', 'eq', [True]),
				resultspec.Filter('results', 'ne', [results.SKIPPED]),
			],
			order=['-complete_at'], limit=1)
	if not completed:
		return

	complete_at = completed[0]['complete_at']

	last_build = yield bldr.master.data.get(
			('builds', ),
			[
				resultspec.Filter('builderid', 'eq', [bldrid]),
			],
			order=['-started_at'], limit=1)

	if last_build and last_build[0]:
		last_complete_at = last_build[0]['complete_at']
		if last_complete_at and (last_complete_at > complete_at):
			return last_complete_at

	return complete_at

@defer.inlineCallbacks
def prioritizeBuilders(master, builders):
	"""Returns sorted list of builders by their last timestamp of completed and
	not skipped build.

	@returns: list of sorted builders
	"""

	def is_building(bldr):
		return bool(bldr.building) or bool(bldr.old_building)

	def bldr_info(bldr):
		d = defer.maybeDeferred(getNewestCompleteTime, bldr)
		d.addCallback(lambda complete_at: (complete_at, bldr))
		return d

	def bldr_sort(item):
		(complete_at, bldr) = item

		if not complete_at:
			date = datetime.min
			complete_at = date.replace(tzinfo=tzutc())

		if is_building(bldr):
			date = datetime.max
			complete_at = date.replace(tzinfo=tzutc())

		return (complete_at, bldr.name)

	results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
	results.sort(key=bldr_sort)

	for r in results:
		log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))

	return [r[1] for r in results]

c['prioritizeBuilders'] = prioritizeBuilders
c['builders'] = []

dlLock = locks.WorkerLock("worker_dl")

workerNames = [ ]

for worker in c['workers']:
	workerNames.append(worker.workername)

force_factory = BuildFactory()

c['builders'].append(BuilderConfig(
	name        = "00_force_build",
	workernames = workerNames,
	factory     = force_factory))

for arch in arches:
	ts = arch[1].split('/')

	factory = BuildFactory()

	# setup shared work directory if required
	factory.addStep(ShellCommand(
		name = "sharedwd",
		description = "Setting up shared work directory",
		command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
		workdir = ".",
		haltOnFailure = True,
		doStepIf = IsSharedWorkdir))

	# find number of cores
	factory.addStep(SetPropertyFromCommand(
		name = "nproc",
		property = "nproc",
		description = "Finding number of CPUs",
		command = ["nproc"]))

	# prepare workspace
	factory.addStep(FileDownload(
		mastersrc = scripts_dir + '/cleanup.sh',
		workerdest = "../cleanup.sh",
		mode = 0o755))

	if not persistent:
		factory.addStep(ShellCommand(
			name = "cleanold",
			description = "Cleaning previous builds",
			command = ["./cleanup.sh", buildbot_url, Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
			workdir = ".",
			haltOnFailure = True,
			timeout = 2400))

		factory.addStep(ShellCommand(
			name = "cleanup",
			description = "Cleaning work area",
			command = ["./cleanup.sh", buildbot_url, Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
			workdir = ".",
			haltOnFailure = True,
			timeout = 2400))

	factory.addStep(ShellCommand(
		name = "mksdkdir",
		description = "Preparing SDK directory",
		command = ["mkdir", "-p", "sdk"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "downloadsdk",
		description = "Downloading SDK archive",
		command = ["rsync"] + rsync_defopts + ["-a", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
		env={'RSYNC_PASSWORD': rsync_sdk_key},
		haltOnFailure = True,
		logEnviron = False))

	factory.addStep(ShellCommand(
		name = "unpacksdk",
		description = "Unpacking SDK archive",
		command = "rm -rf sdk_update && mkdir sdk_update && tar --strip-components=1 -C sdk_update/ -vxf sdk.archive",
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "updatesdk",
		description = "Updating SDK",
		command = "rsync " + (" ").join(rsync_defopts) + " --checksum -a sdk_update/ sdk/ && rm -rf sdk_update",
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "cleancmdlinks",
		description = "Sanitizing host command symlinks",
		command = "find sdk/staging_dir/host/bin/ -type l -exec sh -c 'case $(readlink {}) in /bin/*|/usr/bin/*) true;; /*) rm -vf {};; esac' \\;",
		haltOnFailure = True))

	factory.addStep(StringDownload(
		name = "writeversionmk",
		s = 'TOPDIR:=${CURDIR}\n\ninclude $(TOPDIR)/include/version.mk\n\nversion:\n\t@echo $(VERSION_NUMBER)\n',
		workerdest = "sdk/getversion.mk",
		mode = 0o755))

	factory.addStep(SetPropertyFromCommand(
		name = "getversion",
		property = "release_version",
		description = "Finding SDK release version",
		workdir = "build/sdk",
		command = ["make", "-f", "getversion.mk"]))

	# install build key
	factory.addStep(StringDownload(
		name = "dlkeybuildpub",
		s = UsignSec2Pub(usign_key, usign_comment),
		workerdest = "sdk/key-build.pub",
		mode = 0o600,
		doStepIf = IsUsignEnabled))

	factory.addStep(StringDownload(
		name = "dlkeybuild",
		s = "# fake private key",
		workerdest = "sdk/key-build",
		mode = 0o600,
		doStepIf = IsUsignEnabled))

	factory.addStep(StringDownload(
		name = "dlkeybuilducert",
		s = "# fake certificate",
		workerdest = "sdk/key-build.ucert",
		mode = 0o600,
		doStepIf = IsUsignEnabled))

	factory.addStep(ShellCommand(
		name = "mkdldir",
		description = "Preparing download directory",
		command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "mkconf",
		description = "Preparing SDK configuration",
		workdir = "build/sdk",
		command = ["sh", "-c", "rm -f .config && make defconfig"]))

	factory.addStep(FileDownload(
		mastersrc = scripts_dir + '/ccache.sh',
		workerdest = 'sdk/ccache.sh',
		mode = 0o755))

	factory.addStep(ShellCommand(
		name = "prepccache",
		description = "Preparing ccache",
		workdir = "build/sdk",
		command = ["./ccache.sh"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "feeds-override",
		description = "Creating feeds.conf with host override",
		descriptionDone = "feeds.conf with override created",
		workdir = "build/sdk",
		command = Interpolate(
			"sed -E 's;git.openwrt.org/(feed|project);%(kw:host)s;' feeds.conf.default > feeds.conf",
			host=GetFeedsHostOverride,
		),
		doStepIf = IsFeedsHostOverrideEnabled,
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "updatefeeds",
		description = "Updating feeds",
		workdir = "build/sdk",
		command = ["./scripts/feeds", "update", "-f"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "installfeeds",
		description = "Installing feeds",
		workdir = "build/sdk",
		command = ["./scripts/feeds", "install", "-a"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "feeds-cleanup",
		description = "Removing feeds.conf override",
		descriptionDone = "feeds.conf override removed",
		workdir = "build/sdk",
		command = ["rm", "-f", "feeds.conf"],
		doStepIf = IsFeedsHostOverrideEnabled,
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "logclear",
		description = "Clearing failure logs",
		workdir = "build/sdk",
		command = ["rm", "-rf", "logs/package/error.txt", "faillogs/"],
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = True,
	))

	factory.addStep(ShellCommand(
		name = "compile",
		description = "Building packages",
		workdir = "build/sdk",
		timeout = 3600,
		command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
		env = {'CCACHE_BASEDIR': Interpolate("%(kw:cwd)s", cwd=GetCwd)},
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "mkfeedsconf",
		description = "Generating pinned feeds.conf",
		workdir = "build/sdk",
		command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))

	factory.addStep(ShellCommand(
		name = "checksums",
		description = "Calculating checksums",
		descriptionDone="Checksums calculated",
		workdir = "build/sdk",
		command = "cd bin/packages/%s; " %(arch[0])
		+ "find . -type f -not -name 'sha256sums' -printf \"%P\n\" | "
		+ "sort | xargs -r ../../../staging_dir/host/bin/mkhash -n sha256 | "
		+ r"sed -ne 's!^\(.*\) \(.*\)$!\1 *\2!p' > sha256sums",
		haltOnFailure = True
	))

	factory.addStep(MasterShellCommand(
		name = "signprepare",
		description = "Preparing temporary signing directory",
		command = ["mkdir", "-p", "%s/signing" %(work_dir)],
		haltOnFailure = True,
		doStepIf = IsSignEnabled
	))

	factory.addStep(ShellCommand(
		name = "signpack",
		description = "Packing files to sign",
		workdir = "build/sdk",
		command = "find bin/packages/%s/ -mindepth 1 -maxdepth 2 -type f " %(arch[0])
		+ "-name sha256sums -print0 -or "
		+ "-name Packages -print0 -or "
		+ "-name packages.adb -print0 | "
		+ "xargs -0 tar -czf sign.tar.gz",
		haltOnFailure = True,
		doStepIf = IsSignEnabled
	))

	factory.addStep(FileUpload(
		workersrc = "sdk/sign.tar.gz",
		masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
		haltOnFailure = True,
		doStepIf = IsSignEnabled
	))

	factory.addStep(MasterShellCommand(
		name = "signfiles",
		description = "Signing files",
		command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(work_dir, arch[0])],
		env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
		haltOnFailure = True,
		doStepIf = IsSignEnabled
	))

	factory.addStep(FileDownload(
		mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
		workerdest = "sdk/sign.tar.gz",
		haltOnFailure = True,
		doStepIf = IsSignEnabled
	))

	factory.addStep(ShellCommand(
		name = "signunpack",
		description = "Unpacking signed files",
		workdir = "build/sdk",
		command = ["tar", "-xzf", "sign.tar.gz"],
		haltOnFailure = True,
		doStepIf = IsSignEnabled
	))

	# download remote sha256sums to 'target-sha256sums'
	factory.addStep(ShellCommand(
		name = "target-sha256sums",
		description = "Fetching remote sha256sums for arch",
		command = ["rsync"] + rsync_defopts + ["-z", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/sha256sums", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0]), "arch-sha256sums"],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		logEnviron = False,
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
	))

	factory.addStep(FileDownload(
		name="dlrsync.sh",
		mastersrc = scripts_dir + "/rsync.sh",
		workerdest = "../rsync.sh",
		mode = 0o755
	))

	factory.addStep(FileDownload(
		name = "dlsha2rsyncpl",
		mastersrc = scripts_dir + "/sha2rsync.pl",
		workerdest = "../sha2rsync.pl",
		mode = 0o755,
	))

	factory.addStep(ShellCommand(
		name = "buildlist",
		description = "Building list of files to upload",
		workdir = "build/sdk",
		command = ["../../sha2rsync.pl", "../arch-sha256sums", "bin/packages/%s/sha256sums" %(arch[0]), "rsynclist"],
		haltOnFailure = True,
	))

	factory.addStep(ShellCommand(
		name = "uploadprepare",
		description = "Preparing package directory",
		workdir = "build/sdk",
		command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = True,
		logEnviron = False
	))

	factory.addStep(ShellCommand(
		name = "packageupload",
		description = "Uploading package files",
		workdir = "build/sdk",
		command = ["../../rsync.sh"] + rsync_defopts + ["--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-a", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = True,
		logEnviron = False
	))

	factory.addStep(ShellCommand(
		name = "packageprune",
		description = "Pruning package files",
		workdir = "build/sdk",
		command = ["../../rsync.sh"] + rsync_defopts + ["--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-a", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = True,
		logEnviron = False
	))

	factory.addStep(ShellCommand(
		name = "logprepare",
		description = "Preparing log directory",
		workdir = "build/sdk",
		command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = True,
		logEnviron = False
	))

	factory.addStep(ShellCommand(
		name = "logfind",
		description = "Finding failure logs",
		workdir = "build/sdk/logs/package/feeds",
		command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = True,
	))

	factory.addStep(ShellCommand(
		name = "logcollect",
		description = "Collecting failure logs",
		workdir = "build/sdk",
		command = ["rsync"] + rsync_defopts + ["-a", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = True,
	))

	factory.addStep(ShellCommand(
		name = "logupload",
		description = "Uploading failure logs",
		workdir = "build/sdk",
		command = ["../../rsync.sh"] + rsync_defopts + ["--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-az", "faillogs/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = True,
		logEnviron = False
	))

	if rsync_src_url is not None:
		factory.addStep(ShellCommand(
			name = "sourcelist",
			description = "Finding source archives to upload",
			workdir = "build/sdk",
			command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
			haltOnFailure = True
		))

		factory.addStep(ShellCommand(
			name = "sourceupload",
			description = "Uploading source archives",
			workdir = "build/sdk",
			command = ["../../rsync.sh"] + rsync_defopts + ["--files-from=sourcelist", "--size-only", "--delay-updates",
					Interpolate("--partial-dir=.~tmp~%(kw:archname)s~%(prop:workername)s", archname=arch[0]), "-a", "dl/", "%s/" %(rsync_src_url)],
			env={'RSYNC_PASSWORD': rsync_src_key},
			haltOnFailure = False,
			flunkOnFailure = False,
			warnOnFailure = True,
			logEnviron = False
		))

	factory.addStep(ShellCommand(
		name = "df",
		description = "Reporting disk usage",
		command=["df", "-h", "."],
		env={'LC_ALL': 'C'},
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
		alwaysRun = True
	))

	factory.addStep(ShellCommand(
		name = "du",
		description = "Reporting estimated file space usage",
		command=["du", "-sh", "."],
		env={'LC_ALL': 'C'},
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
		alwaysRun = True
	))

	factory.addStep(ShellCommand(
		name = "ccachestat",
		description = "Reporting ccache stats",
		command=["ccache", "-s"],
		want_stderr = False,
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
		alwaysRun = True,
	))

	c['builders'].append(BuilderConfig(name=arch[0], workernames=workerNames, factory=factory))

	c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ]))
	force_factory.addStep(steps.Trigger(
		name = "trigger_%s" % arch[0],
		description = "Triggering %s build" % arch[0],
		schedulerNames = [ "trigger_%s" % arch[0] ],
		set_properties = { "reason": Property("reason") },
		doStepIf = IsArchitectureSelected(arch[0])
	))

####### STATUS arches

# 'status' is a list of Status arches. The results of each build will be
# pushed to these arches. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

if ini.has_option("phase2", "status_bind"):
	c['www'] = {
		'port': ini.get("phase2", "status_bind"),
		'plugins': {
			'waterfall_view': True,
			'console_view': True,
			'grid_view': True
		}
	}

	if ini.has_option("phase2", "status_user") and ini.has_option("phase2", "status_password"):
		c['www']['auth'] = util.UserPasswordAuth([
			(ini.get("phase2", "status_user"), ini.get("phase2", "status_password"))
		])
		c['www']['authz'] = util.Authz(
			allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
			roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase2", "status_user")]) ]
		)

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = ini.get("general", "title")
c['titleURL'] = ini.get("general", "title_url")

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = buildbot_url

####### DB URL

c['db'] = {
	# This specifies what database buildbot uses to store its state.  You can leave
	# this at its default for all but the largest installations.
	'db_url' : "sqlite:///state.sqlite",
}

c['buildbotNetUsageData'] = None