CTest:Buildserver

Indroduction

In a normal CTest installation you have a central repository for your sourcecode. Each client has a special configuration to know how to get the source, build and test it. After finishing the results will be sent to central CDash. This configuration has to set up at each of the clients. If you create a new branch in your repository and want to get it build by all clients you must change the configuration at each of the clients.

This solution adds the possibility to manage this in a central place (at the BuildMaster).

How it works

Each BuildSlave asks the BuildMaster for a list of buildnames. Then the BuildSlave downloads a CTestScript.cmake for each buildname and executes it.

Requirements

• CMake/CTest
• wget
• webserver with PHP support

Installation

Buildmaster

Change into your document root directory of your webserver and create a direcotry "CDash", if it does not exesit already. Save the following content as generateCTestScript.php in the CDash direcotory:

@$site = $_GET["site"];
@$build_name = $_GET["buildname"];

// Checks
if(!isset($site))
  {
  echo "Not a valid site!";
  return;
  }

// THIS CONFIGURATION SHOULD COME FROM THE DATABASE
$config_from_database = array(
        "BUILDSERVER1" => array(
                "Test1" => array(
                        "@CTS_CMAKE_GENERATOR@" => "Unix Makefiles")),
        "BUILDSERVER2" => array(
                "Test1" => array(
                        "@CTS_CMAKE_GENERATOR@" => "Unix Makefiles")),
        "BUILDSERVER3" => array(
                "Test1" => array(
                        "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles"),
                "Test2" => array(
                        "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles",
                        "@CTS_REPOSITORY_URL@" => "http://VERSIONCONTROL/svn/test2/trunk"),
                "Test3" => array(
                        "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles",
                        "@CTS_REPOSITORY_URL@" => "http://VERSIONCONTROL/svn/test3/branches/next_release")),
        "BUILDSERVER4" => array(
                "Test1" => array(
                        "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles")));


// no build given, so Generate overviewlist
if(!isset($build_name))
  {
  $list = array_keys($config_from_database[$site]);

  header('Content-Type: text/plain');
  echo implode("\n", $list);
  return;
  }

// DEFAULT SETTINGS
$script_parameter = array(
        "@CTS_SITE@" => $site,
        "@CTS_PROJECT_NAME@" => "Test",
        "@CTS_BUILD_NAME@" => $build_name,
        "@CTS_NIGHTLY_START_TIME@" => "22:00:00 UTC",
        "@CTS_CMAKE_GENERATOR@" => "Unix Makefiles",
        "@CTS_BUILD_CONFIGURATION@" => "Release",
        "@CTS_CTEST_MODEL@" => "Continuous",
        "@CTS_BUILD_OPTIONS@" => "-DEXAMPLE_OPTION=1",
        "@CTS_REPOSITORY_URL@" => "http://VERSIONCONTROL/svn/test1");

// template.txt SHOULD ALSO COME FROM THE DATABASE
$ctestscript = strtr(strtr(implode(file("template.txt")), $config_from_database[$site][$build_name]), $script_parameter);

header('Vary: User-Agent');
if(ob_get_contents())
  echo "Some data has already been output";
if(isset($_SERVER['HTTP_USER_AGENT']) && strpos($_SERVER['HTTP_USER_AGENT'],'MSIE'))
  header('Content-Type: application/force-download');
else
  header('Content-Type: application/octet-stream');
if(headers_sent())
  echo "Some data has already been output to browser";

header("Content-Disposition: attachment; filename=\"CTestScript.cmake\"");
header("Content-Transfer-Encoding: binary");
header("Content-Length: ".strlen($ctestscript));
echo $ctestscript;

?>

The following is an example for the template.txt, which also does a upload of a generated NSIS setup:

set(CTEST_SITE "@CTS_SITE@")
set(CTEST_PROJECT_NAME "@CTS_PROJECT_NAME@")
set(CTEST_BUILD_NAME "@CTS_BUILD_NAME@")
set(CTEST_NIGHTLY_START_TIME "@CTS_NIGHTLY_START_TIME@")
set(CTEST_CMAKE_GENERATOR "@CTS_CMAKE_GENERATOR@")
set(CTEST_BUILD_CONFIGURATION "@CTS_BUILD_CONFIGURATION@")
set(MODEL "@CTS_CTEST_MODEL@")

set(BUILD_OPTIONS "@CTS_BUILD_OPTIONS@")
set(REPOSITORY_URL "@CTS_REPOSITORY_URL@")


find_package(Subversion)

set(CTEST_SOURCE_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}/source")
set(CTEST_BINARY_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}/build")

set(CTEST_UPDATE_COMMAND ${Subversion_SVN_EXECUTABLE})

if(NOT DEFINED CTEST_DROP_METHOD)
	set(CTEST_DROP_METHOD "http")
endif()

if(CTEST_DROP_METHOD STREQUAL "http")
	set(CTEST_DROP_SITE "BUILDMASTER")
	set(CTEST_DROP_LOCATION
		"/CDash/submit.php?project=${CTEST_PROJECT_NAME}")
	set(CTEST_DROP_BUILD_LOCATION
		"/CDash/uploadbuild.php?project=${CTEST_PROJECT_NAME}")
	set(CTEST_TRIGGER_SITE "")
endif()


string(COMPARE NOTEQUAL "${MODEL}" "Continuous" SUBMIT_ALWAYS)
set(NEED_REPOSITORY_CHECKOUT 1)

if(EXISTS "${CTEST_SOURCE_DIRECTORY}")
	subversion_wc_info("${CTEST_SOURCE_DIRECTORY}" REPOSITORY)

	string(COMPARE
		NOTEQUAL
		"${REPOSITORY_URL}"
		"${REPOSITORY_WC_URL}"
		NEED_REPOSITORY_CHECKOUT)

	if(${NEED_REPOSITORY_CHECKOUT})
		file(REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}")
	endif()
else()
	set(NEED_REPOSITORY_CHECKOUT 1)
endif()

if(${NEED_REPOSITORY_CHECKOUT})
	set(CTEST_CHECKOUT_COMMAND
		"${CTEST_UPDATE_COMMAND} co ${REPOSITORY_URL} \"${CTEST_SOURCE_DIRECTORY}\" -r HEAD")
else()
	set(CTEST_CHECKOUT_COMMAND "${CTEST_UPDATE_COMMAND} update")
endif()


set(CTEST_CONFIGURE_COMMAND
	"${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=${CTEST_BUILD_CONFIGURATION} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_SOURCE_DIRECTORY}\"")

ctest_start(${MODEL})

if(${CTEST_SCRIPT_ARG} MATCHES ${MODEL})
	ctest_update(SOURCE
		"${CTEST_SOURCE_DIRECTORY}"
		RETURN_VALUE
		NUMBER_FILES)

	if(${SUBMIT_ALWAYS}
		OR
		${NEED_REPOSITORY_CHECKOUT}
		OR
		${NUMBER_FILES}
		GREATER
		0)
		ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}")
		ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}")
		ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}")
		ctest_submit()

		execute_process(COMMAND
			"cpack"
			WORKING_DIRECTORY
			${CTEST_BINARY_DIRECTORY}
			RESULT_VARIABLE
			cpackResult
			OUTPUT_VARIABLE
			cpackLog
			ERROR_VARIABLE
			cpackLog)
		file(WRITE ${CTEST_BINARY_DIRECTORY}/Testing/cpack.log "${cpackLog}")
		file(GLOB
			EXE_FILES
			RELATIVE
			"${CTEST_BINARY_DIRECTORY}"
			"build/*.exe")
		foreach(_currentArg ${EXE_FILES})
			execute_process(COMMAND
				wget
				"${CTEST_DROP_METHOD}://${CTEST_DROP_SITE}${CTEST_DROP_BUILD_LOCATION}&fn=${_currentArg}"
				"--post-file=${CTEST_BINARY_DIRECTORY}/${_currentArg}"
				"-o${CTEST_BINARY_DIRECTORY}/Testing/upload.log"
				"-q")
			file(REMOVE "${CTEST_BINARY_DIRECTORY}/${_currentArg}")
		endforeach()
	endif()
endif()

Linux BuildSlave

Create a a directory "/var/builds" and save the following shell script as make.sh in this directory

cd /var/builds
site=BUILDSLAVE1

wget http://BUILDMASTER/CDash/generateCTestScript.php?site=$site\&buildtype=$1 -O list.txt -q

for d in $(ls -d */); do
  SKIP_DELETE=0

  for p in $(cat list.txt); do
    if [ $p = ${d%%/} ]; then
      SKIP_DELETE=1
    fi
  done

  if [ $SKIP_DELETE = 0 ]; then
    rm -r ${d%%/}
  fi
done

for p in $(cat list.txt); do
  if [ ! -d $p ]; then
    mkdir $p
  fi

  cd $p
  wget http://BUILDMASTER/CDash/generateCTestScript.php?site=$site\&buildname=$p -O CTestScript.cmake -q
  ctest -S CTestScript.cmake,$1
  cd ..
done

rm list.txt

Finaly you have to create a cron job (see CTest Scripting) for each buildtype (Nightly, Continuous), which will execute this shell script:

• For a nightly build create a job to execute "/var/builds/make.sh Nightly" at your CTEST_NIGHTLY_START_TIME.
• To setup a continuous build create a job, which will execute "/var/builds/make.sh Continuous" every minute.

Windows BuildSlave

Create a a directory "C:\builds" and save the following batch script as make.bat in this directory

@echo off
set site=BUILDSLAVE3

call "C:\Program Files\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat"

if not exist %1 mkdir %1
pushd %1

wget http://BUILDMASTER/CDash/generateCTestScript.php?site=%site%^&buildtype=%1 -O list.txt -q

for /D %%d in (*) do (
   if exist skip_delete.flag del skip_delete.flag
      for /f %%p in (list.txt) do (
         if %%p == %%d echo 0 > skip_delete.flag
      )

   if not exist skip_delete.flag rmdir /S /Q %%d
)

if exist skip_delete.flag del skip_delete.flag

for /f %%p in (list.txt) do (
   if not exist %%p mkdir %%p

   if not exist %%p.lock (
      echo 0 > %%p.lock

      pushd %%p
      wget http://BUILDMASTER/CDash/generateCTestScript.php?site=%site%^&buildname=%%p -O CTestScript.cmake -q
      ctest -S CTestScript.cmake,%1
      popd

      del %%p.lock
   )
)

del list.txt
popd

Finaly you have to create a sheduled task (see CTest Scripting) for each buildtype (Nightly, Continuous), which will execute this batch file:

• For a nightly build create a task to execute "C:\builds\make.bat Nightly" at your CTEST_NIGHTLY_START_TIME.
• To setup a continuous build create a task, which will execute "C:\builds\make.bat Continuous" every minute.