The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/tools/remote_build.sh

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10  
    1 #!/bin/bash
    2 #
    3 # Script that rsyncs a source/build tree to a remote server, performs a build,
    4 # and copies the result back
    5 #
    6 
    7 # This script is invoked instead of the initial recursive make(1) in ./Makefile.
    8 # First it must cache all binaries that might be used during the build by
    9 # calling "make print_exports" (any target would work) with an overriden xcrun(1)
   10 # which caches tools an SDKs into ./BUILD/obj/BuildTools. When the combined
   11 # source+build tree is rsync-ed to the remote server, we run a script to
   12 # re-initiate the build using an overriden xcrun(1) which hands back
   13 # cached tools in ./BUILD/obj/BuildTools instead of whatever Xcode tools are on
   14 # the remote system (or if no Xcode tools are installed remotely). Finally,
   15 # the build results are copied back locally.
   16 #
   17 
   18 function die() {
   19     echo "$1" 1>&2
   20     exit 1
   21 }
   22 
   23 
   24 TARGET=
   25 declare -a ARGS
   26 declare -a REMOTEARGS
   27 index=0
   28 for arg in "$@"; do
   29     case $arg in
   30         _REMOTEBUILD_TARGET=*)
   31             TARGET=`echo $arg | awk -F= '{print $2}'`
   32             continue
   33             ;;
   34         _REMOTEBUILD_MAKE=*)
   35             MAKE=`echo $arg | awk -F= '{print $2}'`
   36             continue
   37             ;;
   38         REMOTEBUILD=*)
   39             # Don't restart another remote build remotely
   40             ;;
   41         SRCROOT=*)
   42             continue
   43             ;;
   44         OBJROOT=*)
   45             continue
   46             ;;
   47         SYMROOT=*)
   48             continue
   49             ;;
   50         DSTROOT=*)
   51             continue
   52             ;;
   53         CCHROOT=*)
   54             continue
   55             ;;
   56         RC_XBS=*)
   57             # Remote build isn't chrooted or special in any way
   58             arg="VERBOSE=YES"
   59             continue
   60             ;;
   61         VERBOSE=YES)
   62             set -x
   63             ;;
   64     esac
   65     ARGS[$index]="$arg"
   66     REMOTEARGS[$index]="\"$arg\""
   67     index=$(($index+1))
   68 done
   69 
   70 
   71 RSYNC_ARGS="-azvh"
   72 ARGS[$index]="REMOTEBUILD="
   73 REMOTEARGS[$index]="\"REMOTEBUILD=\""
   74 
   75 # For some targets like installsrc, we can't to a remote build
   76 SKIPREMOTE=0
   77 case $TARGET in
   78     clean)
   79         SKIPREMOTE=1
   80         ;;
   81     installsrc)
   82         SKIPREMOTE=1
   83         ;;
   84     installopensource)
   85         SKIPREMOTE=1
   86         ;;
   87     cscope)
   88         SKIPREMOTE=1
   89         ;;
   90     tags)
   91         SKIPREMOTE=1
   92         ;;
   93     help)
   94         SKIPREMOTE=1
   95         ;;
   96 esac
   97 
   98 if [ $SKIPREMOTE -eq 1 ]; then
   99     exec "$MAKE" "$TARGET" "${ARGS[@]}"
  100 fi
  101 
  102 SRC="$(pwd -P)"
  103 SRCNAME="$(basename $SRC)"
  104 
  105 # Pick up build locations passed in the environment
  106 OBJROOT="${OBJROOT}"
  107 SYMROOT="${SYMROOT}"
  108 DSTROOT="${DSTROOT}"
  109 
  110 if [ -z "${OBJROOT}" ]; then
  111     die "OBJROOT not set in environment"
  112 fi
  113 mkdir -p "${OBJROOT}" || die "Could not create ${OBJROOT}"
  114 
  115 if [ -z "${SYMROOT}" ]; then
  116     die "SYMROOT not set in environment"
  117 fi
  118 mkdir -p "${SYMROOT}" || die "Could not create ${SYMROOT}"
  119 
  120 if [ -z "${DSTROOT}" ]; then
  121     die "DSTROOT not set in environment"
  122 fi
  123 mkdir -p "${DSTROOT}" || die "Could not create ${DSTROOT}"
  124 
  125 if [ "$REMOTEBUILD" = "$SPECIALREMOTEBUILD" ]; then
  126     :
  127 else
  128     DOINSTALLSRC=0
  129     REMOTE_SRCREL="./"
  130     BUILDTOOLSDIR="$OBJROOT"
  131     REMOTE_BUILDTOOLSREL="./BUILD/obj"
  132     BUILDSCRIPTDIR="$OBJROOT"
  133     REMOTE_BUILDSCRIPTREL="./BUILD/obj"
  134     BUILDSCRIPTNAME="build.sh"
  135     if [ ! -d "${OBJROOT}/SETUP" ]; then
  136     RSYNC_DELETE_EXCLUDED="--delete-excluded"
  137     else
  138     RSYNC_DELETE_EXCLUDED=""
  139     fi
  140     if [ ! -e "${SYMROOT}/" ]; then
  141         RSYNC_DELETE_SYMROOT=1
  142     else
  143         RSYNC_DELETE_SYMROOT=0
  144     fi
  145     if [ ! -e "${DSTROOT}/" ]; then
  146         RSYNC_DELETE_DSTROOT=1
  147     else
  148         RSYNC_DELETE_DSTROOT=0
  149     fi
  150     TARBUILDDIRS=0
  151 fi
  152 
  153 echo "Caching build tools..." 1>&2
  154 mkdir -p "${BUILDTOOLSDIR}" || die "Could not create BUILDTOOLSDIR"
  155 $MAKE print_exports "${ARGS[@]}" XCRUN="${SRC}/tools/xcrun_cache.sh -c \"${BUILDTOOLSDIR}\"" >/dev/null || die "Could not cache build tools"
  156 
  157 # Cache the make(1) binary itself
  158 MAKE_SDKROOT=`"${SRC}/tools/xcrun_cache.sh" -u "${BUILDTOOLSDIR}" -sdk / -show-sdk-path`
  159 "${SRC}/tools/xcrun_cache.sh" -c "${BUILDTOOLSDIR}" -sdk "${MAKE_SDKROOT}" -find make >/dev/null || die "Could not cache make"
  160 
  161 # Create a canned build script that can restart the build on the remote server.
  162 mkdir -p "${BUILDSCRIPTDIR}" || die "Could not create BUILDSCRIPTDIR"
  163 cat > "${BUILDSCRIPTDIR}/${BUILDSCRIPTNAME}" <<EOF
  164 #!/bin/sh
  165 mkdir -p /private/tmp
  166 mkdir -p /private/var/tmp
  167 mkdir -p "\${TMPDIR}"
  168 cd "${REMOTE_SRCREL}"
  169 mkdir -p ./BUILD/obj
  170 mkdir -p ./BUILD/sym
  171 mkdir -p ./BUILD/dst
  172 MAKE=\`\$PWD/tools/xcrun_cache.sh -u "\$PWD/${REMOTE_BUILDTOOLSREL}" -sdk / -find make\`
  173 if [ -z "\${MAKE}" ]; then exit 1; fi
  174 \${MAKE} ${TARGET} ${REMOTEARGS[@]} XCRUN="\$PWD/tools/xcrun_cache.sh -u \"\$PWD/${REMOTE_BUILDTOOLSREL}\""
  175 ret=\$?
  176 if [ \$ret -eq 0 ]; then
  177 if [ ${TARBUILDDIRS} -eq 1 ]; then
  178 tar jcf ./BUILD/obj.tar.bz2 --exclude=\*.o --exclude=\*.cpo --exclude=\*.d --exclude=\*.cpd --exclude=\*.non_lto --exclude=\*.ctf --exclude=conf -C ./BUILD/obj . || exit 1
  179 tar jcf ./BUILD/sym.tar.bz2 -C ./BUILD/sym . || exit 1
  180 tar jcf ./BUILD/dst.tar.bz2 -C ./BUILD/dst . || exit 1
  181 fi
  182 fi
  183 exit \$ret
  184 EOF
  185 chmod a+x "${BUILDSCRIPTDIR}/${BUILDSCRIPTNAME}"
  186 #echo "Build script is:"
  187 #cat "${BUILDSCRIPTDIR}/${BUILDSCRIPTNAME}"
  188 
  189 mkdir -p "${BUILDTOOLSDIR}/empty"
  190 
  191 if [ "$REMOTEBUILD" = "$SPECIALREMOTEBUILD" ]; then
  192     :
  193 else
  194 
  195     REMOTEBUILD="$REMOTEBUILD"
  196     REMOTEBUILDPATH="$REMOTEBUILDPATH"
  197 
  198     if [ -z "$REMOTEBUILDPATH" ]; then
  199         WHOAMI=`whoami`
  200         case "${REMOTEBUILD}" in
  201             *@*)
  202                 WHOAMI=`echo "${REMOTEBUILD}" | awk -F@ '{print $1}'`
  203                 ;;
  204         esac
  205         REMOTEBUILDPATH="/tmp/$WHOAMI"
  206     fi
  207 
  208 # Construct a unique remote path
  209     eval `stat -s "${SRC}"`
  210 
  211     REMOTEBUILDPATH="${REMOTEBUILDPATH}/$st_ino/${SRCNAME}/"
  212     echo "Remote path is ${REMOTEBUILD}:${REMOTEBUILDPATH}" 1>&2
  213 
  214     ssh $REMOTEBUILD "mkdir -p \"${REMOTEBUILDPATH}/BUILD/\"{obj,sym,dst}" || die "Could not make remote build directory"
  215 
  216     # Copy source only
  217     rsync $RSYNC_ARGS --delete --exclude=\*~ --exclude=.svn --exclude=.git --exclude=/BUILD . $REMOTEBUILD:"${REMOTEBUILDPATH}" || die "Could not rsync source tree"
  218 
  219     # Copy partial OBJROOT (just build tools and build script), and optionally delete everything else
  220     rsync $RSYNC_ARGS --delete $RSYNC_DELETE_EXCLUDED --include=/build.sh --include=/BuildTools --include=/BuildTools/\*\* --exclude=\* "${OBJROOT}/" $REMOTEBUILD:"${REMOTEBUILDPATH}/BUILD/obj/" || die "Could not rsync build tree"
  221 
  222     # Delete remote SYMROOT if it has been deleted locally
  223     if [ "$RSYNC_DELETE_SYMROOT" -eq 1 ]; then
  224         rsync $RSYNC_ARGS --delete "${BUILDTOOLSDIR}/empty/" $REMOTEBUILD:"${REMOTEBUILDPATH}/BUILD/sym/" || die "Could not rsync delete SYMROOT"
  225     fi
  226 
  227     # Delete remote DSTROOT if it has been deleted locally
  228     if [ "$RSYNC_DELETE_DSTROOT" -eq 1 ]; then
  229         rsync $RSYNC_ARGS --delete "${BUILDTOOLSDIR}/empty/" $REMOTEBUILD:"${REMOTEBUILDPATH}/BUILD/dst/" || die "Could not rsync delete DSTROOT"
  230     fi
  231 
  232     # Start the build
  233     echo ssh $REMOTEBUILD "/bin/bash -c 'cd \"${REMOTEBUILDPATH}\" && ${REMOTE_BUILDSCRIPTREL}/${BUILDSCRIPTNAME}'" 1>&2
  234     ssh $REMOTEBUILD "/bin/bash -c 'cd \"${REMOTEBUILDPATH}\" && ${REMOTE_BUILDSCRIPTREL}/${BUILDSCRIPTNAME}'" || die "Could not complete remote build"
  235 
  236     # Copy back build results except for object files (which might be several GB)
  237     echo "Copying results back..."
  238     rsync $RSYNC_ARGS --no-o --no-g --exclude=\*.o --exclude=\*.cpo --exclude=\*.d --exclude=\*.cpd --exclude=\*.non_lto --exclude=\*.ctf $REMOTEBUILD:"${REMOTEBUILDPATH}/BUILD/obj/" "${OBJROOT}/" || die "Could not rsync build results"
  239     rsync $RSYNC_ARGS --no-o --no-g $REMOTEBUILD:"${REMOTEBUILDPATH}/BUILD/sym/" "${SYMROOT}/" || die "Could not rsync build results"
  240     rsync $RSYNC_ARGS --no-o --no-g $REMOTEBUILD:"${REMOTEBUILDPATH}/BUILD/dst/" "${DSTROOT}/" || die "Could not rsync build results"
  241 
  242 fi
  243 
  244 exit 0

Cache object: a99223d9cd3f3b5dcb46a5eadb8f2a5b

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.