From 86b19b974337abe62001491c073b99cf9c71eadf Mon Sep 17 00:00:00 2001 From: Andrew Kesterson Date: Thu, 23 Aug 2012 21:32:27 -0400 Subject: [PATCH] Changed the requires file to take 'before' and 'after' parameters to declare forward and backward dependencies. Declaring 'before' will FORCE the listed module to be installed along with this one. See the docs. --- balls/entities/requires | 0 balls/groups/requires | 3 +- balls/users/requires | 3 +- client/bin/disco | 35 +++++++++++++++++------ universe/bin/disco-ball | 61 +++++++++++++++++++++++++++++++++++++---- 5 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 balls/entities/requires diff --git a/balls/entities/requires b/balls/entities/requires new file mode 100644 index 0000000..e69de29 diff --git a/balls/groups/requires b/balls/groups/requires index 1485563..a05a4b2 100644 --- a/balls/groups/requires +++ b/balls/groups/requires @@ -1 +1,2 @@ -entities +after entities +before users diff --git a/balls/users/requires b/balls/users/requires index 65a2e2e..7219874 100644 --- a/balls/users/requires +++ b/balls/users/requires @@ -1,2 +1 @@ -entities -groups +after entities diff --git a/client/bin/disco b/client/bin/disco index bd3e6a5..970542f 100755 --- a/client/bin/disco +++ b/client/bin/disco @@ -47,22 +47,39 @@ function dance() { exit 1 fi - # Create the toposort of all the modules - for module in $(disco-param keys $(hostname)/modules) - do - NOOP="true" disco-ball fetch $module - disco-ball requires $module >> /tmp/$$.tsort + # Create the toposort of all the modules and fetch all their metadata + # This also includes fetching modules that aren't explicitly called out in + # our class list, but are required by 'before' statements in the requires files. + rm -f /tmp/$$.tsort + disco-param keys $(hostname)/modules | tac > /tmp/$$.modules.prev + NEWBALLS=$(cat /tmp/$$.modules.prev) + while [ "$NEWBALLS" != "" ]; do + for module in $NEWBALLS + do + NOOP="true" disco-ball fetch $module + disco-ball requires $module >> /tmp/$$.tsort + cat /tmp/$$.tsort | tsort | tac > /tmp/$$.modules.new + done + NEWBALLS=$(diff /tmp/$$.modules.prev /tmp/$$.modules.new | grep "^> " | cut -d " " -f 2-) + cp /tmp/$$.modules.new /tmp/$$.modules.prev + if [ "$NEWBALLS" != "" ]; then + echo "info: Discovered dependency : ${module} : $(echo -n ${NEWBALLS} | sed s/' '/', '/g)" + fi done - for module in $(cat /tmp/$$.tsort | tsort | tac) - do - disco-ball spin ${module} - done + echo "info: Ready to dance :" + cat /tmp/$$.tsort | tsort | tac | sed s/"^"/"info: "/g + +# for module in $(cat /tmp/$$.tsort | tsort | tac) +# do +# disco-ball spin ${module} +# done if [ "$REPORT" != "" ]; then report fi + #rm -f /tmp/$$.* } $1 "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" diff --git a/universe/bin/disco-ball b/universe/bin/disco-ball index e35024f..47ad8d8 100755 --- a/universe/bin/disco-ball +++ b/universe/bin/disco-ball @@ -31,38 +31,86 @@ function fetch_params() { return $RETVAL } +function params() { + if [ "$1" == "" ]; then + echo "Must pass a module name to fetch data for" + return 1 + fi + mkdir -p ${DISCOROOT}/reports/$1 + rm -f ${DISCOROOT}/reports/${1}/params + /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/$1/params \ + rsync -aWH ${STORAGE}/parameters/* ${DISCOROOT}/parameters/$(hostname)/ + return $? +} + function fetch() { if [ "$1" == "" ]; then - echo "Must pass a module name to fetch" + echo "Must pass a module name to fetch data for" return 1 fi + SRC="${SERVERURI}::${1}" + if [ "$2" != "" ]; then + SRC="$2" + fi if [ "$NOOP" == "" ]; then DEST=/var/disco/testfs/real/scratchfs else DEST=/var/disco/testfs/noop/scratchfs fi + rm -rf ${STORAGE}/${1} mkdir -p ${DISCOROOT}/reports/$1 rm -f ${DISCOROOT}/reports/${1}/fetch # Sometimes there's some shell escaping voodoo and rsync doesn't like the command args some people will pass it straight on # the command line; so we wrap it in a little bash script, and everyone's happy. - echo ${RSYNC} ${SERVERURI}::${1}/files/* ${DEST} > /tmp/$$.sh - echo ${RSYNC} --delete ${SERVERURI}::${1}/requires ${SERVERURI}::${1}/scripts ${SERVERURI}::${1}/parameters ${SERVERURI}::${1}/templates ${STORAGE}/${1} >> /tmp/$$.sh + echo ${RSYNC} --delete ${SRC}/requires ${SRC}/scripts ${SRC}/parameters ${SRC}/templates ${STORAGE}/${1} > /tmp/$$.sh echo 'exit $?' >> /tmp/$$.sh - /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/${1}/fetch /tmp/$$.sh 2>/tmp/$$.errors | sed s/"^"/"info: ${1}: "/g + /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/${1}/fetch /bin/bash /tmp/$$.sh 2>/tmp/$$.errors | sed s/"^"/"info: ${1}: "/g cat /tmp/$$.errors | grep -v "some files/attrs were not transferred" | sed s/"^"/"error: ${1}: "/g RETVAL=$? rm -f /tmp/$$.sh return $RETVAL } +function files() { + if [ "$1" == "" ]; then + echo "Must pass a module name to retrieve files for" + return 1 + fi + SRC="${SERVERURI}::${1}" + if [ "$2" != "" ]; then + SRC="$2" + fi + if [ "$NOOP" == "" ]; then + DEST=/var/disco/testfs/real/scratchfs + else + DEST=/var/disco/testfs/noop/scratchfs + fi + + mkdir -p ${DISCOROOT}/reports/$1 + rm -f ${DISCOROOT}/reports/${1}/files + # Sometimes there's some shell escaping voodoo and rsync doesn't like the command args some people will pass it straight on + # the command line; so we wrap it in a little bash script, and everyone's happy. + echo ${RSYNC} ${SRC}/files/* ${DEST} > /tmp/$$.sh + echo 'exit $?' >> /tmp/$$.sh + /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/${1}/files /bin/bash /tmp/$$.sh 2>/tmp/$$.errors | sed s/"^"/"info: ${1}: "/g + cat /tmp/$$.errors | grep -v "some files/attrs were not transferred" | sed s/"^"/"error: ${1}: "/g + RETVAL=$? + rm -f /tmp/$$.sh + return $RETVAL +} + + function requires() { if [ "$1" == "" ]; then echo "Must pass a module name for requirements" return 1 fi - cat ${STORAGE}/${1}/requires | sed s/"^"/"${1} "/g + cat ${STORAGE}/${1}/requires 2>/dev/null |\ + grep -v "^#" |\ + sed -e s/"^ *after"/"${1} "/g \ + -e s/"^ *before *\([a-zA-Z0-9\-_\.]*\)"/"\1 ${1}"/g } function exec() { @@ -117,6 +165,7 @@ function template() { function spin() { module=$1 echo "info: Processing ${module}" + NOOP="true" files $module NOOP="true" template $module if [ "$NOOP" == "" ]; then # We rsync all the configs and files into the 'real' scratchfs so that we can @@ -146,4 +195,4 @@ function spin() { return $? } -$1 $2 +$1 "$2" "$3" "$4" "$5" "$6" "$7" "$8"