Changed the requires file to take 'before' and 'after' parameters to declare forward and backward dependencies. Declaring 'before' will FORCE the listed module to be installed along with this one. See the docs.

Makefile

@@ -14,6 +14,7 @@ install:
 	$(INSTALL_CMD) ./client/bin/disco-fs-diff $(EXEC_PREFIX)/disco-fs-diff
 	$(INSTALL_CMD) ./client/bin/disco-sh-exec $(EXEC_PREFIX)/disco-sh-exec
 	$(INSTALL_CMD) ./client/bin/disco-sh-shell $(EXEC_PREFIX)/disco-sh-shell
+	$(INSTALL_CMD) ./client/bin/disco-shutup $(EXEC_PREFIX)/disco-shutup
 	$(INSTALL_CMD) ./client/bin/disco $(EXEC_PREFIX)/disco
 	$(INSTALL_CMD) ./universe/bin/disco-ball $(EXEC_PREFIX)/disco-ball
 	$(INSTALL_CMD) ./universe/bin/disco-param $(EXEC_PREFIX)/disco-param

README.md

@@ -1,3 +1,9 @@
+Read the Wiki
+=====
+
+This README is only a bare introduction. Please read the wiki at https://github.com/akesterson/disco/wiki
+for more complete documentation.
+
 disco
 =====
 
@@ -60,319 +66,3 @@ environment stops all potentially dangerous commands at the reporting level (pre
 the fetched files are not merged out of the scratchpad onto the live filesystem. 
 
 See the client disco-fs-* and disco-exec-* scripts for more information on how this is done.
-
-A complete example
-=====
-
-Presume we have a server with an incoming user, "disco", who has a home directory like this:
-
-    disco@server:~$ cat rsyncd.conf
-    [parameters]
-    path = /home/disco/parameters
-    read only = true
-    comment = DISCO Parameters
-    list = yes
-    use chroot = false
-   
-    [testmodule-1.0]
-    path = /home/disco/modules/testmodule-1.0
-    read only = true
-    comment = v1.0 of the Test module
-    list = yes
-    use chroot = false
-   
-    [othermodule-3.2]
-    path = /home/disco/modules/othermodule-3.2
-    read only = true
-    comment = v3.2 of othermodule
-    list = yes
-    use chroot = false 
-
-    disco@server:~$ find parameters
-    parameters
-    parameters/localhost.localdomain
-    parameters/localhost.localdomain/parameters
-    parameters/localhost.localdomain/parameters/something
-    parameters/localhost.localdomain/modules
-    parameters/localhost.localdomain/modules/othermodule-3.2
-    parameters/localhost.localdomain/modules/testmodule-1.0
-  
-    disco@server:~$ cat parameters/localhost.localdomain/parameters/something
-    LOLTHISKEYMEANSNOTHING
-  
-    disco@server:~$ find modules
-    modules
-    modules/othermodule-3.2
-    modules/othermodule-3.2/requires
-    modules/othermodule-3.2/parameters
-    modules/othermodule-3.2/parameters/othermodule-3.2
-    modules/othermodule-3.2/scripts
-    modules/othermodule-3.2/templates
-    modules/othermodule-3.2/templates/etc
-    modules/othermodule-3.2/templates/etc/othermodule
-    modules/othermodule-3.2/templates/etc/othermodule/stuff.cfg
-    modules/othermodule-3.2/files
-    modules/testmodule-1.0
-    modules/testmodule-1.0/requires
-    modules/testmodule-1.0/parameters
-    modules/testmodule-1.0/parameters/testmodule-1.0
-    modules/testmodule-1.0/scripts
-    modules/testmodule-1.0/scripts/00-hello.sh
-    modules/testmodule-1.0/scripts/10-service_stop.sh
-    modules/testmodule-1.0/templates
-    modules/testmodule-1.0/files
-   
-    disco@server:~$ cat modules/othermodule-3.2/templates/etc/othermodule/stuff.cfg
-    echo HOST=$(hostname)
-    echo KEY_VALUE=$(cat /var/disco/parameters/$(hostname)/parameters/something)
-  
-    disco@server:~$ cat modules/testmodule-1.0/scripts/00-hello.sh
-    #!/bin/bash
-  
-    echo "Hello, disco"
-  
-    disco@server:~$ cat modules/testmodule-1.0/scripts/10-service_stop.sh
-    #!/bin/bash
-  
-    service postgresql stop
-
-... and that we have, on our client, a disco parameters tree set up like this:
-
-    [disco@client disco]$ disco-param dump
-    disco = {}
-    disco/client = {}
-    disco/client/cmds = {}
-    disco/client/cmds/rsync = rsync -qaWHe "ssh -i /home/disco/.ssh/id_rsa_disco"
-    disco/server = {}
-    disco/server/uri = disco@aklabs.net
-
-... Then we can use disco to configure our host.
-
-First we need to mount and initialize disco's testing/noop filesystem as
-root on the client.
-
-    [root@localhost disco]$ NOOP=true disco-fs-mount
-    [root@localhost disco]$ NOOP=true disco-fs-init
-
-This will take a minute or two, the init does a lot of work. (But you only
-have to run the init once at system start, no matter how many times you 
-run disco.) Now we can do our noop run:
-
-    [disco@localhost disco]$ NOOP=true disco dance
-    error: othermodule-3.2: rsync: link_stat "/files/*" (in othermodule-3.2) failed: No such file or directory (2)
-    error: testmodule-1.0: rsync: link_stat "/files/*" (in testmodule-1.0) failed: No such file or directory (2)
-    info: Processing testmodule-1.0
-    Hello, disco
-    warning: Would execute : service postgresql stop
-    info: Processing othermodule-3.2
-    info: File: file: /etc/othermodule/stuff.cfg : Created : type=[regular file] device=[fd00] mode=[81a4] selinux=[?] md5=[77b20e4840b1be13a577e152edc6b443] perms=[root:root 644]
-    0a1,2
-    > HOST=localhost.localdomain
-    > KEY_VALUE=LOLTHISKEYMEANSNOTHING
-
-Here we can see the noop at work; it is preventing potentially destructive
-commands like 'service' from running, while allowing other harmless commands
-to operate in the noop context so that script logic is not affected. We can
-also see the highly detailed statistics and diffs returned for file 
-modifications. But none of the files actually wind up present on the 
-system, and no running processes were affected:
-
-    [root@client ~]$ ps ax | grep -i postgresql
-    15595 pts/1    S+     0:00 grep -i postgresql
-    24457 ?        S      0:12 /usr/lib/postgresql/8.4/bin/postgres -D /var/lib/postgresql/8.4/main -c config_file=/etc/postgresql/8.4/main/postgresql.conf
-    [root@client ~]$ ls -l /etc/othermodule/stuff.cfg
-    ls: cannot access /etc/othermodule/stuff.cfg: No such file or directory
-
-If we were to turn the NOOP flag off, this would all happen for real:
-
-    [root@client disco]$ disco dance
-    error: othermodule-3.2: rsync: link_stat "/files/*" (in othermodule-3.2) failed: No such file or directory (2)
-    error: testmodule-1.0: rsync: link_stat "/files/*" (in testmodule-1.0) failed: No such file or directory (2)
-    info: Processing testmodule-1.0
-    Hello, disco
-    info: Processing othermodule-3.2
-    info: File: file: /etc/othermodule/stuff.cfg : Created : type=[regular file] device=[fd00] mode=[81a4] selinux=[?] md5=[77b20e4840b1be13a577e152edc6b443] perms=[root:root 644]
-    0a1,2
-    > HOST=localhost.localdomain
-    > KEY_VALUE=LOLTHISKEYMEANSNOTHING
-
-... And we will see that the config file has been installed:
-
-    [root@client ~]$ cat /etc/othermodule/stuff.cfg
-    HOST=localhost.localdomain
-    KEY_VALUE=LOLTHISKEYMEANSNOTHING
-
-... And that postgres has been stopped:
-
-    [root@client ~]# ps ax | grep -i postgresql
-    28394 pts/1    S+     0:00 grep -i postgresql
-
-Hooray!
-
-Disco will report other types of file modifications, as well. If you were to 
-open an interactive shell in the disco chroot, and perform some more interesting
-operations, representing what a more advanced sort of script might do:
-
-    [disco@client disco]$ NOOP=true disco-sh-shell
-    [root@client /]# rm -f /etc/passwd
-    [root@client /]# grep -v root /etc/shadow | tee tmpfile
-    bin:*:15240:0:99999:7:::
-    daemon:*:15240:0:99999:7:::
-    adm:*:15240:0:99999:7:::
-    lp:*:15240:0:99999:7:::
-    sync:*:15240:0:99999:7:::
-    shutdown:*:15240:0:99999:7:::
-    halt:*:15240:0:99999:7:::
-    mail:*:15240:0:99999:7:::
-    uucp:*:15240:0:99999:7:::
-    operator:*:15240:0:99999:7:::
-    games:*:15240:0:99999:7:::
-    gopher:*:15240:0:99999:7:::
-    ftp:*:15240:0:99999:7:::
-    nobody:*:15240:0:99999:7:::
-    dbus:!!:15324::::::
-    usbmuxd:!!:15324::::::
-    avahi-autoipd:!!:15324::::::
-    vcsa:!!:15324::::::
-    rtkit:!!:15324::::::
-    rpc:!!:15324:0:99999:7:::
-    pulse:!!:15324::::::
-    haldaemon:!!:15324::::::
-    avahi:!!:15324::::::
-    saslauth:!!:15324::::::
-    postfix:!!:15324::::::
-    apache:!!:15324::::::
-    ntp:!!:15324::::::
-    rpcuser:!!:15324::::::
-    nfsnobody:!!:15324::::::
-    gdm:!!:15324::::::
-    sshd:!!:15324::::::
-    tcpdump:!!:15324::::::
-    disco:$6$Hv67bVi.$d/EolMfURGTMbq1hBr1QL2HdYMYxAXvruq550Qqgu2HCOKWQ1YptMghLKvOAgr3h0NwzXZwHpXQ6fVLdpYe.9.:15533:0:99999:7:::
-    discostu:!!:15558:0:99999:7:::
-    [root@client /]# mv tmpfile /etc/shadow
-    mv: overwrite `/etc/shadow'? y
-    [root@client /]# echo LOL > /var/lib/p0wnt
-    bash: /var/lib/p0wnt: restricted: cannot redirect output
-    [root@client /]# echo LOL | tee /var/lib/p0wnt
-    LOL
-    [root@client /]# echo > /bin/myhotbash
-    bash: /bin/myhotbash: restricted: cannot redirect output
-    [root@client /]# touch /bin/myhotbash
-    [root@client /]# exit
-
-... Since that was done inside of the noop shell (where all the scripts and 
-templates run during noop), we can easily report on these activities:
-
-    [disco@client disco]$ NOOP=true disco-fs-diff
-    info: File: file: /etc/passwd : Deleted
-    1,35d0
-    < root:x:0:0:root:/root:/bin/bash
-    < bin:x:1:1:bin:/bin:/sbin/nologin
-    < daemon:x:2:2:daemon:/sbin:/sbin/nologin
-    < adm:x:3:4:adm:/var/adm:/sbin/nologin
-    < lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
-    < sync:x:5:0:sync:/sbin:/bin/sync
-    < shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
-    < halt:x:7:0:halt:/sbin:/sbin/halt
-    < mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
-    < uucp:x:10:14:uucp:/var/spool/uucp:/sbin/nologin
-    < operator:x:11:0:operator:/root:/sbin/nologin
-    < games:x:12:100:games:/usr/games:/sbin/nologin
-    < gopher:x:13:30:gopher:/var/gopher:/sbin/nologin
-    < ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
-    < nobody:x:99:99:Nobody:/:/sbin/nologin
-    < dbus:x:81:81:System message bus:/:/sbin/nologin
-    < usbmuxd:x:113:113:usbmuxd user:/:/sbin/nologin
-    < avahi-autoipd:x:170:170:Avahi IPv4LL Stack:/var/lib/avahi-autoipd:/sbin/nologin
-    < vcsa:x:69:69:virtual console memory owner:/dev:/sbin/nologin
-    < rtkit:x:499:496:RealtimeKit:/proc:/sbin/nologin
-    < rpc:x:32:32:Rpcbind Daemon:/var/cache/rpcbind:/sbin/nologin
-    < pulse:x:498:495:PulseAudio System Daemon:/var/run/pulse:/sbin/nologin
-    < haldaemon:x:68:68:HAL daemon:/:/sbin/nologin
-    < avahi:x:70:70:Avahi mDNS/DNS-SD Stack:/var/run/avahi-daemon:/sbin/nologin
-    < saslauth:x:497:76:"Saslauthd user":/var/empty/saslauth:/sbin/nologin
-    < postfix:x:89:89::/var/spool/postfix:/sbin/nologin
-    < apache:x:48:48:Apache:/var/www:/sbin/nologin
-    < ntp:x:38:38::/etc/ntp:/sbin/nologin
-    < rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin
-    < nfsnobody:x:65534:65534:Anonymous NFS User:/var/lib/nfs:/sbin/nologin
-    < gdm:x:42:42::/var/lib/gdm:/sbin/nologin
-    < sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin
-    < tcpdump:x:72:72::/:/sbin/nologin
-    < disco:x:500:10::/home/disco:/bin/bash
-    < discostu:x:501:501::/home/discostu:/bin/bash
-    info: File: file: /bin/myhotbash : Created : type=[regular empty file] device=[fd00] mode=[81a4] selinux=[?] md5=[d41d8cd98f00b204e9800998ecf8427e] perms=[root:root 644]
-    info: File: file: /etc/shadow : Modified : md5=[8b02f6d00dbcd622f869216bb1dbbbf4 => 336d0b913c8f8cd029964afd00357952] perms=[root:root 0 => root:root 644] mode=[8000 => 81a4]
-    1d0
-    < root:$6$57kBYzwRrygFb5op$vghIbLjxmkzTznSbN4kA5fdxFsd1ye7WWe/HFtwMJSTBlDuBcOZISgLKNg/xlA4uAFIBi82yAnW/JajgwhCXY.:15517:0:99999:7:::
-    info: File: file: /root/.bash_history : Modified : md5=[ead812e487da32cb99cebd09ad7f773b => cb0138b7f3c4f48639cafe9f7147413f] selinux=[unconfined_u:object_r:admin_home_t:s0 => ?]
-    228a229,237
-    > exit
-    > rm -f /etc/passwd
-    > grep -v root /etc/shadow | tee tmpfile
-    > mv tmpfile /etc/shadow
-    > echo LOL > /var/lib/p0wnt
-    > echo LOL | tee /var/lib/p0wnt
-    > echo > /bin/myhotbash
-    > touch /bin/myhotbash
-    > exit
-    info: File: file: /var/lib/p0wnt : Created : type=[regular file] device=[fd00] mode=[81a4] selinux=[?] md5=[5732edd7e4e1240b868e15bc95d36339] perms=[root:root 644]
-    0a1
-    > LOL
-
-And here we see some more of Disco's rather extensive noop reporting capabilities.
-
-But let's say that this run took longer than we thought it should. What was taking
-so much time? Disco will tell us.
-
-    [root@disco ~]# disco report
-    report: _internal: diff
-    report:    time_real 0.82 : time_user 0.14 : time_sys 0.66
-    report:    mem_avg 0 : mem_max 5184 : mem_faults_major 0 : mem_faults_minor 18218
-    report:    io_fsin 0 : io_fsout 8 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:    exit: 0
-    report: _internal: fetch_params
-    report:    time_real 1.26 : time_user 0.02 : time_sys 0.08
-    report:    mem_avg 0 : mem_max 11136 : mem_faults_major 0 : mem_faults_minor 1728
-    report:    io_fsin 0 : io_fsout 0 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:    exit: 0
-    report: othermodule-3.2: diff
-    report:    time_real 0.80 : time_user 0.13 : time_sys 0.65
-    report:    mem_avg 0 : mem_max 4816 : mem_faults_major 0 : mem_faults_minor 16448
-    report:    io_fsin 0 : io_fsout 8 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:    exit: 0
-    report: othermodule-3.2: fetch
-    report:    time_real 2.28 : time_user 0.05 : time_sys 0.14
-    report:    mem_avg 0 : mem_max 11152 : mem_faults_major 0 : mem_faults_minor 2843
-    report:    io_fsin 0 : io_fsout 24 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:    exit: 0
-    report: othermodule-3.2: template
-    report:    etc/othermodule/stuff.cfg :
-    report:        time_real 0.56 : time_user 0.04 : time_sys 0.37
-    report:        mem_avg 0 : mem_max 4592 : mem_faults_major 122 : mem_faults_minor 4885
-    report:        io_fsin 25536 : io_fsout 16 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:        exit: 0
-    report: testmodule-1.0: diff
-    report:    time_real 2.56 : time_user 0.42 : time_sys 2.10
-    report:    mem_avg 0 : mem_max 5184 : mem_faults_major 0 : mem_faults_minor 57661
-    report:    io_fsin 0 : io_fsout 32 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:    exit: 0
-    report: testmodule-1.0: exec
-    report:    00-hello.sh :
-    report:        time_real 0.27 : time_user 0.03 : time_sys 0.18
-    report:        mem_avg 0 : mem_max 4672 : mem_faults_major 32 : mem_faults_minor 3783
-    report:        io_fsin 6640 : io_fsout 8 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:        exit: 0
-    report:    10-service_stop.sh :
-    report:        time_real 0.58 : time_user 0.03 : time_sys 0.41
-    report:        mem_avg 0 : mem_max 4960 : mem_faults_major 122 : mem_faults_minor 5462
-    report:        io_fsin 25656 : io_fsout 8 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:        exit: 1
-    report: testmodule-1.0: fetch
-    report:    time_real 2.72 : time_user 0.06 : time_sys 0.15
-    report:    mem_avg 0 : mem_max 11152 : mem_faults_major 0 : mem_faults_minor 2996
-    report:    io_fsin 0 : io_fsout 24 : io_sockin 0 : io_sockout 0 : io_signals 0
-    report:    exit: 0
-
-Happy dancing!

balls/entities/files/usr/sbin/disco-linux-ents

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+DEL=""
+ADD=""
+MOD=""
+GETENT=""
+
+HOSTNAME=$(hostname)
+
+if [ "$1" == "user" ]; then
+    GETENT="passwd"
+    DEL="userdel"
+    ADD="useradd"
+    MOD="usermod"
+elif [ "$1" == "group" ]; then
+    GETENT="group"
+    DEL="groupdel"
+    ADD="groupadd"
+    MOD="groupmod"
+fi
+
+NAME="$2"
+op="$3"
+
+set -x
+
+PARAMS=$(disco-param get ${HOSTNAME}/${GETENT}s/${op}/${NAME})
+BLAH=$(getent ${GETENT} | grep "^${NAME}" 2>&1)
+RETVAL=$?
+if [ $RETVAL -eq 0 ] && [ "$op" == "absent" ]; then
+    ${DEL} ${NAME}
+elif [ $RETVAL -eq 0 ]; then
+    ${MOD} ${PARAMS} ${NAME}
+elif [ "$op" == "present" ]; then
+    ${ADD} ${PARAMS} ${NAME}
+fi
+
+exit $?

balls/groups/requires

@@ -0,0 +1,2 @@
+after entities
+before users

balls/groups/scripts/00-makegroups.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+########################
+# 00-makegroups.sh
+#
+# Make linux groups for the 'group' disco ball
+# Each group is represented as a key under ${HOSTNAME}/groups, with the value
+# of each key being a list of groupadd/groupmod compatible command line flags
+# that are passed, one each, directly into groupmod/groupadd
+########################
+
+HOSTNAME=$(hostname)
+
+RETVAL=0
+
+for op in present absent
+do
+    for groupname in $(disco-param keys ${HOSTNAME}/groups/${op})
+    do
+	disco-linux-ents group $groupname $op
+	RETVAL=$(expr $RETVAL + $?)
+    done
+done
+
+exit $RETVAL
+

balls/packages/parameters/packages/commands/absent

@@ -0,0 +1 @@
+yum remove -y -d 0 -e 0

balls/packages/parameters/packages/commands/present

@@ -0,0 +1 @@
+yum install -y -d 1 -e 0

balls/packages/scripts/00-installpkgs.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#########################
+#
+# 00-installpkgs.sh
+#
+# For each package listed under $(hostname)/packages/[present|absent], 
+# install/remove the package there using the package management 
+# commands under $(hostname)/packages/commands/[present|absent].
+#########################
+
+HOSTNAME=$(hostname)
+RETVAL=0
+
+for op in present absent
+do
+    PKGCMD=$(disco-param get ${HOSTNAME}/packages/commands/${op})
+    for pkgname in $(disco-param keys ${HOSTNAME}/packages/${op})
+    do
+	$PKGCMD $pkgname
+	RETVAL=$(expr $RETVAL + $?)
+    done
+done
+
+exit $RETVAL

balls/users/requires

@@ -0,0 +1 @@
+after entities

balls/users/scripts/00-makeusers.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+########################
+# 00-makeusers.sh
+#
+# Make linux users for the 'users' disco ball
+# Each user is represented as a key under ${HOSTNAME}/users, with the value
+# of each key being a list of useradd/usermod compatible command line flags
+# that are passed, one each, directly into usermod/useradd
+########################
+
+HOSTNAME=$(hostname)
+
+RETVAL=0
+
+for op in present absent
+do
+    for username in $(disco-param keys ${HOSTNAME}/users/${op})
+    do
+	disco-linux-ents user $username $op
+	RETVAL=$(expr $RETVAL + $?)
+    done
+done
+
+exit $RETVAL

client/bin/disco

@@ -18,7 +18,12 @@ function colorize() {
 }
 
 function report() {
-    for module in $(ls /var/disco/reports/)
+    if [ "$1" != "" ]; then
+	MODULELIST="$1"
+    else
+	MODULELIST=$(ls /var/disco/reports/)
+    fi
+    for module in $MODULELIST;
     do
 	BASE="report: $module:"
 	for file in $(ls /var/disco/reports/${module})
@@ -42,33 +47,39 @@ function dance() {
 	exit 1
     fi
 
-    # Create the toposort of all the modules
+    # Create the toposort of all the modules and fetch all their metadata
-    for module in $(disco-param keys $(hostname)/modules)
+    # This also includes fetching modules that aren't explicitly called out in
+    # our class list, but are required by 'before' statements in the requires files.
+    rm -f /tmp/$$.tsort
+    disco-param keys $(hostname)/modules | tac > /tmp/$$.modules.prev
+    NEWBALLS=$(cat /tmp/$$.modules.prev)
+    while [ "$NEWBALLS" != "" ]; do
+	for module in $NEWBALLS
 	do
 	    NOOP="true" disco-ball fetch $module
 	    disco-ball requires $module >> /tmp/$$.tsort
+	    cat /tmp/$$.tsort | tsort | tac > /tmp/$$.modules.new
+	done
+	NEWBALLS=$(diff /tmp/$$.modules.prev /tmp/$$.modules.new | grep "^> " | cut -d " " -f 2-)
+	cp /tmp/$$.modules.new /tmp/$$.modules.prev
+	if [ "$NEWBALLS" != "" ]; then
+	    echo "info: Discovered dependency : ${module} : $(echo -n ${NEWBALLS} | sed s/' '/', '/g)"
+	fi
     done
 
-    for module in $(cat /tmp/$$.tsort | tsort | tac)
+    echo "info: Ready to dance :"
-    do
+    cat /tmp/$$.tsort | tsort | tac | sed s/"^"/"info:        "/g
-	echo "info: Processing ${module}"
+
-	NOOP="true" disco-ball template $module
+#    for module in $(cat /tmp/$$.tsort | tsort | tac)
-	NOOP=true /usr/bin/time -f "$TIME" -o /var/disco/reports/${module}/diff disco-fs-diff
+#    do
-	if [ "$NOOP" == "" ]; then
+#	disco-ball spin ${module}
-	    rsync -aWH /var/disco/testfs/noop/scratchfs/. /.
+#    done
-	fi
-	NOOP="$NOOP" disco-ball exec $module
-	RETVAL=$?
-	rm -rf /var/disco/testfs/noop/scratchfs/*
-	if [ $RETVAL -ne 0 ]; then
-	    echo "error: Failed to apply $module."
-	fi
-    done
 
     if [ "$REPORT" != "" ]; then
 	report
     fi
 
+    #rm -f /tmp/$$.*
 }
 
 $1 "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9"

client/bin/disco-fs-diff

@@ -15,67 +15,19 @@ CREATED=0
 DELETED=0
 MODIFIED=0
 
-rsync --checksum --times --perms --owner --group -ani ./scratchfs/ ./rootfs/ |\
+rsync --checksum --times --perms --owner --group -ani ./scratchfs/ ./rootfs/ --out-format "info: File: %i : %f : mode='%B', size='%l', mtime='%M'" |\
-    grep -v ".unionfs.*/$" |\
+    sed -e s/"scratchfs"/""/ \
-    sed s/"\.\/scratchfs"/""/ |\
+        -e s/"\.unionfs\/"/""/ \
-    sed s/"\.unionfs\/"/""/ |\
+        -e s/"info: File: .* : \(\/.*\)_HIDDEN~ : mode="/"info: File: *_deleted__ : \1 : mode="/|\
-    grep -v "./$" |\
-    sed s/"^[<>ch\.*]d"/"File: directory: "/ |\
-    sed s/"^[<>ch\.*]f"/"File: file: "/ |\
-    sed s/"^[<>ch\.*]L"/"File: symlink: "/ |\
-    sed s/"^[<>ch\.*]D"/"File: device: "/ |\
-    sed s/"^[<>ch\.*]S"/"File: special: "/ |\
-    sed s/": \([\.cstpoguax+]*\) \(.*\)$"/": \2 : \1"/ |\
-    sed s/"+++++++++$"/"Created"/ |\
-    sed s/" [\.cstpoguax]*$"/" Modified"/ |\
-    sed s/"_HIDDEN~ : Created"/" : Deleted"/ |\
-    sed s/"^\(.*\)\$"/"info: \1"/g |\
 while read LINE
 do 
     FNAME=$(echo $LINE | cut -d : -f 4 | sed s/"^ *"/""/ | sed s/" *$"/""/)    
-
+    MD5_NEW=$(md5sum ${DISCOROOT}/scratchfs${FNAME} 2>/dev/null| cut -d " " -f 1)
-    # ----- File modification/creation data
+    STAT_NEW=$(chroot ${DISCOROOT}/chroot stat --format "owner='%G:%U', selinux='%C', md5='${MD5_NEW}'" /${FNAME} 2>/dev/null)
-    MODSTR=""
+    if [ ! -d ${FNAME} ] && [ ! -d ${DISCOROOT}/scratchfs${FNAME} ]; then 
-    MD5_NEW=$(md5sum ${DISCOROOT}/scratchfs/${FNAME} 2>/dev/null| cut -d " " -f 1)
+	diff -N ${FNAME} ${DISCOROOT}/scratchfs${FNAME} 2>/dev/null | grep -v "^Binary file" > /tmp/$$.diff
-    MD5_OLD=$(md5sum /${FNAME} 2>/dev/null| cut -d " " -f 1)
-    if [ "$MD5_NEW" != "$MD5_OLD" ]; then
-	MODSTR="$MODSTR md5=[$MD5_OLD => $MD5_NEW]"
     fi
-    PERMS_NEW=$(stat --format "%G:%U %a" ${DISCOROOT}/scratchfs/${FNAME} 2>/dev/null)
+    echo "${LINE}, ${STAT_NEW}"
-    PERMS_OLD=$(stat --format "%G:%U %a" /${FNAME} 2>/dev/null)
-    if [ "$PERMS_NEW" != "$PERMS_OLD" ]; then
-	MODSTR="$MODSTR perms=[$PERMS_OLD => $PERMS_NEW]"
-    fi
-    SELINUX_NEW=$(stat --format "%C" ${DISCOROOT}/scratchfs/${FNAME} 2>/dev/null)
-    SELINUX_OLD=$(stat --format "%C" /${FNAME} 2>/dev/null)
-    if [ "$SELINUX_NEW" != "$SELINUX_OLD" ]; then
-	MODSTR="$MODSTR selinux=[$SELINUX_OLD => $SELINUX_NEW]"
-    fi
-    TYPE_NEW=$(stat --format "%F" ${DISCOROOT}/scratchfs/${FNAME} 2>/dev/null)
-    TYPE_OLD=$(stat --format "%F" /${FNAME} 2>/dev/null)
-    if [ "$TYPE_NEW" != "$TYPE_OLD" ]; then
-	MODSTR="$MODSTR type=[$TYPE_OLD => $TYPE_NEW]"
-    fi
-    RAWMODE_NEW=$(stat --format "%f" ${DISCOROOT}/scratchfs/${FNAME} 2>/dev/null)
-    RAWMODE_OLD=$(stat --format "%f" /${FNAME} 2>/dev/null)
-    if [ "$RAWMODE_NEW" != "$RAWMODE_OLD" ]; then
-	MODSTR="$MODSTR mode=[$RAWMODE_OLD => $RAWMODE_NEW]"
-    fi
-    DEVICE_NEW=$(stat --format "%D" ${DISCOROOT}/scratchfs/${FNAME} 2>/dev/null)
-    DEVICE_OLD=$(stat --format "%D" /${FNAME} 2>/dev/null)
-    if [ "$DEVICE_NEW" != "$DEVICE_OLD" ]; then
-	MODSTR="$MODSTR device=[$DEVICE_OLD => $DEVICE_NEW]"
-    fi
-    if [ ! -d /${FNAME} ] && [ ! -d ${DISCOROOT}/scratchfs/${FNAME} ]; then 
-	diff -N /${FNAME} ${DISCOROOT}/scratchfs/${FNAME} 2>/dev/null > /tmp/$$.diff
-    fi
-
-    # ------
-
-    echo $LINE |\
-        sed -e s/"Created$"/"Created : type=[${TYPE_NEW}] device=[${DEVICE_NEW}] mode=[${RAWMODE_NEW}] selinux=[${SELINUX_NEW}] md5=[${MD5_NEW}] perms=[${PERMS_NEW}]"/ |\
-        sed -e s/"Modified$"/"Modified :${MODSTR}"/ |\
-        sed s/"^\([a-z]*\): File: \([a-z]*\): "/"\1: File: \2: \/"/
     if [ -f /tmp/$$.diff ]; then
 	cat /tmp/$$.diff
     fi

client/bin/disco-fs-init

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 DISCOCFG=/etc/disco
+
 if [ "$NOOP" != "" ]; then
     DISCOROOT=/var/disco/testfs/noop
 else

client/bin/disco-sh-exec

@@ -8,7 +8,7 @@ fi
 
 mount | grep $DISCOROOT >/dev/null 2>&1
 if [ $? -ne 0 ]; then
-    ./disco-fs-mount || exit 1
+    echo "disco filesystem is not mounted; please mount it and try again."
 fi
 
 # Strip out any shebang and put the script in the root

universe/bin/disco-ball

@@ -11,7 +11,7 @@ mkdir -p ${DISCOROOT}/reports/_internal
 function init() {
     if [ "$1" == "" ]; then
 	echo "Must enter a path to initialize" >&2
-	exit 1
+	return 1
     fi
     mkdir -p $1/templates
     mkdir -p $1/scripts
@@ -28,36 +28,89 @@ function fetch_params() {
     /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/_internal/fetch_params /bin/bash /tmp/$$.sh
     RETVAL=$?
     rm -f /tmp/$$.sh
-    exit $RETVAL
+    return $RETVAL
+}
+
+function params() {
+    if [ "$1" == "" ]; then
+	echo "Must pass a module name to fetch data for"
+	return 1
+    fi
+    mkdir -p ${DISCOROOT}/reports/$1
+    rm -f ${DISCOROOT}/reports/${1}/params
+    /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/$1/params \
+	rsync -aWH ${STORAGE}/parameters/* ${DISCOROOT}/parameters/$(hostname)/
+    return $?
 }
 
 function fetch() {
     if [ "$1" == "" ]; then
-	echo "Must pass a module name to fetch"
+	echo "Must pass a module name to fetch data for"
-	exit 1
+	return 1
     fi
+    SRC="${SERVERURI}::${1}"
+    if [ "$2" != "" ]; then
+	SRC="$2"
+    fi
+    if [ "$NOOP" == "" ]; then
+	DEST=/var/disco/testfs/real/scratchfs
+    else
+	DEST=/var/disco/testfs/noop/scratchfs
+    fi
+
     rm -rf ${STORAGE}/${1}
     mkdir -p ${DISCOROOT}/reports/$1
     rm -f ${DISCOROOT}/reports/${1}/fetch
     # Sometimes there's some shell escaping voodoo and rsync doesn't like the command args some people will pass it straight on
     # the command line; so we wrap it in a little bash script, and everyone's happy.
-    echo ${RSYNC} ${SERVERURI}::${1}/files/* / > /tmp/$$.sh
+    echo ${RSYNC} --delete ${SRC}/requires ${SRC}/scripts ${SRC}/parameters ${SRC}/templates ${STORAGE}/${1} > /tmp/$$.sh
-    echo ${RSYNC} --delete ${SERVERURI}::${1}/requires ${SERVERURI}::${1}/scripts ${SERVERURI}::${1}/parameters ${SERVERURI}::${1}/templates ${STORAGE}/${1} >> /tmp/$$.sh
     echo 'exit $?' >> /tmp/$$.sh
     /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/${1}/fetch /bin/bash /tmp/$$.sh 2>/tmp/$$.errors | sed s/"^"/"info: ${1}: "/g
     cat /tmp/$$.errors | grep -v "some files/attrs were not transferred" | sed s/"^"/"error: ${1}: "/g
     RETVAL=$?
     rm -f /tmp/$$.sh
-    exit $RETVAL
+    return $RETVAL
 }
 
+function files() {
+    if [ "$1" == "" ]; then
+	echo "Must pass a module name to retrieve files for"
+	return 1
+    fi
+    SRC="${SERVERURI}::${1}"
+    if [ "$2" != "" ]; then
+	SRC="$2"
+    fi
+    if [ "$NOOP" == "" ]; then
+	DEST=/var/disco/testfs/real/scratchfs
+    else
+	DEST=/var/disco/testfs/noop/scratchfs
+    fi
+
+    mkdir -p ${DISCOROOT}/reports/$1
+    rm -f ${DISCOROOT}/reports/${1}/files
+    # Sometimes there's some shell escaping voodoo and rsync doesn't like the command args some people will pass it straight on
+    # the command line; so we wrap it in a little bash script, and everyone's happy.
+    echo ${RSYNC} ${SRC}/files/* ${DEST} > /tmp/$$.sh
+    echo 'exit $?' >> /tmp/$$.sh
+    /usr/bin/time -f "$TIME" -o ${DISCOROOT}/reports/${1}/files /bin/bash /tmp/$$.sh 2>/tmp/$$.errors | sed s/"^"/"info: ${1}: "/g
+    cat /tmp/$$.errors | grep -v "some files/attrs were not transferred" | sed s/"^"/"error: ${1}: "/g
+    RETVAL=$?
+    rm -f /tmp/$$.sh
+    return $RETVAL
+}
+
+
 function requires()
 {
     if [ "$1" == "" ]; then
 	echo "Must pass a module name for requirements"
-	exit 1
+	return 1
     fi
-    cat ${STORAGE}/${1}/requires | sed s/"^"/"${1} "/g
+    cat ${STORAGE}/${1}/requires 2>/dev/null |\
+        grep -v "^#" |\
+        sed -e s/"^ *after"/"${1} "/g \
+            -e s/"^ *before *\([a-zA-Z0-9\-_\.]*\)"/"\1 ${1}"/g
 }
 
 function exec() {
@@ -73,13 +126,13 @@ function exec() {
 	cat /tmp/$$.exec.time | sed s/"^"/"    "/g >> ${DISCOROOT}/reports/${1}/exec
     done
     rm -f /tmp/$$.exec.time
-    exit $RETVAL
+    return $RETVAL
 }
 
 function template() {
     if [ "$1" == "" ]; then
 	echo "Must pass a module name to template"
-	exit 1
+	return 1
     fi
     mkdir -p ${DISCOROOT}/reports/$1
     rm -f ${DISCOROOT}/reports/${1}/template
@@ -87,6 +140,9 @@ function template() {
     if [ "$NOOP" != "" ]; then
 	DESTROOT=/var/disco/testfs/noop/scratchfs
     fi
+    if [ ! -d ${STORAGE}/${1}/templates ]; then
+	return 0;
+    fi
     cd ${STORAGE}/${1}/templates
     RETVAL=0
     for file in $(find . -type f | sed s/"^\.\/"//g)
@@ -103,7 +159,40 @@ function template() {
 	fi
     done
     rm -f /tmp/$$.template.time
-    exit $RETVAL
+    return $RETVAL
 }
 
-$1 $2
+function spin() {
+    module=$1
+    echo "info: Processing ${module}"
+    NOOP="true" files $module
+    NOOP="true" template $module
+    if [ "$NOOP" == "" ]; then
+	# We rsync all the configs and files into the 'real' scratchfs so that we can
+	# just do one big diff at the very end, capturing every single disk change,
+	# including ones made by the scripts.
+	rsync -aWH /var/disco/testfs/noop/scratchfs/. /var/disco/testfs/real/scratchfs/
+    fi
+    NOOP="$NOOP" exec $module
+    RETVAL=$?
+    /usr/bin/time -f "$TIME" -o /var/disco/reports/${module}/diff disco-fs-diff
+    if [ "$NOOP" == "" ]; then
+	# Now for the real rsync back home
+	cd /var/disco/testfs/real/scratchfs/.unionfs
+	for file in $(find . -iname "*_HIDDEN~" | sed -e s/"^\."/""/g -e s/"_HIDDEN~$"/""/g); 
+	do 
+	    rm -rf /${file}
+	done
+	cd ..
+	rm -rf ./.unionfs
+	rsync -aWH /var/disco/testfs/real/scratchfs/. /
+    fi
+    rm -rf /var/disco/testfs/noop/scratchfs/*
+    rm -rf /var/disco/testfs/real/scratchfs/*
+    if [ $RETVAL -ne 0 ]; then
+	echo "error: Failed to apply $module."
+    fi
+    return $?
+}
+
+$1 "$2" "$3" "$4" "$5" "$6" "$7" "$8"