Moved disco-ball to universe, added functions necessary for fetching, templating, and executing a given disco ball. Made all existing scripts aware of NOOP and how to change their DISCOROOT accordingly. Added skeleton restricted.d/* files for a semi-safe base system. Added disco-param that allows management of parameters on the client (currently isn't smart enough to manage them on the server, or per-module). Added client/bin/disco that actually allows the entire thing to come together and get executed. Still lots of bugs to work out.

This commit is contained in:
2012-08-08 05:30:03 -04:00
parent f51124252e
commit 7d6d94ae3d
10 changed files with 433 additions and 41 deletions

48
client/bin/disco Executable file
View File

@@ -0,0 +1,48 @@
#!/bin/bash
DISCOCFG=/etc/disco
if [ "$NOOP" != "" ]; then
DISCOROOT=/var/disco/testfs/noop
else
DISCOROOT=/var/disco/testfs/real
fi
function main() {
mount | grep $DISCOROOT >/dev/null 2>&1
if [ $? -ne 0 ]; then
echo "error: disco filesystem does not appear to be mounted, please exec disco-fs-init, disco-fs-mount, and try again."
exit 1
fi
disco-ball fetch_params
if [ $? -ne 0 ]; then
echo "error: Unable to fetch parameters for this host from remote server"
exit 1
fi
# Create the toposort of all the modules
for module in $(disco-param keys $(hostname)/modules)
do
NOOP="true" disco-ball fetch $module
disco-ball requires $module >> /tmp/$$.tsort
done
for module in $(cat /tmp/$$.tsort | tsort | tac)
do
echo "info: Processing ${module}"
NOOP="true" disco-ball template $module
disco-fs-diff
if [ "$NOOP" == "" ]; then
rsync -aWH /var/disco/testfs/noop/* /
fi
NOOP="$NOOP" disco-ball exec $module
RETVAL=$?
rm -rf /var/disco/testfs/noop/scratchfs/*
if [ $RETVAL -ne 0 ]; then
echo "error: Failed to apply $module."
fi
done
}
main $@

View File

@@ -1,8 +1,13 @@
import os import os
import sys import sys
DISCOROOT="/var/disco/testfs/real"
if ("NOOP" in os.environ) and (os.environ["NOOP"] != ""):
DISCOROOT="/var/disco/testfs/noop"
def file_is_text(fname): def file_is_text(fname):
os.system("file %s > /tmp/%s.typeof" % (os.path.abspath("/var/disco/testfs/scratchfs/" + fname), os.getpid())) global DISCOROOT
os.system("file %s > /tmp/%s.typeof" % (os.path.abspath(DISCOROOT + "/scratchfs/" + fname), os.getpid()))
with open("/tmp/%s.typeof" % os.getpid(), "r") as ifile: with open("/tmp/%s.typeof" % os.getpid(), "r") as ifile:
line = ifile.readline() line = ifile.readline()
if "ASCII" in line: if "ASCII" in line:
@@ -10,6 +15,7 @@ def file_is_text(fname):
return False return False
def main(argc, argv): def main(argc, argv):
global DISCOROOT
for line in sys.stdin.readlines(): for line in sys.stdin.readlines():
line = line.strip("\n") line = line.strip("\n")
pid = os.getpid() pid = os.getpid()
@@ -18,13 +24,13 @@ def main(argc, argv):
if file_is_text(fname): if file_is_text(fname):
content = "" content = ""
with open(os.path.abspath("/var/disco/testfs/scratchfs/%s" % fname), "r") as ifile: with open(os.path.abspath(DISCOROOT + "/scratchfs/%s" % fname), "r") as ifile:
content = "> " + "> ".join(ifile.readlines()) content = "> " + "> ".join(ifile.readlines())
line = line.replace("(CONTENT)", "\n%s" % (content)) line = line.replace("(CONTENT)", "\n%s" % (content))
elif os.path.isdir("/var/disco/testfs/scratchfs/" + fname): elif os.path.isdir(DISCOROOT + "/scratchfs/" + fname):
line = line.replace("(CONTENT)", "directory") line = line.replace("(CONTENT)", "directory")
else: else:
os.system("md5sum /var/disco/testfs/scratchfs/%s > /tmp/%s" % (fname, pid)) os.system("md5sum " + os.path.abspath(DISCOROOT + "/scratchfs/" + fname) + " > /tmp/%s" % (pid))
content = "" content = ""
with open("/tmp/%s" % (pid), "r") as ifile: with open("/tmp/%s" % (pid), "r") as ifile:
content = ifile.readline().split(" ")[0] content = ifile.readline().split(" ")[0]
@@ -32,14 +38,14 @@ def main(argc, argv):
line = line.strip("\n") line = line.strip("\n")
if "(OLDMD5SUM)" in line: if "(OLDMD5SUM)" in line:
fname = line.split(" ")[3] fname = line.split(" ")[3]
os.system("md5sum /var/disco/testfs/rootfs/%s > /tmp/%s" % (fname, pid)) os.system("md5sum " + os.path.abspath(DISCOROOT + "/rootfs/" + fname) + " > /tmp/%s" % (pid))
content = "" content = ""
with open("/tmp/%s" % (pid), "r") as ifile: with open("/tmp/%s" % (pid), "r") as ifile:
content = ifile.readline().split(" ")[0] content = ifile.readline().split(" ")[0]
line = line.replace("(OLDMD5SUM)", content).strip("\n") line = line.replace("(OLDMD5SUM)", content).strip("\n")
if "(NEWMD5SUM)" in line: if "(NEWMD5SUM)" in line:
fname = line.split(" ")[3] fname = line.split(" ")[3]
os.system("md5sum /var/disco/testfs/scratchfs/%s > /tmp/%s" % (fname, pid)) os.system("md5sum " + os.path.abspath(DISCOROOT + "/scratchfs/" + fname) + " > /tmp/%s" % (pid))
content = "" content = ""
with open("/tmp/%s" % (pid), "r") as ifile: with open("/tmp/%s" % (pid), "r") as ifile:
content = ifile.readline().split(" ")[0] content = ifile.readline().split(" ")[0]

View File

@@ -1,6 +1,10 @@
#!/bin/bash #!/bin/bash
DISCOROOT=/var/disco/testfs if [ "$NOOP" == "" ]; then
DISCOROOT=/var/disco/testfs/real
else
DISCOROOT=/var/disco/testfs/noop
fi
mount | grep $DISCOROOT > /dev/null 2>&1 mount | grep $DISCOROOT > /dev/null 2>&1
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then

View File

@@ -1,14 +1,18 @@
#!/bin/bash #!/bin/bash
DISCOROOT=/var/disco/testfs if [ "$NOOP" == "" ]; then
DISCOROOT=/var/disco/testfs/real
else
DISCOROOT=/var/disco/testfs/noop
fi
mount | grep $DISCOROOT >/dev/null 2>&1 mount | grep $DISCOROOT >/dev/null 2>&1
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
./disco-fs-mount ./disco-fs-mount || exit 1
fi fi
# Strip out any shebang and put the script in the root # Strip out any shebang and put the script in the root
mkdir -p ${DISCOROOT}/restricted/$(dirname $2) mkdir -p ${DISCOROOT}/restricted/$(dirname $2)
cat $1 | sed s/'^#!.*'/''/g > ${DISCOROOT}/restricted/$2 cat $1 | sed s/'^#!.*'/''/g > ${DISCOROOT}/restricted/$2
$(dirname $0)/disco-sh-shell ${DISCOROOT}/restricted/$2 NOOP="$NOOP" $(dirname $0)/disco-sh-shell ${DISCOROOT}/restricted/$2
exit $? exit $?

View File

@@ -9,13 +9,12 @@ fi
mount | grep $DISCOROOT >/dev/null 2>&1 mount | grep $DISCOROOT >/dev/null 2>&1
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "disco filesystem is not mounted" echo "disco filesystem is not mounted"
exit 0 exit 1
fi fi
if [ "$NOOP" != "" ]; then if [ "$NOOP" != "" ]; then
#chroot ${DISCOROOT}/chroot /bin/env PATH=${DISCOROOT}/restricted/bin:${DISCOROOT}/munge/mungebin /bin/bash --login --restricted $@ chroot ${DISCOROOT}/chroot /bin/env PATH=${DISCOROOT}/restricted/bin:${DISCOROOT}/munge/mungebin /bin/bash --restricted $@
chroot ${DISCOROOT}/chroot //bin/bash --login --restricted $@
else else
chroot ${DISCOROOT}/chroot /bin/bash --login $@ chroot ${DISCOROOT}/chroot /bin/bash $@
fi fi
exit $? exit $?

View File

@@ -0,0 +1,68 @@
/bin/arch
/bin/basename
/bin/cat
/bin/chgrp
/bin/chmod
/bin/chown
/bin/cp
/bin/cpio
/bin/cut
/bin/dash
/bin/date
/bin/dd
/bin/df
/bin/dmesg
/bin/dnsdomainname
/bin/domainname
/bin/dumpkeys
/bin/echo
/bin/ed
/bin/egrep
# This can execute code, but if we wrap it, bash fails to start.
/bin/env
# -----
/bin/find
/bin/false
/bin/fgrep
/bin/findmnt
/bin/grep
/bin/gtar
/bin/gunzip
/bin/gzip
/bin/hostname
/bin/ipcalc
/bin/keyctl
/bin/link
/bin/ln
/bin/loadkeys
/bin/ls
/bin/lsblk
/bin/mkdir
/bin/mknod
/bin/mktemp
/bin/mv
/bin/netstat
/bin/nisdomainname
/bin/ping
/bin/ping6
/bin/ps
/bin/pwd
/bin/readlink
/bin/rm
/bin/rmdir
/bin/sed
/bin/sleep
/bin/sort
/bin/sync
/bin/tar
/bin/touch
/bin/tracepath
/bin/tracepath6
/bin/traceroute
/bin/traceroute6
/bin/true
/bin/uname
/bin/unlink
/bin/usleep
/bin/ypdomainname
/bin/zcat

View File

@@ -0,0 +1,146 @@
/usr/bin/attr
/usr/bin/base64
/usr/bin/bashbug-32
/usr/bin/bc
/usr/bin/bunzip2
/usr/bin/bzcat
/usr/bin/bzcmp
/usr/bin/bzdiff
/usr/bin/bzgrep
/usr/bin/bzip2
/usr/bin/bzip2recover
/usr/bin/bzless
/usr/bin/bzmore
/usr/bin/chacl
/usr/bin/chage
/usr/bin/chattr
/usr/bin/chcon
/usr/bin/chfn
/usr/bin/chvt
/usr/bin/clear
/usr/bin/cmp
/usr/bin/crontab
/usr/bin/cut
/usr/bin/diff
/usr/bin/diff3
/usr/bin/dig
/usr/bin/dir
/usr/bin/dircolors
/usr/bin/dirname
/usr/bin/du
/usr/bin/expand
/usr/bin/expr
/usr/bin/file
/usr/bin/find-repos-of-install
/usr/bin/fmt
/usr/bin/fold
/usr/bin/free
/usr/bin/funzip
/usr/bin/getent
/usr/bin/getfacl
/usr/bin/getfattr
/usr/bin/gethostip
/usr/bin/getkeycodes
/usr/bin/getopt
/usr/bin/gpasswd
/usr/bin/gpg
/usr/bin/gpg2
/usr/bin/gpgconf
/usr/bin/gpg-error
/usr/bin/gpgkey2ssh
/usr/bin/groups
/usr/bin/gunzip
/usr/bin/gzip
/usr/bin/head
/usr/bin/hexdump
/usr/bin/host
/usr/bin/hostid
/usr/bin/iconv
/usr/bin/id
/usr/bin/install
/usr/bin/install-catalog
/usr/bin/iostat
/usr/bin/join
/usr/bin/last
/usr/bin/lastlog
/usr/bin/locale
/usr/bin/localedef
/usr/bin/locate
/usr/bin/logger
/usr/bin/lsattr
/usr/bin/lscpu
/usr/bin/lsusb
/usr/bin/md5sum
/usr/bin/mkfifo
/usr/bin/mkisofs
/usr/bin/needs-restarting
/usr/bin/nslookup
/usr/bin/ntpstat
/usr/bin/openssl
/usr/bin/passwd
/usr/bin/paste
/usr/bin/pidstat
/usr/bin/pr
/usr/bin/printenv
/usr/bin/printf
/usr/bin/pstree
/usr/bin/quota
/usr/bin/rdate
/usr/bin/readelf
/usr/bin/readlink
/usr/bin/rename
/usr/bin/rpm2cpio
/usr/bin/rpmdb
/usr/bin/rpmdumpheader
/usr/bin/rpmquery
/usr/bin/rpmsign
/usr/bin/rpmverify
/usr/bin/rsync
/usr/bin/scp
/usr/bin/script
/usr/bin/scriptreplay
/usr/bin/seq
/usr/bin/sha1sum
/usr/bin/sha224sum
/usr/bin/sha256sum
/usr/bin/sha384sum
/usr/bin/sha512sum
/usr/bin/showkey
/usr/bin/shred
/usr/bin/shuf
/usr/bin/size
/usr/bin/split
/usr/bin/sqlite3
/usr/bin/ssh-keygen
/usr/bin/stat
/usr/bin/sum
/usr/bin/tac
/usr/bin/tail
/usr/bin/tailf
/usr/bin/tee
/usr/bin/test
/usr/bin/tr
/usr/bin/tree
/usr/bin/truncate
/usr/bin/tsort
/usr/bin/tty
/usr/bin/ul
/usr/bin/unexpand
/usr/bin/uniq
/usr/bin/unlzma
/usr/bin/unzip
/usr/bin/unzipsfx
/usr/bin/updatedb
/usr/bin/uptime
/usr/bin/users
/usr/bin/uuidgen
/usr/bin/vmstat
/usr/bin/w
/usr/bin/wc
/usr/bin/whatis
/usr/bin/whereis
/usr/bin/which
/usr/bin/who
/usr/bin/whoami
/usr/bin/wnck-urgency-monitor
/usr/bin/zip

View File

@@ -1,15 +0,0 @@
#!/bin/bash
function init() {
if [ "$1" == "" ]; then
echo "Must enter a path to initialize" >&2
exit 1
fi
mkdir -p $1/templates
mkdir -p $1/scripts
mkdir -p $1/files
touch $1/requires
mkdir -p $1/parameters/$(basename $1)
}
$1 $2

91
universe/bin/disco-ball Executable file
View File

@@ -0,0 +1,91 @@
#!/bin/bash
DISCOROOT=/var/disco
RSYNC=$(disco-param get disco/client/cmds/rsync || exit 1)
SERVERURI=$(disco-param get disco/server/uri || exit 1)
STORAGE=$(disco-param get disco/client/storage 2>/dev/null || echo "/var/disco/localstore")
mkdir -p ${STORAGE}
function init() {
if [ "$1" == "" ]; then
echo "Must enter a path to initialize" >&2
exit 1
fi
mkdir -p $1/templates
mkdir -p $1/scripts
mkdir -p $1/files
touch $1/requires
mkdir -p $1/parameters/$(basename $1)
}
function fetch_params() {
# Sometimes there's some shell escaping voodoo and rsync doesn't like the command args some people will pass it straight on
# the command line; so we wrap it in a little bash script, and everyone's happy.
echo ${RSYNC} --delete ${SERVERURI}::parameters/$(hostname)/* ${DISCOROOT}/parameters/$(hostname)/ > /tmp/$$.sh
echo 'exit $?' >> /tmp/$$.sh
/bin/bash /tmp/$$.sh
RETVAL=$?
rm -f /tmp/$$.sh
exit $RETVAL
}
function fetch() {
if [ "$1" == "" ]; then
echo "Must pass a module name to fetch"
exit 1
fi
rm -rf ${STORAGE}/${1}
# Sometimes there's some shell escaping voodoo and rsync doesn't like the command args some people will pass it straight on
# the command line; so we wrap it in a little bash script, and everyone's happy.
echo ${RSYNC} ${SERVERURI}::${1}/files/* / > /tmp/$$.sh
echo ${RSYNC} --delete ${SERVERURI}::${1}/requires ${SERVERURI}::${1}/scripts ${SERVERURI}::${1}/parameters ${SERVERURI}::${1}/templates ${STORAGE}/${1} >> /tmp/$$.sh
echo 'exit $?' >> /tmp/$$.sh
/bin/bash /tmp/$$.sh
RETVAL=$?
rm -f /tmp/$$.sh
exit $RETVAL
}
function requires()
{
if [ "$1" == "" ]; then
echo "Must pass a module name for requirements"
exit 1
fi
cat ${STORAGE}/${1}/requires | sed s/"^"/"${1} "/g
}
function exec() {
RETVAL=0
for file in $(find ${STORAGE}/${1}/scripts/ -type f | sort -u)
do
NOOP="$NOOP" disco-sh-exec $file /tmp/${1}-$(basename $file)
RETVAL=$(expr $RETVAL + $?)
done
}
function template() {
if [ "$1" == "" ]; then
echo "Must pass a module name to template"
exit 1
fi
DESTROOT=""
if [ "$NOOP" != "" ]; then
DESTROOT=/var/disco/testfs/noop
fi
cd ${STORAGE}/${1}/templates
for file in $(find . -type f | sed s/"^\.\/"//g)
do
NOOP=true disco-sh-exec $(pwd)/$file /$file >/tmp/$$.tmpl
if [ $? -eq 0 ]; then
mkdir -p $(dirname ${DESTROOT}/$file)
mv /tmp/$$.tmpl ${DESTROOT}/$file
else
echo "error: template ${1}/templates/$file failed, not replacing /$file"
exit 1
fi
done
exit 0
}
$1 $2

View File

@@ -40,8 +40,27 @@ function disco_delete() {
function disco_set() { function disco_set() {
if [ "$1" == "--help" ]; then if [ "$1" == "--help" ]; then
echo "info: disco-param set [PATH_NAME] [optarg] ... reads from stdin and sets PATH_NAME, unless [optarg] is {}, in which case a new hierarchy is made" >&2 echo "info: disco-param set [PATH_NAME] [optarg] ... reads from stdin and sets PATH_NAME, unless [optarg] is {}, in which case a new hierarchy is made" >&2
echo "info: disco-param set - .... Reads lines from stdin, with each line being PATH_NAME=VALUE, and performs 'set' for each pair." >&2
exit 1 exit 1
fi fi
if [ "$1" == "-" ]; then
RETVAL=0
while read LINE
do
PATHNAME=$(echo $LINE | cut -d = -f 1)
VALUE=$(echo $LINE | cut -d = -f 2-| sed s/"^\s*"//g)
echo $PATHNAME = $VALUE
if [ "$VALUE" == "{}" ]; then
disco_set $PATHNAME "{}"
else
echo $VALUE | disco_set $PATHNAME
fi
RETVAL=$(expr $RETVAL + $?)
done
exit $RETVAL
fi
check_names_regex "$1" || exit 1 check_names_regex "$1" || exit 1
mkdir -p $(dirname ${PARAMROOT}/${1}) mkdir -p $(dirname ${PARAMROOT}/${1})
if [ "$2" == "" ]; then if [ "$2" == "" ]; then
@@ -58,41 +77,63 @@ function disco_set() {
fi fi
} }
function disco_get() { function disco_keys() {
if [ "$1" == "--help" ]; then if [ "$1" == "--help" ]; then
echo "info: disco-param get [PATH_NAME]" >&2 echo "info: disco-param keys [PATH_NAME] ... returns a list of all keys present in the toplevel of the given directory" >&2
exit 1 exit 1
fi fi
VARPATH=$(echo $1 | sed s/"^\/*"//g) VARPATH=$(echo $1 | sed s/"^\/*"//g)
check_names_regex "$VARPATH" || exit 1 check_names_regex "$VARPATH" || exit 1
if [ -d ${PARAMROOT}/${VARPATH} ]; then if [ -d ${PARAMROOT}/${VARPATH} ]; then
if [ "$RECURSING" == "" ]; then
echo "${VARPATH} = {}"
else
echo '{}'
fi
# Iterate over the path like a key/value dictionary # Iterate over the path like a key/value dictionary
cd ${PARAMROOT}/${VARPATH} cd ${PARAMROOT}/${VARPATH}
for file in $(ls 2>/dev/null); for file in $(ls 2>/dev/null);
do do
echo "${VARPATH}/${file} = $(RECURSING='true' disco_get ${VARPATH}/${file})" | sed s/"^\/*"//g echo $file
done done
exit 0 exit 0
elif [ -e ${PARAMROOT}/${VARPATH} ]; then
cat ${PARAMROOT}/${VARPATH}
exit 0
else else
echo "error: ${VARPATH} is undefined." echo "error: $VARPATH does not exist or is not a directory" >&2
exit 1 exit 1
fi fi
} }
function disco_dump() {
if [ "$1" == "--help" ]; then
echo "info: disco-param dump [PATH_NAME] ... Print each key/value pair in the tree in PATH_NAME = VALUE format, one per line." >&2
exit 1
fi
VARPATH=$(echo $1 | sed s/"^\/*"//g)
check_names_regex "$VARPATH" || exit 1
cd $PARAMROOT
find $VARPATH | sed s/"^\.\/*"//g | grep -v "^$" | while read line; do if [ -d $line ]; then echo $line = '{}' ; else echo $line = $(cat $line); fi; done | sort -u
exit 0
}
function disco_get() {
if [ "$1" == "--help" ]; then
echo "info: disco-param get [PATH_NAME] ... Get the value of the given path name." >&2
exit 1
fi
VARPATH=$(echo $1 | sed s/"^\/*"//g)
check_names_regex "$VARPATH" || exit 1
cd $PARAMROOT
if [ -d ${PARAMROOT}/${VARPATH} ]; then
echo '{}'
else
cat ${PARAMROOT}/${VARPATH}
fi
exit $?
}
if [ "$1" == "--help" ]; then if [ "$1" == "--help" ]; then
echo "info: disco-param [CMD] [OPTIONS] ... execute the given command with the given options, and return zero on success." echo "info: disco-param [CMD] [OPTIONS] ... execute the given command with the given options, and return zero on success."
echo echo
$0 set --help $0 set --help
$0 get --help $0 get --help
$0 dump --help
$0 delete --help $0 delete --help
$0 keys --help
exit 0 exit 0
fi fi