commit 196d70fa434f27ca6cb43ff89af03d058ee2c7a1 Author: Michael Hanselmann Date: Wed Dec 15 18:53:34 2010 +0100 ensure-dirs: Speed up when using big queues The “ensure-dirs” script as included in Ganeti 2.3 is very slow when working with big queues requiring a change of permissions on many or all files. $ find /var/lib/ganeti/queue/ | wc -l 52354 Before this change: $ time /usr/local/lib/ganeti/ensure-dirs -f real 16m4.739s While not adressed in this patch, I'd like to record the overall ineffiency of the “ensure-dirs” script, even after this change: $ time /usr/local/lib/ganeti/ensure-dirs -f real 5m57.362s […] $ strace -e clone,execve -f -c /usr/local/lib/ganeti/ensure-dirs -f % time seconds usecs/call calls errors syscall ------ ----------- ----------- --------- --------- ---------------- 50.08 5.147090 49 104774 clone 49.92 5.131094 49 104739 execve More changes will be needed. Just for comparision, a small Python snippet changing permissions on all files (“ensure-dirs” changes the owner too): $ time python -c 'import os; from ganeti import utils; [os.chmod(i, 0644) for i in utils.ListVisibleFiles("/var/lib/ganeti/queue/archive/big")]' real 0m0.605s […] Signed-off-by: Michael Hanselmann Reviewed-by: Iustin Pop diff --git a/daemons/ensure-dirs.in b/daemons/ensure-dirs.in index 252f24a..4cb4d99 100644 --- a/daemons/ensure-dirs.in +++ b/daemons/ensure-dirs.in @@ -97,14 +97,17 @@ _ensure_datadir() { _ensure_file ${DATADIR}/queue/lock 0600 "$(_fileset_owner masterd)" if ! [[ -z "${full_run}" ]]; then - for file in $(_gather_files ${DATADIR}/queue 0600 @GNTMASTERUSER@ \ - @GNTMASTERDGROUP@); do - _ensure_file "${file}" 0600 "$(_fileset_owner masterd)" + local queue_owner="$(_fileset_owner masterd)" + local ssconf_owner="$(_fileset_owner noded)" + + _gather_files ${DATADIR}/queue 0600 @GNTMASTERUSER@ @GNTMASTERDGROUP@ | \ + while read path; do + _ensure_file "$path" 0600 "$queue_owner" done - for file in $(_gather_files ${DATADIR} 0600 root \ - @GNTMASTERDGROUP@ -name 'ssconf_*'); do - _ensure_file "${file}" 0444 "$(_fileset_owner noded)" + _gather_files ${DATADIR} 0600 root @GNTMASTERDGROUP@ -name 'ssconf_*' | \ + while read path; do + _ensure_file "$path" 0444 "$ssconf_owner" done fi }