Changeset 586

Timestamp:

12/23/2011 00:42:07 (5 months ago)

Author:

blee

Message:

Merge fix for #139 into confman-1.9.

Closes #139

Location:

branches/confman-1.9

Files:

• . (modified) (1 prop)
• confexport.in (modified) (1 diff)
• confman.conf.5.in (modified) (1 diff)
• confman.conf.in (modified) (2 diffs)
• confmancommon.sh.in (modified) (1 diff)
• confmanlib.sh.in (modified) (9 diffs)
• confsync.in (modified) (1 diff)

branches/confman-1.9/confexport.in

@@ -145 +145 @@
 old_conf_export=$(conf_tmp_dir)
 if [ -e "${CONF_EXPORT_FILE}" ]; then
-        mv "${CONF_EXPORT_FILE}" "${old_conf_export}" || exit 1
+    mv "${CONF_EXPORT_FILE}" "${old_conf_export}" || exit 1
 fi
 mv "${temp_conf_export}" "${CONF_EXPORT_FILE}"

branches/confman-1.9/confman.conf.5.in

@@ -121 +121 @@
 .Xr cp 1 .  This option can only be used if temporary files are created
 on the same filesystem partition as CONF_EXPORT_FILE.
+.It CONF_EXPORT_FILEDB
+This option optimizes
+.Xr confexport 8 and
+.Xr confsync 8 performance.  When enabled,
+.Xr confexport 8 will convert repository metadata from Subversion metadata
+into a proprietary file-backed metadata format and exclude all Subversion
+working-copy-related files from exports.  This significantly reduces the
+file size of export tarballs.  This also eliminates the previous requirement
+that the version of Subversion be matched between
+.Xr confexport 8 and
+.Xr confsync 8 machines.
 .It CONF_FETCH_SSH_KEY
 This is a private SSH keyfile for use with "sftp://" and "scp://" URIs above.

branches/confman-1.9/confman.conf.in

@@ -18 +18 @@
 REPO_DOCS_CONFMAN="@pkgdatadir@/confmandoc.sh"
 REPO_DOCS_CONFADMIN="@pkgdatadir@/confadmindoc.sh"
+
+# The location of the filedb library
+FILEDB_LIBRARY="@pkgdatadir@/libfiledb.sh"
 
 # The system lockfile
@@ -82 +85 @@
 CONF_EXPORT_INCREMENTAL="false"
 CONF_EXPORT_INCREMENTAL_LN="false"
+CONF_EXPORT_FILEDB="false"
 EXPORT_REPO_PROTOCOL=""
 EXPORT_REPO_HOSTNAME=""

branches/confman-1.9/confmancommon.sh.in

@@ -133 +133 @@
 fi
 
+# Source the filedb library
+if [ -r "${FILEDB_LIBRARY}" ]; then
+    . "${FILEDB_LIBRARY}"
+else
+    echo "Could not load filedb library: ${FILEDB_LIBRARY}" >&2
+    exit 1
+fi
+
 # Create and/or fix permissions on our local state directory
 if ! [ -d "$REPO_DB" ] ; then

branches/confman-1.9/confmanlib.sh.in

@@ -34 +34 @@
 CONF_EXPORT="false"
 LOG_MESSAGE_SET="false"
+CONF_GET_PROP_USE_FDB="false" # XXX: Until ticket #151 is resolved
 
 VERSION='@VERSION@'
@@ -401 +402 @@
 function conf_status {
         @SVN@ status $*
+}
+
+# XXX: Until we completely change metadata formats (ticket #151), provide
+# a way to convert Subversion properties into an fdb
+function conf_convert_metadata {
+    local module="$1"
+
+    local moduledir
+    declare -A metadata
+
+    metadata["module"]="${module}"
+
+    if [ "${module}" = "meta" ]; then
+        metadata["revision"]=`conf_revision_svn`
+        filedb_write "metadata" "`conf_meta_dir`/${module}.fdb"
+        unset metadata
+        return 0
+    fi
+
+    moduledir="${WORK_PATH}/${module}"
+
+    for directory in `find -L "${moduledir}" -mindepth 1 -type d -not -name '.svn' -not -path '*/.svn/*'`; do
+        owner=`conf_get_prop ${directory} owner`
+        group=`conf_get_prop ${directory} group`
+        mode=`conf_get_prop ${directory} mode`
+
+        directory="${directory#${moduledir}}"
+        metadata["${directory}:owner"]="${owner}"
+        metadata["${directory}:group"]="${group}"
+        metadata["${directory}:mode"]="${mode}"
+    done
+
+    for file in `find -L "${moduledir}" -mindepth 1 -type f -not -path '*/.svn/*'`; do
+        owner=`conf_get_prop ${file} owner`
+        group=`conf_get_prop ${file} group`
+        mode=`conf_get_prop ${file} mode`
+        symlink="`conf_get_prop ${file} symlink`"
+
+        file="${file#${moduledir}}"
+        metadata["${file}:owner"]="${owner}"
+        metadata["${file}:group"]="${group}"
+        metadata["${file}:mode"]="${mode}"
+        metadata["${file}:symlink"]="${symlink}"
+    done
+
+    filedb_write "metadata" "`conf_meta_dir`/${module}.fdb"
+    unset metadata
 }
 
@@ -431 +479 @@
 
                 # Preserve old module exports where possible
-                for module in $(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$'); do
+                for module in $(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|meta|checkpoints)$'); do
                     # If this module has changed, move on to the next one
                     for changed_module in ${changed_modules}; do
@@ -449 +497 @@
 
                 modules="${changed_modules}"
+
+                # XXX: Metadata needs to be regenerated unconditionally for now
+                modules="meta ${modules}"
             else
-                modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$') # until ticket $40 is resolved
+                # XXX: Until ticket #40 is resolved
+                modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$')
             fi
 
-            for module in ${modules}; do
-                echo "Generating new export for module: ${module}"
-                tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" ${module} || return 1
-            done
+            if ${CONF_EXPORT_FILEDB}; then
+                for module in ${modules}; do
+                    echo "Converting metadata for module: ${module}"
+                    conf_convert_metadata "${module}"
+
+                    echo "Generating new export for module: ${module}"
+                    tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" --exclude='*.svn*' ${module} "meta/${module}.fdb" || return 1
+
+                    # XXX: The fdbs aren't versioned yet (ticket #151), so
+                    # it's not necessary to keep them around because they
+                    # will be regenerated anyway when the module changes
+                    rm -f "$(conf_meta_dir)/${module}.fdb" || return 1
+                done
+            else
+                for module in ${modules}; do
+                    echo "Generating new export for module: ${module}"
+                    tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" ${module} || return 1
+                done
+            fi
             ;;
         recipe)
@@ -507 +574 @@
                 conf_debug "updated_recipes: ${updated_recipes}"
 
+                modules="${changed_modules}"
                 recipes="${changed_recipes} ${updated_recipes}"
+
+                # XXX: Metadata needs to be regenerated unconditionally for now
+                modules="meta ${modules}"
             else
+                # XXX: Until ticket #40 is resolved
+                modules=`ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$'`
                 recipes=$(ls -1 "$(conf_recipe_dir)"| egrep -v '^\.svn$')
             fi
 
-            for recipe in ${recipes}; do
-                echo "Generating new export for recipe: ${recipe}"
-                modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe})
-                tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "meta/.svn/entries" "meta/recipes/${recipe}" ${modules} || return 1
-            done
+            if ${CONF_EXPORT_FILEDB}; then
+                for module in ${modules}; do
+                    echo "Regenerating metadata for module: ${module}"
+                    conf_convert_metadata "${module}"
+                done
+
+                for recipe in ${recipes}; do
+                    echo "Generating new export for recipe: ${recipe}"
+                    modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe})
+                    tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" --exclude='*.svn*' "meta/meta.fdb" "meta/recipes/${recipe}" $(for module in ${modules}; do echo "meta/${module}.fdb"; done) ${modules} || return 1
+                done
+            else
+                for recipe in ${recipes}; do
+                    echo "Generating new export for recipe: ${recipe}"
+                    modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe})
+                    tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "meta/.svn/entries" "meta/recipes/${recipe}" ${modules} || return 1
+                done
+            fi
             ;;
         *)
@@ -556 +642 @@
         ${NFS_HACK:-false} && chmod o+rx ${WORK_PATH}
 
-    # Test to see if the SVN working copy is out of date
-    if ! @SVN@ info $moduledir >/dev/null ; then
-        echo "Error: $moduledir isn't usable by svn." >&2
-        exit 1
+    # XXX: Until ticket #151 is resolved, prefer to use the fdb metadata
+    # when it is available
+    if ! ${CONF_GET_PROP_USE_FDB}; then
+        # Test to see if the SVN working copy is out of date
+        if ! @SVN@ info $moduledir >/dev/null ; then
+            echo "Error: $moduledir isn't usable by svn." >&2
+            exit 1
+        fi
     fi
 
@@ -775 +865 @@
                 myfile="${WORK_PATH}/${layer}/${file}-${layer}"
                 if [ -f $myfile ] && $SUDO [ -f $livepart ] ; then
-                                owner=`conf_get_prop ${myfile} owner`
-                                group=`conf_get_prop ${myfile} group`
-                                mode=`conf_get_prop     ${myfile} mode`
-                                comment=`conf_get_prop  ${myfile} comment`
+            eval `$SUDO ${stat_cmd} "${stat_opts}" "${livepart}"`
                         $SUDO cat $livepart >> $tmpfile
                         $SUDO rm $livepart
@@ -901 +988 @@
 # This function will print the value of the specified property to STDOUT
 function conf_get_prop {
-        local file=$1
-        local prop=$2
-        local result=`@SVN@ propget "confman:${prop}" ${file}`
-    if [ -z "$result" ] ; then
-        file=`$readlink_cmd -m ${file}`
-        @SVN@ propget "confman:${prop}" ${file}
-    else
-        echo $result
-    fi
+    local file="$1"
+    local prop="$2"
+    
+    local moduledir
+    local result
+    
+    # XXX: Until ticket #151 is resolved, prefer to use the fdb metadata
+    # when it is available
+    if ${CONF_GET_PROP_USE_FDB}; then
+        moduledir="${WORK_PATH}/${metadata[module]}"
+        file="${file#${moduledir}}"
+        result="${metadata[${file}:${prop}]}"
+    else
+        result=`@SVN@ propget "confman:${prop}" ${file}`
+        if [ -z "$result" ] ; then
+            file=`$readlink_cmd -m ${file}`
+            result=`@SVN@ propget "confman:${prop}" ${file}`
+        fi
+    fi
+    
+    echo "${result}"
 }
 
@@ -1131 +1230 @@
 # Returns the current revision number of the repository on stdout
 function conf_revision {
-    @SVN@ info $(conf_meta_dir) | awk '/^Revision:/ {print $2}'
+    # XXX: Until ticket #151 is resolved, prefer to use the fdb
+    # when it is available
+    if [ -r "$(conf_meta_dir)/meta.fdb" ]; then
+        conf_revision_fdb "$@"
+    else
+        conf_revision_svn "$@"
+    fi
+}
+
+# Returns the current revision number of the repository using Subversion
+function conf_revision_svn {
+    @SVN@ info "$(conf_meta_dir)" | awk '/^Revision:/ {print $2}'
+}
+
+# Returns the current revision number of the repository using filedb
+function conf_revision_fdb {
+    declare -A metadata
+    filedb_load "$(conf_meta_dir)/meta.fdb" metadata
+    echo "${metadata["revision"]}"
+    unset metadata
 }
 

branches/confman-1.9/confsync.in

@@ -134 +134 @@
 for layer in $(conf_get_reverse_recipe) ; do
     echo "Rolling on $layer..."
+
+    # XXX: Until ticket #151 is resolved, prefer to use the fdb metadata
+    # when it is available
+    if [ -r "$(conf_meta_dir)/${layer}.fdb" ]; then
+        declare -A metadata
+        filedb_load "$(conf_meta_dir)/${layer}.fdb" "metadata"
+        CONF_GET_PROP_USE_FDB="true"
+    fi
+
     conf_rollout $layer $statefile || conf_cleanExit
+
+    if ${CONF_GET_PROP_USE_FDB}; then
+        unset metadata
+    fi
 done
 for file in $SINGULARITIES ; do