Changeset 525

Timestamp:

08/14/2011 17:15:31 (9 months ago)

Author:

blee

Message:

Create new CONF_EXPORT_INCREMENTAL configuration option.

When enabled, this optimizes confexport performance by only regenerating
export tarballs that have been changed since the previous confexport run.

This option has no effect when using repository-style exports.

See #140

Location:

trunk

Files:

• confexport.in (modified) (1 diff)
• confman.conf.5.in (modified) (1 diff)
• confman.conf.in (modified) (1 diff)
• confmanlib.sh.in (modified) (1 diff)

trunk/confexport.in

@@ -117 +117 @@
 fi
 
-if ! conf_export "${temp_conf_export}"; then
+if ! conf_export "${temp_conf_export}" "${sys_revision}" "${export_revision}"; then
     echo "Error: The export failed. Exiting." >&2
     rm -rf "${temp_conf_export}"

trunk/confman.conf.5.in

@@ -107 +107 @@
 The export style to be used by
 .Xr confexport 8 . Valid options are: repository, module, recipe.
+.It CONF_EXPORT_INCREMENTAL
+This option optimizes
+.Xr confexport 8 performance.  When enabled,
+.Xr confexport 8 will only export the changes made to the
+.Xr confman 8 repository since the last
+.Xr confexport 8 run.  This option has no effect when using repository-style
+exports.
 .It CONF_FETCH_SSH_KEY
 This is a private SSH keyfile for use with "sftp://" and "scp://" URIs above.

trunk/confman.conf.in

@@ -80 +80 @@
 CONF_EXPORT_WORK_PATH=""
 CONF_EXPORT_STYLE="repository"
+CONF_EXPORT_INCREMENTAL="false"
 EXPORT_REPO_PROTOCOL=""
 EXPORT_REPO_HOSTNAME=""

trunk/confmanlib.sh.in

@@ -402 +402 @@
 function conf_export {
     local exportpath="$1"
+    local sys_revision="$2"
+    local export_revision="$3"
     local modules
     local module
-    local recipefile
+    local changed_recipes
+    local changed_modules
+    local updated_recipes
     local recipe
-
+    local changed_module
+    local changed_recipe
+    local recipes
+    
     case ${CONF_EXPORT_STYLE} in
         repository)
-            tar -czf "$exportpath" -C "${WORK_PATH}" .
+            tar -czf "$exportpath" -C "${WORK_PATH}" . || return 1
             ;;
         module)
-            modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$') # until ticket $40 is resolved
-            for module in $modules; do
-                tar -hczf "$exportpath/$module.tgz" -C "${WORK_PATH}" $module
+            if ! ${FORCE} && ${CONF_EXPORT_INCREMENTAL}; then
+                cd "${WORK_PATH}"
+                changed_modules=$(@SVN@ diff -r ${sys_revision}:${export_revision} --summarize | sed 's/^.[ ]*//' | cut -d '/' -f 1 | sort | uniq | egrep -v '^(meta|checkpoints)$')
+                cd - >/dev/null
+
+                # Preserve old module exports where possible
+                for module in $(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$'); do
+                    # If this module has changed, move on to the next one
+                    for changed_module in ${changed_modules}; do
+                        if [ "${module}" = "${changed_module}" ]; then
+                            continue 2
+                        fi
+                    done
+
+                    # This module was not changed, so preserve its old export
+                    echo "Preserving old export for module: ${module}"
+                    cp -p "${CONF_EXPORT_FILE}/${module}.tgz" "${exportpath}" || return 1
+                done
+
+                modules="${changed_modules}"
+            else
+                modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$') # until ticket $40 is resolved
+            fi
+
+            for module in ${modules}; do
+                echo "Generating new export for module: ${module}"
+                tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" ${module} || return 1
             done
             ;;
         recipe)
-            for recipefile in $(conf_recipe_dir)/*; do
-                recipe=${recipefile#$(conf_recipe_dir)/}
-                modules=$(conf_get_recipe ${recipefile})
-                tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "meta/.svn/entries" "meta/recipes/${recipe}" ${modules}
+            if ! ${FORCE} && ${CONF_EXPORT_INCREMENTAL}; then
+                # Get the list of changed recipes
+                cd "$(conf_recipe_dir)"
+                changed_recipes=$(@SVN@ diff -r ${sys_revision}:${export_revision} --summarize | sed 's/^.[ ]*//' | cut -d '/' -f 1)
+                cd - >/dev/null
+                conf_debug "changed_recipes: ${changed_recipes}"
+
+                # Get the list of changed modules
+                cd "${WORK_PATH}"
+                changed_modules=$(@SVN@ diff -r ${sys_revision}:${export_revision} --summarize | sed 's/^.[ ]*//' | cut -d '/' -f 1 | sort | uniq | egrep -v '^(meta|checkpoints)$')
+                cd - >/dev/null
+                conf_debug "changed_modules: ${changed_modules}"
+
+                # Check all recipes for changes
+                for recipe in $(ls -1 "$(conf_recipe_dir)" | egrep -v '^\.svn$'); do
+                    # If this recipe is in the list of changed recipes,
+                    # there's no need to check it any further since it will
+                    # be regenerated anyway
+                    for changed_recipe in ${changed_recipes}; do
+                        if [ "${recipe}" = "${changed_recipe}" ]; then
+                            continue 2
+                        fi
+                    done
+
+                    # If an unchanged recipe references a changed module,
+                    # its export needs to be regenerated
+                    modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe})
+                    for module in ${modules}; do
+                        for changed_module in ${changed_modules}; do
+                            if [ "${module}" = "${changed_module}" ]; then
+                                updated_recipes="${updated_recipes} ${recipe}"
+                                continue 3
+                            fi
+                        done
+                    done
+
+                    # This recipe has neither changed nor references a
+                    # changed module, so preserve its old export
+                    echo "Preserving old export for recipe: ${recipe}"
+                    cp -p "${CONF_EXPORT_FILE}/${recipe}.tgz" "${exportpath}" || return 1
+                done
+                conf_debug "updated_recipes: ${updated_recipes}"
+
+                recipes="${changed_recipes} ${updated_recipes}"
+            else
+                recipes=$(ls -1 "$(conf_recipe_dir)"| egrep -v '^\.svn$')
+            fi
+
+            for recipe in ${recipes}; do
+                echo "Generating new export for recipe: ${recipe}"
+                modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe})
+                tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "meta/.svn/entries" "meta/recipes/${recipe}" ${modules} || return 1
             done
             ;;