Changeset 563

Timestamp:

12/16/2011 18:15:38 (5 months ago)

Author:

blee

Message:

Merge confexport changes from trunk.

Closes #140 and #148

Location:

branches/confman-1.9

Files:

• . (modified) (1 prop)
• confexport.8 (modified) (2 diffs)
• confexport.in (modified) (4 diffs)
• confman.conf.5.in (modified) (1 diff)
• confman.conf.in (modified) (1 diff)
• confmanlib.sh.in (modified) (5 diffs)

branches/confman-1.9/confexport.8

@@ -45 +45 @@
 repository for use by hosts running
 .Xr confsync 8 .
+.Pp
+If the system is already up to date,
+.Nm
+will exit without making any changes.  This behavior can be overridden with the
+.Fl f
+option, which will force
+.Nm
+to run even if the system revision equals the export revision.  Note that the
+.Fl f
+option cannot be used to export a revision less than the system revision.
 .Pp
 The
@@ -103 +113 @@
 .Xr confman.conf 5 ,
 .Xr cron 8
+.Sh BUGS
+When using
+.Nm
+with the CONF_EXPORT_INCREMENTAL option, changes to files that are
+referenced in other modules by in-repository symlinks do not cause
+all relevant exports to be regenerated.  This can be worked around by
+either disabling the CONF_EXPORT_INCREMENTAL option or running
+.Nm
+with the
+.Fl f
+flag when such changes are made.
 .Sh AUTHORS
 .An Chris Cowart Aq ccowart@timesinks.net

branches/confman-1.9/confexport.in

@@ -29 +29 @@
 # This script prepares an "export" of the confman repository for system
 # processes to be able to pull. It is intended to be prepared regularly
-# by a cron. It may be useful to modify this script to only prepare
-# the export if the repository's most recent commit post-dates the
-# tarball.
+# by an automated process like cron.
 
 if [ -r @pkgdatadir@/confmancommon.sh ] ; then
@@ -41 +39 @@
 
 # This changes the behavior of some library functions (e.g. URIs, locks)
-CONF_EXPORT="true"
+CONF_EXPORT=true
 
 function print_help {
-    echo "usage: $0 [-d]" >&4
+    echo "usage: $0 [-f] [-d]" >&4
     echo "Type '$0 -h' to display this help and exit." >&4
     conf_cleanExit ${1:-0}
 }
 
-while getopts "dh" OPT 2>&4 ; do
+FORCE=false
+DEBUG=false
+
+while getopts "fdh" OPT 2>&4 ; do
     case "$OPT" in
+        f)  FORCE=true ;;
         d)  DEBUG=true ;;
         h)  print_help 0;;
@@ -64 +66 @@
     exec 1>/dev/null
 fi
+
+echo "Export operation started" | conf_logger
 
 confexport_lock_system
@@ -82 +86 @@
 esac
 
-if [ -z "$temp_conf_export" ] ; then
+if [ -z "${temp_conf_export}" ] ; then
     echo "Could not create tempfile. Exiting." >&2
     exit 1
 fi
 
-echo "Beginning export operation." | conf_logger
-if ! conf_export $temp_conf_export ; then
-    echo "The export failed. Exiting." >&2
-    rm -rf $temp_conf_export
-    exit 1
+if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
+    WORK_PATH=$(conf_tmp_dir)
+    conf_debug "Using temporary export working copy at ${WORK_PATH}"
 else
-    # Prepare the new export
-    chown -R ${CONF_EXPORT_USER}:${CONF_EXPORT_GROUP} $temp_conf_export
-    chmod -R ${CONF_EXPORT_MODE} $temp_conf_export
-    # Switch to the new export
-    old_conf_export=$(conf_tmp_dir)
-    mv ${CONF_EXPORT_FILE} $old_conf_export
-    mv $temp_conf_export ${CONF_EXPORT_FILE}
-    # Remove the old export
-    rm -rf $old_conf_export
+    WORK_PATH="${CONF_EXPORT_WORK_PATH}"
+    conf_debug "Using persistent export working copy at ${WORK_PATH}"
 fi
 
-confexport_unlock_system
+conf_checkout_tree
 
-echo "Export complete." | conf_logger
+repo_export="${CONF_EXPORT_STYLE}:$(conf_revision)"
+sys_export="$(conf_export_sysrev)"
+
+conf_debug "repo_export: ${repo_export}"
+conf_debug "sys_export: ${sys_export}"
+
+if [ "${sys_export}" = "${repo_export}" ]; then
+    if ! ${FORCE}; then
+        echo "System is up to date." | conf_logger
+        conf_cleanExit 0
+    fi
+fi
+
+repo_method=$(echo ${repo_export} | cut -d ':' -f 1)
+sys_method=$(echo ${sys_export} | cut -d ':' -f 1)
+repo_revision=$(echo ${repo_export} | cut -d ':' -f 2)
+sys_revision=$(echo ${sys_export} | cut -d ':' -f 2)
+
+if [ "${sys_method}" != "${repo_method}" ]; then
+    echo "Export style change detected: ${sys_method} -> ${repo_method}"
+    if ${CONF_EXPORT_INCREMENTAL}; then
+        echo "Ignoring CONF_EXPORT_INCREMENTAL during export style change"
+        CONF_EXPORT_INCREMENTAL="false"
+    fi
+fi
+
+if [ "${sys_revision}" -gt "${repo_revision}" ]; then
+    echo "Error: The last export was newer than the current repository.  Exiting without making changes." >&2
+    conf_cleanExit 1
+fi
+
+if ! conf_export "${temp_conf_export}" "${sys_revision}" "${repo_revision}"; then
+    echo "Error: The export failed. Exiting." >&2
+    rm -rf "${temp_conf_export}"
+    conf_cleanExit 1
+fi
+
+# Prepare the new export
+chown -R ${CONF_EXPORT_USER}:${CONF_EXPORT_GROUP} "${temp_conf_export}" || exit 1
+chmod -R ${CONF_EXPORT_MODE} "${temp_conf_export}" || exit 1
+
+# Switch to the new export
+old_conf_export=$(conf_tmp_dir)
+if [ -e "${CONF_EXPORT_FILE}" ]; then
+        mv "${CONF_EXPORT_FILE}" "${old_conf_export}" || exit 1
+fi
+mv "${temp_conf_export}" "${CONF_EXPORT_FILE}"
+
+conf_record_export
+
+echo "Export operation finished successfully" | conf_logger
+
+confexport_unlock_system "${system_lock}"
+
+# Remove the old export
+rm -rf "${old_conf_export}"
+
+# If CONF_EXPORT_WORK_PATH is not set, we were using a temporary
+# directory, so clean it up
+if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
+    rm -rf "${WORK_PATH}"
+fi
 
 conf_cleanExit 0

branches/confman-1.9/confman.conf.5.in

@@ -107 +107 @@
 The export style to be used by
 .Xr confexport 8 . Valid options are: repository, module, recipe.
+.It CONF_EXPORT_INCREMENTAL
+This option optimizes
+.Xr confexport 8 performance.  When enabled,
+.Xr confexport 8 will only export the changes made to the
+.Xr confman 8 repository since the last
+.Xr confexport 8 run.  This option has no effect when using repository-style
+exports.
+.It CONF_EXPORT_INCREMENTAL_LN
+This option further optimizes
+.Xr confexport 8 performance when using CONF_EXPORT_INCREMENTAL.  When enabled,
+.Xr confexport 8 will use
+.Xr ln 1 to copy unchanged exports into the new export rather than
+.Xr cp 1 .  This option can only be used if temporary files are created
+on the same filesystem partition as CONF_EXPORT_FILE.
 .It CONF_FETCH_SSH_KEY
 This is a private SSH keyfile for use with "sftp://" and "scp://" URIs above.

branches/confman-1.9/confman.conf.in

@@ -80 +80 @@
 CONF_EXPORT_WORK_PATH=""
 CONF_EXPORT_STYLE="repository"
+CONF_EXPORT_INCREMENTAL="false"
+CONF_EXPORT_INCREMENTAL_LN="false"
 EXPORT_REPO_PROTOCOL=""
 EXPORT_REPO_HOSTNAME=""

branches/confman-1.9/confmanlib.sh.in

@@ -28 +28 @@
 
 REPO_ACTION="${REPO_DB}/last_action"
+EXPORT_REV_FILE="${REPO_DB}/last_export"
 REPO_SYNC_STATE="${REPO_DB}/state"
 RECIPE_NAME="$(cat "$RECIPE_FILE" 2>/dev/null)"
@@ -377 +378 @@
 function conf_export {
     local exportpath="$1"
+    local sys_revision="$2"
+    local export_revision="$3"
     local modules
-
-    if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
-        WORK_PATH=$(conf_tmp_dir)
-    else
-        WORK_PATH="${CONF_EXPORT_WORK_PATH}"
-    fi
-
-    conf_checkout_tree
-
+    local module
+    local changed_recipes
+    local changed_modules
+    local updated_recipes
+    local recipe
+    local changed_module
+    local changed_recipe
+    local recipes
+    
     case ${CONF_EXPORT_STYLE} in
         repository)
-            tar -czf "$exportpath" -C "${WORK_PATH}" .
+            tar -czf "$exportpath" -C "${WORK_PATH}" . || return 1
             ;;
         module)
-            modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$') # until ticket $40 is resolved
-            for module in $modules; do
-                tar -hczf "$exportpath/$module.tgz" -C "${WORK_PATH}" $module
+            if ! ${FORCE} && ${CONF_EXPORT_INCREMENTAL}; then
+                # Get the list of changed modules
+                # TODO: Catch in-repository symlinks, ticket #144
+                cd "${WORK_PATH}"
+                changed_modules=$(@SVN@ diff -r ${sys_revision}:${export_revision} --summarize | egrep -v '^D[^/]*$' | sed 's/^[^ ][ ]*//' | cut -d '/' -f 1 | sort | uniq | egrep -v '^(meta|checkpoints)$')
+                cd - >/dev/null
+
+                # Preserve old module exports where possible
+                for module in $(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$'); do
+                    # If this module has changed, move on to the next one
+                    for changed_module in ${changed_modules}; do
+                        if [ "${module}" = "${changed_module}" ]; then
+                            continue 2
+                        fi
+                    done
+
+                    # This module was not changed, so preserve its old export
+                    echo "Preserving old export for module: ${module}"
+                    if ${CONF_EXPORT_INCREMENTAL_LN}; then
+                        ln "${CONF_EXPORT_FILE}/${module}.tgz" "${exportpath}" || return 1
+                    else
+                        cp -p "${CONF_EXPORT_FILE}/${module}.tgz" "${exportpath}" || return 1
+                    fi
+                done
+
+                modules="${changed_modules}"
+            else
+                modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$') # until ticket $40 is resolved
+            fi
+
+            for module in ${modules}; do
+                echo "Generating new export for module: ${module}"
+                tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" ${module} || return 1
             done
             ;;
         recipe)
-            for recipefile in $(conf_recipe_dir)/*; do
-                modules="meta $(conf_get_recipe $recipefile)"
-                tar -hczf "$exportpath/${recipefile#$(conf_recipe_dir)/}.tgz" -C "${WORK_PATH}" $modules
+            if ! ${FORCE} && ${CONF_EXPORT_INCREMENTAL}; then
+                # Get the list of changed recipes
+                cd "$(conf_recipe_dir)"
+                changed_recipes=$(@SVN@ diff -r ${sys_revision}:${export_revision} --summarize | grep -v '^D' | sed 's/^[^ ]*[ ]*//' | cut -d '/' -f 1)
+                cd - >/dev/null
+                conf_debug "changed_recipes: ${changed_recipes}"
+
+                # Get the list of changed modules
+                # TODO: Catch in-repository symlinks, ticket #144
+                cd "${WORK_PATH}"
+                changed_modules=$(@SVN@ diff -r ${sys_revision}:${export_revision} --summarize | sed 's/^[^ ]*[ ]*//' | cut -d '/' -f 1 | sort | uniq | egrep -v '^(meta|checkpoints)$')
+                cd - >/dev/null
+                conf_debug "changed_modules: ${changed_modules}"
+
+                # Check all recipes for changes
+                for recipe in $(ls -1 "$(conf_recipe_dir)" | egrep -v '^\.svn$'); do
+                    # If this recipe is in the list of changed recipes,
+                    # there's no need to check it any further since it will
+                    # be regenerated anyway
+                    for changed_recipe in ${changed_recipes}; do
+                        if [ "${recipe}" = "${changed_recipe}" ]; then
+                            continue 2
+                        fi
+                    done
+
+                    # If an unchanged recipe references a changed module,
+                    # its export needs to be regenerated
+                    modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe})
+                    for module in ${modules}; do
+                        for changed_module in ${changed_modules}; do
+                            if [ "${module}" = "${changed_module}" ]; then
+                                updated_recipes="${updated_recipes} ${recipe}"
+                                continue 3
+                            fi
+                        done
+                    done
+
+                    # This recipe has neither changed nor references a
+                    # changed module, so preserve its old export
+                    echo "Preserving old export for recipe: ${recipe}"
+                    if ${CONF_EXPORT_INCREMENTAL_LN}; then
+                        ln "${CONF_EXPORT_FILE}/${recipe}.tgz" "${exportpath}" || return 1
+                    else
+                        cp -p "${CONF_EXPORT_FILE}/${recipe}.tgz" "${exportpath}" || return 1
+                    fi
+                done
+                conf_debug "updated_recipes: ${updated_recipes}"
+
+                recipes="${changed_recipes} ${updated_recipes}"
+            else
+                recipes=$(ls -1 "$(conf_recipe_dir)"| egrep -v '^\.svn$')
+            fi
+
+            for recipe in ${recipes}; do
+                echo "Generating new export for recipe: ${recipe}"
+                modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe})
+                tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "meta/.svn/entries" "meta/recipes/${recipe}" ${modules} || return 1
             done
             ;;
@@ -409 +496 @@
     esac
 
-    if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
-        rm -rf ${WORK_PATH}
-    fi
-    
     return 0
 }
@@ -999 +1082 @@
 }
 
+# Returns the revision of the last export operation on stdout
+function conf_export_sysrev {
+    local export_sysrev
+    local rv
+
+    export_sysrev="none:0"
+    rv=0
+
+    if [ -f "${EXPORT_REV_FILE}" ]; then
+        if ! export_sysrev=$(cat "${EXPORT_REV_FILE}"); then
+            echo "Could not read file: ${EXPORT_REV_FILE}" >&2
+            export_sysrev="none:0"
+            rv=1
+        fi
+    fi
+
+    echo "${export_sysrev}"
+    return ${rv}
+}
+
 # Returns the last action of the repository on stdout
 function conf_lastact {
@@ -1013 +1116 @@
     local revision=`conf_revision`
     $SUDO sh -c "echo '${action}:${revision}' > $REPO_ACTION"
+}
+
+# Records the export operation
+function conf_record_export {
+    local revision=$(conf_revision)
+
+    if ! echo "${CONF_EXPORT_STYLE}:${revision}" > "${EXPORT_REV_FILE}"; then
+            echo "Could not record export to file: ${EXPORT_REV_FILE}" >&2
+            return 1
+    fi
 }