Changeset 522 for trunk

Timestamp:

08/12/2011 19:39:18 (9 months ago)

Author:

blee

Message:

First pass at optimizing confexport performance

• Keep state between confexport runs to avoid unnecessary export regeneration
• In recipe-style exports, include only the relevant recipe metadata

See #140

Location:

trunk

Files:

• confexport.in (modified) (4 diffs)
• confmanlib.sh.in (modified) (6 diffs)

trunk/confexport.in

@@ -29 +29 @@
 # This script prepares an "export" of the confman repository for system
 # processes to be able to pull. It is intended to be prepared regularly
-# by a cron. It may be useful to modify this script to only prepare
-# the export if the repository's most recent commit post-dates the
-# tarball.
+# by an automated process like cron.
 
 if [ -r @pkgdatadir@/confmancommon.sh ] ; then
@@ -41 +39 @@
 
 # This changes the behavior of some library functions (e.g. URIs, locks)
-CONF_EXPORT="true"
+CONF_EXPORT=true
 
 function print_help {
-    echo "usage: $0 [-d]" >&4
+    echo "usage: $0 [-f] [-d]" >&4
     echo "Type '$0 -h' to display this help and exit." >&4
     conf_cleanExit ${1:-0}
 }
 
-while getopts "dh" OPT 2>&4 ; do
+FORCE=false
+DEBUG=false
+
+while getopts "fdh" OPT 2>&4 ; do
     case "$OPT" in
+        f)  FORCE=true ;;
         d)  DEBUG=true ;;
         h)  print_help 0;;
@@ -64 +66 @@
     exec 1>/dev/null
 fi
+
+echo "Export operation started" | conf_logger
 
 confexport_lock_system
@@ -82 +86 @@
 esac
 
-if [ -z "$temp_conf_export" ] ; then
+if [ -z "${temp_conf_export}" ] ; then
     echo "Could not create tempfile. Exiting." >&2
     exit 1
 fi
 
-echo "Beginning export operation." | conf_logger
-if ! conf_export $temp_conf_export ; then
-    echo "The export failed. Exiting." >&2
-    rm -rf $temp_conf_export
-    exit 1
+if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
+    WORK_PATH=$(conf_tmp_dir)
+    conf_debug "Using temporary export working copy at ${WORK_PATH}"
 else
-    # Prepare the new export
-    chown -R ${CONF_EXPORT_USER}:${CONF_EXPORT_GROUP} $temp_conf_export
-    chmod -R ${CONF_EXPORT_MODE} $temp_conf_export
-    # Switch to the new export
-    old_conf_export=$(conf_tmp_dir)
-    mv ${CONF_EXPORT_FILE} $old_conf_export
-    mv $temp_conf_export ${CONF_EXPORT_FILE}
-    # Remove the old export
-    rm -rf $old_conf_export
+    WORK_PATH="${CONF_EXPORT_WORK_PATH}"
+    conf_debug "Using persistent export working copy at ${WORK_PATH}"
 fi
+
+conf_checkout_tree
+
+export_revision=$(conf_revision)
+sys_revision=$(conf_export_sysrev)
+
+conf_debug "export_revision is ${export_revision}"
+conf_debug "sys_revision is ${sys_revision}"
+
+if [ "${sys_revision}" -gt "${export_revision}" ]; then
+    echo "Error: The last export was newer than the current repository.  Exiting without making changes." >&2
+    conf_cleanExit 1
+elif [ "${sys_revision}" -eq "${export_revision}" ]; then
+    if ! ${FORCE}; then
+        echo "System is up to date." | conf_logger
+        conf_cleanExit 0
+    fi
+fi
+
+if ! conf_export "${temp_conf_export}"; then
+    echo "Error: The export failed. Exiting." >&2
+    rm -rf "${temp_conf_export}"
+    conf_cleanExit 1
+fi
+
+# Prepare the new export
+chown -R ${CONF_EXPORT_USER}:${CONF_EXPORT_GROUP} "${temp_conf_export}" || exit 1
+chmod -R ${CONF_EXPORT_MODE} "${temp_conf_export}" || exit 1
+
+# Switch to the new export
+old_conf_export=$(conf_tmp_dir)
+mv "${CONF_EXPORT_FILE}" "${old_conf_export}" || exit 1
+mv "${temp_conf_export}" "${CONF_EXPORT_FILE}"
+
+conf_record_export
+
+echo "Export operation finished successfully" | conf_logger
 
 confexport_unlock_system "${system_lock}"
 
-echo "Export complete." | conf_logger
+# Remove the old export
+rm -rf "${old_conf_export}"
+
+# If CONF_EXPORT_WORK_PATH is not set, we were using a temporary
+# directory, so clean it up
+if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
+    rm -rf "${WORK_PATH}"
+fi
 
 conf_cleanExit 0

trunk/confmanlib.sh.in

@@ -28 +28 @@
 
 REPO_ACTION="${REPO_DB}/last_action"
+EXPORT_REV_FILE="${REPO_DB}/last_export"
 REPO_SYNC_STATE="${REPO_DB}/state"
 RECIPE_NAME="$(cat "$RECIPE_FILE" 2>/dev/null)"
@@ -402 +403 @@
     local exportpath="$1"
     local modules
-
-    if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
-        WORK_PATH=$(conf_tmp_dir)
-    else
-        WORK_PATH="${CONF_EXPORT_WORK_PATH}"
-    fi
-
-    conf_checkout_tree
+    local module
+    local recipefile
+    local recipe
 
     case ${CONF_EXPORT_STYLE} in
@@ -423 +419 @@
         recipe)
             for recipefile in $(conf_recipe_dir)/*; do
-                modules="meta $(conf_get_recipe $recipefile)"
-                tar -hczf "$exportpath/${recipefile#$(conf_recipe_dir)/}.tgz" -C "${WORK_PATH}" $modules
+                recipe=${recipefile#$(conf_recipe_dir)/}
+                modules=$(conf_get_recipe ${recipefile})
+                tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "${recipefile}" ${modules}
             done
             ;;
@@ -433 +430 @@
     esac
 
-    if [ -z "${CONF_EXPORT_WORK_PATH}" ]; then
-        rm -rf ${WORK_PATH}
-    fi
-    
     return 0
 }
@@ -1023 +1016 @@
 }
 
+# Returns the revision of the last export operation on stdout
+function conf_export_sysrev {
+    local export_sysrev
+    local rv
+
+    export_sysrev=0
+    rv=0
+
+    if [ -f "${EXPORT_REV_FILE}" ]; then
+        if ! export_sysrev=$(cat "${EXPORT_REV_FILE}"); then
+            echo "Could not read file: ${EXPORT_REV_FILE}" >&2
+            export_sysrev=0
+            rv=1
+        fi
+    fi
+
+    echo ${export_sysrev}
+    return ${rv}
+}
+
 # Returns the last action of the repository on stdout
 function conf_lastact {
@@ -1037 +1050 @@
     local revision=`conf_revision`
     $SUDO sh -c "echo '${action}:${revision}' > $REPO_ACTION"
+}
+
+# Records the export operation
+function conf_record_export {
+    local revision=$(conf_revision)
+
+    if ! echo "${revision}" > "${EXPORT_REV_FILE}"; then
+            echo "Could not record export to file: ${EXPORT_REV_FILE}" >&2
+            return 1
+    fi
 }