Changeset 585 for trunk


Ignore:
Timestamp:
12/23/2011 00:31:34 (5 months ago)
Author:
blee
Message:

Introduce new option CONF_EXPORT_FILEDB.

This allows confexport to convert repository metadata into a
proprietary file-backed metadata format and exclude all Subversion
working-copy-related files from exports.

confsync has been modified to automatically detect the presence of
the new metadata format and use it if it is available.

This has multiple benefits:

  • Significantly reduces the size of export tarballs
  • Reduces the time to generate export tarballs
  • Eliminates the requirement that confexport/confsync have matching versions of Subversion
  • Significantly improves confsync performance by eliminating svn forks

There will be even further benefits when ticket #151 is closed because
the repository metadata wil not need to be converted during confexport.

Fixes #139

Location:
trunk
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/confexport.in

    r539 r585  
    145145old_conf_export=$(conf_tmp_dir) 
    146146if [ -e "${CONF_EXPORT_FILE}" ]; then 
    147         mv "${CONF_EXPORT_FILE}" "${old_conf_export}" || exit 1 
     147    mv "${CONF_EXPORT_FILE}" "${old_conf_export}" || exit 1 
    148148fi 
    149149mv "${temp_conf_export}" "${CONF_EXPORT_FILE}" 

  • trunk/confman.conf.5.in

    r542 r585  
    121121.Xr cp 1 .  This option can only be used if temporary files are created 
    122122on the same filesystem partition as CONF_EXPORT_FILE. 
     123.It CONF_EXPORT_FILEDB 
     124This option optimizes 
     125.Xr confexport 8 and 
     126.Xr confsync 8 performance.  When enabled, 
     127.Xr confexport 8 will convert repository metadata from Subversion metadata 
     128into a proprietary file-backed metadata format and exclude all Subversion 
     129working-copy-related files from exports.  This significantly reduces the 
     130file size of export tarballs.  This also eliminates the previous requirement 
     131that the version of Subversion be matched between 
     132.Xr confexport 8 and 
     133.Xr confsync 8 machines. 
    123134.It CONF_FETCH_SSH_KEY 
    124135This is a private SSH keyfile for use with "sftp://" and "scp://" URIs above. 

  • trunk/confman.conf.in

    r542 r585  
    1818REPO_DOCS_CONFMAN="@pkgdatadir@/confmandoc.sh" 
    1919REPO_DOCS_CONFADMIN="@pkgdatadir@/confadmindoc.sh" 
     20 
     21# The location of the filedb library 
     22FILEDB_LIBRARY="@pkgdatadir@/libfiledb.sh" 
    2023 
    2124# The system lockfile 
    … …  
    8285CONF_EXPORT_INCREMENTAL="false" 
    8386CONF_EXPORT_INCREMENTAL_LN="false" 
     87CONF_EXPORT_FILEDB="false" 
    8488EXPORT_REPO_PROTOCOL="" 
    8589EXPORT_REPO_HOSTNAME="" 

  • trunk/confmancommon.sh.in

    r506 r585  
    133133fi 
    134134 
     135# Source the filedb library 
     136if [ -r "${FILEDB_LIBRARY}" ]; then 
     137    . "${FILEDB_LIBRARY}" 
     138else 
     139    echo "Could not load filedb library: ${FILEDB_LIBRARY}" >&2 
     140    exit 1 
     141fi 
     142 
    135143# Create and/or fix permissions on our local state directory 
    136144if ! [ -d "$REPO_DB" ] ; then 

  • trunk/confmanlib.sh.in

    r572 r585  
    3434CONF_EXPORT="false" 
    3535LOG_MESSAGE_SET="false" 
     36CONF_GET_PROP_USE_FDB="false" # XXX: Until ticket #151 is resolved 
    3637 
    3738VERSION='@VERSION@' 
    … …  
    401402function conf_status { 
    402403        @SVN@ status $* 
     404} 
     405 
     406# XXX: Until we completely change metadata formats (ticket #151), provide 
     407# a way to convert Subversion properties into an fdb 
     408function conf_convert_metadata { 
     409    local module="$1" 
     410 
     411    local moduledir 
     412    declare -A metadata 
     413 
     414    metadata["module"]="${module}" 
     415 
     416    if [ "${module}" = "meta" ]; then 
     417        metadata["revision"]=`conf_revision_svn` 
     418        filedb_write "metadata" "`conf_meta_dir`/${module}.fdb" 
     419        unset metadata 
     420        return 0 
     421    fi 
     422 
     423    moduledir="${WORK_PATH}/${module}" 
     424 
     425    for directory in `find -L "${moduledir}" -mindepth 1 -type d -not -name '.svn' -not -path '*/.svn/*'`; do 
     426        owner=`conf_get_prop ${directory} owner` 
     427        group=`conf_get_prop ${directory} group` 
     428        mode=`conf_get_prop ${directory} mode` 
     429 
     430        directory="${directory#${moduledir}}" 
     431        metadata["${directory}:owner"]="${owner}" 
     432        metadata["${directory}:group"]="${group}" 
     433        metadata["${directory}:mode"]="${mode}" 
     434    done 
     435 
     436    for file in `find -L "${moduledir}" -mindepth 1 -type f -not -path '*/.svn/*'`; do 
     437        owner=`conf_get_prop ${file} owner` 
     438        group=`conf_get_prop ${file} group` 
     439        mode=`conf_get_prop ${file} mode` 
     440        symlink="`conf_get_prop ${file} symlink`" 
     441 
     442        file="${file#${moduledir}}" 
     443        metadata["${file}:owner"]="${owner}" 
     444        metadata["${file}:group"]="${group}" 
     445        metadata["${file}:mode"]="${mode}" 
     446        metadata["${file}:symlink"]="${symlink}" 
     447    done 
     448 
     449    filedb_write "metadata" "`conf_meta_dir`/${module}.fdb" 
     450    unset metadata 
    403451} 
    404452 
    … …  
    431479 
    432480                # Preserve old module exports where possible 
    433                 for module in $(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$'); do 
     481                for module in $(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|meta|checkpoints)$'); do 
    434482                    # If this module has changed, move on to the next one 
    435483                    for changed_module in ${changed_modules}; do 
    … …  
    449497 
    450498                modules="${changed_modules}" 
     499 
     500                # XXX: Metadata needs to be regenerated unconditionally for now 
     501                modules="meta ${modules}" 
    451502            else 
    452                 modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$') # until ticket $40 is resolved 
     503                # XXX: Until ticket #40 is resolved 
     504                modules=$(ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$') 
    453505            fi 
    454506 
    455             for module in ${modules}; do 
    456                 echo "Generating new export for module: ${module}" 
    457                 tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" ${module} || return 1 
    458             done 
     507            if ${CONF_EXPORT_FILEDB}; then 
     508                for module in ${modules}; do 
     509                    echo "Converting metadata for module: ${module}" 
     510                    conf_convert_metadata "${module}" 
     511 
     512                    echo "Generating new export for module: ${module}" 
     513                    tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" --exclude='*.svn*' ${module} "meta/${module}.fdb" || return 1 
     514 
     515                    # XXX: The fdbs aren't versioned yet (ticket #151), so 
     516                    # it's not necessary to keep them around because they 
     517                    # will be regenerated anyway when the module changes 
     518                    rm -f "$(conf_meta_dir)/${module}.fdb" || return 1 
     519                done 
     520            else 
     521                for module in ${modules}; do 
     522                    echo "Generating new export for module: ${module}" 
     523                    tar -hczf "${exportpath}/${module}.tgz" -C "${WORK_PATH}" ${module} || return 1 
     524                done 
     525            fi 
    459526            ;; 
    460527        recipe) 
    … …  
    507574                conf_debug "updated_recipes: ${updated_recipes}" 
    508575 
     576                modules="${changed_modules}" 
    509577                recipes="${changed_recipes} ${updated_recipes}" 
     578 
     579                # XXX: Metadata needs to be regenerated unconditionally for now 
     580                modules="meta ${modules}" 
    510581            else 
     582                # XXX: Until ticket #40 is resolved 
     583                modules=`ls -1 ${WORK_PATH} | egrep -v '^(\.svn|checkpoints)$'` 
    511584                recipes=$(ls -1 "$(conf_recipe_dir)"| egrep -v '^\.svn$') 
    512585            fi 
    513586 
    514             for recipe in ${recipes}; do 
    515                 echo "Generating new export for recipe: ${recipe}" 
    516                 modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe}) 
    517                 tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "meta/.svn/entries" "meta/recipes/${recipe}" ${modules} || return 1 
    518             done 
     587            if ${CONF_EXPORT_FILEDB}; then 
     588                for module in ${modules}; do 
     589                    echo "Regenerating metadata for module: ${module}" 
     590                    conf_convert_metadata "${module}" 
     591                done 
     592 
     593                for recipe in ${recipes}; do 
     594                    echo "Generating new export for recipe: ${recipe}" 
     595                    modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe}) 
     596                    tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" --exclude='*.svn*' "meta/meta.fdb" "meta/recipes/${recipe}" $(for module in ${modules}; do echo "meta/${module}.fdb"; done) ${modules} || return 1 
     597                done 
     598            else 
     599                for recipe in ${recipes}; do 
     600                    echo "Generating new export for recipe: ${recipe}" 
     601                    modules=$(conf_get_recipe $(conf_recipe_dir)/${recipe}) 
     602                    tar -hczf "${exportpath}/${recipe}.tgz" -C "${WORK_PATH}" "meta/.svn/entries" "meta/recipes/${recipe}" ${modules} || return 1 
     603                done 
     604            fi 
    519605            ;; 
    520606        *) 
    … …  
    556642        ${NFS_HACK:-false} && chmod o+rx ${WORK_PATH} 
    557643 
    558     # Test to see if the SVN working copy is out of date 
    559     if ! @SVN@ info $moduledir >/dev/null ; then 
    560         echo "Error: $moduledir isn't usable by svn." >&2 
    561         exit 1 
     644    # XXX: Until ticket #151 is resolved, prefer to use the fdb metadata 
     645    # when it is available 
     646    if ! ${CONF_GET_PROP_USE_FDB}; then 
     647        # Test to see if the SVN working copy is out of date 
     648        if ! @SVN@ info $moduledir >/dev/null ; then 
     649            echo "Error: $moduledir isn't usable by svn." >&2 
     650            exit 1 
     651        fi 
    562652    fi 
    563653 
    … …  
    775865                myfile="${WORK_PATH}/${layer}/${file}-${layer}" 
    776866                if [ -f $myfile ] && $SUDO [ -f $livepart ] ; then 
    777                                 owner=`conf_get_prop ${myfile} owner` 
    778                                 group=`conf_get_prop ${myfile} group` 
    779                                 mode=`conf_get_prop     ${myfile} mode` 
    780                                 comment=`conf_get_prop  ${myfile} comment` 
     867            eval `$SUDO ${stat_cmd} "${stat_opts}" "${livepart}"` 
    781868                        $SUDO cat $livepart >> $tmpfile 
    782869                        $SUDO rm $livepart 
    … …  
    901988# This function will print the value of the specified property to STDOUT 
    902989function conf_get_prop { 
    903         local file=$1 
    904         local prop=$2 
    905         local result=`@SVN@ propget "confman:${prop}" ${file}` 
    906     if [ -z "$result" ] ; then 
    907         file=`$readlink_cmd -m ${file}` 
    908         @SVN@ propget "confman:${prop}" ${file} 
    909     else 
    910         echo $result 
    911     fi 
     990    local file="$1" 
     991    local prop="$2" 
     992     
     993    local moduledir 
     994    local result 
     995     
     996    # XXX: Until ticket #151 is resolved, prefer to use the fdb metadata 
     997    # when it is available 
     998    if ${CONF_GET_PROP_USE_FDB}; then 
     999        moduledir="${WORK_PATH}/${metadata[module]}" 
     1000        file="${file#${moduledir}}" 
     1001        result="${metadata[${file}:${prop}]}" 
     1002    else 
     1003        result=`@SVN@ propget "confman:${prop}" ${file}` 
     1004        if [ -z "$result" ] ; then 
     1005            file=`$readlink_cmd -m ${file}` 
     1006            result=`@SVN@ propget "confman:${prop}" ${file}` 
     1007        fi 
     1008    fi 
     1009     
     1010    echo "${result}" 
    9121011} 
    9131012 
    … …  
    11311230# Returns the current revision number of the repository on stdout 
    11321231function conf_revision { 
    1133     @SVN@ info $(conf_meta_dir) | awk '/^Revision:/ {print $2}' 
     1232    # XXX: Until ticket #151 is resolved, prefer to use the fdb 
     1233    # when it is available 
     1234    if [ -r "$(conf_meta_dir)/meta.fdb" ]; then 
     1235        conf_revision_fdb "$@" 
     1236    else 
     1237        conf_revision_svn "$@" 
     1238    fi 
     1239} 
     1240 
     1241# Returns the current revision number of the repository using Subversion 
     1242function conf_revision_svn { 
     1243    @SVN@ info "$(conf_meta_dir)" | awk '/^Revision:/ {print $2}' 
     1244} 
     1245 
     1246# Returns the current revision number of the repository using filedb 
     1247function conf_revision_fdb { 
     1248    declare -A metadata 
     1249    filedb_load "$(conf_meta_dir)/meta.fdb" metadata 
     1250    echo "${metadata["revision"]}" 
     1251    unset metadata 
    11341252} 
    11351253 

  • trunk/confsync.in

    r550 r585  
    134134for layer in $(conf_get_reverse_recipe) ; do 
    135135    echo "Rolling on $layer..." 
     136 
     137    # XXX: Until ticket #151 is resolved, prefer to use the fdb metadata 
     138    # when it is available 
     139    if [ -r "$(conf_meta_dir)/${layer}.fdb" ]; then 
     140        declare -A metadata 
     141        filedb_load "$(conf_meta_dir)/${layer}.fdb" "metadata" 
     142        CONF_GET_PROP_USE_FDB="true" 
     143    fi 
     144 
    136145    conf_rollout $layer $statefile || conf_cleanExit 
     146 
     147    if ${CONF_GET_PROP_USE_FDB}; then 
     148        unset metadata 
     149    fi 
    137150done 
    138151for file in $SINGULARITIES ; do 
Note: See TracChangeset for help on using the changeset viewer.