yum repo sync script

sync-all.sh

#!/usr/bin/env bash

EL_VERSION=7

LOG=/tmp/.sync-attempt-out.$$
TIMESTAMP_FILE=/var/tmp/.sync-newer.$$

YUM_BASE=/data/yum${EL_VERSION}
YUM_REPOS_DIR=/etc/yum${EL_VERSION}.repos.d
YUM_CONF=/etc/yum_${EL_VERSION}.conf

cd $YUM_BASE

echo "=> checking for all repos from $YUM_REPOS_DIR that have enabled=1; will sync:"
REPOS=($(egrep '^\[|enabled' $YUM_REPOS_DIR/*.repo | egrep -B 1 '1$' | grep '\[' | cut -d: -f2- | sed -e 's/[][]//g' -e 's/ /\n/g' | sort))
for REPO in ${REPOS[@]}; do
  echo "- $REPO"
done

echo
echo "=> re-creating cache of repo metadata.. "
rm -fr /var/cache/yum/* # per https://access.redhat.com/articles/1320623
yum -c $YUM_CONF clean all
yum -c $YUM_CONF makecache

for i in ${REPOS[@]}; do
  echo
  echo "=> syncing repo: $i"

  touch $TIMESTAMP_FILE
  reposync -d -c $YUM_CONF -r $i .

  if [[ $(find $i -cnewer $TIMESTAMP_FILE|wc -l) = 0 ]]; then
    echo "==> no new files pulled down, skipping repo metadata creation"

  else # new files fetched, need to process them
    echo "==> generating new packages' metadata"
    pushd $i

    SHOULD_CREATEREPO=1
    while [[ $SHOULD_CREATEREPO > 0 ]]; do
      createrepo --update $(pwd)
      if ! [[ $? = 0 ]]; then
        echo "===> error was encountered, checking packages fetched.. "

        echo "====> checking for empty files.. "
        NUM_EMPTY=$(find . -cnewer $TIMESTAMP_FILE -size 0|wc -l)
        if [[ $NUM_EMPTY -ne 0 ]]; then
          echo "====> removing empty files:"
          find . -cnewer $TIMESTAMP_FILE -size 0 -print -delete|cut -d/ -f2-
        else
          echo "====> none found!"
        fi

        echo "====> also removing corrupt RPMs:"
        find . -cnewer $TIMESTAMP_FILE -name \*.rpm -print0|xargs -0 -r file|awk '/rpm:\s+data/ {print substr($1, 0, length($1)-1)}'|tee $LOG
        NUM_CORRUPT=$(<$LOG wc -l)
        if [[ $NUM_CORRUPT -gt 0 ]]; then
          cat $LOG
          cat $LOG|xargs rm -f
        else
          echo "====> no corrupt RPMs found"
        fi

        if [[ $NUM_CORRUPT -ne 0 ]] && [[ $NUM_EMPTY -ne 0 ]]; then
          echo "===> trying createrepo again... "
        else
          echo "===> ERROR: unknown failure of create-repo, no empty nor corrupt files found, skipping retry.. "
          SHOULD_CREATEREPO=0
        fi
      else
        SHOULD_CREATEREPO=0
      fi
    done # successfully generated metadata
    popd
  fi # done processing new files
done
rm -f $TIMESTAMP_FILE