manage

#!/bin/sh

##H Usage: manage ACTION [ARG] [SECURITY-STRING]
##H
##H Available actions:
##H   help            show this help
##H   version         get current version of the service
##H   status          show current service's status
##H   sysboot         start server from crond if not running
##H   restart         (re)start the service
##H   start           (re)start the service
##H   stop            stop the service
##H   pushapps        push couch applications
##H   pushreps        push couch replications
##H   compact         compact database ARG
##H   compactviews    compact database views for design doc ARG ARG
##H   cleanviews      clean view named ARG
##H   backup          rsync databases to ARG (i.e. [user@]host:path)
##H   archive         archive backups to ARG area in castor
##H
##H For more details please refer to operations page:
##H   https://twiki.cern.ch/twiki/bin/view/CMS/CouchDB

[ $(id -un) != cmsweb -o "$1" = "backup" -o "$1" = "archive" ] ||
  { echo "ERROR: please use another account" 1>&2; exit 1; }

case $(uname) in Darwin )
  md5sum() { md5 -r ${1+"$@"}; }
  ;;
esac

ME=$(basename $(dirname $0))
TOP=$(cd $(dirname $0)/../../.. && pwd)
ROOT=$(cd $(dirname $0)/../.. && pwd)
CFGDIR=$(cd $(dirname $0) && pwd)
LOGDIR=$TOP/logs/$ME
STATEDIR=$TOP/state/$ME
KEYFILE=$ROOT/auth/$ME/hmackey.ini
COLOR_OK="\\033[0;32m"
COLOR_WARN="\\033[0;31m"
COLOR_NORMAL="\\033[0;39m"
PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/kerberos/bin
export STAGE_HOST=${STAGE_HOST:-castorcms}
export RFIO_USE_CASTOR_V2=YES

. $ROOT/apps/$ME/etc/profile.d/init.sh

# Start service conditionally on crond restart.
sysboot()
{
  if couchdb -p $STATEDIR/couchdb.pid -s; then :; else
    start
  fi
}

# Start the service.
start()
{
  cd $STATEDIR
  couchdb -A $CFGDIR/ -a $KEYFILE -b \
    -o $LOGDIR/$(date +%Y%m%d-%H%M%S).stdout \
    -e $LOGDIR/$(date +%Y%m%d-%H%M%S).stderr \
    -p $STATEDIR/couchdb.pid </dev/null >/dev/null 2>&1

  push_apps
  replications push
}

# Stop the service.
stop()
{
  couchdb -d -p $STATEDIR/couchdb.pid
}

# Check if the server is running.
status()
{
  couchdb -p $STATEDIR/couchdb.pid -s

  curl -s localhost:5984 | grep -q '{"couchdb":"Welcome","version":"1.[0-9.]*"}' ||
    { echo "CouchDB is not correctly responding to requests"; return; }

  local TASKS=$(curl -s localhost:5984/_active_tasks)
  [ "$TASKS" = '[]' ] && TASKS="No active tasks (e.g. compactions)"
  echo $TASKS

  replications status
}

# When a view is changed, such as a new app version is deployed,
# invoke this to clean up the views in that database.
clean_views()
{
  local database=$1
  [ -n "$database" ] ||
    { echo "You must specify the database you wish to clean the views "; exit 1; }

  curl -s -H "Content-Type: application/json" -X POST http://localhost:5984/$database/_view_cleanup | \
       grep -q '{"ok":true}' ||
    { echo "An error occured while cleaning the views. Please look in the CouchDB logs."; exit 3; }
}

# Push applications from staging area into couchdb.
push_apps()
{
  n=0 started=false
  while [ $n -le 100 ]; do
    couchdb -p $STATEDIR/couchdb.pid -s &> /dev/null &&
      curl -s localhost:5984 | grep -q '{"couchdb":"Welcome","version":"1.[0-9.]*"}' && 
      started=true && break
    echo "waiting for couchdb..."
    sleep 1
    n=$(expr $n + 1)
  done

  if $started; then
    for APP in $STATEDIR/stagingarea/*; do
      [ -f $APP ] || continue
      . $APP
      for DB in $(egrep -o '5984/.*$' $APP | cut -d/ -f2); do
        clean_views $DB
      done
    done
  else
    echo "couchdb did not start, not pushing application"
    exit 1
  fi
}

replications()
{
  [ "$1" = "push" ] && local status=false || local status=true
  local all_reps=$(curl -s localhost:5984/_replicator/_all_docs?include_docs=true \
                   | grep '^{"id":"[^_]' | awk -F\" '{print $4,$14,$28,$32,$38,$42}')
  local req_reps=$(cat $STATEDIR/replication/* 2>/dev/null || echo "")

  if [ -n "$all_reps" ]; then
    echo "$all_reps" | while read ID REV SRC DST FILTER STATE; do
      if echo "$req_reps" | grep -q "$SRC $DST $FILTER"; then
        echo "Replication 'id=$ID source=$SRC target=$DST filter=$FILTER' $STATE."
      else
        echo "Replication 'id=$ID source=$SRC target=$DST filter=$FILTER' unknown."
        if ! $status; then
          echo -n "Removing it... "
          curl -s -X DELETE localhost:5984/_replicator/$ID?rev=$REV
        fi
      fi
    done
  fi

  if [ -n "$req_reps" ]; then
    local IDS=$(echo "$all_reps" | awk '{print $1}')
    echo "$req_reps" | while read SRC DST FILTER; do
      local ID=$(echo "$SRC $DST $FILTER" | md5sum | awk '{print $1}')
      if ! echo "$IDS" | grep -q "$ID"; then
        echo "Replication 'id=$ID source=$SRC target=$DST filter=$FILTER' not pushed."
        if ! $status; then
          echo -n "Pushing it... "
          curl -s -X PUT localhost:5984/_replicator/$ID \
               -d "{\"source\":\"$SRC\", \"target\":\"$DST\", \"continuous\":true, \"filter\":\"$FILTER\"}"
        fi
      fi
    done
  fi
}

# Trigger a database compaction. If "all", then compact all databases
# and all the views of each database.
compact()
{
  local database=$1
  [ -n "$database" ] ||
    { echo "You must specify a database to compact"; exit 3; }

  if [ "$database" = all ]; then
    for database in $STATEDIR/database/[^_]*.couch; do
      [ -f $database ] || continue
      database=${database##*/}
      database=${database%.couch}
      compact_database $database

      curl -s "localhost:5984/$database/_all_docs?startkey=%22_design%22&endkey=%22_design/zzzzzzzz%22" |
        awk -F\" '/"id":"_design/ {print $4}' |
	while read dbview; do
	  compact_views $database ${dbview#*/}
	done
    done
  else
    compact_database $database
  fi
}

compact_database()
{
  local database=$1
  curl -s localhost:5984/$database | grep -q '"compact_running":true' &&
    { echo "$database is already compacting"; exit 5; }

  curl -s -H "Content-Type: application/json" -X POST http://localhost:5984/$database/_compact | \
        grep -q '{"ok":true}' ||
    { echo "An error occured triggering compaction. Please look in the CouchDB logs."; exit 7; }
}

# Trigger a view compaction
compact_views()
{
  local database=$1
  local designdoc=$2
  [ -n "$database" ] ||
    { echo "You must specify a database to compact its views"; exit 3; }
  [ -n "$designdoc" ] ||
    { echo "You must specify a design doc to compact its views"; exit 3; }

  curl -s localhost:5984/$database/_design/$designdoc/_info | grep -q '"compact_running":true' &&
    { echo "$database/$designdoc is already compacting"; exit 5; }

  curl -s -H "Content-Type: application/json" -X POST http://localhost:5984/$database/_compact/$designdoc | \
        grep -q '{"ok":true}' ||
    { echo "An error occured triggering view compaction. Please look in the CouchDB logs."; exit 7; }
}

# Rsync couchdb databases to elsewhere (can be used to restore a backup too)
backup()
{
  local hostdir=$1
  [ -n "$hostdir" ] ||
    { echo "You must specify a destination [user@]host:path."; exit 9; }
  rsync --delete --delete-excluded --exclude "*.compact.*" -au -e 'ssh -c arcfour' $STATEDIR/database/ $hostdir/ ||
    { echo "failed to synchronise databases to $hostdir: $?" 1>&2; exit 11; }
}

# Archive couchdb backups to castor
archive()
{
  local archdir=$1
  klist -s ||
    { echo "You must have a valid kerberos token to run the archive." 1>&2; exit 20; }
  rfstat $archdir >/dev/null ||
    { echo "Could not stat $archdir in the castor archive" 1>&2; exit 19; }

  echo "Checking for couchdb archives not yet staged out to tape:"
  stager_qry -M $archdir | grep couchdb | grep -v STAGED || echo "all staged out."

  echo
  echo "Starting to archive couchdb backups on $(date)"
  for bkp in $STATEDIR/backup/*; do
    [ -d $bkp ] || continue;
    tarfile=couchdb_${bkp##*/}_$(date +%Y%m%d-%H%M).tar
    echo "$archdir/$tarfile"
    tar cf - -C $bkp . | rfcp - $archdir/$tarfile &
  done
  wait;
  [ -n "$tarfile" ] && echo "Archive copy to castor completed on $(date)." ||
    echo "No backups found on $STATEDIR/backup. Nothing to archive."

  #FIXME: do some sanity checks on castor to make sure new archives are genuine
  #FIXME: cleanup old archives from castor
}

# Verify the security string.
check()
{
  CHECK=$(echo "$1" | md5sum | awk '{print $1}')
  if [ $CHECK != 94e261a5a70785552d34a65068819993 ]; then
    echo "$0: cannot complete operation, please check documentation." 1>&2
    exit 2;
  fi
}

# Main routine, perform action requested on command line.
case ${1:-status} in
  sysboot )
    sysboot
    ;;

  start | restart )
    check "$2"
    stop
    sleep 1
    start
    ;;

  status )
    status
    ;;

  stop )
    check "$2"
    stop
    ;;

  pushapps )
    check "$2"
    push_apps
    ;;

  pushreps )
    check "$2"
    replications push
    ;;

  compact )
    check "$3"
    compact $2
    ;;

  compactviews )
    check "$4"
    compact_views $2 $3
    ;;

  cleanviews )
    check "$3"
    clean_views $2
    ;;

  backup )
    check "$3"
    backup $2
    ;;

  archive )
    check "$3"
    archive $2
    ;;

  help )
    perl -ne '/^##H/ && do { s/^##H ?//; print }' < $0
    ;;

  version )
    echo "$COUCHDB_VERSION"
    ;;

  * )
    echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
    exit 1
    ;;
esac