Drop collections older than 30 days on MongoDB

1#!/bin/bash
2set -e
3
4# Format: impressions-020418
5
6# AWS Config
7export AWS_ACCESS_KEY_ID="XXX"
8export AWS_SECRET_ACCESS_KEY="yyy"
9
10# Dump from mongodb host into backup directory
11FULL_MONGO_DB_CONN_STRING="mongodb://XXX.mlab.com:YYYYY"
12DB_HOST="xxxx.mlab.com:YYYYY"
13DB_USERNAME="xxx"
14DB_PASSWORD="zzzzz"
15DB_COLLECTION_NAME="impressions"
16DB_NAME="xxxxxx"
17BUCKET="my-fancy-bucket"
18
19BACKUP_LOCAL_DIR="/opt/backups"
20OUTDIR="$BACKUP_LOCAL_DIR/$DB_NAME"
21
22# Provision space and folders
23mkdir -p $OUTDIR
24ls $OUTDIR
25sudo chown -R $(whoami) $BACKUP_LOCAL_DIR
26
27
28# Dynamic Collection name Generation
29#BEGIN_OF_LAST_MONTH=`date -d "-1 month -$(($(date +%d)-1)) days" +"%Y-%m-%d"`
30#END_OF_LAST_MONTH=`date -d "-$(date +%d) days + 1 day" +"%Y-%m-%d"`
31THIRTY_DAYS_AGO=`date -d "29 days ago" +"%Y-%m-%d"`
32SIXTY_DAYS_AGO=`date -d "60 days ago" +"%Y-%m-%d"`
33
34start=$SIXTY_DAYS_AGO
35end=$THIRTY_DAYS_AGO
36
37   while [[ $start < $end ]]
38     do
39       cleanDate=`date -d $start +"%d%m%y"`
40       collection="impressions-"$cleanDate
41       echo "Copying collection" $collection " to : "$OUTDIR
42
43       files=$(ls $OUTDIR/$DB_NAME/$collection* 2> /dev/null | wc -l)
44
45       echo $files
46       if [ "$files" != "0" ]
47       then
48          echo "$0: File $collection exists."
49          jsquery="db["\"$collection"\"].drop()"
50          echo "$jsquery"
51          mongo $FULL_MONGO_DB_CONN_STRING --eval $jsquery
52       else
53          # Dump Collection to this machine
54          echo "Copying collection" $collection " to : "$OUTDIR
55           mongodump --archive=$OUTDIR/$DB_NAME/$collection.gz --gzip -h $DB_HOST -d $DB_NAME -c $collection -u $DB_USERNAME -p $DB_PASSWORD
56          if [ $? -eq 0 ]; then
57             echo OK dump complete
58
59             # Upload to S3
60             # sudo chown -R $(whoami) $OUTDIR/$DB_NAME/
61             # tar cvzf $OUTDIR/$DB_NAME/$collection.tar.gz $OUTDIR/$DB_NAME/$collection*
62             if [ $? -eq 0 ]; then
63                echo OK compressed
64                  aws s3 cp $OUTDIR/$DB_NAME/$collection.gz s3://$BUCKET --region eu-central-1
65
66                   if [ $? -eq 0 ]; then
67                      echo OK uploaded
68
69                      # Remove collection
70                         jsquery="db["\"$collection"\"].drop()"
71                         echo "$jsquery"
72                         mongo $FULL_MONGO_DB_CONN_STRING --eval $jsquery
73                         if [ $? -eq 0 ]; then
74                            echo OK removed collection
75                         else
76                            echo FAIL to remove collection
77                         fi
78                   else
79                      echo FAIL to upload
80                   fi
81             else
82               echo FAIL to compress
83             fi
84          else
85            echo FAIL to dump
86          fi
87       fi
88       start=$(date -d "$start + 1 day" +"%Y-%m-%d")
89     done