Shell实现定期删除HDFS下的过期文件

代码思路:获取文件夹下的文件最后改动日期,与当前时间戳对比,删除不满足条件的所有文件。

#!/bin/bash
source ~/.bashrc



removeOutDate(){
hadoop fs -ls afs://xxxxxxxxxxxx > temp.txt
today_timestamp=$(date -d "$(date +"%Y-%m-%d %H:%M")" +%s) 
cat temp.txt | while read quanxian temp user group size day hour filepath
do
    file_time="$day $hour"
    file_timestamp=$(date -d "$file_time" +%s)
#测试时间转换,原始为  $((3*3600*24))   换成231min前
    if [ $(($today_timestamp-$current_file_timestamp)) -ge $((60*231)) ];then
        echo "$(date +'%Y-%m-%d %H:%M:%S') $filepath"
        hadoop fs -rm -r $filepath > /dev/null 2>&1
    fi
done

}

execute(){
echo -e "\n\n"
echo "$(date +'%Y-%m-%d %H:%M:%S') start to remove outdate files in hdfs/afs"

removeOutDate 

echo "$(date +'%Y-%m-%d %H:%M:%S') remove outdate files in hdfs/afs finished"
echo -e "\n\n"

rm temp.txt
echo "temp.txt is cleaned!"
}

#开始执行
execute

猜你喜欢

转载自blog.csdn.net/Daverain/article/details/81016729