本文主要是为了方便自己以后查找相关知识使用!
#!bin/bash ##### ## 编写:zhangqm ## 日期:2018-01-30 ## 简述:胶子月程序流程 ## 调用方式:nohup sh jz_process.sh click_start_time click_end_time adcookie_start_time adcookie_end_time > ${log}/jz_process.log 2>&1 & ## 备注: ## 参数必须是4个!前两个参数是跑click表的开始日期和结束日期,后两个参数是跑adcookie表的开始日期和结束日期 ##### ##日志目录 log='/data/u_lx_data/zhangqm/jiaozi/log' ##补数据的存放目录 add_data='/data/u_lx_data/zhangqm/jiaozi/add_data' ##脚本路径 shell_dir='/data/u_lx_data/zhangqm/jiaozi' ##判断程序是否成功的标志 flag='true' ##获取click当月 click_date=$2 click_month=${click_date:4:2} ##存放当月的一些结果数据 jz_result=`mkdir $shell_dir/jz${click_month}yue` ##获取adcook当月 adcook_date=$4 adcook_month=${adcook_date:4:2} ##稽核语句 ##Check_num 'adcookie' 12 function Check_num() { num=$(hadoop fs -du -h /user/u_lx_data/private/liyahong/JZ/Jz_${1}_${2}_01/|awk -F" " '{if ($1==0){print $1}}'|sed 's/ //g'|wc -l) echo $num } ##找出第一步的缺失数据,并补全数据 ##Check_Data "adcookie" ${adcook_month} ${add_data} ${shell_dir} ${log} function Check_Data() { while flag do num=$(Check_num $1 $2) if [ $num > 0 ]; then hadoop fs -du -h /user/u_lx_data/private/liyahong/JZ/Jz_${1}_${2}_01/|awk -F" " '{if ($1==0){print $3}}'> ${3}/Jz_${1}_${2}_01.txt if [ -f ${3}/Jz_${1}_${2}_01.txt ];then cat ${3}/Jz_${1}_${2}_01.txt | while read line do addtime=${line:0-8} nohup sh ${4}/Jz_${1}_01.sh $addtime $addtime> ${5}/Jz_${1}_01_add_$addtime.log 2>&1 & done fi else flag='false' fi done } if [ $# != 4 ]; then echo "参数必须是4个!前两个参数是跑click表的开始日期和结束日期,后两个参数是跑adcookie表的开始日期和结束日期!" exit 1 fi ##############################第一步跑基础数据 ##跑adcookie数据 if [ $3 -le $4 ]; then nohup sh $shell_dir/Jz_adcookie_01.sh $3 $4> $log/Jz_adcookie_01.log 2>&1 & else echo "第三个参数为跑adcookie表开始日期,第四个参数为结束日期,请重新输入!" exit 1 fi ##跑click数据 if [ $1 -le $2 ]; then nohup sh $shell_dir/Jz_newclick_01.sh $1 $2> $log/Jz_newclick_01.log 2>&1 & else echo "第一个参数为跑click表开始日期,第二个参数为结束日期,请重新输入!" exit 1 fi ##检查第一步是否全部完成并补缺数据 while flag do ps_num1=$(ps -ef |grep Jz_adcookie_01.sh | grep -v grep |sed 's/ //g'|wc -l) ps_num2=$(ps -ef |grep Jz_newclick_01.sh | grep -v grep |sed 's/ //g'|wc -l) if [ ${ps_num1} == 0 && ${ps_num2} == 0 ];then ##找出第一步adcookie的缺失数据,并补全数据 Check_Data "adcookie" ${adcook_month} ${add_data} ${shell_dir} ${log} ##找出第一步click的缺失数据,并补全数据 Check_Data "click" ${click_month} ${add_data} ${shell_dir} ${log} flag='false' fi done ####################跑第二步数据并再次核查第一步数据是否完成 click_num=$(Check_num "click" ${click_month}) adcook_num=$(Check_num "adcookie" ${click_month}) if [ $click_num == 0 && $ $adcook_num == 0 ]; then nohup sh $shell_dir/Jz_newclick_02.sh > $log/Jz_newclick_02.log 2>&1 & nohup sh $shell_dir/Jz_adcookie_02.sh > $log/Jz_adcookie_02.log 2>&1 & else echo "click和adcookie数据未全部完成,请检查!" fi ####################跑OTV数据 ##根据第一步click跑出来的结果帅选出数据第一列为Otv的数据 hadoop fs -cat /user/u_lx_data/private/liyahong/JZ/Jz_click_$click_month_01/*/* | grep -i ^otv | grep -v -i ^otv_days > ${jz_result}/Otv_${click_month}yue.txt file_size_otv=`du ${jz_result}/Otv_${click_month}yue.txt|awk -F" " '{print $1}'|sed 's/ //g'` ##判断数据是否存在且大小至少大于1G if [ -f ${jz_result}/Otv_${click_month}yue.txt ];then if [ ${file_size_otv} > 1048576 ];then hadoop fs -put ${jz_result}/Otv_${click_month}yue.txt /user/u_lx_data/private/liyahong/JZ/ hadoop jar /data/u_lx_data/zhangqm/test_mr/Jiaozi_otv_result.jar /user/u_lx_data/private/baiqian/ItemMaster.txt /user/u_lx_data/private/liyahong/JZ/Otv_${click_month}yue.txt /user/u_lx_data/private/liyahong/Jiaozi_otv_result${click_month}/ else echo "${jz_result}/Otv_${click_month}yue.txt文件的大小小于1G,请核查!" fi else echo "${jz_result}/Otv_${click_month}yue.txt文件没有产生,请检查!" fi ####################跑DS数据 ##根据第一步click跑出来的结果筛选出数据第一列为Ds的数据。 hadoop fs -cat /user/u_lx_data/private/liyahong/JZ/Jz_click_$click_month_01/*/* | grep -i ^Ds > ${jz_result}/Ds_${click_month}yue.txt file_size_ds=`du ${jz_result}/Ds_${click_month}yue.txt|awk -F" " '{print $1}'|sed 's/ //g'` ##判断数据是否存在且大小至少大于1G if [ -f ${jz_result}/Ds_${click_month}yue.txt ];then if [ ${file_size_ds} > 1048576 ];then ##跑电商脚本 nohup sh jz_ds_shell.sh > ${log}/jz_ds_shell_${click_month}.log 2>&1 & else echo "${jz_result}/Ds_${click_month}yue.txt文件的大小小于1G,请核查!" fi else echo "${jz_result}/Ds_${click_month}yue.txt文件没有产生,请检查!" fi
版权声明:本文为博主原创文章,未经博主允许不得转载。