将脚本依赖关系放到表中
使用shell解析脚本依赖关系,递归的计算各个脚本。
#!/bin/bash # 2017.08.31 dm 补数 basepath=$(cd `dirname $0`; pwd) cd $basepath source /etc/profile source ../etc/env.ini if [[ $# -ne 3 ]]; then echo "参数格式错误!" echo "$0 [运算日期,格式:'yyyy-mm-dd'] [运算类型:down/up] [表名列表] " echo "注:运算类型:down:表名是dm结果表,首先查找并调用其依赖的脚本,然后调用结果表。up:根据传入的前置表,查找其影响的结果表,然后执行。" echo "注:表名列表 (如果传入一个以上的表,需要用\" \"分割,\"\"包裹):单表,例如“dm_event_meterial”;多表,例如“\dm_event_meterial,dm_voice_meterial\"”" exit 2 fi #运算类型 run_type down up run_type=$2 #脚本名 shellNames=$3 #运算日期 sdate=$(date -d "-1 day" "+%Y-%m-%d") if [[ $# -eq 3 ]]; then sdate=$(date -d "$1" "+%Y-%m-%d") fi #映射关系缓存 dmDic="" #刷新映射关系缓存 function dic(){ local dm=$1 dmDic=$(spark-sql -e " use $HIVE_DB_ODS_S; select concat_ws(\",\",tmp.dm,tmp.shell) from ( select dm,shell from yq_control_execute where dm=\"$dm\" union select c2.dm as dm,c2.shell as shell from yq_control_execute c1 inner join yq_control_execute c2 on c1.shell = c2.dm where c1.dm=\"$dm\" union select c3.dm as dm,c3.shell as shell from yq_control_execute c1 inner join yq_control_execute c2 on c1.shell = c2.dm inner join yq_control_execute c3 on c2.shell = c3.dm where c1.dm=\"$dm\" ) tmp where tmp.shell!=tmp.dm ; ") } #将参数的shell列表转化为dm列表 function getDms(){ local first=1 shellNameSqlPart="" for _shellName in $shellNames do if [[ "$first" -ne "1" ]]; then shellNameSqlPart=$shellNameSqlPart",\"$_shellName\"" else shellNameSqlPart=\"$_shellName\" ((first++)) fi done echo $(spark-sql -S -e " use $HIVE_DB_ODS_S; select dm from ( select dm from yq_control_execute where shell in ($shellNameSqlPart) union select c1.dm from yq_control_execute c1 inner join yq_control_execute c2 on c1.shell = c2.dm where c2.shell in ($shellNameSqlPart) union select c1.dm from yq_control_execute c1 inner join yq_control_execute c2 on c1.shell = c2.dm inner join yq_control_execute c3 on c2.shell = c3.dm where c3.shell in ($shellNameSqlPart) ) tmp ; ") } #根据dm返回其依赖的shell列表 function getShellS() { if [[ $# -ne 1 ]]; then echo "$0 parameter length error." exit 2 fi local target=$1 local shells="" for line in $dmDic do local __dm=$(echo $line|cut -d ',' -f 1) local __shell=$(echo $line|cut -d ',' -f 2) if [[ "$target" == "$__dm" ]]; then local shells=$shells" "$__shell fi done echo $shells } runnedArray="" runCount=0 #检查并执行脚本 #只执行dm开头并且没有执行过的脚本,执行过后存入已执行列表 function checkAndRun(){ local flag=0 local c_shell=$1 for record in $runnedArray do if [[ "$record" == "$c_shell" ]]; then local flag=1 fi done # echo "已执行的脚本"$runnedArray if [[ "$flag" == "0" && "${c_shell:0:2}" == "dm" ]]; then runnedArray="$runnedArray ${c_shell}" ((runCount++)) echo "【$runCount】sh ./deal/${c_shell}.sh" sh ./deal/${c_shell}.sh $sdate sh ./export/main.sh $sdate ${c_shell} fi } #递归执行dm function run(){ local _dm=$1 for shell in $(getShellS $_dm) do # echo $shell"==>"$_dm if [[ "${shell:0:2}" == "dm" ]]; then run $shell checkAndRun $shell fi done if [[ "${_dm:0:2}" == "dm" ]]; then checkAndRun $_dm fi } if [[ "$run_type" == "up" ]]; then #找出其对应的结果表 遍历执行 for dm in $(getDms $shellNames) do #执行前 更新依赖关系 dic $dm run $dm done else #遍历执行传入的dm列表 for dm in $shellNames do #执行前 更新依赖关系 dic $dm run $dm done fi
脚本依赖表的格式:
这个依赖表除了有脚本的源表和目标表的关系,还有数据的来源类型(平台和渠道);如果要实现基本功能的话,如下几列即可。
列 | 类型 | 注释 |
from | string | 源表 |
to | string | 目标表 |