1 #!bin/bash
  2 
  3 DEPLOY_PATH=$(cd "$(dirname "$0")"; pwd)
  4 INI_FILE=$DEPLOY_PATH‘/cnedmp_etl.ini‘
  5 . $INI_FILE
  6 BASE_PATH=$(cd "$(dirname "$DEPLOY_PATH")"; pwd)
  7 MAIN_PATH=$(cd "$(dirname "$BASE_PATH")"; pwd)
  8 CURRRENT_DATE=$(date "+%Y-%m-%d")
  9 CURRENT_TIME=`date -u +\%Y\%m\%d`
 10 JAR_FILE=$DEPLOY_PATH‘/dailydata-1.0-SNAPSHOT.jar‘
 11 LOG_PATH=$BASE_PATH‘/log/‘
 12 LOAD_CHECK_LOG=$BASE_PATH‘/log/Load_Check.‘$CURRENT_TIME‘.log‘
 13 PASSWORD=`yarn jar $JAR_FILE com.hypers.etl.file.specialFileProcess.BlowFish`
 14 
 15 logging(){
 16     logTime=`date "+%y/%m/%d %T"`
 17     echo "$logTime $1 $2" >> $LOAD_CHECK_LOG
 18 }
 19 
 20 #Check whether this is the first time running
 21 test -e $LOAD_CHECK_LOG || touch $LOAD_CHECK_LOG
 22 
 23 #mysql parameters
 24 mysql_user=$USERNAME
 25 mysql_password=$PASSWORD
 26 mysql_host=$HOSTNAME
 27 mysql_port=$PORT
 28 mysql_database=$DBNAME
 29 
 30 #Get WARNING LIST
 31 sql="select File_Name,Transfer_Status,Load_Status,Job_Status \
 32 from $JOBNAME  33 where File_Date=‘$CURRRENT_DATE‘  34 and Hive_Partition != ‘none‘  35 and (Transfer_Status!=2 or Load_Status!=2 or Job_Status!=2)
 36 and Load_Status!=3
 37 and Job_Status!=3;"
 38 
 39 WARN_list=$(printf "%s\n"  40     "[client]"  41     "user=${mysql_user}"  42     "password=${mysql_password}"  43     "host=${mysql_host}"  44     "port=${mysql_port}"  45     "database=${mysql_database}"  46 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
 47 
 48 #Use to store email cotent
 49 #email_message="File_Name, Transfer_Status, Load_Status, Job_Status"
 50 
 51 while read line
 52 do
 53         test -z "${line}"
 54         if [ $? -eq 0 ]
 55         then
 56             continue
 57         fi
 58         exist=`cat $LOAD_CHECK_LOG | grep "Job doesn‘t finish." | grep "$line"`
 59         test -z "${exist}"
 60         if [ $? -eq 0 ]
 61         then
 62 #            echo $line >> $LOAD_CHECK_LOG
 63 #            email_message=$email_message"\\n"$line
 64             logging WARN "Job doesn‘t finish. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line"
 65         fi
 66 done <<EOF
 67 $WARN_list
 68 EOF
 69 
 70 #Get ERROR LIST
 71 sql="select File_Name,Transfer_Status,Load_Status,Job_Status \
 72 from $JOBNAME  73 where File_Date=‘$CURRRENT_DATE‘  74 and (Load_Status=3 or Job_Status=3);"
 75 
 76 ERROR_list=$(printf "%s\n"  77     "[client]"  78     "user=${mysql_user}"  79     "password=${mysql_password}"  80     "host=${mysql_host}"  81     "port=${mysql_port}"  82     "database=${mysql_database}"  83 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
 84 
 85 while read line
 86 do
 87         test -z "${line}"
 88         if [ $? -eq 0 ]
 89         then
 90             continue
 91         fi
 92         exist=`cat $LOAD_CHECK_LOG | grep "Job run ERROR." | grep "$line"`
 93         test -z "${exist}"
 94         if [ $? -eq 0 ]
 95         then
 96 #            echo $line >> $LOAD_CHECK_LOG
 97 #            email_message=$email_message"\\n"$line
 98             logging FATAL "Job run ERROR. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line"
 99         fi
100 done <<EOF
101 $ERROR_list
102 EOF
103 
104 #Get Row Fail list
105 sql="select File_Name,Transfer_Status,Load_Status,Job_Status, Rows_In_Total, Rows_Succeed, Rows_Fail \
106 from $JOBNAME 107 where File_Date=‘$CURRRENT_DATE‘ 108 and Rows_Fail!=0 109 and Transfer_Status=2 and Load_Status=2 and Job_Status=2;"
110 
111 RF_list=$(printf "%s\n" 112     "[client]" 113     "user=${mysql_user}" 114     "password=${mysql_password}" 115     "host=${mysql_host}" 116     "port=${mysql_port}" 117     "database=${mysql_database}" 118 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
119 
120 while read line
121 do
122         test -z "${line}"
123         if [ $? -eq 0 ]
124         then
125             continue
126         fi
127         exist=`cat $LOAD_CHECK_LOG | grep "Job finished with Row Fail." | grep "$line"`
128         test -z "${exist}"
129         if [ $? -eq 0 ]
130         then
131 #            echo $line >> $LOAD_CHECK_LOG
132 #            email_message=$email_message"\\n"$line
133             logging WARN "Job finished with Row Fail. | Status: File_Name, Transfer_Status, Load_Status, Job_Status, Total Row, Succeed Row, Fail Row | $line"
134         fi
135 done <<EOF
136 $RF_list
137 EOF
138 #echo -e $email_message >> testoutput.txt;
139 #$echo -e "hello jason \\n jason" | wc -l