1 #! /bin/bash
 2 
 3 #/python2292/CNEDMp/ETLscript/CNEDMp_etl_v1.0/testhis
 4 LOAD_PATH=$(cd $(dirname $0); pwd)
 5 #/python2292/CNEDMp/ETLscript/CNEDMp_etl_v1.0
 6 DEPLOY_PATH=$(cd $(dirname $LOAD_PATH); pwd)
 7 #/python2292/CNEDMp/ETLscript
 8 BASE_PATH=$(cd "$(dirname "$DEPLOY_PATH")"; pwd)
 9 #/python2292/CNEDMp
10 MAIN_PATH=$(cd "$(dirname "$BASE_PATH")"; pwd)
11 
12 FILE_LANDING=$DEPLOY_PATH‘/File_Landing.sh‘
13 LOAD_HDFS=$DEPLOY_PATH‘/Load_hdfs.sh‘
14 LOAD_HIVE=$DEPLOY_PATH‘/Load_hive.sh‘
15 LANDING_PATH=$MAIN_PATH‘/landing/‘
16 JAR_FILE=$DEPLOY_PATH‘/dailydata-1.0-SNAPSHOT.jar‘
17 INI_FILE=$DEPLOY_PATH‘/cnedmp_etl.ini‘
18 LOG_FILE=$DEPLOY_PATH‘/shellrun.log‘
19 
20 CODING=‘GB2312‘
21 
22 #Funtion for Log
23 function logging(){
24     logTime=$(date +"%Y-%m-%d %T")
25     echo "$logTime $1 $2"
26 }
27 
28 
29 source $INI_FILE
30 
31 cd $LOAD_PATH
32 rm -r dataspace
33 mkdir dataspace
34 cd dataspace
35 ziplist=$(cat $LOAD_PATH/ziplist.txt)
36 logging INFO "Get zip file list $ziplist"
37 for zipname in $ziplist
38 do
39     logging INFO "-------------------Handling zipfile $zipname----------------------------"
40     cp /home/infra1/jason/DataFiles/$zipname ./
41     foldername=$(echo $zipname|awk -F .  ‘{print $1}‘)
42     echo $foldername
43     mkdir $foldername
44     unzip -d ./$foldername $zipname
45     cd $foldername
46     partition=$foldername
47     filelist=$(cat $LOAD_PATH/filelist.txt)
48     logging INFO "Get file list $filelist"
49     for filename in $filelist
50     do
51         cp $filename $LANDING_PATH && logging INFO "$filename copied to $LANDING_PATH"
52     done
53     cd ..
54     sh $LOAD_PATH/Loading_Meta_Init_history.sh $partition
55     sh $LOAD_PATH/schedule_history.sh $partition
56     cd $LOAD_PATH
57     rm -r $LOAD_PATH/dataspace
58     logging INFO "-------------------zipfile $zipname finished----------------------------"
59 done