CNDBA社区
cat collectDataxLog.sh
#!/bin/sh
###############################################################################################################
#Script name: collectDataxLog.sh
#Script description: for Ansible Server collect Datax Server Log
#Current Release Version: 1.0.0
#Script Owner: He ,Haibo
#Latest editor: He, Haibo
#Support platform: Linux OS for redhat and centos.
#Date: 2021/11/8---first Version for collectDataxLog.sh.
#Change log:
#Descript:sh collectDataxLog.sh
#
###############################################################################################################
excuteUser(){
excuteCurrentUser=`whoami`
if [[ $excuteCurrentUser == "sysinsp" ]];then
continue
else
echo "Current excute user is $excuteCurrentUser,scripts will exit."
exit 1
fi
}
mkdirLogPath(){
if [[ ! -d /home/sysinsp/dataxLog ]];then
mkdir -p /home/sysinsp/dataxLog
chmod 767 /home/sysinsp/dataxLog
fi
ansibleLinuxLogPath="/home/sysinsp/dataxLog"
ansibleLinuxLog=ansible_datax_$(date +%y%m%d).log
}
checkPing(){
ansible -i hosts datax -m ping >> "$ansibleLinuxLogPath/$ansibleLinuxLog" 2>&1
if [[ $? == 0 ]];then
echo "ansible -i hosts all -m ping success." >> "$ansibleLinuxLogPath/$ansibleLinuxLog" 2>&1
else
echo "ansible -i hosts all -m ping failed." >> "$ansibleLinuxLogPath/$ansibleLinuxLog" 2>&1
exit 1
fi
}
copyLogFileToLocal(){
ansible datax -i hosts -m synchronize -a "mode=pull src=/datax/logs/*_datax_*_$(date +%y%m%d --date '1 days ago').log dest=$ansibleLinuxLogPath" >> "$ansibleLinuxLogPath/$ansibleLinuxLog" 2>&1
if [[ $? == 0 ]];then
echo "ansible datax -i hosts -m synchronize -a 'mode=pull src=/datax/logs/*_datax_*_$(date +%y%m%d --date '1 days ago').log dest=$ansibleLinuxLogPath' successfully." >> "$ansibleLinuxLogPath/$ansibleLinuxLog" 2>&1
else
echo "ansible datax -i hosts -m synchronize -a 'mode=pull src=/datax/logs/*_datax_*_$(date +%y%m%d --date '1 days ago').log dest=$ansibleLinuxLogPath' failed." >> "$ansibleLinuxLogPath/$ansibleLinuxLog" 2>&1
fi
}
moveLogs(){
if [[ ! -d "$ansibleLinuxLogPath"/$(date +%y%m%d) ]];then
mkdir "$ansibleLinuxLogPath"/$(date +%y%m%d) && cd "$ansibleLinuxLogPath" && mv *_datax_*_$(date +%y%m%d --date '1 days ago').log "$ansibleLinuxLogPath"/$(date +%y%m%d)
else
cd "$ansibleLinuxLogPath" && mv *_datax_*_$(date +%y%m%d --date '1 days ago').log "$ansibleLinuxLogPath"/$(date +%y%m%d)
fi
}
main(){
excuteUser
mkdirLogPath
checkPing
copyLogFileToLocal
moveLogs
exit 0
}
main
cat checkDataxLog.sh
#!/bin/sh
###############################################################################################################
#Script name: checkDataxLog.sh
#Script description: check Datax log
#Current Release Version: 1.0.0
#Script Owner: He ,Haibo
#Latest editor: He, Haibo
#Support platform: Linux OS for redhat and centos.
#Date: 2021/11/8---first Version for checkDataxLog.
#Change log:
#Descript:checkDataxLog.sh
#
###############################################################################################################
export LANG=en_US
osbox=`uname`
RHversion=$(cat /proc/version | sed 's/[^0-9]//g' | cut -b -3)
###打印日志函数
log_info(){
DATE_N=`date "+%Y-%m-%d %H:%M:%S"`
USER_N=`whoami`
echo "${DATE_N} ${USER_N} execute $0 [INFO] $@"
}
log_error(){
DATE_N=`date "+%Y-%m-%d %H:%M:%S"`
USER_N=`whoami`
echo -e "/033[41;37m ${DATE_N} ${USER_N} execute $0 [ERROR] $@ /033[0m"
}
fn_log(){
if [[ $? -eq 0 ]];then
log_info "$@ sucessed."
echo -e "/033[32m $@ sucessed. /033[0m"
else
log_error "$@ failed."
echo -e "/033[41;37m $@ failed. /033[0m"
fi
}
checkExecuteUser(){
username=`id | awk -F ' ' {'print $1'} | awk -F '(' {'print $2'} | awk -F ')' {'print $1'}`
if [[ ${username} == "sysinsp" ]];then
continue
else
fn_log "Current excute user is not sysinsp ,shell will exist."
exit 1
fi
}
###Create /tmp/log if not exist.
mkdirLogPath(){
if [[ ! -d /home/sysinsp/dataxLog ]];then
mkdir -p /home/sysinsp/dataxLog
chmod 767 /home/sysinsp/dataxLog
fi
CheckLinuxLogPath="/home/sysinsp/dataxLog"
CheckLinuxLog=checkDatax_$(date +%y%m%d).log
}
###Get OS Arch Linux or not
getOsArch(){
if [[ "$osbox" == "Linux" ]];then
continue
else
fn_log "Current OS is $osbox,shell is exit now."
echo 0
exit 0
fi
}
###Get redhat or centos
getOsCentosOrRedhat(){
cat /proc/version | grep -iE "redhat|centos" > /dev/null
if [[ $? == 0 ]];then
continue
else
echo "Current OS is not centos or redhat."
echo 1
exit 1
fi
}
###Get OS Version
getOsVerion(){
###判断系统是否为Redhat,如果不是则退出，支持Redhat|Centos 7版本
cat /proc/version | grep -i redhat > /dev/null
if [[ $? == 0 ]];then
if [[ "$RHversion" -ge 310 ]];then
OSVersion="redhat7"
else
echo "Current Rehat Version will not support."
echo 1
exit 1
fi
else
if [[ "$RHversion" -ge 310 ]];then
OSVersion="centos7"
else
echo "Current Rehat Version will not support."
echo 1
exit 1
fi
fi
}
checkDataxLog(){
if [[ ! -d ${CheckLinuxLogPath}/$(date +%y%m%d) ]];then
echo "Does not have ${CheckLinuxLogPath}/$(date +%y%m%d)"
exit 2
fi
cd ${CheckLinuxLogPath}/$(date +%y%m%d)
for dataxLogFile in `ls ${CheckLinuxLogPath}/$(date +%y%m%d)/*_datax_*_$(date +%y%m%d --date '1 days ago').log | xargs`
do
# cat datax_*_$(date +%y%m%d --date '1 days ago').log
echo "###################################################################" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
echo "dataxLog is $dataxLogFile" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
for errorNum in `cat $dataxLogFile | grep -i "读写失败总数" | awk -F ':' {'print $2'} | sed 's/[ /t]//g' | xargs`
do
if [[ ${errorNum} -gt 0 ]];then
echo "Check 读写失败总数 Number is ${errorNum},Check failed" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
else
echo "Check 读写失败总数 Number is ${errorNum},Check success" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
fi
done
##Oracle resource busy
for ORA00054 in `cat $dataxLogFile | grep -i "ORA-00054" | wc -l`
do
if [[ ${ORA00054} -gt 0 ]];then
echo "Check ORA-00054 Number is ${ORA00054},Check failed" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
else
echo "Check ORA-00054 Number is ${ORA00054},Check success" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
fi
done
##oracle user or password does not correct
for ORA01017 in `cat $dataxLogFile | grep -i "ORA-01017" | wc -l`
do
if [[ ${ORA01017} -gt 0 ]];then
echo "Check ORA-01017 Number is ${ORA01017},Check failed" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
else
echo "Check ORA-01017 Number is ${ORA01017},Check success" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
fi
done
##ORA-00942: table or view does not exist
for ORA00942 in `cat $dataxLogFile | grep -i "ORA-00942" | wc -l`
do
if [[ ${ORA00942} -gt 0 ]];then
echo "Check ORA-00942 Number is ${ORA00942},Check failed" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
else
echo "Check ORA-00942 Number is ${ORA00942},Check success" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
fi
done
##ORA-01045: user DATAX lacks CREATE SESSION privilege; logon denied
for ORA01045 in `cat $dataxLogFile | grep -i "ORA-01045" | wc -l`
do
if [[ ${ORA01045} -gt 0 ]];then
echo "Check ORA-01045 Number is ${ORA01045},Check failed" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
else
echo "Check ORA-01045 Number is ${ORA01045},Check success" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
fi
done
##ORA-01031: insufficient privileges
for ORA01030 in `cat $dataxLogFile | grep -i "ORA-01030" | wc -l`
do
if [[ ${ORA01030} -gt 0 ]];then
echo "Check ORA-01030 Number is ${ORA01030},Check failed" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
else
echo "Check ORA-01030 Number is ${ORA01030},Check success" >> ${CheckLinuxLogPath}/${CheckLinuxLog}
fi
done
done
}
main(){
checkExecuteUser
mkdirLogPath
getOsArch
getOsCentosOrRedhat
getOsVerion
checkDataxLog
}
main
python /datax/bin/datax.py /datax/job/t1.json >> /datax/logs/`hostname`_datax_shot_$(date +%y%m%d).log 2>&1
版权声明：本文为博主原创文章，未经博主允许不得转载。
DATAX
