webgame 日志解析的过程
CSDN版 2011-08-19
awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4}' gamelogin.log.2011-08-15-23 > tmp.1
awk 'BEGIN {FS="|"}; {print $1}' tmp.1 | awk 'BEGIN {FS=":"}; {print $1}' > tmp.2
合并
awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; {print $1 | awk 'BEGIN {FS=":"}; {print $1} "|" $2 "|" $3}' > tmp.3
awk 'BEGIN {FS="|"}; [{print awk 'BEGIN {FS=":"}; {print $1}' > tmp.3
-----------------------------------
awk 'BEGIN {FS="|"}; {print ($1 || substr($1, 1, index($1,":")-1 )) "|" $2 "|" $3}' tmp.1 > tmp.4
awk 'BEGIN {FS="|"}; {print substr($1, 1, 9) "|" $2 "|" $3}' tmp.1 > tmp.4
----------------------------------------------------------------
awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' tmp.1 > tmp.4 OK
合并：
清理gamelogin.log数据,按时间分：
格式
23|255123768|mszj|1001
awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3 "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' > tmp.6
awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' tmp.10 | sort| uniq -c > tmp.7
------------------------------
合并：
awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3 "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' | awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' | sort| uniq -c | sort -r -n > tmp.7
逆序输出：
awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3 "|" $4}' gamelogin.log.2011-08-15-23 | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' | awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' | sort| uniq -c |awk 'BEGIN {FS=" "}; {print $3 " " $1 " " $5 " " $4 " " $2 }' | sort -r -n > tmp.7
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
promitioin.log的处理
awk 'BEGIN {FS=","}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3 "|" $4 "|" $5 "|" $6 "|" $7 "|" $8 }' promition.log.2011-08-15-23 > log.1
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $2 " " $3 " " $4 " " $5 " " $8 " "substr($1,index($1, " ") + 1, 2) }' promition.log.2011-08-15-23 | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $2 " " $3 " " $1 " " $5 " " $4 " " $6 " " $7 " " $8 " " $9}'| sort -r -n >log.2cat test.txt test1.txt | sort | uniq -d
vimdiff tmp1 tem2
cat tmp1 tmp2 | uniq -d
awk '{print $7}' log.2 | sort | uniq > log2.log
合并文件
cat file1.out file2.out ＞ file2.out
cat promition.success.log.2011-08-15-* > promition.success.log
gamelogin.log 16日
awk 'BEGIN {FS="|"}; {print substr($1,index($1, " ") + 1, 2) "|" $2 "|" $3 "|" $4}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($2,":")>0) {print $1 "|" substr($2, 1, index($2,":")-1) "|" $3 "|" $4 } else {print $1 "|" $2 "|" $3 "|" $4} }' | awk 'BEGIN {FS="|"}; {print $1 " " $3 " " $2 " " $4 }' | sort| uniq -c |awk 'BEGIN {FS=" "}; {print $3 " " $1 " " $5 " " $4 " " $2 }' | sort -r -n > tmp
-----------------------------------------------------------------------
awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4}' gamelogin.log > tmp.1
awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' tmp.1 > tmp.2
awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' > tmp.3
awk 'BEGIN {FS="|"}; {print $1 " " $2 " " $3 }' tmp.3 | sort| uniq -c > tmp.4
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > tmp.5
awk '{ sub(/^[ \t]+/,""); print }' tmp.5 > tmp.7
awk 'gsub(/^ *| *$/,"")'
awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}' tmp.5 > tmp.6
sort tmp.5 > tmp.6
-----------------------------------------------------------------
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > tmp.5
awk 'BEGIN {FS="|"}; {print $2}' tmp.5 | uniq | sort > tmp-uid
===================================================================
promition.success.log 15日
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $2 " " $3 " " $4 " " $5 " " $8 " "substr($1,index($1, " ") + 1, 2) }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $2 " " $3 " " $1 " " $5 " " $4 " " $6 " " $7 " " $8 " " $9}'| sort -r -n >log.1
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort >log.1
awk 'BEGIN {FS="|"}; {print $2}' log.1 | uniq | sort > log-uid
//多个文件合并为一行：
// paste glogin-uid-count.log prom-suc-uid-count.log log-rate.log > flux.log
----------------------------------------------------------------------
算登录率
cat log-uid tmp-uid | sort | uniq -d|wc -l >test.log
最终整理如下：
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log
awk 'BEGIN {FS="|"}; {print $2}' glogin.log | uniq | sort > glogin-uid
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log | uniq | sort > prom-suc-uid
计算前一天的次登录率
cat glogin-uid | wc -l > tmp-glog-prom-uid.log
cat prom-suc-uid | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid glogin-uid | sort | uniq -d|wc -l >> tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
单个文件合并为一行
计算登录率
awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2}' tmp-flux-rate.log > flux-rate.log
rm -f tmp*
-----------------------------------------------------------------
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log
awk 'BEGIN {FS="|"}; {print $2}' glogin.log | uniq | sort > glogin-uid
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log | uniq | sort > prom-suc-uid
cat glogin-uid | wc -l > tmp-glog-prom-uid.log
cat prom-suc-uid | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid glogin-uid | sort | uniq -d|wc -l >> tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2}' tmp-flux-rate.log > flux-rate.log
rm -f tmp*
结果 532853|36842|15950|0.43293
----------------------------------
简化版
touch test.c 就创建了一个新文件 test.c获取前一天的时间 cd /data/remote-log-server/log/promition/2011-08-17 cat promition.success.log.2011-08-17-* > promition.success.log.2011-08-17 scp promition.success.log.2011-08-17 [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/ rm -f promition.success.log.2011-08-17 cd /data/remote-log-server/log/gamelogin/2011-08-17 scp gamelogin.log [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.2011-08-17 echo `date -d"300 days ago" +%Y-%m-%d` ------------------------------------------------ 单日拷贝数据 echo "************start**************" dd=`date +%Y-%m-%d -d " 1 days ago " ` cd /data/remote-log-server/log/promition/$dd cat promition.success.log.$dd-* > promition.success.log.$dd scp promition.success.log.$dd [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/ rm -f promition.success.log.$dd echo "************ $dd promition.success end**************" cd /data/remote-log-server/log/gamelogin/$dd scp gamelogin.log [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.$dd echo "************ $dd gamelogin end**************" 循环拷贝数据 10.3.17.26 /data/remote-log-server/run/wqo sh testfor.log k=0 while((k++<=2)) do echo "************start**************" dd=`date +%Y-%m-%d -d " $k days ago " ` cd /data/remote-log-server/log/promition/$dd cat promition.success.log.$dd-* > promition.success.log.$dd scp promition.success.log.$dd [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/ rm -f promition.success.log.$dd echo "************ $dd promition.success end**************" cd /data/remote-log-server/log/gamelogin/$dd scp gamelogin.log [email protected]:/data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.$dd echo "************ $dd gamelogin end**************" done -------------------------------------------- #!/bin/bash # This script prints a message about your weight if you give it your # weight in kilos and hight in centimeters. if [ ! $# == 2 ]; then echo "Usage: $0 weight_in_kilos length_in_centimeters" exit fi weight="$1" height="$2" idealweight=$[$height - 110] if [ $weight -le $idealweight ] ; then echo "You should eat a bit more fat." else echo "You should eat a bit more fruit." fi # weight.sh 70 150 You should eat a bit more fruit. # weight.sh 70 150 33 --------------------------------------------------------- awk '{FS="|"}; {print $6 "\t" $5 "\t" $4 "\t" $3 "\t" $2 "\t" $1 }' flux-rate.log | sort -k 1 > fff.log cat flux-rate.log | sort -k 1 > fff.log cat flux-rate.log | sort > fff.log ps aux | grep testfor.sh
---------------------------------------------
#!/bin/sh
echo "*********start********"
dd=`date +%Y-%m-%d -d " 1 days ago " `
if [ ! -f "gamelogin.log.$dd" ] ; then
echo "*****gamelogin.log.$dd not existed*******"
exit
fi
if [ ! -f "glogin.log.$dd" ] ; then
echo "***********glogin.log.$dd not exist, start to create it**************"
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd
echo "***********glogin.log.$dd end***********"
else
echo "************glogin.log.$dd is existed********************"
fi
awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd
cat glogin-uid.$dd | wc -l > tmp-glog-prom-uid.log
################################################################
tt=`date +%Y-%m-%d -d " 2 days ago " `
if [ -f "promition.success.log.$tt" ] ; then
if [ ! -f "prom-suc.log.$tt" ] ; then
echo "************prom-suc.log.$tt not exist , start to create it***********************"
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt
echo "***********prom-suc.log.$tt end***************"
else
echo "***********prom-suc.log.$tt is existed**********************"
fi
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt
cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d | wc -l >> tmp-glog-prom-uid.log
echo $dd >>tmp-glog-prom-uid.log
echo $tt >>tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
awk 'BEGIN {FS=" "}; {print $1 "|" $2 "|" $3 "|" $3/$2 "|" $4 "|" $5 }' tmp-flux-rate.log >> flux-rate.log
echo "***********flux-rate.log end************"
rm -f tmp*
else
echo "********promition.success.log.$tt not exit!***************"
exit
fi
echo "*****************end**********************"-------------------------------------------------------------
#!/bin/sh
echo "*********start********"
m=0
while((m++<=30))
do
dd=`date +%Y-%m-%d -d " $m days ago " `
if [ ! -f "gamelogin.log.$dd" ] ; then
echo "*****gamelogin.log.$dd not existed*******"
exit
fi
if [ ! -f "glogin.log.$dd" ] ; then
echo "***********glogin.log.$dd not exist, start to create it**************"
awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd
echo "***********glogin.log.$dd end***********"
else
echo "************glogin.log.$dd is existed********************"
fi
if [ ! -f "glogin-uid.$dd" ] ; then
echo "********************glogin-uid.$dd not exist, create id***********"
awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd
else
echo "********************glogin-uid.$dd is existed******"
fi
################################################################
echo "****for promition*****"
k=0
while((k++<30))
do
((n=k+m))
tt=`date +%Y-%m-%d -d " $n days ago " `
if [ -f "promition.success.log.$tt" ] ; then
if [ ! -f "prom-suc.log.$tt" ] ; then
echo "************prom-suc.log.$tt not exist , start to create it***********************"
awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt
echo "***********prom-suc.log.$tt end***************"
else
echo "***********prom-suc.log.$tt is existed**********************"
fi
if [ ! -f "prom-suc-uid.$tt" ] ; then
echo "***************prom-suc-uid.$tt is not exited, create**********************"
awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt
else
echo "****************prom-suc-uid.$tt is existed********************"
fi
cat glogin-uid.$dd | wc -l > tmp-glog-prom-uid.log
cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log
cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d | wc -l >> tmp-glog-prom-uid.log
echo $dd >>tmp-glog-prom-uid.log
echo $tt >>tmp-glog-prom-uid.log
cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log
awk 'BEGIN {FS=" "}; {print $5 "|" $4 "|" $3/$2 "|" $3 "|" $2 "|" $1 }' tmp-flux-rate.log >> flux-rate.log
echo "***********flux-rate.log end************"
rm -f tmp*
else
echo "********promition.success.log.$tt not exit!***************"
fi
done
done
echo "*****************end**********************" 
相关推荐
chenpro 0喜欢 / 0评论 2020-07-04
fendou00sd 0喜欢 / 0评论 2020-06-16
RealJianyuan 0喜欢 / 0评论 2020-06-14
cwgxiaoguizi 0喜欢 / 0评论 2020-06-05
chenpro 0喜欢 / 0评论 2020-06-02
Neptune 0喜欢 / 0评论 2020-05-31
老谢的自留地 0喜欢 / 0评论 2020-05-09
YukiRain 0喜欢 / 0评论 2020-05-08
baobaozai 0喜欢 / 0评论 2020-04-29
Proudoffaith 0喜欢 / 0评论 2020-04-08
fenxinzi 0喜欢 / 0评论 2020-03-01
zhiliang 0喜欢 / 0评论 2020-01-31
wannagonna 0喜欢 / 0评论 2020-01-13
wandererdl 0喜欢 / 0评论 2019-12-25
chenchuang 0喜欢 / 0评论 2020-01-25
jyj00 0喜欢 / 0评论 2020-01-09
fendou00sd 0喜欢 / 0评论 2020-01-07
fendou00sd 0喜欢 / 0评论 2020-01-06
PHP学习笔记 0喜欢 / 0评论 2020-01-06
AndesStay 0喜欢 / 0评论 2020-01-03
xiaopc 0喜欢 / 0评论 2013-06-19
applecarelte 0喜欢 / 0评论 2013-06-12
misszc 0喜欢 / 0评论 2013-06-12
PHP学习笔记 0喜欢 / 0评论 2019-12-31
88281640 0喜欢 / 0评论 2019-12-17
susmote 0喜欢 / 0评论 2019-12-09
eroshn 0喜欢 / 0评论 2019-12-08
mlujian 0喜欢 / 0评论 2019-12-08
Darklovy 0喜欢 / 0评论 2019-12-03
liiuweii 0喜欢 / 0评论 2019-11-18
伏雌摘星阁 0喜欢 / 0评论 2013-09-16
数据齿轮 0喜欢 / 0评论 2019-11-19
yoshubom 0喜欢 / 0评论 2015-01-18
LINUXHome 0喜欢 / 0评论 2015-01-15
andyzhaojianhui 0喜欢 / 0评论 2019-11-17
qidu 0喜欢 / 0评论 2019-11-10
陈伟堂 0喜欢 / 0评论 2011-01-22
nuligannima 0喜欢 / 0评论 2010-12-28
xiaohouye 0喜欢 / 0评论 2019-11-08
JESSIEfoo 0喜欢 / 0评论 2019-11-07
jiazhou 0喜欢 / 0评论 2019-11-06
唐门问心 0喜欢 / 0评论 2011-07-25
amethystqi 0喜欢 / 0评论 2011-05-18
GeorgeTH 0喜欢 / 0评论 2019-11-02
Yyqingmofeige 0喜欢 / 0评论 2019-10-30
MJlife 0喜欢 / 0评论 2011-05-18
zaqwsx0 0喜欢 / 0评论 2017-02-10
hyfcomeon 0喜欢 / 0评论 2017-02-23
征服Linux 0喜欢 / 0评论 2017-03-22
xiaonamylove 0喜欢 / 0评论 2019-07-31