首页 > 代码库 > 用wget+awk处理腾讯股票js数据
用wget+awk处理腾讯股票js数据
因深交所原始dbf数据源停止更新，所以需要寻找临时深交所数据，并延迟15分钟展示。
现在需要获取深圳股票数据的代码：000002，000423，000810，000999，399001，399107，399108
一、获取腾讯股票js数据
参考连接：http://hunkz.blog.51cto.com/6157447/1869450
如：
wget http://qt.gtimg.cn/q=sz000002 -O szhq_js/000002_20161104080001.txt
cat szhq_js/000002_20161104080001.txt v_sz000002="51~β >000002~24.87~24.90~24.77~519229~226426~292802~24.87~7945~24.86~1880~24.85~3300~24.84~562~24.83~1208~24.88~2260~24.89~1561~24.90~1353~24.91~1177~24.92~403~15:00:03/24.87/12452/S/30968148/15151|14:57:00/24.87/195/S/485070/15059|14:56:57/24.87/47/S/116918/15055|14:56:54/24.88/234/B/582007/15051|14:56:51/24.88/232/B/576987/15048|14:56:48/24.88/492/B/1224164/15045~20161103150130~-0.03~-0.12~25.19~24.71~24.87/506777/1264439662~519229~129541~0.53~14.06~~25.19~24.71~1.93~2414.41~2745.45~2.73~27.39~22.41~0.65";
二、通过awk处理下载后的js数据
/bin/awk -f sz000002 szhq_js/000002_20161104080001.txt > szhq_js/sz000002_20161104080001.txt
cat sz000002
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"万科A\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"}cat szhq_js/sz000002_20161104080001.txt
,{id:"sz000002",code:"000002",name:"万科A",price:"24.87",lastCP:"24.90",diff:"-0.03",percent:"-0.12",hp:"25.19",lp:"24.71",op:"24.77",bpl:"24.87",spl:"24.88",ta:"519229",tm:"129541",hqtime:"2016-11-03 15:01:30"}三、每分钟将15分钟前的复制出来使用
假设现在时间是201611040815
/bin/cp szhq_js/sz000002_20161104080001.txt 000002.txt
000002.txt就是我们需要的文件，放置线上。
四，下面是我线上的 主要脚本
1，获取股票就是数据，并做处理
# cat szhq_wget.sh
#!/bin/bash
DIR=/usr/local/scripts/szhq
Datetime=`date +"%Y%m%d%H%M%S"`
#Delaytime=`date -d -15min +%Y%m%d%H%M`
for i in `cat ${DIR}/szcode.txt`
do
wget http://qt.gtimg.cn/q=sz${i} -O ${DIR}/szhq_js/${i}_${Datetime}.txt >/dev/null 2>&1
#/bin/cp ${DIR}/szhq_js/${i}_${Delaytime}*.txt ${DIR}/szhq_js/${i}.txt
/bin/awk -f ${DIR}/sz${i} ${DIR}/szhq_js/${i}_${Datetime}.txt > ${DIR}/szhq_js/sz${i}_${Datetime}.txt
done2，对线上hs.txt文件处理
# cat szhq.sh
#!/bin/bash
#Hunk
#2016-11-01
LOCAL_DIR=/usr/local/scripts/szhq
DIR=/usr/local/scripts/szhq
Delaytime=`date -d -15min +%Y%m%d%H%M`
for i in `cat ${DIR}/szcode.txt`
do
#/bin/cp ${DIR}/szhq_js/sz${i}_${Delaytime}*.txt /tmp/${i}.txt
/bin/cp ${DIR}/szhq_js/sz${i}_${Delaytime}*.txt /${LOCAL_DIR}/${i}.txt
done
/bin/cp /usr/local/tomcat/webapps/TodayInvestment/WEB-INF/hs.txt ${LOCAL_DIR}
#cd ${LOCAL_DIR}
/bin/awk -f ${LOCAL_DIR}/sz.txt ${LOCAL_DIR}/hs.txt > ${LOCAL_DIR}/now/hs.txt
/bin/chown tomcat.tomcat ${LOCAL_DIR}/now/hs.txt
/bin/chmod -x ${LOCAL_DIR}/now/hs.txt
#/bin/cp -ar ${LOCAL_DIR}/now/hs.txt /tmp
/bin/cp -ar ${LOCAL_DIR}/now/hs.txt /usr/local/tomcat/webapps/TodayInvestment/WEB-INF/3，awk的指定配置文件
# cat sz000002
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"万科A\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}# cat sz000423
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"东阿阿胶\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}# cat sz000810
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"创维数字\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}# cat sz000999
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"华润三九\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}# cat sz399001
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"深证成指\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}# cat sz399107
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"深证A指\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}# cat sz399108
BEGIN {FS="~"}
{
## define var
Year=substr($31,0,4);Mouth=substr($31,5,2);Day=substr($31,7,2)
Hour=substr($31,9,2);Min=substr($31,11,2);Sec=substr($31,13,2)
DateTime=Year"-"Mouth"-"Day" "Hour":"Min":"Sec
##
print ",{id:\"sz"$3"\",code:\""$3"\",name:\"深证B指\",price:\""$4"\",lastCP:\""$5"\",diff:\""$32"\",percent:\""$33"\",hp:\""$34"\",lp:\""$35"\",op:\""$6"\",bpl:\""$10"\",spl:\""$20"\",ta:\""$37"\",tm:\""$38"\",hqtime:\""DateTime"\"}"
}# cat sz.txt
BEGIN {FS=","}
{if(NR==1){print "["}
if($2 ~ "sz399001"){cmd ="cat /usr/local/scripts/szhq/399001.txt";cmd|getline;print $0}
if($2 ~ "sz399107"){cmd ="cat /usr/local/scripts/szhq/399107.txt";cmd|getline;print $0}
if($2 ~ "sz399108"){cmd ="cat /usr/local/scripts/szhq/399108.txt";cmd|getline;print $0}
if($2 ~ "sz000002"){cmd ="cat /usr/local/scripts/szhq/000002.txt";cmd|getline;print $0}
if($2 ~ "sz000999"){cmd ="cat /usr/local/scripts/szhq/000999.txt";cmd|getline;print $0}
if($2 ~ "sz000810"){cmd ="cat /usr/local/scripts/szhq/000810.txt";cmd|getline;print $0}
if($2 ~ "sz000423"){cmd ="cat /usr/local/scripts/szhq/000423.txt";cmd|getline;print $0}
#if($2 !~ "sz399001"&&$2 !~ "sz399107"&&$2 !~ "sz399108"&&$2 !~ "sz000002" &&$2 !~ "sz000810"&&$2 !~ "sz000423" ){print $0}
if( NR != 1 && $2 !~ "sz399001"&&$2 !~ "sz399107"&&$2 !~ "sz399108"&&$2 !~ "sz000002"&&$2 !~ "sz000999" &&$2 !~ "sz000810"&&$2 !~ "sz000423" ){print $0}
}# cat hs.txt
[
{id:"sh000001",code:"000001",name:"上证指数",price:"3125.317",lastCP:"3128.936",diff:"-3.619",percent:"-0.116",hp:"3141.334",lp:"3119.535",op:"3126.349",bpl:"0.000",spl:"0.000",ta:"197683197",tm:"23904325.728",hqtime:"2016-11-04 15:01:14"}
,{id:"sz000002",code:"000002",name:"万科A",price:"24.59",lastCP:"24.87",diff:"-0.28",percent:"-1.13",hp:"25.00",lp:"24.46",op:"24.84",bpl:"24.59",spl:"24.60",ta:"582099",tm:"143475",hqtime:"2016-11-04 15:01:36"}
,{id:"sz200002",code:"200002",name:"万科B",price:"12.410",lastCP:"12.200",diff:"+0.210",percent:"+1.721",hp:"12.440",lp:"12.250",op:"12.290",bpl:"12.410",spl:"12.420",ta:"83971",tm:"10401.116",hqtime:"2014-06-03 15:33:17"}
,{id:"sz000999",code:"000999",name:"华润三九",price:"27.08",lastCP:"27.48",diff:"-0.40",percent:"-1.46",hp:"27.66",lp:"27.06",op:"27.50",bpl:"27.08",spl:"27.09",ta:"45135",tm:"12314",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000810",code:"000810",name:"创维数字",price:"16.91",lastCP:"17.00",diff:"-0.09",percent:"-0.53",hp:"17.15",lp:"16.86",op:"17.01",bpl:"16.91",spl:"16.92",ta:"57331",tm:"9736",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000423",code:"000423",name:"东阿阿胶",price:"58.20",lastCP:"57.14",diff:"1.06",percent:"1.86",hp:"59.47",lp:"57.50",op:"57.50",bpl:"58.19",spl:"58.20",ta:"104522",tm:"61229",hqtime:"2016-11-04 15:01:36"}
,{id:"sh600062",code:"600062",name:"双鹤药业",price:"23.580",lastCP:"23.900",diff:"-0.320",percent:"-1.339",hp:"23.970",lp:"23.400",op:"23.900",bpl:"23.560",spl:"23.580",ta:"47197",tm:"11177.127",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600055",code:"600055",name:"万东医疗",price:"19.520",lastCP:"19.280",diff:"+0.240",percent:"+1.245",hp:"19.540",lp:"19.200",op:"19.260",bpl:"19.510",spl:"19.520",ta:"36881",tm:"7153.988",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600893",code:"600893",name:"航空动力",price:"35.000",lastCP:"35.000",diff:"0.000",percent:"0.000",hp:"0.000",lp:"0.000",op:"0.000",bpl:"0.000",spl:"0.000",ta:"0",tm:"0.000",hqtime:"2016-11-04 15:34:02"}
,# cat szcode.txt 000002 000423 000810 000999 399001 399107 399108
5，最终目标文件
# cat now/hs.txt
[
{id:"sh000001",code:"000001",name:"上证指数",price:"3125.317",lastCP:"3128.936",diff:"-3.619",percent:"-0.116",hp:"3141.334",lp:"3119.535",op:"3126.349",bpl:"0.000",spl:"0.000",ta:"197683197",tm:"23904325.728",hqtime:"2016-11-04 15:01:14"}
,{id:"sz000002",code:"000002",name:"万科A",price:"24.59",lastCP:"24.87",diff:"-0.28",percent:"-1.13",hp:"25.00",lp:"24.46",op:"24.84",bpl:"24.59",spl:"24.60",ta:"582099",tm:"143475",hqtime:"2016-11-04 15:01:36"}
,{id:"sz200002",code:"200002",name:"万科B",price:"12.410",lastCP:"12.200",diff:"+0.210",percent:"+1.721",hp:"12.440",lp:"12.250",op:"12.290",bpl:"12.410",spl:"12.420",ta:"83971",tm:"10401.116",hqtime:"2014-06-03 15:33:17"}
,{id:"sz000999",code:"000999",name:"华润三九",price:"27.08",lastCP:"27.48",diff:"-0.40",percent:"-1.46",hp:"27.66",lp:"27.06",op:"27.50",bpl:"27.08",spl:"27.09",ta:"45135",tm:"12314",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000810",code:"000810",name:"创维数字",price:"16.91",lastCP:"17.00",diff:"-0.09",percent:"-0.53",hp:"17.15",lp:"16.86",op:"17.01",bpl:"16.91",spl:"16.92",ta:"57331",tm:"9736",hqtime:"2016-11-04 15:01:36"}
,{id:"sz000423",code:"000423",name:"东阿阿胶",price:"58.20",lastCP:"57.14",diff:"1.06",percent:"1.86",hp:"59.47",lp:"57.50",op:"57.50",bpl:"58.19",spl:"58.20",ta:"104522",tm:"61229",hqtime:"2016-11-04 15:01:36"}
,{id:"sh600062",code:"600062",name:"双鹤药业",price:"23.580",lastCP:"23.900",diff:"-0.320",percent:"-1.339",hp:"23.970",lp:"23.400",op:"23.900",bpl:"23.560",spl:"23.580",ta:"47197",tm:"11177.127",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600055",code:"600055",name:"万东医疗",price:"19.520",lastCP:"19.280",diff:"+0.240",percent:"+1.245",hp:"19.540",lp:"19.200",op:"19.260",bpl:"19.510",spl:"19.520",ta:"36881",tm:"7153.988",hqtime:"2016-11-04 15:34:02"}
,{id:"sh600893",code:"600893",name:"航空动力",price:"35.000",lastCP:"35.000",diff:"0.000",percent:"0.000",hp:"0.000",lp:"0.000",op:"0.000",bpl:"0.000",spl:"0.000",ta:"0",tm:"0.000",hqtime:"2016-11-04 15:34:02"}
,6，linux 任务计划
# crontab -l ##szhq */1 09-17 * * 1-5 /bin/bash /usr/local/scripts/szhq/szhq.sh > /dev/null 2>&1 */1 08-17 * * 1-5 /bin/bash /usr/local/scripts/szhq/szhq_wget.sh > /dev/null 2>&1
五，小结
重要的命令：
awk中可以通过getline读取执行linux命令返回的结果，并赋值给$0
if($2 ~ "sz399001"){cmd ="cat /usr/local/scripts/szhq/399001.txt";cmd|getline;print $0}用wget+awk处理腾讯股票js数据
声明：以上内容来自用户投稿及互联网公开渠道收集整理发布，本网站不拥有所有权，未作人工编辑处理，也不承担相关法律责任，若内容有误或涉及侵权可进行投诉： 投诉/举报 工作人员会在5个工作日内联系你，一经查实，本站将立刻删除涉嫌侵权内容。