Shell鑴氭湰
View more presentations from liukaiyi.
]]>
鏁版嵁璇存槑:
knnuu_...txt 鏂囦歡澶у皬 3.2G 鏁版嵁鏍煎紡鏄?nbsp;
user1 user2 score
..
usern userm score
鎴戣繖閲屽笇鏈涢氳繃娓呮礂寰楀埌錛?br /> 涓?user1 鍏崇郴鏈榪戠殑 top 100 浜?br />
鐢變簬鏁版嵁騫墮潪闇瑕佺櫨鍒嗕箣鐧懼噯紜紝鎴戞斁寮冨湪鍒嗛殧鍑虹殑鏁版嵁
if len(dr)!=3 : continue
寮浜?7 涓嚎紼?涔熷氨鏄?浼氭湁 7 涓?鐢ㄦ埛 鐨?nbsp; uid 瀵?top 100 uid 浼氬嚭鐜伴棶棰?
瀵瑰簲 鎬葷敤鎴鋒暟鍑犲崄涓囨潵璇?nbsp; 鍛靛懙 ! 鎴戝氨鐢ㄨ繖 瀹屽杽7涓壒孌婁漢鐨勫垪琛ㄦ椂闂村啓涓?blog 鍚?br />
騫剁粨鍚?linux split , awk 絳?蹇熷疄鐜扮殑 鐚ョ悙 澶氱嚎紼?鍝堝搱!!
鎬庝箞淇敼涓?nbsp; 閫熷害鎻愬崌 5鍊嶏紝鍘熸潵鐨?涓灝忔椂 鍒?10澶氬垎閽?銆傘傘傘傘?br />
# split --bytes=500m knnuu_20091123.txt knnuu/
# ls a* | awk '{system( " python uu.py "$0" & " )}'
import bsddb,sys
db = bsddb.hashopen('../id-item-y-09-10-11.db','c')
uid = -1
arr=[]
arrsc=[]
fw = open('tc/'+sys.argv[1]+'uid-uid-sc.txt','w')
ii=0
def insertion_sort(arr,arrsc,uid,sc):
ls = min(100,len(arrsc))
if ls!=0 and sc < arrsc[ls-1] : return
for i in xrange(ls):
if arrsc[i]<=sc :
arrsc.insert(i,sc)
arr.insert(i,uid)
return
elif arrsc[i] > sc : continue
if ls < 99 :
arr.append(uid)
arrsc.append(sc)
#for row in open('knnuu_20091123.txt') :
for row in open(sys.argv[1]):
dr = row.split('\n')[0].split('\t')
if len(dr)!=3 : continue
u1,u2,strsc = dr[0],dr[1],dr[2]
sc = float(strsc)
if uid == -1 : uid = u1
if u1 != uid :
for c in xrange( min(100,len(arrsc)) ):
tu = arr[c]
ts = arrsc[c]
print >>fw,"%s\t%s\t%s" % ( db[u1],db[tu],ts )
print uid
fw.flush()
arr=[u1]
arrsc=[sc]
uid=u1
else :
insertion_sort(arr,arrsc,u2,sc)
ii+=1
#print ii,u1,uid,u2,strsc,len(arr),len(arrsc)
#if ii>10 : break
fw.close()
# ls a* | awk '{system( " python uu.py "$0" & " )}'
import bsddb,sys
db = bsddb.hashopen('../id-item-y-09-10-11.db','c')
uid = -1
arr=[]
arrsc=[]
fw = open('tc/'+sys.argv[1]+'uid-uid-sc.txt','w')
ii=0
def insertion_sort(arr,arrsc,uid,sc):
ls = min(100,len(arrsc))
if ls!=0 and sc < arrsc[ls-1] : return
for i in xrange(ls):
if arrsc[i]<=sc :
arrsc.insert(i,sc)
arr.insert(i,uid)
return
elif arrsc[i] > sc : continue
if ls < 99 :
arr.append(uid)
arrsc.append(sc)
#for row in open('knnuu_20091123.txt') :
for row in open(sys.argv[1]):
dr = row.split('\n')[0].split('\t')
if len(dr)!=3 : continue
u1,u2,strsc = dr[0],dr[1],dr[2]
sc = float(strsc)
if uid == -1 : uid = u1
if u1 != uid :
for c in xrange( min(100,len(arrsc)) ):
tu = arr[c]
ts = arrsc[c]
print >>fw,"%s\t%s\t%s" % ( db[u1],db[tu],ts )
print uid
fw.flush()
arr=[u1]
arrsc=[sc]
uid=u1
else :
insertion_sort(arr,arrsc,u2,sc)
ii+=1
#print ii,u1,uid,u2,strsc,len(arr),len(arrsc)
#if ii>10 : break
fw.close()
]]>