tf(w,d) = count(w, d) / size(d)

idf = log(n / docs(w, D))

tf-idf(q, d)= sum { i = 1..k | tf-idf(w[i], d) }= sum { i = 1..k | tf(w[i], d) * idf(w[i]) }

sudo apt-get install build-essential python-dev python-setuptools                      python-numpy python-scipy                      libatlas-dev libatlas3gf-base

sudo apt-get install python-sklearn

sudo apt-get install python-pip 
sudo pip install -U scikit-learn

pip list

sudo pip install jieba 

>>> from sklearn.feature_extraction.text import CountVectorizer
>>> vectorizer = CountVectorizer()>>> corpus = [...     ‘This is the first document.‘,...     ‘This is the second second document.‘,...     ‘And the third one.‘,...     ‘Is this the first document?‘,... ]>>> X = vectorizer.fit_transform(corpus)>>> X.toarray()           array([[0, 1, 1, 1, 0, 0, 1, 0, 1],       [0, 1, 0, 1, 0, 2, 1, 0, 1],       [1, 0, 0, 0, 1, 0, 1, 1, 0],       [0, 1, 1, 1, 0, 0, 1, 0, 1]]...)

>>> vectorizer.get_feature_names()
([‘and‘, ‘document‘, ‘first‘, ‘is‘, ‘one‘, ‘second‘, ‘the‘, ‘third‘, ‘this‘])

>>> from sklearn.feature_extraction.text import CountVectorizer
>>> transformer = TfidfTransformer()

>>> counts = [[3, 0, 1],...           [2, 0, 0],...           [3, 0, 0],...           [4, 0, 0],...           [3, 2, 0],...           [3, 0, 2]]>>> tfidf = transformer.fit_transform(counts)

>>> tfidf.toarray()                        array([[ 0.85...,  0.  ...,  0.52...],       [ 1.  ...,  0.  ...,  0.  ...],       [ 1.  ...,  0.  ...,  0.  ...],       [ 1.  ...,  0.  ...,  0.  ...],       [ 0.55...,  0.83...,  0.  ...],       [ 0.63...,  0.  ...,  0.77...]])

import osimport jiebaimport jieba.posseg as psegimport sysimport stringfrom sklearn import feature_extractionfrom sklearn.feature_extraction.text import TfidfTransformerfrom sklearn.feature_extraction.text import CountVectorizerreload(sys)sys.setdefaultencoding(‘utf8‘)#获取文件列表（该目录下放着100份文档）def getFilelist(argv) :    path = argv[1]    filelist = []    files = os.listdir(path)    for f in files :        if(f[0] == ‘.‘) :            pass        else :            filelist.append(f)    return filelist,path#对文档进行分词处理def fenci(argv,path) :    #保存分词结果的目录    sFilePath = ‘./segfile‘    if not os.path.exists(sFilePath) :         os.mkdir(sFilePath)    #读取文档    filename = argv    f = open(path+filename,‘r+‘)    file_list = f.read()    f.close()        #对文档进行分词处理，采用默认模式    seg_list = jieba.cut(file_list,cut_all=True)    #对空格，换行符进行处理    result = []    for seg in seg_list :　　　　 seg = ‘‘.join(seg.split())        if (seg != ‘‘ and seg != "\n" and seg != "\n\n") :            result.append(seg)    #将分词后的结果用空格隔开，保存至本地。比如"我来到北京清华大学"，分词结果写入为："我 来到 北京 清华大学"    f = open(sFilePath+"/"+filename+"-seg.txt","w+")    f.write(‘ ‘.join(result))    f.close()#读取100份已分词好的文档，进行TF-IDF计算def Tfidf(filelist) :　　path = ‘./segfile／‘    corpus = []  #存取100份文档的分词结果，其格式类似于：　    for ff in filelist :        fname = path + ff        f = open(fname,‘r+‘)        content = f.read()        f.close()        corpus.append(content)        vectorizer = CountVectorizer()        transformer = TfidfTransformer()    tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus))        word = vectorizer.get_feature_names() #所有文本的关键字    weight = tfidf.toarray()              #对应的tfidf矩阵        sFilePath = ‘./tfidffile‘    if not os.path.exists(sFilePath) :         os.mkdir(sFilePath)    # 这里将每份文档词语的TF-IDF写入tfidffile文件夹中保存    for i in range(len(weight)) :print u"--------Writing all the tf-idf in the",i,u" file into ",sFilePath+‘/‘+string.zfill(i,5)+‘.txt‘,"--------"        f = open(sFilePath+‘/‘+string.zfill(i,5)+‘.txt‘,‘w+‘)        for j in range(len(word)) :            f.write(word[j]+"    "+str(weight[i][j])+"\n")        f.close()            if __name__ == "__main__" :     (allfile,path) = getFilelist(sys.argv)　　for ff in allfile :        print "Using jieba on "+ff        fenci(ff,path)    Tfidf(allfile)