1     path = r‘C:\Users\user\Desktop\heilongjiang.dbw.cn\\‘
 2     documents = []
 3     num = 0
 4     for file in os.listdir(path)[0:]:
 5         file = path + file
 6         if not file.endswith(‘.json‘): continue
 7         f = open(file, ‘rt‘, encoding=‘utf-8‘)
 8         data =http://www.mamicode.com/ f.read()
 9         body = json.loads(data)
10 
11         content = body[‘content‘]
12         # print(content)
13         num += 1
14         print(num, ‘ ‘, file)
15         documents.append(fenci(content))

 1     corpus_path = "train_corpus_small/"  # 未分词分类语料库路径
 2     seg_path = "train_corpus_seg/"  # 分词后分类语料库路径
 4     catelist = os.listdir(corpus_path)  # 获取corpus_path下的所有子目录
 5 
 6     # 获取每个目录下所有的文件
 7     for mydir in os.listdir(corpus_path):
 8         class_path = corpus_path + mydir + "/"  # 拼出分类子目录的路径
 9         seg_dir = seg_path + mydir + "/"  # 拼出分词后语料分类目录
10         if not os.path.exists(seg_dir):  # 是否存在目录，如果没有创建
11             os.makedirs(seg_dir)
12         file_list = os.listdir(class_path)  # 获取class_path下的所有文件
13         for file_path in file_list:  # 遍历类别目录下文件
14             fullname = class_path + file_path  # 拼出文件名全路径
15             content = readfile(fullname).strip()  # 读取文件内容
16             content = content.replace("\r\n", "")  # 删除换行和多余的空格 \r\n
17             content_seg = jieba.cut(content.strip())  # 为文件内容分词
18             savefile(seg_dir + file_path, " ".join(content_seg))  # 将处理后的文件保存到分词后语料目录

 1     corpus = []
 2     tfidfdict = {}
 3     for file_path in os.listdir(start_path):
 4         fullname = start_path + file_path
 5         content = readfile(fullname).strip()
 6         #content = content.decode(‘gbk‘,‘ignore‘).encode(‘utf-8‘)
 7         content = content.replace("\r\n", "")
 8         content = content.decode(‘utf-8‘)
 9         content_seg = jieba.cut(content.strip())
10         savefile(end_path + file_path, " ".join(content_seg))
11     for file_path in os.listdir(end_path):
12         fullname = end_path + file_path
13         content = readfile(fullname).strip()
14         content = content.decode(‘utf-8‘)
15         content = content.replace("\r\n", "").strip()
16         corpus.append(content)
17         #print corpus
18     for i in range(len(corpus)):
19         print str(corpus[i])

1 import sys
2 reload(sys)
3 sys.setdefaultencoding(‘utf8‘)