首页 > 代码库 > 大文件读写
大文件读写
import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; public class FileSplit { public static void main(String[] args) throws IOException { long timer = System.currentTimeMillis(); int bufferSize = 20 * 1024 * 1024;//设读取文件的缓存为20MB //建立缓冲文本输入流 File file = new File("/media/Data/毕业设计/kdd cup/数据/userid_profile.txt"); FileInputStream fileInputStream = new FileInputStream(file); BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream); InputStreamReader inputStreamReader = new InputStreamReader(bufferedInputStream); BufferedReader input = new BufferedReader(inputStreamReader, bufferSize); int splitNum = 112-1;//要分割的块数减一 int fileLines = 23669283;//输入文件的行数 long perSplitLines = fileLines / splitNum;//每个块的行数 for (int i = 0; i <= splitNum; ++i) { //分割 //每个块建立一个输出 FileWriter output = new FileWriter("/home/haoqiong/part" + i + ".txt"); String line = null; //逐行读取,逐行输出 for (long lineCounter = 0; lineCounter < perSplitLines && (line = input.readLine()) != null; ++lineCounter) { output.append(line + "\r"); } output.flush(); output.close(); output = null; } input.close(); timer = System.currentTimeMillis() - timer; System.out.println("处理时间:" + timer); } }
方法2
void largeFileIO(String inputFile, String outputFile) { try { BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(inputFile))); BufferedReader in = new BufferedReader(new InputStreamReader(bis, "utf-8"), 10 * 1024 * 1024);//10M缓存 FileWriter fw = new FileWriter(outputFile); while (in.ready()) { String line = in.readLine(); fw.append(line + " "); } in.close(); fw.flush(); fw.close(); } catch (IOException ex) { ex.printStackTrace(); }
大文件读写
声明:以上内容来自用户投稿及互联网公开渠道收集整理发布,本网站不拥有所有权,未作人工编辑处理,也不承担相关法律责任,若内容有误或涉及侵权可进行投诉: 投诉/举报 工作人员会在5个工作日内联系你,一经查实,本站将立刻删除涉嫌侵权内容。