首页 > 代码库 > Yarn下分片和分块源代码分析

Yarn下分片和分块源代码分析

public class FileSplit extends InputSplit implements Writable {	private Path file;	private long start;	private long length;	private String[] hosts;	public FileSplit() {	}	public FileSplit(Path file, long start, long length, String[] hosts) {		this.file = file;		this.start = start;		this.length = length;		this.hosts = hosts;	}	public Path getPath() {		return this.file;	}	public long getStart() {		return this.start;	}	public long getLength() {		return this.length;	}	public String toString() {		return this.file + ":" + this.start + "+" + this.length;	}	public void write(DataOutput out) throws IOException {		Text.writeString(out, this.file.toString());		out.writeLong(this.start);		out.writeLong(this.length);	}	public void readFields(DataInput in) throws IOException {		this.file = new Path(Text.readString(in));		this.start = in.readLong();		this.length = in.readLong();		this.hosts = null;	}	public String[] getLocations() throws IOException {		if (this.hosts == null) {			return new String[0];		}		return this.hosts;	}}

代码比较简单, 四部分组成  文件路径 ,启始位置,长度,Host列表

Host为什么是个列表

看分片的时候创建函数

splits.add(makeSplit(path, length - bytesRemaining,
                                splitSize, blkLocations[blkIndex].getHosts()));

再来看块的源代码

public class BlockLocation {    private String[] hosts;    private String[] names;    private String[] topologyPaths;    private long offset;    private long length;    private boolean corrupt;    public BlockLocation() {        this(new String[0], new String[0], 0L, 0L);    }    public BlockLocation(String[] names, String[] hosts, long offset,            long length) {        this(names, hosts, offset, length, false);    }    public BlockLocation(String[] names, String[] hosts, long offset,            long length, boolean corrupt) {        if (names == null)            this.names = new String[0];        else {            this.names = names;        }        if (hosts == null)            this.hosts = new String[0];        else {            this.hosts = hosts;        }        this.offset = offset;        this.length = length;        this.topologyPaths = new String[0];        this.corrupt = corrupt;    }    public BlockLocation(String[] names, String[] hosts,            String[] topologyPaths, long offset, long length) {        this(names, hosts, topologyPaths, offset, length, false);    }    public BlockLocation(String[] names, String[] hosts,            String[] topologyPaths, long offset, long length, boolean corrupt) {        this(names, hosts, offset, length, corrupt);        if (topologyPaths == null)            this.topologyPaths = new String[0];        else            this.topologyPaths = topologyPaths;    }    public String[] getHosts() throws IOException {        if ((this.hosts == null) || (this.hosts.length == 0)) {            return new String[0];        }        return this.hosts;    }    public String[] getNames() throws IOException {        if ((this.names == null) || (this.names.length == 0)) {            return new String[0];        }        return this.names;    }    public String[] getTopologyPaths() throws IOException {        if ((this.topologyPaths == null) || (this.topologyPaths.length == 0)) {            return new String[0];        }        return this.topologyPaths;    }    public long getOffset() {        return this.offset;    }    public long getLength() {        return this.length;    }    public boolean isCorrupt() {        return this.corrupt;    }    public void setOffset(long offset) {        this.offset = offset;    }    public void setLength(long length) {        this.length = length;    }    public void setCorrupt(boolean corrupt) {        this.corrupt = corrupt;    }    public void setHosts(String[] hosts) throws IOException {        if (hosts == null)            this.hosts = new String[0];        else            this.hosts = hosts;    }    public void setNames(String[] names) throws IOException {        if (names == null)            this.names = new String[0];        else            this.names = names;    }    public void setTopologyPaths(String[] topologyPaths) throws IOException {        if (topologyPaths == null)            this.topologyPaths = new String[0];        else            this.topologyPaths = topologyPaths;    }    public String toString() {        StringBuilder result = new StringBuilder();        result.append(this.offset);        result.append(‘,‘);        result.append(this.length);        if (this.corrupt) {            result.append("(corrupt)");        }        for (String h : this.hosts) {            result.append(‘,‘);            result.append(h);        }        return result.toString();    }}