首页 > 代码库 > Hadoop源码分析(2)——Configuration类

Hadoop源码分析(2)——Configuration类

这篇文章主要介绍Hadoop的系统配置类Configuration。

 

接着上一篇文章介绍,上一篇文章中Hadoop Job的main方法为:

public static void main(String[] args) throws Exception {     int res = ToolRunner.run(new Configuration(), new CalculateSumJob(),args);     System.exit(res);}

其中ToolRunner.run方法传入的第一个变量为new Configuration(),我们在core.org.apache.hadoop.conf包中找到Configuration类,该类主要是根据配置文件中的相应配置项来配置Job参数。Configuration类实现了Java中的Iterable接口。Configuration类的几个要点介绍如下:

 1. addDefaultResource()

  public static synchronized void addDefaultResource(String name) {    if(!defaultResources.contains(name)) {      defaultResources.add(name);      for(Configuration conf : REGISTRY.keySet()) {        if(conf.loadDefaults) {          conf.reloadConfiguration();        }      }    }  }

该方法将默认的配置文件名装入defaultResources这个List,默认配置文件为hadoop-default.xml。

2. loadResources() 参数为ArrayList

private void loadResources(Properties properties,                             ArrayList resources,                             boolean quiet) {    if(loadDefaults) {      for (String resource : defaultResources) {        loadResource(properties, resource, quiet);      }          //support the hadoop-site.xml as a deprecated case      if(getResource("hadoop-site.xml")!=null) {        loadResource(properties, "hadoop-site.xml", quiet);      }    }    else{         for (Object resource : resources) {            loadResource(properties, resource, quiet);         }    }  }

该方法首先判断loadDefaults这个bool变量,默认为true,则调用默认的配置文件,否则调用指定的配置文件,包括core-default.xml,core-site.xml,hdfs-site.xml,mapred-site.xml等。
3. loadResource() 参数为Obeject

 private void loadResource(Properties properties, Object name, boolean quiet) {    try {      DocumentBuilderFactory docBuilderFactory         = DocumentBuilderFactory.newInstance();      //ignore all comments inside the xml file      docBuilderFactory.setIgnoringComments(true);      //allow includes in the xml file      docBuilderFactory.setNamespaceAware(true);      try {          docBuilderFactory.setXIncludeAware(true);      } catch (UnsupportedOperationException e) {        LOG.error("Failed to set setXIncludeAware(true) for parser "                + docBuilderFactory                + ":" + e,                e);      }      DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();      Document doc = null;      Element root = null;      if (name instanceof URL) {                  // an URL resource        URL url = (URL)name;        if (url != null) {          if (!quiet) {            LOG.info("parsing " + url);          }          doc = builder.parse(url.toString());        }      } else if (name instanceof String) {        // a CLASSPATH resource        URL url = getResource((String)name);        if (url != null) {          if (!quiet) {            LOG.info("parsing " + url);          }          doc = builder.parse(url.toString());        }      } else if (name instanceof Path) {          // a file resource        // Can‘t use FileSystem API or we get an infinite loop        // since FileSystem uses Configuration API.  Use java.io.File instead.        File file = new File(((Path)name).toUri().getPath())          .getAbsoluteFile();        if (file.exists()) {          if (!quiet) {            LOG.info("parsing " + file);          }          InputStream in = new BufferedInputStream(new FileInputStream(file));          try {            doc = builder.parse(in);          } finally {            in.close();          }        }      } else if (name instanceof InputStream) {        try {          doc = builder.parse((InputStream)name);        } finally {          ((InputStream)name).close();        }      } else if (name instanceof Element) {        root = (Element)name;      }      if (doc == null && root == null) {        if (quiet)          return;        throw new RuntimeException(name + " not found");      }      if (root == null) {        root = doc.getDocumentElement();      }      if (!"configuration".equals(root.getTagName()))        LOG.fatal("bad conf file: top-level element not <configuration>");      NodeList props = root.getChildNodes();      for (int i = 0; i < props.getLength(); i++) {        Node propNode = props.item(i);        if (!(propNode instanceof Element))          continue;        Element prop = (Element)propNode;        if ("configuration".equals(prop.getTagName())) {          loadResource(properties, prop, quiet);          continue;        }        if (!"property".equals(prop.getTagName()))          LOG.warn("bad conf file: element not <property>");        NodeList fields = prop.getChildNodes();        String attr = null;        String value = null;        boolean finalParameter = false;        for (int j = 0; j < fields.getLength(); j++) {          Node fieldNode = fields.item(j);          if (!(fieldNode instanceof Element))            continue;          Element field = (Element)fieldNode;          if ("name".equals(field.getTagName()) && field.hasChildNodes())            attr = ((Text)field.getFirstChild()).getData().trim();          if ("value".equals(field.getTagName()) && field.hasChildNodes())            value = ((Text)field.getFirstChild()).getData();          if ("final".equals(field.getTagName()) && field.hasChildNodes())            finalParameter = "true".equals(((Text)field.getFirstChild()).getData());        }                // Ignore this parameter if it has already been marked as ‘final‘        if (attr != null) {          if (value != null) {            if (!finalParameters.contains(attr)) {              properties.setProperty(attr, value);              if (storeResource) {                updatingResource.put(attr, name.toString());              }            } else if (!value.equals(properties.getProperty(attr))) {              LOG.warn(name+":a attempt to override final parameter: "+attr                     +";  Ignoring.");            }          }          if (finalParameter) {            finalParameters.add(attr);          }        }      }            } catch (IOException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    } catch (DOMException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    } catch (SAXException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    } catch (ParserConfigurationException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    }  }

该方法将配置文件的具体信息读入properties全局变量。

4. getProps()

private synchronized Properties getProps() {    if (properties == null) {      properties = new Properties();      loadResources(properties, resources, quietmode);      if (overlay!= null) {        properties.putAll(overlay);        if (storeResource) {          for (Map.Entry<Object,Object> item: overlay.entrySet()) {            updatingResource.put((String) item.getKey(), "Unknown");          }        }      }    }    return properties;  }

该方法中的properties.putAll(overlay)properties中的配置项在overlay在overlay中也存在的话,则用overlay的配置项覆盖掉properties的配置项。在整个MapReduce任务中,如果需要用到配置项信息,则直接从Properties中读取相应的参数值。