// thin device数据结构
type DevInfo struct {
	Hash          string     `json:"-"`
	DeviceId      int        `json:"device_id"`
	Size          uint64     `json:"size"`
	TransactionId uint64     `json:"transaction_id"`
	Initialized   bool       `json:"initialized"`
	devices       *DeviceSet `json:"-"`

	mountCount int    `json:"-"`
	mountPath  string `json:"-"`
	lock sync.Mutex `json:"-"`
}

// thin pool数据结构
type DeviceSet struct {
	MetaData
	//根目录，默认为/var/lib/docker/devicemapper
	root             string
	//创建thin device名字使用的前缀，默认使用`docker-${major}:${minor}-${inode}-XXX-pool`
	devicePrefix     string

	TransactionId    uint64
	NewTransactionId uint64
	nextDeviceId     int

	//选项
	dataLoopbackSize     int64 	///var/lib/docker /devicemapper/devicemapper/data稀疏文件大小
	metaDataLoopbackSize int64	///var/lib/docker/devicemapper/devicemapper/metadata稀疏文件大小
	baseFsSize           uint64 	//base image之上格式化的文件系统大小
	filesystem           string	//base image之上格式化的文件系统类型
	mountOptions         string	
	mkfsArgs             []string	//格式化base image文件系统时的选项
	dataDevice           string	//指定使用哪个设备作为data device，eg，/dev/sda
	metadataDevice       string	//指定使用哪个设备作为metadata device，eg，/dev/sda
	doBlkDiscard         bool	
	thinpBlockSize       uint32	//thin pool block size
}

// devmapper的driver数据结构
type Driver struct {
	*DeviceSet
	home string 		//home默认为/var/lib/docker/devicemapper
}

docker使用device mapper的架构方式：

//初始化devicemapper driver
//	home=/var/lib/docker/devicemapper
//	options=device mapper的选项
//	调用路径：newdevice->initfunc
1.1 func Init(home string, options []string) (graphdriver.Driver, error) {
	//初始化deviceset
	deviceSet, err := NewDeviceSet(home, true, options)
	if err != nil {
		return nil, err
	}
	...
	d := &Driver{
		DeviceSet: deviceSet,
		home:      home,
	}
	return d, nil
}

//初始化deviceset
//	device set root=/var/lib/docker/devicemapper
//	调用路径：Init->NewDeviceSet
1.2 func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error) {
	SetDevDir("/dev")

	devices := &DeviceSet{
		root:                 root,
		//metaData通过deviceID存放thin device的配置信息
		MetaData:             MetaData{Devices: make(map[string]*DevInfo)},
		dataLoopbackSize:     DefaultDataLoopbackSize,
		metaDataLoopbackSize: DefaultMetaDataLoopbackSize,
		baseFsSize:           DefaultBaseFsSize,
		filesystem:           "ext4",
		doBlkDiscard:         true,
		thinpBlockSize:       DefaultThinpBlockSize,
	}

	//初始化deviceset选项参数
	for _, option := range options {
		key, val, err := utils.ParseKeyValueOpt(option)
		if err != nil {
			return nil, err
		}
		key = strings.ToLower(key)
		switch key {
		case "dm.basesize":
			size, err := units.RAMInBytes(val)
			if err != nil {
				return nil, err
			}
			devices.baseFsSize = uint64(size)
		...
		default:
			return nil, fmt.Errorf("Unknown option %s\n", key)
		}
	}

	//由deviceset继续完成初始化
	if err := devices.initDevmapper(doInit); err != nil {
		return nil, err
	}

	return devices, nil
}

//	初始化thin pool
//		调用路径：NewDeviceSet->initDevmapper
1.3 func (devices *DeviceSet) initDevmapper(doInit bool) error {
	logInit(devices)


	//创建/var/lib/docker/devicemapper/metadata目录
	if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) {
		return err
	}

	//获取/var/lib/docker目录所在设备的 inode
	st, err := os.Stat(devices.root)
	if err != nil {
		return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
	}
	sysSt := st.Sys().(*syscall.Stat_t)


	//thin device取名规则docker-$major:$minor-$inode-$imageid/$containerid
	//thin poll取名为docker-$major:$minor-$inode-pool
	devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)


	//如果thin pool device存在，获取device的信息
	utils.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
	info, err := getInfo(devices.getPoolName())
	if info == nil {
		utils.Debugf("Error device getInfo: %s", err)
		return err
	}

	setCloseOnExec("/dev/mapper/control")
	createdLoopback := false

	//创建thin pool
	if info.Exists == 0 {
		utils.Debugf("Pool doesn't exist. Creating it.")

		var (
			dataFile     *os.File
			metadataFile *os.File
		)

		//没有指定datadevice设备
		if devices.dataDevice == "" {

			//检查/var/lib/docker/devicemapper/devicemapper/data文件是否存在
			hasData := devices.hasImage("data")

			//既不要求初始化新的devicemapper，又没有旧的data文件
			if !doInit && !hasData {
				//返回错误
				return errors.New("Loopback data file not found")
			}

			//创建data loopdevice
			if !hasData {
				createdLoopback = true
			}

			//创建/var/lib/docker/devicemapper/devicemapper/data 稀疏文件
			data, err := devices.ensureImage("data", devices.dataLoopbackSize)
			if err != nil {
				utils.Debugf("Error device ensureImage (data): %s\n", err)
				return err
			}

			//data文件与loopback device关联
			dataFile, err = attachLoopDevice(data)
			if err != nil {
				return err
			}
		} else {
			//如果指定了data device，则打开
			dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
			if err != nil {
				return err
			}
		}
		defer dataFile.Close()

	//通过同样的办法初始化metadata device
	...


	//没有创建新loopback device，则从目录/var/lib/docker/devicemapper/metadata/$ids
	//加载旧的metadata
	if !createdLoopback {
		if err = devices.initMetaData(); err != nil {
			return err
		}
	}

	//初始化一个新的空镜像文件，作为所有镜像的祖先镜像
	if doInit {
		if err := devices.setupBaseImage(); err != nil {
			utils.Debugf("Error device setupBaseImage: %s\n", err)
			return err
		}
	}
	return nil
}

//	创建祖先镜像
1.4 func (devices *DeviceSet) setupBaseImage() error {
	//祖先镜像的描述信息存放在/var/lib/docker/devicemapper/metadata/base
	oldInfo, _ := devices.lookupDevice("")
	//之前已经创建，并完成了初始化，则直接成功返回
	if oldInfo != nil && oldInfo.Initialized {
		return nil
	}
	//已创建，但未完成初始化，删除base device
	if oldInfo != nil && !oldInfo.Initialized {
		utils.Debugf("Removing uninitialized base image")
		if err := devices.deleteDevice(oldInfo); err != nil {
			return err
		}
	}

	//下一个可用的deviceid
	id := devices.nextDeviceId

	//创建base device
	if err := createDevice(devices.getPoolDevName(), &id); err != nil {
		return err
	}

	devices.nextDeviceId = (id + 1) & 0xffffff

	//向thin pool注册base device
	utils.Debugf("Registering base device (id %v) with FS size %v", id, devices.baseFsSize)
	info, err := devices.registerDevice(id, "", devices.baseFsSize)
	if err != nil {
		_ = deleteDevice(devices.getPoolDevName(), id)
		return err
	}

	//激活base device
	if err = devices.activateDeviceIfNeeded(info); err != nil {
		return err
	}

	//在base device之上格式化新文件系统
	if err := devices.createFilesystem(info); err != nil {
		return err
	}

	//完成初始化，保存metadata到/var/lib/docker/devicemapper/metadata/base中
	info.Initialized = true
	if err = devices.saveMetadata(info); err != nil {
		info.Initialized = false
		return err
	}

	return nil
}