首页 > 代码库 > csync+lsync 实时双向批量同步小文件
csync+lsync 实时双向批量同步小文件
cysnc+lsync配置安装
环境 ubuntu 12.04
1) 两台机器安装配置:
apt-get install xinetd csync2 lsyncd
cat /etc/xinetd.d/csync2 service csync2 { disable = no port = 30865 socket_type = stream protocol = tcp wait = no user = root passenv = PATH server = /usr/sbin/csync2 server_args = -i }
2) node2配置:
在主机node2上面执行csync2 -k /etc/csync2.key (时间长, 耐心等待)
在主机node2上面csync配置下面三个文件:
cat /etc/csync2.cfg nossl * *; group WEB { host node2.wenjun.com; host node3.wenjun.com; key /etc/csync2.key; include /home/ubuntu; exclude *~ .*; auto none; }
cat /etc/csync2_web1.cfg nossl * *; group web1 { host node2.wenjun.com; host (node3.wenjun.com); #()向这台推送数据 key /etc/csync2.key; include /home/ubuntu; exclude *~ .*; auto none; }
cat /etc/csync2_web2.cfg nossl * *; group web2 { host (node2.wenjun.com); host node3.wenjun.com; key /etc/csync2.key; include /home/ubuntu; exclude *~ .*; auto none; }
lsyncd node2配置
cat /etc/lsyncd/lsyncd.conf.lua settings = { logident = "lsyncd", logfacility = "user", logfile = "/var/log/lsyncd.log", statusFile = "/var/log/lsyncd.stat", statusInterval = 1, } initSync = { delay = 1, maxProcesses = 1, action = function(inlet) local config = inlet.getConfig() local elist = inlet.getEvents(function(event) return event.etype ~= "Blanket" end) local directory = string.sub(config.source, 1, -2) local paths = elist.getPaths(function(etype, path) return "\t" .. config.syncid .. ":" .. directory .. path end) log("Normal", "Processing syncing list:\n", table.concat(paths, "\n")) spawn(elist, "/usr/sbin/csync2", "-C", config.syncid, "-x") end, collect = function(agent, exitcode) local config = agent.config if not agent.isList and agent.etype == "Blanket" then if exitcode == 0 then log("Normal", "Startup of ‘", config.syncid, "‘ instance finished.") elseif config.exitcodes and config.exitcodes[exitcode] == "again" then log("Normal", "Retrying startup of ‘", config.syncid, "‘ instance.") return "again" else log("Error", "Failure on startup of ‘", config.syncid, "‘ instance.") terminate(-1) end return end local rc = config.exitcodes and config.exitcodes[exitcode] if rc == "die" then return rc end if agent.isList then if rc == "again" then log("Normal", "Retrying events list on exitcode = ", exitcode) else log("Normal", "Finished events list = ", exitcode) end else if rc == "again" then log("Normal", "Retrying ", agent.etype, " on ", agent.sourcePath, " = ", exitcode) else log("Normal", "Finished ", agent.etype, " on ", agent.sourcePath, " = ", exitcode) end end return rc end, init = function(inlet) local config = inlet.getConfig() local event = inlet.createBlanketEvent() log("Normal", "Recursive startup sync: ", config.syncid, ":", config.source) spawn(event, "/usr/sbin/csync2", "-C", config.syncid, "-x") end, prepare = function(config) if not config.syncid then error("Missing ‘syncid‘ parameter.", 4) end local c = "csync2_" .. config.syncid .. ".cfg" local f, err = io.open("/etc/" .. c, "r") if not f then error("Invalid ‘syncid‘ parameter: " .. err, 4) end f:close() end } local sources = { ["/home/ubuntu"] = "web1" } for key, value in pairs(sources) do sync {initSync, source=key, syncid=value} end
3) node3配置
同步node2 csync配置到node3
scp node2:/etc/csync.* /etc/
同步node2 lsyncd 配置到node3
scp -r node2:/etc/lsyncd /etc/
修改下面部分:
local sources = { ["/home/ubuntu"] = "web2" }
4) 两台机器都执行
csync2 -xv # 初始化csync2 的sqlite数据库 csync2 -vvv -T # 测试csync2 /etc/init.d/lsyncd start #启动lsyncd
tail -f /var/log/lsyncd.log #查看日志输出显示如下为正常
Tue Jul 29 17:25:23 2014 Normal: Recursive startup sync: web1:/home/ubuntu/wenjun1/ Tue Jul 29 17:25:23 2014 Normal: Startup of ‘web1‘ instance finished. Tue Jul 29 17:25:33 2014 Normal: Processing syncing list:
Wed Jul 30 10:56:28 2014 Normal: Recursive startup sync: web2:/home/ubuntu/wenjun1/
Wed Jul 30 10:56:28 2014 Normal: Startup of ‘web2‘ instance finished.
Wed Jul 30 10:56:39 2014 Normal: Processing syncing list:
5) 测试同步
测试1:
#!/bin/sh for ((i=0; i<100; i++)); do touch /home/ubuntu/${i}.html done
测试2:
#!/bin/bash for ((i=0;i<10;i++)) do number=$((1000000 + ($(od -An -N2 -i /dev/random)) % (10000 + 1000))) for ((j=0; j<1000; j++)) do touch /home/ubuntu/${i}-${j}.html done sleep ${number} done
6) 适用环境
此方案也可做成链式同步;N台相同资源机器同步
参考: http://oss.linbit.com/csync2/paper.pdf
本文出自 “willard_SA” 博客,请务必保留此出处http://374400.blog.51cto.com/364400/1532772