首页 > 代码库 > 【原创】Scrapyd 的 .net 客户端
【原创】Scrapyd 的 .net 客户端
最近项目需要部署Scrapy爬虫，采用最简单的Scrapyd服务进行部署，基于.net core 进行了客户端的封装。
1）Scrapyd API文档：http://scrapyd.readthedocs.io/en/latest/api.html
2）使用HttpClient进行交互
比较麻烦的是部署爬虫所需要的使用 HttpClient 上传文件的代码，使用了 “MultipartFormDataContent” 对象进行上传，添加字符串内容，及StreamContent，直接可以将MVC中的File对象传入，不用自行处理临时文件。
var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", "")); content.Add(new StringContent(project), "\"project\""); content.Add(new StringContent(version), "\"version\""); content.Add(new StreamContent(eggStream), "\"egg\"", "\"" + project + ".egg" + "\""); var resp = client.PostAsync("addversion.json", content).Result; var strResult = resp.Content.ReadAsStringAsync().Result; return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);
完整代码
1 using System; 2 using System.Collections.Generic; 3 using System.IO; 4 using System.Linq; 5 using System.Net.Http; 6 using System.Text; 7 using Newtonsoft.Json; 8 using Scrapyd.Client.Response; 9 10 namespace Scrapyd.Client 11 { 12 /// <summary> 13 /// A Client used to communicate with the Scrapyd Daemon 14 /// </summary> 15 public class ScrapydClient 16 { 17 private readonly string _host; 18 private readonly int _port; 19 20 public ScrapydClient(string host, int port) 21 { 22 _host = host; 23 _port = port; 24 } 25 26 /// <summary> 27 /// 查询状态 28 /// </summary> 29 /// <returns></returns> 30 public DaemonStatusResponse QueryStatus() 31 { 32 HttpClient client = new HttpClient() 33 { 34 BaseAddress = new Uri($"http://{_host}:{_port}") 35 }; 36 37 try 38 { 39 var strResult = client.GetStringAsync("daemonstatus.json").Result; 40 41 var jresult = JsonConvert.DeserializeObject<DaemonStatusResponse>(strResult); 42 return jresult; 43 } 44 catch (Exception e) 45 { 46 return new DaemonStatusResponse { status = "failed", message = e.Message }; 47 } 48 } 49 50 public AddVersionResponse AddVersion(string project, string version, Stream eggStream) 51 { 52 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project)); 53 if (eggStream == null) throw new ArgumentNullException(nameof(eggStream)); 54 55 var client = new HttpClient() 56 { 57 BaseAddress = new Uri($"http://{_host}:{_port}") 58 }; 59 60 try 61 { 62 var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", "")); 63 64 content.Add(new StringContent(project), "\"project\""); 65 content.Add(new StringContent(version), "\"version\""); 66 content.Add(new StreamContent(eggStream), "\"egg\"", "\"" + project + ".egg" + "\""); 67 68 var resp = client.PostAsync("addversion.json", content).Result; 69 70 var strResult = resp.Content.ReadAsStringAsync().Result; 71 72 return JsonConvert.DeserializeObject<AddVersionResponse>(strResult); 73 } 74 catch (Exception e) 75 { 76 return new AddVersionResponse { status = "failed", message = e.Message }; 77 } 78 } 79 80 public ScheduleResponse ScheduleJob(string project, string spider, 81 string setting = "", 82 string jobid = "", 83 string version = "", 84 Dictionary<string, string> arguments = null) 85 { 86 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project)); 87 if (string.IsNullOrEmpty(spider)) throw new ArgumentNullException(nameof(spider)); 88 89 90 var client = new HttpClient() 91 { 92 BaseAddress = new Uri($"http://{_host}:{_port}") 93 }; 94 95 try 96 { 97 var data = http://www.mamicode.com/new Dictionary<string, string>() 98 { 99 {"project", project},100 {"spider", spider}101 };102 103 if (!string.IsNullOrEmpty(jobid))104 {105 data.Add("jobid", jobid);106 }107 if (!string.IsNullOrEmpty(setting))108 {109 data.Add("setting", setting);110 }111 112 if (!string.IsNullOrEmpty(version))113 {114 data.Add("_version", version);115 }116 117 if (arguments != null && arguments.Count > 0)118 {119 foreach (var argument in arguments)120 {121 data.Add(argument.Key, argument.Value);122 }123 }124 125 var strResult =126 client.PostAsync("schedule.json", new FormUrlEncodedContent(data))127 .Result.Content.ReadAsStringAsync()128 .Result;129 130 return JsonConvert.DeserializeObject<ScheduleResponse>(strResult);131 132 }133 catch (Exception e)134 {135 return new ScheduleResponse { status = "failed", message = e.Message };136 }137 }138 139 public CancelResponse CancelJob(string project, string jobid)140 {141 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));142 if (string.IsNullOrEmpty(jobid)) throw new ArgumentNullException(nameof(jobid));143 144 145 var client = new HttpClient()146 {147 BaseAddress = new Uri($"http://{_host}:{_port}")148 };149 150 try151 {152 var content = new FormUrlEncodedContent(new Dictionary<string, string>()153 {154 {"project", project},155 {"job", jobid}156 });157 var strResult = client.PostAsync($"/cancel.json", content).Result.Content.ReadAsStringAsync().Result;158 159 return JsonConvert.DeserializeObject<CancelResponse>(strResult);160 161 }162 catch (Exception e)163 {164 return new CancelResponse { status = "failed", message = e.Message };165 }166 }167 168 public ListProjectsResponse ListProjects()169 {170 try171 {172 var client = new HttpClient173 {174 BaseAddress = new Uri($"http://{_host}:{_port}")175 };176 var result = client.GetStringAsync("listprojects.json").Result;177 return JsonConvert.DeserializeObject<ListProjectsResponse>(result);178 }179 catch (Exception e)180 {181 return new ListProjectsResponse { status = "failed", message = e.Message };182 }183 }184 185 public ListVersionsResponse ListVersions(string project)186 {187 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));188 189 try190 {191 var client = new HttpClient192 {193 BaseAddress = new Uri($"http://{_host}:{_port}")194 };195 196 var result = client.GetStringAsync($"listversions.json?project={project}").Result;197 return JsonConvert.DeserializeObject<ListVersionsResponse>(result);198 }199 catch (Exception e)200 {201 return new ListVersionsResponse { status = "failed", message = e.Message };202 }203 }204 205 public ListSpidersResponse ListSpiders(string project, string version = "")206 {207 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));208 209 try210 {211 var client = new HttpClient212 {213 BaseAddress = new Uri($"http://{_host}:{_port}")214 };215 216 var result = client.GetStringAsync($"listspiders.json?project={project}&_version={version}").Result;217 return JsonConvert.DeserializeObject<ListSpidersResponse>(result);218 }219 catch (Exception e)220 {221 return new ListSpidersResponse { status = "failed", message = e.Message };222 }223 }224 225 public ListJobsResponse ListJobs(string project)226 {227 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));228 229 try230 {231 var client = new HttpClient232 {233 BaseAddress = new Uri($"http://{_host}:{_port}")234 };235 236 var result = client.GetStringAsync($"listjobs.json?project={project}").Result;237 return JsonConvert.DeserializeObject<ListJobsResponse>(result);238 }239 catch (Exception e)240 {241 return new ListJobsResponse { status = "failed", message = e.Message };242 }243 }244 245 public ScrapydResponse DeleteVersion(string project, string version)246 {247 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));248 if (string.IsNullOrEmpty(version)) throw new ArgumentNullException(nameof(version));249 250 var client = new HttpClient()251 {252 BaseAddress = new Uri($"http://{_host}:{_port}")253 };254 255 try256 {257 var data = http://www.mamicode.com/new Dictionary<string, string>()258 {259 {"project", project},260 {"version", version}261 };262 263 var strResult =264 client.PostAsync("delversion.json", new FormUrlEncodedContent(data))265 .Result.Content.ReadAsStringAsync()266 .Result;267 268 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);269 270 }271 catch (Exception e)272 {273 return new ScrapydResponse { status = "failed", message = e.Message };274 }275 }276 277 public ScrapydResponse DeleteProject(string project)278 {279 if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));280 281 282 var client = new HttpClient()283 {284 BaseAddress = new Uri($"http://{_host}:{_port}")285 };286 287 try288 {289 var data = http://www.mamicode.com/new Dictionary<string, string>()290 {291 {"project", project}292 };293 294 var strResult =295 client.PostAsync("delproject.json", new FormUrlEncodedContent(data))296 .Result.Content.ReadAsStringAsync()297 .Result;298 299 return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);300 301 }302 catch (Exception e)303 {304 return new ScrapydResponse { status = "failed", message = e.Message };305 }306 }307 308 }309 310 }
博客地址 : http://www.cnblogs.com/mobwiz/
【原创】Scrapyd 的 .net 客户端
声明：以上内容来自用户投稿及互联网公开渠道收集整理发布，本网站不拥有所有权，未作人工编辑处理，也不承担相关法律责任，若内容有误或涉及侵权可进行投诉： 投诉/举报 工作人员会在5个工作日内联系你，一经查实，本站将立刻删除涉嫌侵权内容。