ASP.NET Core 中使用 Hangfire 定時啓動 Scrapyd 爬蟲

用 Scrapy 作好的爬蟲使用 Scrapyd 來管理髮布啓動等工做，每次手動執行也很繁瑣;考慮能夠使用 Hangfire 集成在 web 工程裏。

Scrapyd 中啓動爬蟲的請求以下:

curl http://172.0.0.1:8081/schedule.json -d project=spider  -d spider=jrj_spider -u name:pwd
{"node_name": "iZbp1gf15gbzzqwvxbj18jZ", "status": "ok", "project": "spider", "spiders": 1, "version": "1492884063"}

修改:

 /// <summary>
        /// 執行方法 /// </summary>
        public async Task SchedulePollingBackgroundJob() { try { var response = await @"http://172.0.0.1:8081/schedule.json" .WithBasicAuth("name", "pwd") .PostUrlEncodedAsync(new { project = "spider", spider = "jrj_spider" }) .ReceiveString(); } catch (Exception) { } }

整個過程仍是比較簡單的，完整的代碼:

/// <summary>
        /// 配置接口 /// </summary>
        public IConfigurationRoot Configuration { get; } /// <summary>
        /// Redis 服務 /// </summary>
        public static ConnectionMultiplexer Redis; /// <summary>
        /// 構造方法 /// </summary>
        /// <param name="env"></param>
        public Startup(IHostingEnvironment env) { var builder = new ConfigurationBuilder() .SetBasePath(env.ContentRootPath) .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true) .AddEnvironmentVariables(); Configuration = builder.Build(); Redis = ConnectionMultiplexer.Connect(Configuration.GetConnectionString("Redis")); } // This method gets called by the runtime. Use this method to add services to the container.
        public void ConfigureServices(IServiceCollection services) { //自定義的配置
            services.Configure<DbSetting>(Configuration.GetSection("ConnectionStrings")); //注入 Hangfire服務
            services.AddHangfire(config => config.UseRedisStorage(Redis)); // services.AddHangfire(config => config.UseSqlServerStorage("<connection string>")); //添加 cookie 中間件
            services.AddAuthentication(sharedOptions => { sharedOptions.DefaultScheme = CookieAuthenticationDefaults.AuthenticationScheme; // sharedOptions.DefaultChallengeScheme = OpenIdConnectDefaults.AuthenticationScheme;
 }) //.AddAzureAdB2C(opts => Configuration.Bind("AzureAdB2C", opts))
            .AddCookie(opts => { opts.LoginPath = new PathString("/account/login"); opts.AccessDeniedPath = new PathString("/account/denied"); }); //返回大小寫問題
 services.AddMvc() .AddJsonOptions(option => option.SerializerSettings.ContractResolver = new Newtonsoft.Json.Serialization.DefaultContractResolver()); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
        public void Configure(IApplicationBuilder app, IHostingEnvironment env) { if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); app.UseBrowserLink(); } else { app.UseExceptionHandler("/Home/Error"); } app.UseStaticFiles(); //添加驗證中間件
 app.UseAuthentication(); //Hangfire //http://docs.hangfire.io/en/latest/configuration/using-dashboard.html#configuring-authorization
            app.UseHangfireDashboard("/hangfire", new DashboardOptions { Authorization = new[] { new HangfireDashboardAuthorizationFilter() } }); app.UseHangfireServer(); app.UseHangfireDashboard(); //http://cron.qqe2.com
            //0 0 * * MON-FRI At 00:00, Monday through Friday //Cron.Daily(16, 30) 30 16 * * * //30 16,17,18 * * * //30 16,17,18 * * MON-FRI //https://github.com/HangfireIO/Cronos
            RecurringJob.AddOrUpdate(() => SchedulePollingBackgroundJob(), @"30 16,17,18 * * *", TimeZoneInfo.Local); app.UseMvc(routes => { routes.MapRoute( name: "default", template: "{controller=Hq}/{action=Index}/{id?}"); }); }

 

Attention: If you are using Microsoft.Extensions.Caching.Redis package, you will need to use Hangfire.Redis.StackExchange.StrongName instead, because the former package requires StackExchange.Redis.StrongName instead of StackExchange.Redis!

REFER:

https://github.com/marcoCasamento/Hangfire.Redis.StackExchange