Logger, crontab

Gemfile

@@ -64,4 +64,5 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
 
 
 ##
 ##
 gem "nokogiri"
 gem "nokogiri"
+gem 'whenever', require: false
 ##
 ##

Gemfile.lock

@@ -64,6 +64,7 @@ GEM
     chromedriver-helper (2.1.1)
     chromedriver-helper (2.1.1)
       archive-zip (~> 0.10)
       archive-zip (~> 0.10)
       nokogiri (~> 1.8)
       nokogiri (~> 1.8)
+    chronic (0.10.2)
     coffee-rails (4.2.2)
     coffee-rails (4.2.2)
       coffee-script (>= 2.2.0)
       coffee-script (>= 2.2.0)
       railties (>= 4.0.0)
       railties (>= 4.0.0)
@@ -187,6 +188,8 @@ GEM
     websocket-driver (0.7.3)
     websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     websocket-extensions (0.1.5)
+    whenever (1.0.0)
+      chronic (>= 0.6.3)
     xpath (3.2.0)
     xpath (3.2.0)
       nokogiri (~> 1.8)
       nokogiri (~> 1.8)
 
 
@@ -214,6 +217,7 @@ DEPENDENCIES
   tzinfo-data
   tzinfo-data
   uglifier (>= 1.3.0)
   uglifier (>= 1.3.0)
   web-console (>= 3.3.0)
   web-console (>= 3.3.0)
+  whenever
 
 
 RUBY VERSION
 RUBY VERSION
    ruby 2.6.6p146
    ruby 2.6.6p146

config/schedule.rb

+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+#   command "/usr/bin/some_great_command"
+#   runner "MyModel.some_method"
+#   rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+#   runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+env :PATH, ENV['PATH']
+every 5.minutes do
+  rake 'crawl:crawl_industries_locations_jobs[1]'
+end
\ No newline at end of file

lib/tasks/crawler.rake

 require "open-uri"
 require "open-uri"
+@logger ||= Logger.new("#{Rails.root}/log/crawler.log")
 
 
 namespace :crawl do
 namespace :crawl do
   desc "crawl industries locations jobs"
   desc "crawl industries locations jobs"
@@ -7,7 +8,7 @@ namespace :crawl do
     crawl_industries_and_locations
     crawl_industries_and_locations
     job_links = get_job_links(args[:page].to_i, args[:link])
     job_links = get_job_links(args[:page].to_i, args[:link])
     job_links.each do |link|
     job_links.each do |link|
-      break if link.empty?
+      next if link.empty?
       crawl_job(link)
       crawl_job(link)
     end
     end
   end
   end
@@ -43,6 +44,7 @@ def crawl_company(company_link)
     end
     end
   rescue => exception
   rescue => exception
     puts exception
     puts exception
+    @logger.error "#{exception.message} - Company link: #{uri}"
     return
     return
   end
   end
 end
 end
@@ -83,6 +85,7 @@ def crawl_job(job_link)
     puts job_title
     puts job_title
   rescue => exception
   rescue => exception
     puts exception
     puts exception
+    @logger.error "#{exception.message} - Job link: #{uri}"
     return exception
     return exception
   end
   end
 end
 end