refactor code

lib/tasks/crawler.rake

 require "open-uri"
 require "open-uri"
 
 
 namespace :crawl do
 namespace :crawl do
-  desc "crawl industries and locations"
-  task crawl_industries_locations: :environment do
+  desc "crawl industries locations jobs"
+  task :crawl_industries_locations_jobs, [:page, :link] => [:environment] do |task, args|
+    args.with_defaults(link: 'https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-vi.html')
     crawl_industries_and_locations
     crawl_industries_and_locations
+    job_links = get_job_links(args[:page].to_i, args[:link])
+    job_links.each do |link|
+      break if link.empty?
+      crawl_job(link)
     end
     end
-
-  desc "crawl jobs" 
-  task crawl_jobs: :environment do
-    job_links = get_job_links(1)
-    crawl_jobs(job_links)
   end
   end
-
-  desc "crawl all"
-  task all: [:crawl_industries_locations, :crawl_jobs]
 end
 end
 
 
-def get_job_links(page)
+def get_job_links(page, link)
   job_links = []
   job_links = []
-  page.times do |i|
-    document = Nokogiri::HTML(open("https://careerbuilder.vn/viec-lam/tat-ca-viec-lam-trang-#{i}-vi.html"))
+  page.times do
+    document = Nokogiri::HTML(open(link))
     jobs_xml = document.xpath('//div/a[@class="job_link"]/@href')
     jobs_xml = document.xpath('//div/a[@class="job_link"]/@href')
     jobs_xml.each { |item| job_links << item.value}
     jobs_xml.each { |item| job_links << item.value}
+    next_page = document.at_css('.next-page a')
+    break if next_page.nil?
+    link = next_page[:href]
   end
   end
   job_links
   job_links
 end
 end
@@ -31,25 +31,19 @@ def crawl_company(company_link)
     uri = URI.parse(URI.escape(company_link)) #fix error: uri must be ascii only
     uri = URI.parse(URI.escape(company_link)) #fix error: uri must be ascii only
     document = Nokogiri::HTML(open(uri))
     document = Nokogiri::HTML(open(uri))
     company_name = document.css(".content .name").text
     company_name = document.css(".content .name").text
-    return nil if company_name.empty?
+    return if company_name.empty?
 
 
     puts company_name
     puts company_name
     company_address = document.css(".content p")[1].text
     company_address = document.css(".content p")[1].text
     company_description = document.css(".main-about-us").css('.content').text
     company_description = document.css(".main-about-us").css('.content').text
 
 
-    Company.find_or_create_by(name: company_name) do |c|
-      c.address = company_address
-      c.description = company_description
+    Company.find_or_create_by(name: company_name) do |company|
+      company.address = company_address
+      company.description = company_description
     end
     end
   rescue => exception
   rescue => exception
     puts exception
     puts exception
-    return nil
-  end
-end
-
-def crawl_jobs(job_links)
-  job_links.each do |link|
-    crawl_job(link)
+    return
   end
   end
 end
 end
 
 
@@ -64,41 +58,24 @@ def crawl_job(job_link)
     job_company = crawl_company(job_company_link)
     job_company = crawl_company(job_company_link)
     return if job_company.nil?
     return if job_company.nil?
 
 
-    ## parse data
-    job_detail = document.css('.job-detail-content div.detail-box')
-
-    job_location_name = job_detail[0].css('p a').map{ |val| val.text.strip }
+    job_location_name = document.css('.map p a').map{ |val| val.text.strip }
     job_locations = Location.where(city: job_location_name)
     job_locations = Location.where(city: job_location_name)
 
 
-    job_industry_names = job_detail[1].css('ul li')[1].css('p a').map{ |val| val.text.strip }
+    job_industry_names = document.at_xpath('//li[./strong/em[contains(@class, "mdi mdi-briefcase")]]').css('p a').map{ |val| val.text.strip }
     job_industries = Industry.where(name: job_industry_names)
     job_industries = Industry.where(name: job_industry_names)
 
 
-    job_created_at = job_detail[1].css('ul li')[0].css('p').text.strip
-
-    job_salary = ""
-    job_level = ""
-    job_experience = ""
-    job_expiration_date = ""
-    
-    info_detail = job_detail[2].css('ul li')
-    info_detail.count.times do |i|
-      data = info_detail[i].css('p').text.strip
-
-      job_salary = data if info_detail[i].css('.fa.fa-usd').present?
-      job_experience = data if info_detail[i].css('.fa.fa-briefcase').present?
-      job_level = data if info_detail[i].css('.mdi.mdi-account').present?
-      job_expiration_date = data if info_detail[i].css('.mdi.mdi-calendar-check').present?
-    end
+    job_salary = document.at_xpath('//li[./strong/i[contains(@class, "fa fa-usd")]]').css('p').text
+    job_level = document.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-account")]]').css('p').text
+    job_experience = document.at_xpath('//li[./strong/i[contains(@class, "fa fa-briefcase")]]').css('p').text
+    job_expiration_date = document.at_xpath('//li[./strong/i[contains(@class, "mdi mdi-calendar-check")]]').css('p').text
 
 
     job_description = document.css('.job-detail-content .detail-row').to_s
     job_description = document.css('.job-detail-content .detail-row').to_s
-    ##
 
 
     Job.find_or_create_by(title: job_title, company_id: job_company.id) do |job|
     Job.find_or_create_by(title: job_title, company_id: job_company.id) do |job|
       job.salary = job_salary
       job.salary = job_salary
       job.experience = job_experience
       job.experience = job_experience
       job.level = job_level
       job.level = job_level
       job.expiration_date = job_expiration_date
       job.expiration_date = job_expiration_date
-      job.created_at = job_created_at
       job.description = job_description
       job.description = job_description
       job.industries << job_industries
       job.industries << job_industries
       job.locations << job_locations
       job.locations << job_locations